1. Bug fixed; 2. Support abs, sign, convert data type(float <->int) ;3. Merge…

1. Bug fixed; 2. Support abs, sign, convert data type(float <->int) ;3. Merge with xu and xiao branch

source/XLink.cpp

-/* NiuTrans.Tensor - an open-source tensor library
- * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
- * All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-04
- */
-
-#include <stdio.h>
-#include "XLink.h"
-#include "XName.h"
-
-namespace nts{ // namespace nts(NiuTrans.Tensor)
-
-int XLink::paramSize = 64;
-
-/* constuctor */
-XLink::XLink()
-{
-    head   = NULL;
-    tails  = NULL;
-    params = NULL;
-    tailNum  = 0;
-    paramNum = 0;
-    type[0] = 0;
-    typeID = 0;
-}
-    
-/* deconstructor */
-XLink::~XLink()
-{
-    delete[] tails;
-    delete[] (char*)params;
-}
-
-/* reset it */
-void XLink::Reset()
-{
-    delete[] tails;
-    delete[] (char*)params;
-    head   = NULL;
-    tails  = NULL;
-    params = NULL;
-    tailNum  = 0;
-    paramNum = 0;
-    type[0] = 0;
-}
-
-/* 
-set edge type name 
->> id - id of the type
-*/
-void XLink::SetType(int id)
-{
-    type[0] = 0;
-    strcpy(type, GetOPName(id));
-    typeID = id;
-    CheckNTErrors(!strcmp(type, "NULL"), "illegal edge type name!");
-}
-
-/* 
-set head 
->> h - pointer to the head tensor
-*/
-void XLink::SetHead(XTensor * h)
-{
-    head = h;
-}
-
-/* 
-add a tail
->> t - pointer to the tail tensor
-*/
-void XLink::AddTail(XTensor * t)
-{
-    XTensor ** ts = tails;
-    tails = new XTensor*[tailNum + 1];
-    memcpy(tails, ts, sizeof(XTensor*) * tailNum);
-    tails[tailNum++] = t;
-    delete[] ts;
-}
-
-/* 
-add two tails in one time 
->> t1 - pointer to the tail tensor
->> t2 - pointer to another tail tensor
-*/
-void XLink::AddTwoTails(XTensor * t1, XTensor * t2)
-{
-    XTensor ** ts = tails;
-    tails = new XTensor*[tailNum + 2];
-    memcpy(tails, ts, sizeof(XTensor*) * tailNum);
-    tails[tailNum++] = t1;
-    tails[tailNum++] = t2;
-    delete[] ts;
-}
-
-/* 
-add a parameter 
->> param - parameter in default type
-*/
-void XLink::AddParam(DTYPE param)
-{
-    void * ps = params;
-    params = new char[paramNum + 1];
-    memcpy(params, ps, paramNum * paramSize);
-    DTYPE * p = (DTYPE*)((char*)params + paramNum * paramSize);
-    *p = param;
-    paramNum++;
-    delete[] (char*)ps;
-}
-
-/* 
-add a parameter 
->> param - pointer to the parameter
->> size - size of the parameter
-*/
-void XLink::AddParam(void * param, int size)
-{
-    void * ps = params;
-    params = new char[paramNum + 1];
-    memcpy(params, ps, paramNum * paramSize);
-    char * p = (char*)params + paramNum * paramSize;
-    memcpy(p, param, size);
-    paramNum++;
-    delete[] (char*)ps;
-}
-/* 
-create a hyperedge with two input tensors and a output tensor 
->> t1 - a tail tensor
->> t2 - another tail tensor
->> h - head tensor
->> id - id of the edge type
-*/
-void XLink::MakeLink(XTensor * t1, XTensor * t2, XTensor * h, int id)
-{
-    if(h != NULL)
-        return;
-
-    /* forward */
-    XLink &income = h->income;
-    income.Reset();
-    income.SetHead(h);
-    if(t1 != NULL && t2 != NULL)
-        income.AddTwoTails(t1, t2);
-    else if(t1 != NULL)
-        income.AddTail(t1);
-    else{
-        ShowNTErrors("TODO!");
-    }
-    income.SetType(id);
-
-    /* backward for t1 */
-    if(t1 != NULL){
-        XLink &outgo = t1->outgo;
-        CheckNTErrors(outgo.head != t1, "Wrong head of the hyperedge!");
-        outgo.AddTail(h);
-    }
-
-    /* backward for t2 */
-    if(t2 != NULL){
-        XLink &outgo = t2->outgo;
-        CheckNTErrors(outgo.head != t2, "Wrong head of the hyperedge!");
-        outgo.AddTail(h);
-    }
-}
-
-/* 
-create a hyper edge with a list of tensors and a output tensor 
->> list - a list of input tensors
->> h - head tensor
->> id - id of the edge type
-*/
-void XLink::MakeLink(XList * list, XTensor * h, int id)
-{
-    /* forward */
-    XLink &income = h->income;
-    income.Reset();
-    income.SetHead(h);
-    income.SetType(id);
-
-    for(int i = 0; i < list->count; i++){
-        XTensor * t = (XTensor*)list->GetItem(i);
-        income.AddTail(t);
-    }
-
-    /* backward */
-    for(int i = 0; i < list->count; i++){
-        XTensor * t = (XTensor*)list->GetItem(i);
-        XLink &outgo = t->outgo;
-        CheckNTErrors(outgo.head != t, "Wrong head of the hyperedge!");
-        outgo.AddTail(h);
-    }
-}
-
-/* 
-add parameters 
->> h - head
->> param - parameter we want introduce
-*/
-void XLink::AddParamToHead(XTensor * h, DTYPE param)
-{
-    if(h != NULL)
-        return;
-    h->income.AddParam(param);
-}
-
-/* 
-add an integer parameter 
->> h - head
->> param - parameter we want introduce
-*/
-void XLink::AddParamToHeadInt(XTensor * h, int param)
-{
-    if(h != NULL)
-        return;
-    h->income.AddParam(&param, sizeof(int));
-}
-    
-} // namespace nts(NiuTrans.Tensor)
-

source/network/Main.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-10
+ */
+
+#include <stdio.h>
+#include "../tensor/XTensor.h"
+
+//#define CRTDBG_MAP_ALLOC
+//#include <stdlib.h>
+//#include <crtdbg.h>
+
+using namespace nts;
+
+
+int main( int argc, const char ** argv )
+{
+    
+    if(argc > 1 && !strcmp(argv[1], "-test"))
+        1;//Test();
+    else{
+        fprintf(stderr, "Thanks for using NiuTrans.Network! This is a library for building\n");
+        fprintf(stderr, "neural networks in an easy way. \n\n");
+        fprintf(stderr, "Run this program with \"-test\" for unit test!\n");
+    }
+    
+    //_CrtDumpMemoryLeaks();
+    
+    return 0;
+}

source/Main.cpp → source/tensor/Main.cpp

@@ -29,8 +29,7 @@
 #include "XTensor.h"
 #include "XTensor.h"
 #include "XDevice.h"
 #include "XDevice.h"
 #include "./sample/fnnlm/FNNLM.h"
 #include "./sample/fnnlm/FNNLM.h"
-
-#include "test/Test.h"
+#include "./test/Test.h"
 
 
 //#define CRTDBG_MAP_ALLOC
 //#define CRTDBG_MAP_ALLOC
 //#include <stdlib.h>  
 //#include <stdlib.h>  
@@ -39,8 +38,16 @@
 using namespace nts;
 using namespace nts;
 using namespace samplefnnlm;
 using namespace samplefnnlm;
 
 
+void SmallTest();
+
 int main( int argc, const char ** argv )
 int main( int argc, const char ** argv )
 {
 {
+    //_CrtSetBreakAlloc(78);
+
+    /* a tiny test */
+    //if(1)
+    //    SmallTest();
+
     if(argc > 1 && !strcmp(argv[1], "-test"))
     if(argc > 1 && !strcmp(argv[1], "-test"))
         Test();
         Test();
     else if(argc > 1 && !strcmp(argv[1], "-fnnlm"))
     else if(argc > 1 && !strcmp(argv[1], "-fnnlm"))
@@ -56,3 +63,30 @@ int main( int argc, const char ** argv )
 
 
     return 0;
     return 0;
 }
 }
+
+void SmallTest()
+{
+    XTensor a;
+    XTensor b;
+
+    InitTensor2D(&a, 2, 2);
+    a.SetZeroAll();
+    a.Set2D(1.0F, 0, 0);
+    a.Set2D(2.0F, 1, 1);
+
+    b = Sum(a, Multiply(a, a));
+
+    XTensor c = a * b + a;
+
+    int nnn = 1;
+
+    XTensor d = a + b + c.Lin(0.5F);
+
+    XLink::CheckNetwork(&d);
+    XLink::ShowNetwork(stderr, &b);
+        
+    a.Dump(stderr, "a: ");
+    b.Dump(stderr, "b: ");
+    c.Dump(stderr, "c: ");
+    d.Dump(stderr, "d: ");
+}

source/XGlobal.h → source/tensor/XGlobal.h

@@ -74,7 +74,7 @@ namespace nts {
 { \
 { \
     if(!(x)) \
     if(!(x)) \
     { \
     { \
-        fprintf(stderr, "Error! calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__, msg); \
+        fprintf(stderr, "[ERROR] calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__, msg); \
         exit(1); \
         exit(1); \
     } \
     } \
 } \
 } \
@@ -83,7 +83,7 @@ namespace nts {
 { \
 { \
     if(!(x)) \
     if(!(x)) \
     { \
     { \
-        fprintf(stderr, "Error! calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__); \
+        fprintf(stderr, "[ERROR] calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__); \
         exit(1); \
         exit(1); \
     } \
     } \
 } \
 } \
@@ -91,7 +91,7 @@ namespace nts {
 #define ShowNTErrors(msg) \
 #define ShowNTErrors(msg) \
 { \
 { \
     { \
     { \
-        fprintf(stderr, "Error! (%s line %d): %s\n", __FILENAME__, __LINE__, msg); \
+        fprintf(stderr, "[ERROR] (%s line %d): %s\n", __FILENAME__, __LINE__, msg); \
         exit(1); \
         exit(1); \
     } \
     } \
 } \
 } \

source/tensor/XLink.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-04
+ */
+
+#include <stdio.h>
+#include "XLink.h"
+#include "XName.h"
+
+namespace nts{ // namespace nts(NiuTrans.Tensor)
+
+int XLink::paramSize = 64;
+
+/* constuctor */
+XLink::XLink()
+{
+    head   = NULL;
+    tails  = NULL;
+    params = NULL;
+    tailNum  = 0;
+    paramNum = 0;
+    type[0] = 0;
+    typeID = 0;
+}
+    
+/* deconstructor */
+XLink::~XLink()
+{
+    delete[] tails;
+    delete[] (char*)params;
+}
+
+/* reset it */
+void XLink::Reset()
+{
+    delete[] tails;
+    delete[] (char*)params;
+    head   = NULL;
+    tails  = NULL;
+    params = NULL;
+    tailNum  = 0;
+    paramNum = 0;
+    type[0]  = 0;
+}
+
+/* clear it */
+void XLink::Clear()
+{
+    head   = NULL;
+    tailNum  = 0;
+    paramNum = 0;
+    type[0]  = 0;
+}
+
+/* reset tails */
+void XLink::ClearTail()
+{
+    tailNum = 0;
+}
+
+/*
+clear the incoming node list of tensor node
+>> node - the node to be cleared
+*/
+void XLink::ClearIncoming(XTensor * node)
+{
+    if(node == NULL)
+        return;
+
+    XLink &income = node->income;
+
+    for(int i = 0; i < income.tailNum; i++){
+
+        /* for a incoming node */
+        XTensor * child = income.tails[i];
+        XLink &childOutgo = child->outgo;
+
+        CheckNTErrors(childOutgo.tailNum > 0, "The node must have outgoing edges!");
+
+        /* we check for each child node and remove the link to current node */
+        for(int j = 0; j < childOutgo.tailNum; j++){
+            if(childOutgo.tails[j] == node){
+                memcpy(childOutgo.tails + j, 
+                       childOutgo.tails + j + 1, 
+                      (childOutgo.tailNum - 1 - j) * sizeof(XTensor*));
+                childOutgo.tailNum--;
+                break;
+            }
+        }
+
+        if(child->isTmp && childOutgo.tailNum == 0)
+            delete child;
+    }
+
+    income.ClearTail();
+    income.tailNum = 0;
+}
+
+/* 
+set edge type name 
+>> id - id of the type
+*/
+void XLink::SetType(int id)
+{
+    type[0] = 0;
+    strcpy(type, GetOPName(id));
+    typeID = id;
+    CheckNTErrors(strcmp(type, "NULL"), "illegal edge type name!");
+}
+
+/* 
+set head 
+>> h - pointer to the head tensor
+*/
+void XLink::SetHead(XTensor * h)
+{
+    head = h;
+}
+
+/* 
+add a tail
+>> t - pointer to the tail tensor
+*/
+void XLink::AddTail(XTensor * t)
+{
+    XTensor ** ts = tails;
+    tails = new XTensor*[tailNum + 1];
+    memcpy(tails, ts, sizeof(XTensor*) * tailNum);
+    tails[tailNum++] = t;
+    delete[] ts;
+}
+
+/* 
+add two tails in one time 
+>> t1 - pointer to the tail tensor
+>> t2 - pointer to another tail tensor
+*/
+void XLink::AddTwoTails(XTensor * t1, XTensor * t2)
+{
+    XTensor ** ts = tails;
+    tails = new XTensor*[tailNum + 2];
+    memcpy(tails, ts, sizeof(XTensor*) * tailNum);
+    tails[tailNum++] = t1;
+    tails[tailNum++] = t2;
+    delete[] ts;
+}
+
+/* 
+add a parameter 
+>> param - parameter in default type
+*/
+void XLink::AddParam(DTYPE param)
+{
+    void * ps = params;
+    params = new char[paramNum + 1];
+    memcpy(params, ps, paramNum * paramSize);
+    DTYPE * p = (DTYPE*)((char*)params + paramNum * paramSize);
+    *p = param;
+    paramNum++;
+    delete[] (char*)ps;
+}
+
+/* 
+add a parameter 
+>> param - pointer to the parameter
+>> size - size of the parameter
+*/
+void XLink::AddParam(void * param, int size)
+{
+    void * ps = params;
+    params = new char[paramNum + 1];
+    memcpy(params, ps, paramNum * paramSize);
+    char * p = (char*)params + paramNum * paramSize;
+    memcpy(p, param, size);
+    paramNum++;
+    delete[] (char*)ps;
+}
+/* 
+create a hyperedge with two input tensors and a output tensor 
+>> t1 - a tail tensor
+>> t2 - another tail tensor
+>> h - head tensor
+>> id - id of the edge type
+*/
+void XLink::MakeLink(const XTensor * t1, const XTensor * t2, XTensor * h, int id)
+{
+    if(h == NULL)
+        return;
+    
+    XList list(2);
+    list.Add(t1);
+    list.Add(t2);
+
+    MakeLink(&list, h, id);
+}
+
+/* 
+create a hyper edge with a list of tensors and a output tensor 
+>> list - a list of input tensors
+>> h - head tensor
+>> id - id of the edge type
+*/
+void XLink::MakeLink(XList * list, XTensor * h, int id)
+{
+    /* forward */
+    XLink &income = h->income;
+    income.Reset();
+    income.SetHead(h);
+    income.SetType(id);
+
+    for(int i = 0; i < list->count; i++){
+        XTensor * t = (XTensor*)list->GetItem(i);
+        if(t == NULL)
+            continue;
+        income.AddTail(t);
+    }
+
+    /* backward */
+    for(int i = 0; i < list->count; i++){
+        XTensor * t = (XTensor*)list->GetItem(i);
+        if(t == NULL)
+            continue;
+        XLink &outgo = t->outgo;
+        CheckNTErrors(outgo.head == NULL || outgo.head == t, 
+                      "Wrong head of the hyperedge!");
+        outgo.AddTail(h);
+    }
+}
+
+/* 
+add parameters 
+>> h - head
+>> param - parameter we want introduce
+*/
+void XLink::AddParamToHead(XTensor * h, DTYPE param)
+{
+    if(h != NULL)
+        return;
+    h->income.AddParam(param);
+}
+
+/* 
+add an integer parameter 
+>> h - head
+>> param - parameter we want introduce
+*/
+void XLink::AddParamToHeadInt(XTensor * h, int param)
+{
+    if(h != NULL)
+        return;
+    h->income.AddParam(&param, sizeof(int));
+}
+
+/* 
+replace a node with another, i.e., we redirect the links to the new node 
+>> oldOne - the node to be replaced
+>> newOne - the new node
+*/
+void XLink::Replace(const XTensor * oldOne, XTensor * newOne)
+{
+    if(oldOne == NULL || newOne == NULL)
+        return;
+
+    XLink::ClearIncoming(newOne);
+
+    XLink &newIncome = newOne->income;
+    XLink &newOutgo  = newOne->outgo;
+    delete[] newIncome.tails;
+
+    /* incoming nodes for the new node */
+    newIncome.head = newOne;
+    newIncome.tailNum = oldOne->income.tailNum;
+    newIncome.tails = new XTensor*[newIncome.tailNum];
+    memcpy(newIncome.tails, oldOne->income.tails, sizeof(XTensor*) * newIncome.tailNum);
+
+    /* update the link to each child node */
+    for(int i = 0; i < newIncome.tailNum; i++){
+        XTensor * child = newIncome.tails[i];
+        XLink &childOutgo = child->outgo;
+        bool hit = false;
+        for(int j = 0; j < childOutgo.tailNum; j++){
+            if(childOutgo.tails[j] == oldOne){
+                childOutgo.tails[j] = newOne;
+                hit = true;
+                break;
+            }
+        }
+
+        if(childOutgo.tailNum > 0){
+            CheckNTErrors(hit, "No proper node found in child.outgo edge!");
+        }
+    }
+
+    /* outgoing nodes for the new node */
+    newOutgo.head = newOne;
+    newOutgo.tailNum = oldOne->outgo.tailNum;
+    newOutgo.tails = new XTensor*[newOutgo.tailNum];
+    memcpy(newOutgo.tails, oldOne->outgo.tails, sizeof(XTensor*) * newOutgo.tailNum);
+
+    /* update the link to each parent node */
+    for(int i = 0; i < newOutgo.tailNum; i++){
+        XTensor * parent = newOutgo.tails[i];
+        XLink &parentIncome = parent->income;
+        bool hit = false;
+        for(int j = 0; j < parentIncome.tailNum; j++){
+            if(parentIncome.tails[j] == oldOne){
+                parentIncome.tails[j] = newOne;
+                hit = true;
+            }
+        }
+
+        if(parentIncome.tailNum > 0){
+            CheckNTErrors(hit, "No proper node found in parent.income edge!");
+        }
+    }
+}
+
+/* 
+copy incoming edges of a given node
+>> reference - the node we copy from
+>> target - where we copy to
+*/
+void XLink::CopyIncoming(const XTensor * reference, XTensor * target)
+{
+    CheckNTErrors(reference && target, "Empty input tensors!");
+
+    ClearIncoming(target);
+
+    int tailNum = reference->income.tailNum;
+    XList tails(tailNum);
+    for(int i = 0; i < tailNum; i++){
+        XTensor * tail = (XTensor*)reference->income.tails[i];
+        tails.Add(tail);
+    }
+
+    MakeLink(&tails, target, reference->id);
+
+    int paraNum = reference->income.paramNum;
+    target->income.paramNum = paraNum;
+
+    delete[] (char*)target->income.params;
+    int size = paraNum * reference->income.paramSize;
+    target->income.params = new char[size];
+    memcpy(target->income.params, reference->income.params, size);
+}
+
+/* 
+check the correctness of the network encoded in a root node (tensor) 
+>> root - pointer to the root node
+*/
+void XLink::CheckNetwork(XTensor * root)
+{
+    XLink &income = root->income;
+    if(income.head == NULL){
+        CheckNTErrors(income.tailNum == 0, "Wrong number of the incoming edge tails!");
+    }
+    else{
+        for(int i = 0; i < income.tailNum; i++){
+            XTensor * child = income.tails[i];
+            if(child == NULL)
+                continue;
+            XLink & childOutgo = child->outgo;
+            bool hit = false;
+            for(int j = 0; j < childOutgo.tailNum; j++){
+                if(childOutgo.tails[j] == root){
+                    hit = true;
+                    break;
+                }
+            }
+            CheckNTErrors(hit, "Wrong outgoing edge!");
+        }
+
+    }
+
+    XLink &outgo = root->outgo;
+    if(outgo.head == NULL){
+        CheckNTErrors(outgo.tailNum == 0, "Wrong number of the incoming edge tails!");
+    }
+    else{
+        for(int i = 0; i < outgo.tailNum; i++){
+            XTensor * parent = outgo.tails[i];
+            if(parent == NULL)
+                continue;
+            XLink & parentOutgo = parent->outgo;
+            bool hit = false;
+            for(int j = 0; j < parentOutgo.tailNum; j++){
+                if(parentOutgo.tails[j] == root){
+                    hit = true;
+                    break;
+                }
+            }
+            CheckNTErrors(hit, "Wrong outgoing edge!");
+        }
+    }
+
+    for(int i = 0; i < income.tailNum; i++){
+        XTensor * child = income.tails[i];
+        CheckNetwork(child);
+    }
+}
+
+/* 
+show the network encoded in a root node (tensor) 
+>> file - file to dump information
+>> root - pointer to the root node
+*/
+void XLink::ShowNetwork(FILE * file, XTensor * root)
+{
+    fprintf(file, "node %d - ", root->id);
+
+    XLink &income = root->income;
+    if(income.head == NULL){
+        fprintf(file, "income[%d]: null ", income.tailNum);
+    }
+    else{
+        fprintf(file, "income[%d]: ", income.tailNum);
+        for(int i = 0; i < income.tailNum; i++){
+            XTensor * child = income.tails[i];
+            if(child == NULL)
+                fprintf(file, "na ");
+            else
+                fprintf(file, "%d ", child->id);
+        }
+    }
+
+    XLink &outgo = root->outgo;
+    if(outgo.head == NULL){
+        fprintf(file, "outgo[%d]: null ", outgo.tailNum);
+    }
+    else{
+        fprintf(file, "outgo[%d]: ", income.tailNum);
+        for(int i = 0; i < outgo.tailNum; i++){
+            XTensor * parent = outgo.tails[i];
+            if(parent == NULL)
+                fprintf(file, "na ");
+            else
+                fprintf(file, "%d ", parent->id);
+        }
+    }
+
+    fprintf(stderr, "\n");
+
+    for(int i = 0; i < income.tailNum; i++){
+        XTensor * child = income.tails[i];
+        ShowNetwork(file, child);
+    }
+}
+    
+} // namespace nts(NiuTrans.Tensor)
+

source/XLink.h → source/tensor/XLink.h

@@ -86,6 +86,16 @@ struct XLink
     /* reset it */
     /* reset it */
     void Reset();
     void Reset();
 
 
+    /* clear it */
+    void Clear();
+
+    /* clear tails */
+    void ClearTail();
+
+    /* clear the incoming node list of tensor node */
+    static
+    void ClearIncoming(XTensor * node);
+
     /* set edge type id and name */
     /* set edge type id and name */
     void SetType(int id);
     void SetType(int id);
 
 
@@ -106,7 +116,7 @@ struct XLink
 
 
     /* create a hyper edge with two input tensors and a output tensor */
     /* create a hyper edge with two input tensors and a output tensor */
     static
     static
-    void MakeLink(XTensor * t1, XTensor * t2, XTensor * h, int id);
+    void MakeLink(const XTensor * t1, const XTensor * t2, XTensor * h, int id);
 
 
     /* create a hyper edge with a list of input tensors and a output tensor */
     /* create a hyper edge with a list of input tensors and a output tensor */
     static
     static
@@ -119,6 +129,22 @@ struct XLink
     /* add an integer parameter */
     /* add an integer parameter */
     static
     static
     void AddParamToHeadInt(XTensor * h, int param);
     void AddParamToHeadInt(XTensor * h, int param);
+
+    /* replace a node with another, i.e., we redirect the links to the new node */
+    static 
+    void Replace(const XTensor * oldOne, XTensor * newOne);
+
+    /* copy links of a given node */
+    static
+    void CopyIncoming(const XTensor * reference, XTensor * target);
+
+    /* check the correctness of the network encoded in a root node (tensor) */
+    static
+    void CheckNetwork(XTensor * root);
+
+    /* show the network encoded in a root node (tensor) */
+    static
+    void ShowNetwork(FILE * file, XTensor * root);
 };
 };
     
     
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)

source/XList.cpp → source/tensor/XList.cpp

@@ -111,7 +111,7 @@ void XList::Create(int myMaxNum, XMem * myMem)
 add an item into the list
 add an item into the list
 >> item - pointer to the item
 >> item - pointer to the item
 */
 */
-void XList::Add(void * item)
+void XList::Add(const void * item)
 {
 {
     if( count == maxNum ){
     if( count == maxNum ){
         void ** newItems;
         void ** newItems;
@@ -126,7 +126,8 @@ void XList::Add(void * item)
         maxNum = maxNum * 2 + 1;
         maxNum = maxNum * 2 + 1;
     }
     }
     
     
-    items[count++] = item;
+    MTYPE p = (MTYPE)item;
+    items[count++] = (MTYPE*)p;
 
 
 }
 }
 
 
@@ -355,4 +356,4 @@ void XList::Shuffle(int nround, int beg, int len)
 }
 }
 
 
 } 
 } 
-/* end of the nts (NiuTrans.Tensor) namespace */
+/* end of the nts (NiuTrans.Tensor) namespace */
\ No newline at end of file

source/XList.h → source/tensor/XList.h

@@ -69,7 +69,7 @@ public:
 
 
     /* utilities */
     /* utilities */
     void Create(int myMaxNum, XMem * myMem);
     void Create(int myMaxNum, XMem * myMem);
-    void Add(void * item);
+    void Add(const void * item);
     void Add(void ** inputItems, int inputItemCount);
     void Add(void ** inputItems, int inputItemCount);
     void AddList(XList * l);
     void AddList(XList * l);
     void AddInt(int i);
     void AddInt(int i);
@@ -99,4 +99,4 @@ public:
 } 
 } 
 /* end of the nts (NiuTrans.Tensor) namespace */
 /* end of the nts (NiuTrans.Tensor) namespace */
 
 
-#endif
+#endif
\ No newline at end of file

source/XName.cpp → source/tensor/XName.cpp

@@ -19,15 +19,10 @@
  * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-05
  * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-05
  */
  */
 
 
-#ifndef __XNAME_H__
-#define __XNAME_H__
+#include "XName.h"
 
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 namespace nts { // namespace nts(NiuTrans.Tensor)
     
     
-#define MATH_ARITHMETIC     0x00001000
-#define MATH_SUM            MATH_ARITHMETIC + 1
-#define MATH_MULTIPLY       MATH_SUM + 1
-    
 /* get operator name */
 /* get operator name */
 const char * GetOPName(int type)
 const char * GetOPName(int type)
 {
 {
@@ -36,6 +31,8 @@ const char * GetOPName(int type)
             return "M_SUM";
             return "M_SUM";
         else if(type == MATH_MULTIPLY)
         else if(type == MATH_MULTIPLY)
             return "M_MULTIPLY";
             return "M_MULTIPLY";
+        else if(type == MATH_SCALEANDSHIFT)
+            return "M_SCALEANDSHIFT";
     }
     }
     
     
     return "NULL";
     return "NULL";
@@ -43,4 +40,3 @@ const char * GetOPName(int type)
     
     
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
 
 
-#endif // __XNAME_H__

source/XName.h → source/tensor/XName.h

@@ -31,6 +31,7 @@ namespace nts { // namespace nts(NiuTrans.Tensor)
 #define MATH_ARITHMETIC     10000
 #define MATH_ARITHMETIC     10000
 #define MATH_SUM            MATH_ARITHMETIC + 1
 #define MATH_SUM            MATH_ARITHMETIC + 1
 #define MATH_MULTIPLY       MATH_SUM + 1
 #define MATH_MULTIPLY       MATH_SUM + 1
+#define MATH_SCALEANDSHIFT  MATH_MULTIPLY + 1
 
 
 /* get operator name */
 /* get operator name */
 const char * GetOPName(int type);
 const char * GetOPName(int type);

source/XTensor.cpp → source/tensor/XTensor.cpp

@@ -39,6 +39,10 @@
 #include "XHeap.h"
 #include "XHeap.h"
 #include "XBLAS.h"
 #include "XBLAS.h"
 #include "core/shape/MergeBlockLists.h"
 #include "core/shape/MergeBlockLists.h"
+#include "core/movement/CopyValues.h"
+#include "core/arithmetic/Sum.h"
+#include "core/arithmetic/Multiply.h"
+#include "core/math/ScaleAndShift.h"
 
 
 #ifdef USE_CUDA
 #ifdef USE_CUDA
 
 
@@ -55,6 +59,23 @@
 /* the nts (NiuTrans.Tensor) namespace */
 /* the nts (NiuTrans.Tensor) namespace */
 namespace nts{
 namespace nts{
 
 
+int tensorIDGlobal = 0;
+MUTEX_HANDLE tensorMutex;
+XTensor firstTensor;
+
+/* generate a tensor id */
+int MakeTensorID()
+{
+    if(tensorIDGlobal == 0)
+        MUTEX_INIT(tensorMutex);
+
+    MUTEX_LOCK(tensorMutex);
+    int id = tensorIDGlobal++;    
+    MUTEX_UNLOCK(tensorMutex);
+
+    return id;
+}
+
 /* 
 /* 
 constructor 
 constructor 
 >> myOrder - order of the tensor
 >> myOrder - order of the tensor
@@ -63,7 +84,9 @@ constructor
 XTensor::XTensor()
 XTensor::XTensor()
 {
 {
     memset(this, 0, sizeof(XTensor));
     memset(this, 0, sizeof(XTensor));
+    SetDataPointer();
 
 
+    id = MakeTensorID();
     order = -1;
     order = -1;
     memset(dimSize, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSize, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSizeRDI, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSizeRDI, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
@@ -81,18 +104,22 @@ XTensor::XTensor()
     isDefaultDType = true;
     isDefaultDType = true;
     isInGlobalMem  = false;
     isInGlobalMem  = false;
     isInit = false;
     isInit = false;
+    isTmp =  false;
 }
 }
 
 
 /* constructor */
 /* constructor */
-XTensor::XTensor(XTensor * reference)
+XTensor::XTensor(const XTensor * reference)
 {
 {
     memset(this, 0, sizeof(XTensor));
     memset(this, 0, sizeof(XTensor));
+    SetDataPointer();
 
 
+    id = MakeTensorID();
     dataType = DEFAULT_DTYPE;
     dataType = DEFAULT_DTYPE;
     devID = -1;
     devID = -1;
     denseRatio = 1.0F;
     denseRatio = 1.0F;
     isDefaultDType = true;
     isDefaultDType = true;
     isInit = false;
     isInit = false;
+    isTmp  = false;
 
 
     InitTensor(this, reference);
     InitTensor(this, reference);
 }
 }
@@ -107,6 +134,8 @@ XTensor::XTensor(const int myOrder, int myDevID, XMem * myMem)
 {
 {
     CheckNTErrors((myOrder > 0), "Illegal tensor order1");
     CheckNTErrors((myOrder > 0), "Illegal tensor order1");
 
 
+    SetDataPointer();
+    id = MakeTensorID();
     order = myOrder;
     order = myOrder;
     memset(dimSize, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSize, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSizeRDI, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSizeRDI, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
@@ -127,6 +156,7 @@ XTensor::XTensor(const int myOrder, int myDevID, XMem * myMem)
     isDefaultDType = true;
     isDefaultDType = true;
     isInGlobalMem  = false;
     isInGlobalMem  = false;
     isInit = false;
     isInit = false;
+    isTmp  = false;
 }
 }
 
 
 /* 
 /* 
@@ -142,6 +172,8 @@ XTensor::XTensor(const int myOrder, const int * myDimSize, const TENSOR_DATA_TYP
 {
 {
     CheckNTErrors((myOrder > 0), "Illegal tensor order1");
     CheckNTErrors((myOrder > 0), "Illegal tensor order1");
 
 
+    SetDataPointer();
+    id = MakeTensorID();
     order = myOrder;
     order = myOrder;
     memset(dimSize, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSize, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSizeRDI, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
     memset(dimSizeRDI, 0, sizeof(int) * MAX_TENSOR_DIM_NUM);
@@ -157,10 +189,50 @@ XTensor::XTensor(const int myOrder, const int * myDimSize, const TENSOR_DATA_TYP
     isDefaultDType = true;
     isDefaultDType = true;
     isInGlobalMem  = false;
     isInGlobalMem  = false;
     isInit = false;
     isInit = false;
+    isTmp  = false;
 
 
     Resize(myOrder, myDimSize, myDataType, myDenseRatio);
     Resize(myOrder, myDimSize, myDataType, myDenseRatio);
 }
 }
 
 
+/* copy constructor */
+XTensor::XTensor(const XTensor &reference)
+{
+    SetDataPointer();
+    id = MakeTensorID();
+    ShallowCopy(reference);
+    data = NULL;
+    dataHost = NULL;
+    
+    if(reference.isTmp){
+        devID = reference.devID;
+        mem = reference.mem;
+        data = reference.data;
+        
+        /* what we really want to do is "reference.data = NULL;"
+           As "reference" is constant, we cannot reset reference.data
+           here. So we save the ADDRESS of reference.data in
+           reference.dataP, and do this work by updating "*reference.dataP".
+           This is VERY trick and might not be the best solution :) */
+        *reference.dataP = NULL;
+    }
+    else{
+        devID = reference.devID;
+        mem = reference.mem;
+        InitTensor(this, &reference);
+        CopyValues(&reference, this);
+    }
+
+    if(reference.isTmp)
+        XLink::Replace(&reference, this);
+    else{
+        CheckNTErrors(outgo.tailNum == 0, "The node has outgoing edge to other nodes!");
+        XLink::CopyIncoming(&reference, this);
+    }
+
+    isInit = false;
+    isTmp  = false;
+}
+
 /* de-constructor */
 /* de-constructor */
 XTensor::~XTensor()
 XTensor::~XTensor()
 {
 {
@@ -168,6 +240,7 @@ XTensor::~XTensor()
     data = NULL;
     data = NULL;
     dataHost = NULL;
     dataHost = NULL;
     mem = NULL;
     mem = NULL;
+    XLink::ClearIncoming(this);
 }
 }
 
 
 /* delete data arrays */
 /* delete data arrays */
@@ -186,9 +259,32 @@ void XTensor::DestroyData()
     dataHost = NULL;
     dataHost = NULL;
 }
 }
 
 
+/* 
+shallow copy of tensor
+Note that we do not copy data array here
+>> tensor - the source tensor
+*/
+void XTensor::ShallowCopy(const XTensor &tensor)
+{
+    order = tensor.order;
+    memcpy(dimSize, tensor.dimSize, sizeof(int) * MAX_TENSOR_DIM_NUM);
+    memcpy(dimSizeRDI, tensor.dimSizeRDI, sizeof(int) * MAX_TENSOR_DIM_NUM);
+    dataType = tensor.dataType;
+    unitSize = tensor.unitSize;
+    unitNum = tensor.unitNum;
+    isSparse = tensor.isSparse;
+    unitNumNonZero = tensor.unitNumNonZero;
+    denseRatio =  tensor.denseRatio;
+    isShared = tensor.isShared;
+    isDefaultDType = tensor.isDefaultDType;
+    isInGlobalMem = tensor.isInGlobalMem;
+    memcpy(isAllValued, tensor.isAllValued, sizeof(bool) * MAX_TENSOR_DIM_NUM);
+}
+
 /* overloading of the equal-sign */
 /* overloading of the equal-sign */
-XTensor& XTensor::operator = (const XTensor& tensor)
+XTensor& XTensor::operator= (const XTensor& tensor)
 {
 {
+    /* hard copy of data array */
     int size = unitNum * unitSize;
     int size = unitNum * unitSize;
     if( isInit && !isSparse && !tensor.isSparse &&
     if( isInit && !isSparse && !tensor.isSparse &&
         size == tensor.unitNum * tensor.unitSize &&
         size == tensor.unitNum * tensor.unitSize &&
@@ -201,40 +297,51 @@ XTensor& XTensor::operator = (const XTensor& tensor)
     }
     }
     else{
     else{
         DestroyData();
         DestroyData();
-        if(!isInit){
+        if(isInit){
             devID = tensor.devID;
             devID = tensor.devID;
             mem = tensor.mem;
             mem = tensor.mem;
         }
         }
 
 
         Resize(tensor.order, tensor.dimSize, tensor.dataType, tensor.denseRatio);
         Resize(tensor.order, tensor.dimSize, tensor.dataType, tensor.denseRatio);
-
-        if(tensor.isSparse) {
-            int num = int(tensor.unitNum * tensor.denseRatio + 1);
-            int tupleSize = sizeof(int)+sizeof(DTYPE);
-            size = sizeof(int) + tupleSize * num;
-        }
-
-        XMemCopy(data, devID, tensor.data, tensor.devID, size);
+        CopyValues(&tensor, this);
     }
     }
 
 
-    order = tensor.order;
-    memcpy(dimSize, tensor.dimSize, sizeof(int) * MAX_TENSOR_DIM_NUM);
-    memcpy(dimSizeRDI, tensor.dimSizeRDI, sizeof(int) * MAX_TENSOR_DIM_NUM);
-    dataType = tensor.dataType;
-    unitSize = tensor.unitSize;
-    unitNum = tensor.unitNum;
-    isSparse = tensor.isSparse;
-    unitNumNonZero = tensor.unitNumNonZero;
-    denseRatio =  tensor.denseRatio;
-    isShared = tensor.isShared;
-    isDefaultDType = tensor.isDefaultDType;
-    isInGlobalMem = tensor.isInGlobalMem;
-    memcpy(isAllValued, tensor.isAllValued, sizeof(bool) * MAX_TENSOR_DIM_NUM);
+    /* copy member variables */
+    ShallowCopy(tensor);
+
     isInit = true;
     isInit = true;
+    isTmp  = false;
+
+    CheckNTErrors(outgo.tailNum == 0, "The node has outgoing edge to other nodes!");
+
+    /* create tensor links for the new tensor */
+    XLink::Replace(&tensor, this);
 
 
     return *this;
     return *this;
 }
 }
 
 
+/* overloading of the plus-sign */
+XTensor XTensor::operator+ (const XTensor& tensor)
+{
+    return Sum(*this, tensor);
+}
+
+/* overloading of the multiply-sign */
+XTensor XTensor::operator* (const XTensor& tensor)
+{
+    return Multiply(*this, tensor);
+}
+
+/* 
+linear transformation b = a * \scale + \shift
+>> scale - the slope
+>> shift - the intercept
+*/
+XTensor XTensor::Lin(DTYPE scale, DTYPE shift)
+{
+    return Linear(*this, scale, shift);
+}
+
 /* 
 /* 
 judge whether the two matrices are in the same type and size 
 judge whether the two matrices are in the same type and size 
 >> a - input tensor
 >> a - input tensor
@@ -555,6 +662,12 @@ bool XTensor::CheckData(const void * d, int num, int beg)
 #endif
 #endif
     return true;
     return true;
 }
 }
+    
+/* set the pointer to "data" */
+void XTensor::SetDataPointer()
+{
+    dataP = &data;
+}
 
 
 bool XTensor::CheckData(const void * d, int num, float tolerance, int beg)
 bool XTensor::CheckData(const void * d, int num, float tolerance, int beg)
 {
 {
@@ -815,7 +928,7 @@ set the value of a cell
 */
 */
 bool XTensor::Set(DTYPE value, int index[], int size)
 bool XTensor::Set(DTYPE value, int index[], int size)
 {
 {
-    CheckNTErrors((dataType == DEFAULT_DTYPE), "The tensor is not in default type.");
+	CheckNTErrors((dataType == DEFAULT_DTYPE), "The tensor is not in default type.");
 
 
     return SetToDevice(devID, GetCell(index, size), value);
     return SetToDevice(devID, GetCell(index, size), value);
 }
 }
@@ -932,6 +1045,15 @@ int XTensor::GetNonzeroSize()
 }
 }
 
 
 /* 
 /* 
+set the tensor as "temporary" 
+>> myIsTMP - flag
+*/
+void XTensor::SetTMP(bool myIsTmp)
+{
+    isTmp = myIsTmp;
+}
+
+/* 
 resize a tensor with a specified tensor size
 resize a tensor with a specified tensor size
 >> myOrder - order of the tensor
 >> myOrder - order of the tensor
 >> myDimSize - the size of each dimension
 >> myDimSize - the size of each dimension
@@ -1664,7 +1786,7 @@ initialize a tensor with a reference tensor
 >> tensor - the tensor we intend to initialize
 >> tensor - the tensor we intend to initialize
 >> reference - the reference tensor
 >> reference - the reference tensor
 */
 */
-void InitTensor(XTensor * tensor, XTensor * reference)
+void InitTensor(XTensor * tensor, const XTensor * reference)
 {
 {
     if(reference->order < 0)
     if(reference->order < 0)
         return;
         return;

source/XTensor.h → source/tensor/XTensor.h

@@ -55,12 +55,13 @@ struct XLink;
 #define UNSAFE_BUT_FAST_MEM
 #define UNSAFE_BUT_FAST_MEM
 #define FAST_MATRIX
 #define FAST_MATRIX
 
 
-/* 
-We implemente the tensor class here though we have defined the class of XMatrix. It
-is the parent class of XMatrix.
-*/
+/* XTensor is a class to do everything a tensor can do :) */
 struct XTensor
 struct XTensor
 {
 {
+public:
+    /* id */
+    int id;
+
     /* memory pool */
     /* memory pool */
     XMem * mem;
     XMem * mem;
 
 
@@ -70,6 +71,10 @@ struct XTensor
     /* copy of data on the host memory. It is only activated 
     /* copy of data on the host memory. It is only activated 
        when the matrix is operated on GPUs */
        when the matrix is operated on GPUs */
     void * dataHost;
     void * dataHost;
+    
+    /* a pointer to data (i.e., a pointer to the address of "data".
+       This is for reset "data" when XTensor is used as a const variable. */
+    void ** dataP;
 
 
     /* 
     /* 
     device id 
     device id 
@@ -130,6 +135,9 @@ struct XTensor
 
 
     /* indicates whether the tensor is initialized or not */
     /* indicates whether the tensor is initialized or not */
     bool isInit;
     bool isInit;
+
+    /* indicates whether the tensor is created temporarily */
+    bool isTmp;
     
     
     /*
     /*
     the link used to form networks. Note that when we compute on tensors, we actually create a
     the link used to form networks. Note that when we compute on tensors, we actually create a
@@ -152,7 +160,7 @@ struct XTensor
     XTensor();
     XTensor();
 
 
     /* constructor */
     /* constructor */
-    XTensor(XTensor * reference);
+    XTensor(const XTensor * reference);
 
 
     /* constructor */
     /* constructor */
     XTensor(const int myOrder, int myDevID, XMem * myMem);
     XTensor(const int myOrder, int myDevID, XMem * myMem);
@@ -161,14 +169,29 @@ struct XTensor
     XTensor(const int myOrder, const int * myDimSize, const TENSOR_DATA_TYPE myDataType, 
     XTensor(const int myOrder, const int * myDimSize, const TENSOR_DATA_TYPE myDataType, 
             const float myDenseRatio, XMem * myMem);
             const float myDenseRatio, XMem * myMem);
 
 
+    /* copy constructor */
+    XTensor(const XTensor &reference);
+
     /* de-constructor */
     /* de-constructor */
     ~XTensor();
     ~XTensor();
 
 
     /* delete data arrays */
     /* delete data arrays */
     void DestroyData();
     void DestroyData();
 
 
+    /* shallow copy of tensor */
+    void ShallowCopy(const XTensor &tensor);
+
     /* overloading of the equal-sign */
     /* overloading of the equal-sign */
-    XTensor& operator = (const XTensor& tensor);
+    XTensor& operator= (const XTensor &tensor);
+
+    /* overloading of the plus-sign */
+    XTensor  operator+ (const XTensor &tensor);
+
+    /* overloading of the multiply-sign */
+    XTensor  operator* (const XTensor &tensor);
+
+    /* linear transformation */
+    XTensor Lin(DTYPE scale, DTYPE shift = 0);
 
 
     /* judge whether the two matrices are in the same type and size */
     /* judge whether the two matrices are in the same type and size */
     static
     static
@@ -213,6 +236,9 @@ struct XTensor
 
 
     /* check whether the data array is the same as the answer */
     /* check whether the data array is the same as the answer */
     bool CheckData(const void * answer, int num, float tolerance, int beg = 0);
     bool CheckData(const void * answer, int num, float tolerance, int beg = 0);
+    
+    /* set the pointer to "data" */
+    void SetDataPointer();
 
 
     /* set the cell to the ascending order along a given dimension */
     /* set the cell to the ascending order along a given dimension */
     void SetAscendingOrder(int dim);
     void SetAscendingOrder(int dim);
@@ -265,6 +291,9 @@ struct XTensor
     /* get the number of non-zero elements (in a sparse tensor) */
     /* get the number of non-zero elements (in a sparse tensor) */
     int GetNonzeroSize();
     int GetNonzeroSize();
 
 
+    /* set the tensor as "temporary" */
+    void SetTMP(bool myIsTmp = true);
+
     /* resize a matrix with a specified matrix size */
     /* resize a matrix with a specified matrix size */
     bool Resize(const int myOrder, const int * myDimSize,
     bool Resize(const int myOrder, const int * myDimSize,
                 const TENSOR_DATA_TYPE myDataType = DEFAULT_DTYPE,
                 const TENSOR_DATA_TYPE myDataType = DEFAULT_DTYPE,
@@ -299,6 +328,12 @@ struct XTensor
     void FreeData(XTensor * matrix, XMem * myMem = NULL, bool useBuf = false);
     void FreeData(XTensor * matrix, XMem * myMem = NULL, bool useBuf = false);
 };
 };
 
 
+/* we make a unique id for every tensor */
+extern int tensorIDGlobal;
+extern MUTEX_HANDLE tensorMutex;
+extern XTensor firstTensor;
+extern int MakeTensorID();
+
 /************************************************
 /************************************************
 * we define the "new and delete" functions below
 * we define the "new and delete" functions below
 */
 */
@@ -329,7 +364,7 @@ void InitTensor5D(XTensor * tensor, const int d0, const int d1, const int d2, co
                   const TENSOR_DATA_TYPE myDataType = X_FLOAT, const int myDevID = -1, XMem * myMem = NULL);
                   const TENSOR_DATA_TYPE myDataType = X_FLOAT, const int myDevID = -1, XMem * myMem = NULL);
 
 
 /* initialize a tensor with a reference tensor */
 /* initialize a tensor with a reference tensor */
-void InitTensor(XTensor * tensor, XTensor * reference);
+void InitTensor(XTensor * tensor, const XTensor * reference);
 
 
 /* generate a XTensor */
 /* generate a XTensor */
 XTensor * NewTensor(const int myOrder, const int * myDimSize, const TENSOR_DATA_TYPE myDataType = X_FLOAT,
 XTensor * NewTensor(const int myOrder, const int * myDimSize, const TENSOR_DATA_TYPE myDataType = X_FLOAT,

source/tensor/core/arithmetic/Absolute.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "../../XTensor.h"
+#include "Absolute.h"
+#include "Absolute.cuh"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/*
+set every entry to its absolute value
+>> a - the tensor we are processing
+*/
+void Absolute(XTensor * a)
+{
+#ifdef USE_CUDA
+    /* run it on GPUs */
+    if (a->devID >= 0) {
+        CudaAbsolute(a);
+    return;
+}
+#endif
+
+    CheckNTErrors((a->dataType == DEFAULT_DTYPE), "TODO!");
+    DTYPE * d = (DTYPE*)a->data;
+    for (int i = 0; i < a->unitNum; i++)
+        d[i] = (DTYPE)fabs(d[i]);
+}
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/tensor/core/arithmetic/Absolute.cu

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "../../XDevice.h"
+#include "../../XTensor.h"
+#include "Absolute.h"
+#include "Absolute.cuh"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+#ifdef USE_CUDA
+/*
+set each entry to its absolute value (CUDA Kernel)
+>> d - pointer to the data array
+>> size - size of the data array
+*/
+__global__
+void KernelAbsolute(DTYPE * d, int size)
+{
+    int i = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (i < size)
+        d[i] = fabs(d[i]);
+}
+
+/*
+set each entry to its absolute value (CUDA Kernel)
+This is for float16 computation
+>> d - pointer to the data array
+>> size - size of the data array
+*/
+__global__
+void KernelAbsolute(__half * d, int size)
+{
+    return;
+}
+
+/*
+set each entry to its  with float16 data type value
+>> a - the tensor
+*/
+extern "C"
+void CudaAbsolute(XTensor * a)
+{
+    CheckNTErrors((a->isSparse == false), "TODO!");
+
+    int gridSize[3];
+    int blockSize[3];
+
+    GDevs.GetCudaThread(a->devID, a->unitNum, gridSize, blockSize);
+
+    dim3 blocks(gridSize[0]);
+    dim3 threads(blockSize[0]);
+
+    int devIDBackup;
+    ProtectCudaDev(a->devID, devIDBackup);
+
+    if (a->dataType == DEFAULT_DTYPE) {
+        KernelAbsolute << <blocks, threads >> >((DTYPE*)a->data, a->unitNum);
+    }
+    else if (a->dataType == X_FLOAT16) {
+        KernelAbsolute << <blocks, threads >> >((__half*)a->data, a->unitNum);
+    }
+    else {
+        ShowNTErrors("TODO!");
+    }
+
+    BacktoCudaDev(a->devID, devIDBackup);
+}
+
+#endif // USE_CUDA
+} // namespace nts(NiuTrans.Tensor)

source/tensor/core/arithmetic/Absolute.cuh

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "Absolute.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+#ifdef USE_CUDA
+
+/* set each entry to its absolute value (CUDA Kernel) */
+__global__
+void KernelAbsolute(DTYPE * d, int size);
+
+/* set each entry to its absolute value (CUDA Kernel) with float16 data type*/
+__global__
+void KernelAbsolute(__half * d, int size);
+
+/* set each entry to its absolute value */
+extern "C"
+void CudaAbsolute(XTensor * a);
+
+#endif // USE_CUDA
+
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/core/arithmetic/Multiply.h → source/tensor/core/arithmetic/Absolute.h

@@ -16,20 +16,20 @@
 */
 */
 
 
 /*
 /*
-* $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-04-24
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
 */
 */
 
 
-#ifndef __MULTIPLY_H__
-#define __MULTIPLY_H__
+#ifndef __ABSOLUTE_H__
+#define __ABSOLUTE_H__
 
 
 #include "../../XTensor.h"
 #include "../../XTensor.h"
 
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
-/* element-wise product of two tensors */
+/* set every entry to its absolute value */
 extern "C"
 extern "C"
-void Multiply(XTensor * a, XTensor * b, XTensor * c, int leadingDim = 0, DTYPE alpha = 0);
+void Absolute(XTensor * a);
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
 
 
-#endif // __MULTIPLY_H__
+#endif // __ABSOLUTE_H__
\ No newline at end of file

source/core/arithmetic/Multiply.cpp → source/tensor/core/arithmetic/Multiply.cpp

@@ -34,18 +34,20 @@ where i is the index of the item
 >> b - matrix b
 >> b - matrix b
 >> c - result matrix
 >> c - result matrix
 >> alpha - the coefficient
 >> alpha - the coefficient
+>> leadingDim - the dimension along which we perform broadcasting
 >>
 >>
 */
 */
-void Multiply(XTensor * a, XTensor * b, XTensor * c, int leadingDim, DTYPE alpha)
+void _Multiply(const XTensor * a, const XTensor * b, XTensor * c, DTYPE alpha, int leadingDim)
 {
 {
 	int leadingDimRDI = a->order - leadingDim - 1;
 	int leadingDimRDI = a->order - leadingDim - 1;
     CheckNTErrors((a->unitNum <= c->unitNum && b->unitNum <= c->unitNum),
     CheckNTErrors((a->unitNum <= c->unitNum && b->unitNum <= c->unitNum),
-        "Unmatched tensors in multiplication!");
-    CheckNTErrors((a->order == b->order && a->order == c->order), "Unmatched tensors!");
+                  "Unmatched tensors in multiplication!");
+    CheckNTErrors((a->order == b->order && a->order == c->order), 
+                  "Unmatched tensors!");
 
 
 #ifdef USE_CUDA
 #ifdef USE_CUDA
     if (a->devID >= 0 || b->devID >= 0 || c->devID >= 0) {
     if (a->devID >= 0 || b->devID >= 0 || c->devID >= 0) {
-        CudaMultiply(a, b, c, leadingDim, alpha);
+        _CudaMultiply(a, b, c, alpha, leadingDim);
         return;
         return;
     }
     }
 #endif
 #endif
@@ -118,4 +120,46 @@ void Multiply(XTensor * a, XTensor * b, XTensor * c, int leadingDim, DTYPE alpha
     }
     }
 }
 }
 
 
+/*
+element-wise product of two tensors and keep the result in the input
+a(i) = a(i)*b(i) + \alpha * a(i)
+where i is the index of the item
+>> a - tensor a (where keep the result)
+>> b - tensor b
+>> alpha - the coefficient
+>> leadingDim - the dimension along which we perform broadcasting
+*/
+void _MultiplyMe(XTensor * a, const XTensor * b, DTYPE alpha, int leadingDim)
+{
+    _Multiply(a, b, a, alpha, leadingDim);
+}
+
+/*
+make a tensor of the element-wise product for two input tensors: 
+c(i) = a(i)*b(i) + \alpha * c(i)
+where i is the index of the item
+>> a - tensor a
+>> b - tensor b
+>> alpha - the coefficient
+>> leadingDim - the dimension along which we perform broadcasting
+<< return - the product of the tensors
+*/
+XTensor Multiply(const XTensor &a, const XTensor &b, DTYPE alpha, int leadingDim)
+{
+    CheckNTErrors(a.dimSize[leadingDim] == b.dimSize[leadingDim], "TODO!");
+
+    XTensor c(&a);
+    c.SetTMP();
+    
+    /* computation */
+    _Multiply(&a, &b, &c, alpha, leadingDim);
+    
+    /* tensor connections */
+    XLink::MakeLink(&a, &b, &c, MATH_MULTIPLY);
+    XLink::AddParamToHead(&c, alpha);
+    XLink::AddParamToHeadInt(&c, leadingDim);
+    
+    return c;
+}
+
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)

source/core/arithmetic/Multiply.cu → source/tensor/core/arithmetic/Multiply.cu

@@ -117,15 +117,15 @@ where i is the item index
 >> a - tensor a
 >> a - tensor a
 >> b - tensor b
 >> b - tensor b
 >> c - result tensor
 >> c - result tensor
->> leadingDim - leading dimension
 >> alpha - the coefficient
 >> alpha - the coefficient
+>> leadingDim - dimension along which we perform broadcasting
 */
 */
 extern "C"
 extern "C"
-void CudaMultiply(XTensor * a, XTensor * b, XTensor * c, int leadingDim, DTYPE alpha)
+void _CudaMultiply(const XTensor * a, const XTensor * b, XTensor * c, DTYPE alpha, int leadingDim)
 {
 {
 	int leadingDimRDI = a->order - leadingDim - 1;
 	int leadingDimRDI = a->order - leadingDim - 1;
     CheckNTErrors((a->unitNum <= c->unitNum && b->unitNum <= c->unitNum),
     CheckNTErrors((a->unitNum <= c->unitNum && b->unitNum <= c->unitNum),
-        "Unmatched tensors in multiplication!");
+                  "Unmatched tensors in multiplication!");
     CheckNTErrors((a->order == b->order && a->order == c->order), "Unmatched tensors!");
     CheckNTErrors((a->order == b->order && a->order == c->order), "Unmatched tensors!");
 
 
     int stride = 1;
     int stride = 1;
@@ -138,8 +138,8 @@ void CudaMultiply(XTensor * a, XTensor * b, XTensor * c, int leadingDim, DTYPE a
     for (int i = 0; i < a->order; i++) {
     for (int i = 0; i < a->order; i++) {
         if (i != leadingDimRDI) {
         if (i != leadingDimRDI) {
             CheckNTErrors((a->dimSizeRDI[i] == b->dimSizeRDI[i] &&
             CheckNTErrors((a->dimSizeRDI[i] == b->dimSizeRDI[i] &&
-                a->dimSizeRDI[i] == c->dimSizeRDI[i]),
-                "Unmatched tensors!");
+                           a->dimSizeRDI[i] == c->dimSizeRDI[i]),
+                          "Unmatched tensors!");
         }
         }
         if (i < leadingDimRDI)
         if (i < leadingDimRDI)
             stride *= a->dimSizeRDI[i];
             stride *= a->dimSizeRDI[i];

source/core/arithmetic/Multiply.cuh → source/tensor/core/arithmetic/Multiply.cuh

@@ -42,7 +42,7 @@ void KernelMulElementWiseTensorDynamic(DTYPE * a, DTYPE * b, DTYPE * c, DTYPE al
 
 
 /* element-wise product of two tensors */
 /* element-wise product of two tensors */
 extern "C"
 extern "C"
-void CudaMultiply(XTensor * a, XTensor * b, XTensor * c, int leadingDim = 0, DTYPE alpha = 0);
+void _CudaMultiply(const XTensor * a, const XTensor * b, XTensor * c, DTYPE alpha = 0, int leadingDim = 0);
 
 
 #endif // USE_CUDA
 #endif // USE_CUDA
 
 

source/tensor/core/arithmetic/Multiply.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-04-24
+*/
+
+#ifndef __MULTIPLY_H__
+#define __MULTIPLY_H__
+
+#include "../../XTensor.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* element-wise product of two tensors: 
+   c(i) = a(i)*b(i) + \alpha * c(i) 
+   where i is the index of the element */
+void _Multiply(const XTensor * a, const XTensor * b, XTensor * c, DTYPE alpha = 0, int leadingDim = 0);
+
+/* element-wise product of two tensors and keep the result in the input tensor: 
+   a(i) = a(i)*b(i) + \alpha * a(i) 
+   where i is the index of the element */
+void _MultiplyMe(XTensor * a, const XTensor * b, DTYPE alpha = 0, int leadingDim = 0);
+
+/* make a tensor of the element-wise product for two input tensors: 
+   c(i) = a(i)*b(i) + \alpha * c(i) 
+   where i is the index of the element */
+XTensor Multiply(const XTensor &a, const XTensor &b, DTYPE alpha = 0, int leadingDim = 0);
+
+} // namespace nts(NiuTrans.Tensor)
+
+#endif // __MULTIPLY_H__
\ No newline at end of file

source/tensor/core/arithmetic/Sign.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "../../XTensor.h"
+#include "Sign.h"
+#include "Sign.cuh"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/*
+set every entry to its sign value
+>> a - the tensor we are processing
+*/
+void Sign(XTensor * a)
+{
+#ifdef USE_CUDA
+    /* run it on GPUs */
+    if (a->devID >= 0) {
+        CudaSign(a);
+    return;
+}
+#endif
+
+    CheckNTErrors((a->dataType == DEFAULT_DTYPE), "TODO!");
+    DTYPE * d = (DTYPE*)a->data;
+    for (int i = 0; i < a->unitNum; i++) {
+        if (d[i] > 0)
+            d[i] = 1.0F;
+        else if (d[i] == 0)
+            d[i] = 0.0F;
+        else
+            d[i] = -1.0F;
+    }
+}
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/tensor/core/arithmetic/Sign.cu

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "../../XDevice.h"
+#include "../../XTensor.h"
+#include "Sign.h"
+#include "Sign.cuh"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+#ifdef USE_CUDA
+/*
+set each entry to its sign value (CUDA Kernel)
+>> d - pointer to the data array
+>> size - size of the data array
+*/
+__global__
+void KernelSign(DTYPE * d, int size)
+{
+    int i = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (i < size) {
+        if (d[i] > 0)
+            d[i] = 1.0F;
+        else if (d[i] == 0)
+            d[i] = 0.0F;
+        else
+            d[i] = -1.0F;
+    }
+}
+
+/*
+set each entry to its sign value (CUDA Kernel)
+This is for float16 computation
+>> d - pointer to the data array
+>> size - size of the data array
+*/
+__global__
+void KernelSign(__half * d, int size)
+{
+    return;
+}
+
+/*
+set each entry to its  with float16 data type value
+>> a - the tensor
+*/
+extern "C"
+void CudaSign(XTensor * a)
+{
+    CheckNTErrors((a->isSparse == false), "TODO!");
+
+    int gridSize[3];
+    int blockSize[3];
+
+    GDevs.GetCudaThread(a->devID, a->unitNum, gridSize, blockSize);
+
+    dim3 blocks(gridSize[0]);
+    dim3 threads(blockSize[0]);
+
+    int devIDBackup;
+    ProtectCudaDev(a->devID, devIDBackup);
+
+    if (a->dataType == DEFAULT_DTYPE) {
+        KernelSign << <blocks, threads >> >((DTYPE*)a->data, a->unitNum);
+    }
+    else if (a->dataType == X_FLOAT16) {
+        KernelSign << <blocks, threads >> >((__half*)a->data, a->unitNum);
+    }
+    else {
+        ShowNTErrors("TODO!");
+    }
+
+    BacktoCudaDev(a->devID, devIDBackup);
+}
+
+#endif // USE_CUDA
+} // namespace nts(NiuTrans.Tensor)

source/tensor/core/arithmetic/Sign.cuh

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "Sign.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+#ifdef USE_CUDA
+
+/* set each entry to its sign value (CUDA Kernel) */
+__global__
+void KernelSign(DTYPE * d, int size);
+
+/* set each entry to its sign value (CUDA Kernel) with float16 data type*/
+__global__
+void KernelSign(__half * d, int size);
+
+/* set each entry to its sign value */
+extern "C"
+void CudaSign(XTensor * a);
+
+#endif // USE_CUDA
+
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/tensor/core/arithmetic/Sign.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#ifndef __SIGN_H__
+#define __SIGN_H__
+
+#include "../../XTensor.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* set every entry to its sign value */
+extern "C"
+void Sign(XTensor * a);
+
+} // namespace nts(NiuTrans.Tensor)
+
+#endif // __SIGN_H__

source/core/arithmetic/Sum.cpp → source/tensor/core/arithmetic/Sum.cpp

@@ -35,12 +35,9 @@ return a pointer
 >> c - where we put a+b*\beta. we save it in a if c is NULL
 >> c - where we put a+b*\beta. we save it in a if c is NULL
 >> beta - the scaling factor
 >> beta - the scaling factor
 */
 */
-void _Sum(XTensor * a, XTensor * b, XTensor * c, DTYPE beta)
+void _Sum(const XTensor * a, const XTensor * b, XTensor * c, DTYPE beta)
 {
 {
-    if (c == NULL)
-        c = a;
-
-    CheckNTErrors(a && b && c, "Empty tensors in addition!");
+    CheckNTErrors(a && b && c, "Empty tensor input!");
     CheckNTErrors(a->unitNum == b->unitNum && a->unitNum == c->unitNum,
     CheckNTErrors(a->unitNum == b->unitNum && a->unitNum == c->unitNum,
                   "Unmatched tensors in addition!");
                   "Unmatched tensors in addition!");
     CheckNTErrors(a->dataType == b->dataType && a->dataType == c->dataType,
     CheckNTErrors(a->dataType == b->dataType && a->dataType == c->dataType,
@@ -121,7 +118,7 @@ do it on site
 >> b - another tensor
 >> b - another tensor
 >> beta - the scaling factor
 >> beta - the scaling factor
 */
 */
-void _SumMe(XTensor * a, XTensor * b, DTYPE beta)
+void _SumMe(XTensor * a, const XTensor * b, DTYPE beta)
 {
 {
     _Sum(a, b, a, beta);
     _Sum(a, b, a, beta);
 }
 }
@@ -133,16 +130,17 @@ return a XTensor structure
 >> b - another tensor
 >> b - another tensor
 >> beta - the scaling factor
 >> beta - the scaling factor
 */
 */
-XTensor Sum(XTensor &a, XTensor &b, DTYPE beta)
+XTensor Sum(const XTensor &a, const XTensor &b, DTYPE beta)
 {
 {
     XTensor c(&a);
     XTensor c(&a);
+    c.SetTMP();
     
     
     /* computation */
     /* computation */
     _Sum(&a, &b, &c, beta);
     _Sum(&a, &b, &c, beta);
     
     
     /* tensor connections */
     /* tensor connections */
-    //XLink::MakeLink(&a, &b, &c, MATH_SUM);
-    //XLink::AddParamToHead(&c, beta);
+    XLink::MakeLink(&a, &b, &c, MATH_SUM);
+    XLink::AddParamToHead(&c, beta);
     
     
     return c;
     return c;
 }
 }

source/core/arithmetic/Sum.cu → source/tensor/core/arithmetic/Sum.cu

@@ -51,11 +51,9 @@ tensor summation c = a + b * \beta (cuda version)
 >> c - where we put a+b*\beta. we save it in a if c is NULL
 >> c - where we put a+b*\beta. we save it in a if c is NULL
 >> beta - the scaling factor
 >> beta - the scaling factor
 */
 */
-void _CudaSum(XTensor * a, XTensor * b, XTensor * c, DTYPE beta)
+void _CudaSum(const XTensor * a, const XTensor * b, XTensor * c, DTYPE beta)
 {
 {
-    if (c == NULL)
-        c = a;
-
+    CheckNTErrors(a && b && c, "Empty tensor input!");
     CheckNTErrors((a->unitNum == b->unitNum && a->unitNum == c->unitNum),
     CheckNTErrors((a->unitNum == b->unitNum && a->unitNum == c->unitNum),
                   "Unmatched tensors in addition!");
                   "Unmatched tensors in addition!");
     CheckNTErrors((a->dataType == b->dataType && a->dataType == c->dataType),
     CheckNTErrors((a->dataType == b->dataType && a->dataType == c->dataType),

source/core/arithmetic/Sum.cuh → source/tensor/core/arithmetic/Sum.cuh

@@ -34,7 +34,7 @@ void KernelADD(DTYPE * a, DTYPE * b, DTYPE * c, int size, DTYPE beta = (DTYPE)1.
 
 
 /* tensor summation c = a + b * \beta (cuda version) */
 /* tensor summation c = a + b * \beta (cuda version) */
 extern "C"
 extern "C"
-void _CudaSum(XTensor * a, XTensor * b, XTensor * c = NULL, DTYPE beta = (DTYPE)1.0);
+void _CudaSum(const XTensor * a, const XTensor * b, XTensor * c = NULL, DTYPE beta = (DTYPE)1.0);
 
 
 /*  tensor summation c = a + b * \beta (cuda version) with an input handle */
 /*  tensor summation c = a + b * \beta (cuda version) with an input handle */
 extern "C"
 extern "C"

source/core/arithmetic/Sum.h → source/tensor/core/arithmetic/Sum.h

@@ -27,13 +27,13 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
 /* tensor summation c = a + b * \beta */
 /* tensor summation c = a + b * \beta */
-void _Sum(XTensor * a, XTensor * b, XTensor * c = NULL, DTYPE beta = (DTYPE)1.0);
+void _Sum(const XTensor * a, const XTensor * b, XTensor * c, DTYPE beta = (DTYPE)1.0);
 
 
 /* tensor summation a = a + b * \beta (return a pointer) */
 /* tensor summation a = a + b * \beta (return a pointer) */
-void _SumMe(XTensor * a, XTensor * b, DTYPE beta = (DTYPE)1.0);
+void _SumMe(XTensor * a, const XTensor * b, DTYPE beta = (DTYPE)1.0);
     
     
 /* tensor summation c = a + b * \beta (return a structure) */
 /* tensor summation c = a + b * \beta (return a structure) */
-XTensor Sum(XTensor &a, XTensor &b, DTYPE beta = (DTYPE)1.0);
+XTensor Sum(const XTensor &a, const XTensor &b, DTYPE beta = (DTYPE)1.0);
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
 
 

source/tensor/core/getandset/ConvertDataType.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "../../XTensor.h"
+#include "ConvertDataType.h"
+#include "ConvertDataType.cuh"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/*
+convert data type
+>> input - input tensor
+>> output - output tensor
+*/
+void ConvertTensorDataType(XTensor * input, XTensor * output)
+{
+    CheckNTErrors(XTensor::IsIdentical(input, output), "Input and Output are different in type or size!");
+    if (input->dataType == output->dataType)
+        return;
+#ifdef USE_CUDA
+    /* run it on GPUs */
+    if (input->devID >= 0) {
+        CudaConvertDataType(input, output);
+    return;
+}
+#endif
+
+    if (input->dataType == X_FLOAT && output->dataType == X_INT) {
+        float * inputData = (float*)input->data;
+        int * outputData = (int*)output->data;
+        for (int i = 0; i < input->unitNum; i++) 
+            outputData[i] = (int)inputData[i];
+    }
+    else if (input->dataType == X_INT && output->dataType == X_FLOAT) {
+        int * inputData = (int*)input->data;
+        float * outputData = (float*)output->data;
+        for (int i = 0; i < input->unitNum; i++) 
+            outputData[i] = (float)inputData[i];
+    }
+    else
+        ShowNTErrors("Unsupported data types for conversion!");
+
+}
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/core/getandset/ConvertDataType.cu → source/tensor/core/getandset/ConvertDataType.cu

@@ -21,6 +21,7 @@
 
 
 #include "../../XTensor.h"
 #include "../../XTensor.h"
 #include "../../XDevice.h"
 #include "../../XDevice.h"
+#include "ConvertDataType.cuh"
 
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
@@ -49,6 +50,24 @@ void KernelFloat16ToFloat(__half * s, float * t, int size)
     }
     }
 }
 }
 
 
+__global__ 
+void KernelFloatToInt(float * inputData, int * outputData, int size)
+{
+    int i = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (i < size){
+        outputData[i] = (int)(inputData[i]);
+    }
+}
+
+__global__ 
+void KernelIntToFloat(int * inputData, float * outputData, int size)
+{
+    int i = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if (i < size){
+        outputData[i] = (float)(inputData[i]);
+    }}
 
 
 /* 
 /* 
 data conversion (cuda code) 
 data conversion (cuda code) 
@@ -88,6 +107,39 @@ void CudaConvertDataType(int devID, void * s, TENSOR_DATA_TYPE typeS, void * t,
     ProtectCudaDev(devID, devIDBackup);
     ProtectCudaDev(devID, devIDBackup);
 }
 }
 
 
+/*
+convert data type (cuda code) 
+>> input - input tensor
+>> output - output tensor
+*/
+void CudaConvertDataType(XTensor * input, XTensor * output)
+{
+    CheckNTErrors(XTensor::IsIdentical(input, output), "Input and Output are different in type or size!");
+    if (input->dataType == output->dataType)
+        return;
+
+    int gridSize[3];
+    int blockSize[3];
+
+    GDevs.GetCudaThread(input->devID, input->unitNum, gridSize, blockSize);
+
+    dim3 blocks(gridSize[0]);
+    dim3 threads(blockSize[0]);
+
+    int devIDBackup;
+    ProtectCudaDev(input->devID, devIDBackup);
+
+    if(input->dataType == X_FLOAT && output->dataType == X_INT)
+        KernelFloatToInt<<<blocks, threads>>>((float*)input->data, (int*)output->data, input->unitNum);
+    else if(input->dataType == X_INT && output->dataType == X_FLOAT)
+        KernelIntToFloat<<<blocks, threads>>>((int*)input->data, (float*)output->data, input->unitNum);
+    else{
+        ShowNTErrors("Unsupported data types for conversion!");
+    }
+
+    ProtectCudaDev(input->devID, devIDBackup);
+}
+
 #endif // USE_CUDA
 #endif // USE_CUDA
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/tensor/core/getandset/ConvertDataType.cuh

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#include "ConvertDataType.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+#ifdef USE_CUDA
+
+/* convert data type from X_FLOAT to X_FLOAT16 (CUDA Kernel) */
+__global__
+void KernelFloatToFloat16(float * s, __half * t, int size);
+
+/* convert data type from X_FLOAT16 to X_FLOAT (CUDA Kernel) */
+__global__
+void KernelFloat16ToFloat(__half * s, float * t, int size);
+
+/* convert data type from X_FLOAT to X_INT (CUDA Kernel) */
+__global__
+void KernelFloatToInt(float * inputData, int * outputData, int size);
+
+/* convert data type from X_INT to X_FLOAT (CUDA Kernel) */
+__global__
+void KernelIntToFloat(int * inputData, float * outputData, int size);
+
+/* convert data type */
+void CudaConvertDataType(XTensor * input, XTensor * output);
+
+#endif // USE_CUDA
+
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/tensor/core/getandset/ConvertDataType.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-7-11
+*/
+
+#ifndef __CONVERTDATATYPE_H__
+#define __CONVERTDATATYPE_H__
+
+#include "../../XTensor.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* convert data type */
+void ConvertDataType(XTensor * input, XTensor * output);
+
+} // namespace nts(NiuTrans.Tensor)
+
+#endif // __CONVERTDATATYPE_H__

source/core/math/ScaleAndShift.cpp → source/tensor/core/math/ScaleAndShift.cpp

@@ -19,24 +19,28 @@
 * $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-04-24
 * $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-04-24
 */
 */
 
 
+#include "../../XTensor.h"
+#include "../../XName.h"
+#include "../../XUtility.h"
 #include "ScaleAndShift.h"
 #include "ScaleAndShift.h"
 #include "ScaleAndShift.cuh"
 #include "ScaleAndShift.cuh"
 
 
 namespace nts{ // namespace nts(NiuTrans.Tensor)
 namespace nts{ // namespace nts(NiuTrans.Tensor)
 
 
 /* 
 /* 
-scale and shift all tensor entires
-p = p * scale + shift
->> a - the tensor
+scale and shift all tensor entires b = a * scale + shift
+b = a * scale + shift
+>> a - the input tensor
+>> b - the output tensor
 >> scale - the scaler factor
 >> scale - the scaler factor
 >> shift - the shift factor
 >> shift - the shift factor
 */
 */
-void ScaleAndShift(XTensor * a, DTYPE scale, DTYPE shift)
+void _ScaleAndShift(const XTensor * a, XTensor * b, DTYPE scale, DTYPE shift)
 {
 {
 #ifdef USE_CUDA
 #ifdef USE_CUDA
     /* run it on GPUs */
     /* run it on GPUs */
     if(a->devID >= 0){
     if(a->devID >= 0){
-        CudaScaleAndShift(a, scale, shift);
+        _CudaScaleAndShift(a, b, scale, shift);
         return;
         return;
     }
     }
 #endif
 #endif
@@ -46,23 +50,65 @@ void ScaleAndShift(XTensor * a, DTYPE scale, DTYPE shift)
 
 
     /* sparse tensor */
     /* sparse tensor */
     if(a->isSparse){
     if(a->isSparse){
-        int num = a->GetNonzeroSize();
+        int num = a->unitNumNonZero;
         char * d = (char*)a->data + sizeof(int);
         char * d = (char*)a->data + sizeof(int);
         char * f = d + (sizeof(int) + sizeof(DTYPE)) * 0 + sizeof(int);
         char * f = d + (sizeof(int) + sizeof(DTYPE)) * 0 + sizeof(int);
+        char * db = (char*)b->data + sizeof(int);
+        char * fb = db + (sizeof(int) + sizeof(DTYPE)) * 0 + sizeof(int);
         for(int i = 0; i < num; i++){
         for(int i = 0; i < num; i++){
             DTYPE * v = (DTYPE*)f;
             DTYPE * v = (DTYPE*)f;
-            *v = *v * scale + shift;
+            DTYPE * vb = (DTYPE*)fb;
+            *vb = *v * scale + shift;
             f += sizeof(int) + sizeof(DTYPE);
             f += sizeof(int) + sizeof(DTYPE);
+            fb += sizeof(int) + sizeof(DTYPE);
         }
         }
     }
     }
     /* dense tensor */
     /* dense tensor */
     else{
     else{
-        DTYPE * v = (DTYPE*)a->data;
-        for(int i = 0; i < a->unitNum; i++){
-            *v = *v * scale + shift;
-            v++;
+        DTYPE * va = (DTYPE*)a->data;
+        DTYPE * vb = (DTYPE*)b->data;
+        for(int i = 0; i < b->unitNum; i++){
+            *vb = *va * scale + shift;
+            va++;
+            vb++;
         }
         }
     }
     }
 }
 }
 
 
+/* 
+scale and shift all tensor entires on site b = a * scale + shift
+b = a * scale + shift
+>> a - the input/output tensor
+>> scale - the scaler factor
+>> shift - the shift factor
+*/
+void _ScaleAndShiftMe(XTensor * a, DTYPE scale, DTYPE shift)
+{
+    _ScaleAndShift(a, a, scale, shift);
+}
+
+/* 
+scale and shift all tensor entires b = a * scale + shift
+b = a * scale + shift
+>> a - the input tensor
+>> b - the output tensor
+>> scale - the scaler factor
+>> shift - the shift factor
+*/
+XTensor ScaleAndShift(const XTensor &a, DTYPE scale, DTYPE shift)
+{
+    XTensor b(&a);
+    b.SetTMP();
+    
+    /* computation */
+    _ScaleAndShift(&a, &b, scale, shift);
+    
+    /* tensor connections */
+    XLink::MakeLink(&a, NULL, &b, MATH_SCALEANDSHIFT);
+    XLink::AddParamToHead(&b, scale);
+    XLink::AddParamToHead(&b, shift);
+    
+    return b;
+}
+
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)

source/core/math/ScaleAndShift.cu → source/tensor/core/math/ScaleAndShift.cu

@@ -19,7 +19,6 @@
 * $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-04-24
 * $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-04-24
 */
 */
 
 
-#include "ScaleAndShift.h"
 #include "ScaleAndShift.cuh"
 #include "ScaleAndShift.cuh"
 #include "../../XDevice.h"
 #include "../../XDevice.h"
 
 
@@ -28,68 +27,71 @@ namespace nts{ // namespace nts(NiuTrans.Tensor)
 #ifdef USE_CUDA
 #ifdef USE_CUDA
 
 
 /* 
 /* 
-scale and shift all tensor entires p = p * scale + shift (CUDA Kernel) 
->> d - the data array
+scale and shift all tensor entires b = a * scale + shift (CUDA Kernel) 
+>> a - the input data array
+>> b - the output data array
 >> size - the size of d
 >> size - the size of d
 >> scale - how much we want to scale it
 >> scale - how much we want to scale it
 >> shift - how much we want to shift it
 >> shift - how much we want to shift it
 */
 */
 template<bool isUnitScale, bool isZeroShift>
 template<bool isUnitScale, bool isZeroShift>
 __global__ 
 __global__ 
-void KernelScaleAndShift(DTYPE * d, int size, DTYPE scale, DTYPE shift)
+void KernelScaleAndShift(DTYPE * a, DTYPE * b, int size, DTYPE scale, DTYPE shift)
 {
 {
     int i = blockDim.x * blockIdx.x + threadIdx.x;
     int i = blockDim.x * blockIdx.x + threadIdx.x;
 
 
     if (i < size){
     if (i < size){
         if (isUnitScale && !isZeroShift){
         if (isUnitScale && !isZeroShift){
-            d[i] = d[i] + shift;
+            b[i] = a[i] + shift;
         }
         }
         else if (isUnitScale && isZeroShift) {
         else if (isUnitScale && isZeroShift) {
-            d[i] = d[i];
+            b[i] = a[i];
         }
         }
         else if (!isUnitScale && isZeroShift) {
         else if (!isUnitScale && isZeroShift) {
-            d[i] = d[i] * scale;
+            b[i] = a[i] * scale;
         }
         }
         else {
         else {
-            d[i] = d[i] * scale + shift;
+            b[i] = a[i] * scale + shift;
         }
         }
     }
     }
 }
 }
 
 
 /* 
 /* 
-scale and shift all matrix entires p = p * scale + shift (CUDA Kernel) 
+scale and shift all tensor entires p = p * scale + shift (CUDA Kernel) 
 This is for float16 computation
 This is for float16 computation
->> d - the data array
+>> a - the input data array
+>> b - the output data array
 >> size - the size of d
 >> size - the size of d
 >> scale - how much we want to scale it
 >> scale - how much we want to scale it
 >> shift - how much we want to shift it
 >> shift - how much we want to shift it
 */
 */
 __global__ 
 __global__ 
-void KernelScaleAndShift(__half * d, int size, __half scale, __half shift)
+void KernelScaleAndShift(__half * a, __half * b, int size, __half scale, __half shift)
 {
 {
 
 
     int i = blockDim.x * blockIdx.x + threadIdx.x;
     int i = blockDim.x * blockIdx.x + threadIdx.x;
 #if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
 #if __CUDA_ARCH__ >= 530 || !defined(__CUDA_ARCH__)
     if(i < size)
     if(i < size)
-        d[i] = __hadd(__hmul(d[i], scale), shift);
+        b[i] = __hadd(__hmul(a[i], scale), shift);
 #else
 #else
     if (i < size)
     if (i < size)
-        d[i] = __float2half(__half2float(d[i]) * __half2float(scale) + __half2float(shift));
+        b[i] = __float2half(__half2float(a[i]) * __half2float(scale) + __half2float(shift));
 #endif
 #endif
 }
 }
 
 
 /* 
 /* 
-scale and shift all matrix entires
+scale and shift all tensor entires
 p = p * scale + shift
 p = p * scale + shift
->> a - the tensor
+>> a - the input tensor
+>> b - the output tensor
 >> scale - the scaler factor
 >> scale - the scaler factor
 >> shift - the shift factor
 >> shift - the shift factor
 */
 */
-void CudaScaleAndShift(XTensor * a, DTYPE scale, DTYPE shift)
+void _CudaScaleAndShift(const XTensor * a, XTensor * b, DTYPE scale, DTYPE shift)
 {
 {
     /* sparse tensor */
     /* sparse tensor */
     if(a->isSparse){
     if(a->isSparse){
-        // TODO
+        ShowNTErrors("TODO!");
     }
     }
     /* dense tensor */
     /* dense tensor */
     else{
     else{
@@ -106,20 +108,20 @@ void CudaScaleAndShift(XTensor * a, DTYPE scale, DTYPE shift)
 
 
         if(a->dataType == DEFAULT_DTYPE){
         if(a->dataType == DEFAULT_DTYPE){
             if(scale == 1.0F && shift == 0)
             if(scale == 1.0F && shift == 0)
-                KernelScaleAndShift<true, true> <<<blocks, threads>>>((DTYPE*)a->data, a->unitNum, scale, shift);
+                KernelScaleAndShift<true, true> <<<blocks, threads>>>((DTYPE*)a->data, (DTYPE*)b->data, a->unitNum, scale, shift);
             else if (scale == 1.0F && shift != 0)
             else if (scale == 1.0F && shift != 0)
-                KernelScaleAndShift<true, false> << <blocks, threads >> >((DTYPE*)a->data, a->unitNum, scale, shift);
+                KernelScaleAndShift<true, false> << <blocks, threads >> >((DTYPE*)a->data, (DTYPE*)b->data, a->unitNum, scale, shift);
             else if(scale != 1.0F && shift == 0)
             else if(scale != 1.0F && shift == 0)
-                KernelScaleAndShift<false, true> << <blocks, threads >> >((DTYPE*)a->data, a->unitNum, scale, shift);
+                KernelScaleAndShift<false, true> << <blocks, threads >> >((DTYPE*)a->data, (DTYPE*)b->data, a->unitNum, scale, shift);
             else
             else
-                KernelScaleAndShift<false, false> << <blocks, threads >> >((DTYPE*)a->data, a->unitNum, scale, shift);
+                KernelScaleAndShift<false, false> << <blocks, threads >> >((DTYPE*)a->data, (DTYPE*)b->data, a->unitNum, scale, shift);
         }
         }
         else if(a->dataType == X_FLOAT16){
         else if(a->dataType == X_FLOAT16){
             unsigned short scale2 = FloatToFloat16(scale);
             unsigned short scale2 = FloatToFloat16(scale);
             unsigned short shift2 = FloatToFloat16(shift);
             unsigned short shift2 = FloatToFloat16(shift);
             __half * scaleft16p = (__half*)&scale2;
             __half * scaleft16p = (__half*)&scale2;
             __half * shiftft16p = (__half*)&shift2;
             __half * shiftft16p = (__half*)&shift2;
-            KernelScaleAndShift<<<blocks, threads>>>((__half*)a->data, a->unitNum, *scaleft16p, *shiftft16p);
+            KernelScaleAndShift<<<blocks, threads>>>((__half*)a->data, (__half*)b->data, a->unitNum, *scaleft16p, *shiftft16p);
         }
         }
         else{
         else{
             ShowNTErrors("TODO!");
             ShowNTErrors("TODO!");

source/core/math/ScaleAndShift.cuh → source/tensor/core/math/ScaleAndShift.cuh

@@ -22,23 +22,23 @@
 #ifndef __SCALEANDSHIFT_CUH__
 #ifndef __SCALEANDSHIFT_CUH__
 #define __SCALEANDSHIFT_CUH__
 #define __SCALEANDSHIFT_CUH__
 
 
-#include "../../XTensor.h"
+#include "ScaleAndShift.h"
 
 
 namespace nts{ // namespace nts(NiuTrans.Tensor)
 namespace nts{ // namespace nts(NiuTrans.Tensor)
 
 
 #ifdef USE_CUDA
 #ifdef USE_CUDA
 
 
-/* scale and shift all tensor entires p = p * scale + shift (CUDA Kernel) */
+/* scale and shift all tensor entires b = a * scale + shift (CUDA Kernel) */
 __global__ 
 __global__ 
-void KernelScaleAndShift(DTYPE * d, int size, DTYPE scale, DTYPE shift);
+void KernelScaleAndShift(DTYPE * a, DTYPE * b, int size, DTYPE scale, DTYPE shift);
 
 
-/* scale and shift all tensor entires p = p * scale + shift (CUDA Kernel) with float16 data type */
+/* scale and shift all tensor entires b = a * scale + shift (CUDA Kernel) with float16 data type */
 __global__ 
 __global__ 
-void KernelScaleAndShift(__half * d, int size, __half scale, __half shift);
+void KernelScaleAndShift(__half * a, __half * b, int size, __half scale, __half shift);
 
 
-/* scale and shift all tensor entires (cuda version) */
+/* scale and shift all tensor entires b = a * scale + shift (cuda version) */
 extern "C" 
 extern "C" 
-void CudaScaleAndShift(XTensor * a, DTYPE scale, DTYPE shift);
+void _CudaScaleAndShift(const XTensor * a, XTensor * b, DTYPE scale, DTYPE shift);
 
 
 #endif // USE_CUDA
 #endif // USE_CUDA
 
 

source/core/math/ScaleAndShift.h → source/tensor/core/math/ScaleAndShift.h

@@ -26,9 +26,18 @@
 
 
 namespace nts{ // namespace nts(NiuTrans.Tensor)
 namespace nts{ // namespace nts(NiuTrans.Tensor)
 
 
-/* scale and shift all tensor entires */
-extern "C" 
-void ScaleAndShift(XTensor * a, DTYPE scale, DTYPE shift);
+#define _Linear   _ScaleAndShift   
+#define _LinearMe _ScaleAndShiftMe
+#define  Linear    ScaleAndShift
+
+/* scale and shift all tensor entires b = a * scale + shift */
+void _ScaleAndShift(const XTensor * a, XTensor * b, DTYPE scale, DTYPE shift = 0);
+
+/* scale and shift all tensor entires on site a = a * scale + shift */
+void _ScaleAndShiftMe(XTensor * a, DTYPE scale, DTYPE shift = 0);
+
+/* scale and shift all tensor entires b = a * scale + shift, and return the result tensor b */
+XTensor ScaleAndShift(const XTensor &a, DTYPE scale, DTYPE shift = 0);
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
 
 

source/core/movement/CopyValues.cpp → source/tensor/core/movement/CopyValues.cpp

@@ -32,7 +32,7 @@ copy s to t
 >> stream - the stream for creating the job pipeline
 >> stream - the stream for creating the job pipeline
 << return - succeeded or not
 << return - succeeded or not
 */
 */
-bool CopyValues(XTensor * s, XTensor * t, XStream * stream)
+bool CopyValues(const XTensor * s, XTensor * t, XStream * stream)
 {
 {
     if (s == NULL || t == NULL)
     if (s == NULL || t == NULL)
         return false;
         return false;
@@ -60,10 +60,10 @@ bool CopyValues(XTensor * s, XTensor * t, XStream * stream)
         memcpy((char*)t->data, (char*)s->data, s->unitSize * s->unitNum);
         memcpy((char*)t->data, (char*)s->data, s->unitSize * s->unitNum);
     }
     }
     else if (s->isSparse && t->isSparse) {
     else if (s->isSparse && t->isSparse) {
-        int d = s->GetNonzeroSize();
+        int d = s->unitNumNonZero;
         t->Resize(s);
         t->Resize(s);
         t->unitNumNonZero = d;
         t->unitNumNonZero = d;
-        memcpy((char*)t->data, (char*)s->data, sizeof(int) + d *(sizeof(int) + sizeof(DTYPE)));
+        memcpy((char*)t->data, (char*)s->data, sizeof(int) + d *(sizeof(int) + t->unitSize));
     }
     }
     else {
     else {
         ShowNTErrors("TODO!");
         ShowNTErrors("TODO!");

source/core/movement/CopyValues.cu → source/tensor/core/movement/CopyValues.cu

@@ -35,7 +35,7 @@ copy a range of elements from a source vector to a target vector
 >> stream - the stream for creating the job pipeline
 >> stream - the stream for creating the job pipeline
 << return - succeed or not
 << return - succeed or not
 */
 */
-bool CudaCopyValues(XTensor * s, XTensor * t, XStream * stream)
+bool CudaCopyValues(const XTensor * s, XTensor * t, XStream * stream)
 {
 {
     if (s == NULL || t == NULL)
     if (s == NULL || t == NULL)
         return false;
         return false;
@@ -67,10 +67,10 @@ bool CudaCopyValues(XTensor * s, XTensor * t, XStream * stream)
     }
     }
     /* sparse -> sparse */
     /* sparse -> sparse */
     else if (s->isSparse && t->isSparse &&
     else if (s->isSparse && t->isSparse &&
-        s->dataType == DEFAULT_DTYPE &&
-        t->dataType == DEFAULT_DTYPE)
+             s->dataType == DEFAULT_DTYPE &&
+             t->dataType == DEFAULT_DTYPE)
     {
     {
-        int num = s->GetNonzeroSize();
+        int num = s->unitNumNonZero;
         int size = sizeof(int) + num * (s->unitSize + sizeof(int));
         int size = sizeof(int) + num * (s->unitSize + sizeof(int));
 
 
         if (stream == NULL)
         if (stream == NULL)

source/core/movement/CopyValues.cuh → source/tensor/core/movement/CopyValues.cuh

@@ -30,7 +30,7 @@ namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
 /* copy all elements from a source matrix to a target matrix */
 /* copy all elements from a source matrix to a target matrix */
 extern "C"
 extern "C"
-bool CudaCopyValues(XTensor * s, XTensor * t, XStream * stream = NULL);
+bool CudaCopyValues(const XTensor * s, XTensor * t, XStream * stream = NULL);
 
 
 #endif // USE_CUDA
 #endif // USE_CUDA
 
 

source/core/movement/CopyValues.h → source/tensor/core/movement/CopyValues.h

@@ -28,7 +28,7 @@ namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
 /* copy s to t */
 /* copy s to t */
 extern "C"
 extern "C"
-bool CopyValues(XTensor * s, XTensor * t, XStream * stream = NULL);
+bool CopyValues(const XTensor * s, XTensor * t, XStream * stream = NULL);
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
 
 

source/core/reduce/ReduceMean.cpp → source/tensor/core/reduce/ReduceMean.cpp

@@ -40,7 +40,7 @@ void ReduceMean(XTensor * input, XTensor * output, int dim)
     int num = input->dimSizeRDI[dimRDI];
     int num = input->dimSizeRDI[dimRDI];
 
 
     ReduceSum(input, output, dim);
     ReduceSum(input, output, dim);
-    ScaleAndShift(output, (DTYPE)1/num, 0);
+    _ScaleAndShiftMe(output, (DTYPE)1/num, 0);
 }
 }
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/core/reduce/ReduceVariance.cpp → source/tensor/core/reduce/ReduceVariance.cpp

@@ -39,7 +39,7 @@ void ReduceVariance(XTensor * input, XTensor * output, int dim, XTensor * mean)
 	int dimRDI = input->order - dim - 1;
 	int dimRDI = input->order - dim - 1;
     int num = input->dimSizeRDI[dimRDI];
     int num = input->dimSizeRDI[dimRDI];
     ReduceSum(input, output, dim, mean, 2.0F);
     ReduceSum(input, output, dim, mean, 2.0F);
-    ScaleAndShift(output, (DTYPE)1 / num, 0);
+    _ScaleAndShiftMe(output, (DTYPE)1 / num, 0);
 }
 }
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)

source/function/Loss.cu → source/tensor/function/Loss.cu

@@ -24,7 +24,7 @@
 #include "../XDevice.h"
 #include "../XDevice.h"
 #include "../core/math/Power.h"
 #include "../core/math/Power.h"
 #include "../core/math/ScaleAndShift.h"
 #include "../core/math/ScaleAndShift.h"
-#include "../core/arithmetic/Log.h"
+#include "../core/math/Log.h"
 #include "../core/arithmetic/Negate.h"
 #include "../core/arithmetic/Negate.h"
 #include "../core/arithmetic/Sum.h"
 #include "../core/arithmetic/Sum.h"
 #include "../core/arithmetic/Multiply.h"
 #include "../core/arithmetic/Multiply.h"
@@ -78,7 +78,7 @@ DTYPE CudaLossCompute(XTensor * gold, XTensor * y, LOSS_FUNCTION_NAME LFName,
         XTensor * diff = NewTensor(gold->order, gold->dimSize, gold->dataType, gold->denseRatio, gold->devID, gold->mem);
         XTensor * diff = NewTensor(gold->order, gold->dimSize, gold->dataType, gold->denseRatio, gold->devID, gold->mem);
         _Sum(gold, y, diff, -1.0F);
         _Sum(gold, y, diff, -1.0F);
         Power(diff, 2.0F);
         Power(diff, 2.0F);
-        ScaleAndShift(diff, 0.5F, 0.0F);
+        _ScaleAndShiftMe(diff, 0.5F, 0.0F);
 
 
         int reduceTimes = diff->order;
         int reduceTimes = diff->order;
         for (int i = 0; i < reduceTimes; i++) {
         for (int i = 0; i < reduceTimes; i++) {
@@ -111,7 +111,7 @@ DTYPE CudaLossCompute(XTensor * gold, XTensor * y, LOSS_FUNCTION_NAME LFName,
         XTensor * diff = NewTensor(y->order, y->dimSize, y->dataType, y->denseRatio, y->devID, y->mem);
         XTensor * diff = NewTensor(y->order, y->dimSize, y->dataType, y->denseRatio, y->devID, y->mem);
         CopyValues(y, diff);
         CopyValues(y, diff);
         Log(diff);
         Log(diff);
-        Multiply(gold, diff, diff);
+        _Multiply(gold, diff, diff);
         Negate(diff);
         Negate(diff);
 
 
         int reduceTimes = diff->order;
         int reduceTimes = diff->order;
@@ -146,10 +146,10 @@ DTYPE CudaLossCompute(XTensor * gold, XTensor * y, LOSS_FUNCTION_NAME LFName,
         XTensor * diff = NewTensor(gold->order, gold->dimSize, gold->dataType, gold->denseRatio, gold->devID, gold->mem);
         XTensor * diff = NewTensor(gold->order, gold->dimSize, gold->dataType, gold->denseRatio, gold->devID, gold->mem);
         XTensor * yOnehot = NewTensor(y->order, y->dimSize, y->dataType, y->denseRatio, y->devID, y->mem);
         XTensor * yOnehot = NewTensor(y->order, y->dimSize, y->dataType, y->denseRatio, y->devID, y->mem);
         CopyValues(y, yOnehot);
         CopyValues(y, yOnehot);
-        Multiply(gold, y, yOnehot);
+        _Multiply(gold, y, yOnehot);
         _Sum(gold, yOnehot, diff, -1.0F);
         _Sum(gold, yOnehot, diff, -1.0F);
         Power(diff, 2.0F);
         Power(diff, 2.0F);
-        ScaleAndShift(diff, 0.5F, 0.0F);
+        _ScaleAndShiftMe(diff, 0.5F, 0.0F);
 
 
         int reduceTimes = diff->order;
         int reduceTimes = diff->order;
         for (int i = 0; i < reduceTimes; i++) {
         for (int i = 0; i < reduceTimes; i++) {

source/function/Softmax.cu → source/tensor/function/Softmax.cu

@@ -248,7 +248,6 @@ void CudaSoftmaxBackward(XTensor * gold, XTensor * y, XTensor * x,
                        "Unknown loss function.");
                        "Unknown loss function.");
 
 
         if(lossName == CROSSENTROPY || lossName == SQUAREDERROR){
         if(lossName == CROSSENTROPY || lossName == SQUAREDERROR){
-            //ShowNTErrors("TODO!");
             _Sum(y, gold, dedx, -1.0F);
             _Sum(y, gold, dedx, -1.0F);
         }
         }
         else if(lossName == ONEHOTERROR){
         else if(lossName == ONEHOTERROR){
@@ -289,7 +288,7 @@ void CudaSoftmaxBackward(XTensor * gold, XTensor * y, XTensor * x,
             beta->data = mem->AllocBuf(mem->devID, beta->unitNum * beta->unitSize);
             beta->data = mem->AllocBuf(mem->devID, beta->unitNum * beta->unitSize);
 
 
             /* \beta = \sum_i (dE/dy_i * y_i) */
             /* \beta = \sum_i (dE/dy_i * y_i) */
-            Multiply(dedy, y, ytmp, 0);
+            _Multiply(dedy, y, ytmp, 0, 0);
             ReduceSum(ytmp, beta, leadDim);
             ReduceSum(ytmp, beta, leadDim);
 
 
             /* ytmp = dE/dy_j - \beta */
             /* ytmp = dE/dy_j - \beta */
@@ -297,7 +296,7 @@ void CudaSoftmaxBackward(XTensor * gold, XTensor * y, XTensor * x,
             _Sum(dedy, ytmp, ytmp, -1.0F);
             _Sum(dedy, ytmp, ytmp, -1.0F);
 
 
             /* dE/ds_j = y_j * ytmp = y_j * (dE/dy_j - \beta) */
             /* dE/ds_j = y_j * ytmp = y_j * (dE/dy_j - \beta) */
-            Multiply(y, ytmp, dedx, 0);
+            _Multiply(y, ytmp, dedx, 0, 0);
 
 
             mem->ReleaseBuf(mem->devID, y->unitNum * y->unitSize);
             mem->ReleaseBuf(mem->devID, y->unitNum * y->unitSize);
             mem->ReleaseBuf(mem->devID, beta->unitNum * beta->unitSize);
             mem->ReleaseBuf(mem->devID, beta->unitNum * beta->unitSize);

source/sample/fnnlm/FNNLM.cpp → source/tensor/sample/fnnlm/FNNLM.cpp

@@ -485,7 +485,7 @@ float GetProb(XTensor &output, XTensor &gold, XTensor * wordProbs)
     InitTensor(&probs, &output);
     InitTensor(&probs, &output);
     
     
     /* probs[i,j] = output[i,j] * gold[i,j] */
     /* probs[i,j] = output[i,j] * gold[i,j] */
-    Multiply(&output, &gold, &probs, 0);
+    _Multiply(&output, &gold, &probs);
 
 
     /* probability of each word */
     /* probability of each word */
     XTensor wprobs;
     XTensor wprobs;

source/test/TCopyIndexed.cpp → source/tensor/test/TCopyIndexed.cpp

@@ -20,7 +20,6 @@
 */
 */
 
 
 #include "TCopyIndexed.h"
 #include "TCopyIndexed.h"
-#include "../xc/Mycode.h"
 
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
@@ -176,7 +175,6 @@ bool TestCopyIndexed2()
     int tgtIndex[2] = {1, 0};
     int tgtIndex[2] = {1, 0};
     int copyNum = 1;
     int copyNum = 1;
 
 
-
     /* CPU test */
     /* CPU test */
     bool cpuTest = true;
     bool cpuTest = true;
 
 

source/tensor/test/TCopyValues.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#include "../XUtility.h"
+#include "TCopyValues.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* case 1: copy tensor s to tensor t */
+bool TestCopyValues1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+
+    DTYPE scaleFactor = 2.0F;
+    DTYPE shiftFactor = 0.5F;
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(sOrder, sDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t->SetZeroAll();
+
+    /* call CopyValues function */
+    CopyValues(s, t);
+
+    /* check results */
+    cpuTest = t->CheckData(s->data, sUnitNum);
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU->SetData(sData, sUnitNum);
+
+    /* call CopyValues function */
+    CopyValues(sGPU, tGPU);
+
+    /* check results */
+    DTYPE * dataGPU = (DTYPE*)sGPU->data;
+    int size = sUnitNum * sGPU->unitSize;
+    char * dataCPU = new char[size];
+    XMemCopy(dataCPU, -1, dataGPU, sGPU->devID, size);
+
+    gpuTest = tGPU->CheckData(dataCPU, sUnitNum);
+
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete sGPU;
+    delete tGPU;
+    delete[] sDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete[] sDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for CopyValues Function */
+bool TestCopyValues()
+{
+    XPRINT(0, stdout, "[TEST CopyValues] copy tensor s to tensor t \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestCopyValues1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/THardTanH.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-20
+*/
+
+#include "../XTensor.h"
+#include "THardTanH.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test HardTanH function 
+y =  1    if x > 1
+     x    if -1 <= x <= 1
+    -1    if x < -1
+*/
+bool TestHardTanH1()
+{
+	/* a tensor of size (2, 3) */
+	int order = 2;
+	int * dimSize = new int[order];
+	dimSize[0] = 2;
+	dimSize[1] = 3;
+
+	int unitNum = 1;
+	for (int i = 0; i < order; i++)
+		unitNum *= dimSize[i];
+
+	DTYPE xData[2][3] = { {0.5F, -1.0F, 2.0F},
+	                      {3.5F, -4.5F, 1.0F} };
+	DTYPE answer[2][3] = { {0.5F, -1.0F, 1.0F},
+	                       {1.0F, -1.0F, 1.0F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * x = NewTensor(order, dimSize);
+	XTensor * y = NewTensor(order, dimSize);
+
+	/* initialize variables */
+	x->SetData(xData, unitNum);
+	y->SetZeroAll();
+
+	/* call hardtanh function */
+	HardTanH(x, y);
+
+	/* check results */
+	cpuTest = y->CheckData(answer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensor */
+	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+	/* Initialize variables */
+	xGPU->SetData(xData, unitNum);
+	yGPU->SetZeroAll();
+
+	/* call hardtanh function */
+	HardTanH(xGPU, yGPU);
+
+	/* check results */
+	gpuTest = yGPU->CheckData(answer, unitNum, 1e-4F);
+
+	/* destroy variables */
+	delete x;
+    delete y;
+    delete xGPU;
+    delete yGPU;
+	delete[] dimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete x;
+    delete y;
+	delete[] dimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/*
+case 2: test backward computation of HardTanH function.
+dE/dx = dE/dy * dy/dx
+hard tanh: y =  1    if x > 1
+                x    if -1 <= x <= 1
+               -1    if x< -1
+
+   and dy/dx =  1    if -1 <= x <= 1
+                0    otherwise
+In this case, lossName=SQUAREDERROR.
+*/
+bool TestHardTanH2()
+{
+	/* a tensor of size (2, 3) */
+	int order = 2;
+	int * dimSize = new int[order];
+	dimSize[0] = 2;
+	dimSize[1] = 3;
+
+	int unitNum = 1;
+	for (int i = 0; i < order; i++)
+		unitNum *= dimSize[i];
+
+	DTYPE xData[2][3] = { {0.5F, -1.0F, 2.0F},
+	                      {3.5F, -4.5F, 1.0F} };
+	DTYPE goldData[2][3] = { {1.0F, 1.0F, 1.0F},
+	                         {1.0F, 1.0F, 1.0F} };
+	DTYPE yAnswer[2][3] = { {0.5F, -1.0F, 1.0F},
+	                        {1.0F, -1.0F, 1.0F} };
+    DTYPE dedyAnswer[2][3] = { {-0.5F, -2.0F, 0.0F},
+	                           {0.0F, -2.0F, 0.0F} };
+	DTYPE dedxAnswer[2][3] = { {-0.5F, -2.0F, 0.0F},
+	                           {0.0F, 0.0F, -0.0F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * x = NewTensor(order, dimSize);
+	XTensor * y = NewTensor(order, dimSize);
+	XTensor * gold = NewTensor(order, dimSize);
+	XTensor * dedy = NewTensor(order, dimSize);
+	XTensor * dedx = NewTensor(order, dimSize);
+
+	/* initialize variables */
+	x->SetData(xData, unitNum);
+	gold->SetData(goldData, unitNum);
+    y->SetZeroAll();
+    dedy->SetZeroAll();
+	dedx->SetZeroAll();
+
+    /* call HardTanH function */
+    HardTanH(x, y);
+
+	/* call HardTanHBackward function */
+	HardTanHBackward(gold, y, x, dedy, dedx, SQUAREDERROR);
+
+	/* check results */
+	cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F) 
+              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensors */
+	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+	/* initialize variables */
+	xGPU->SetData(xData, unitNum);
+	goldGPU->SetData(goldData, unitNum);
+    yGPU->SetZeroAll();
+    dedyGPU->SetZeroAll();
+	dedxGPU->SetZeroAll();
+
+    /* call HardTanH function */
+    HardTanH(xGPU, yGPU);
+
+	/* call hardtanhbackward function */
+	HardTanHBackward(goldGPU, yGPU, xGPU, dedyGPU, dedxGPU, SQUAREDERROR);
+
+	/* check results */
+	gpuTest = y->CheckData(yAnswer, unitNum, 1e-4F) 
+              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+	/* destroy variables */
+    delete x;
+    delete y;
+    delete gold;
+    delete dedx;
+    delete dedy;
+    delete xGPU;
+    delete yGPU;
+    delete goldGPU;
+    delete dedxGPU;
+    delete dedyGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete gold;
+    delete dedx;
+    delete dedy;
+    delete[] dimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for HardTanH Function */
+bool TestHardTanH()
+{
+	XPRINT(0, stdout, "[TEST HARDTANH] test hardtanh and its backward computation \n");
+	bool returnFlag = true, caseFlag = true;
+
+	/* case 1 test */
+	caseFlag = TestHardTanH1();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 1 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 1 passed!\n");
+
+	/* case 2 test */
+	caseFlag = TestHardTanH2();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 2 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 2 passed!\n");
+
+	/* other cases test */
+	/*
+	TODO!!
+	*/
+
+	if (returnFlag) {
+		XPRINT(0, stdout, ">> All Passed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> Failed!\n");
+
+	XPRINT(0, stdout, "\n");
+
+	return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TIdentity.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-29
+*/
+
+#include "../XUtility.h"
+#include "TIdentity.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/*
+case 1: test Identity function.
+Identity function: y = x 
+*/
+bool TestIdentity1()
+{
+    /* a tensor of size (2, 3) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[2][3] = { {0.0F, 1.0F, 2.0F}, 
+                          {0.5F, 0.7F, 1.4F} };
+    DTYPE answer[2][3] = { {0.0F, 1.0F, 2.0F}, 
+                           {0.5F, 0.7F, 1.4F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * x = NewTensor(order, dimSize);
+    XTensor * y = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    x->SetData(xData, unitNum);
+    y->SetZeroAll();
+
+    /* call Identity function */
+    Identity(x, y);
+    
+    /* check result */
+    cpuTest = y->CheckData(answer, unitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    xGPU->SetData(xData, unitNum);
+    yGPU->SetZeroAll();
+
+    /* call Identity function */
+    Identity(xGPU, yGPU);
+    
+    /* check result */
+    gpuTest = yGPU->CheckData(answer, unitNum);
+
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete xGPU;
+    delete yGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: test IdentityBackward function.
+IdentityBackward function: dE/dx = dE/dy * dy/dx = dE/dy
+In this case, lossName=CROSSENTROPY.
+*/
+bool TestIdentity2()
+{
+    /* a tensor of size (2, 3) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 1;
+    dimSize[1] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[3] = {1.0F, 1.0F, 2.0F};
+    DTYPE gData[3] = {0.0F, 0.0F, 1.0F};
+    DTYPE yAnswer[3] = {1.0F, 1.0F, 2.0F};
+    DTYPE dedyAnswer[3] = {0.0F, 0.0F, -0.5F};
+    DTYPE dedxAnswer[3] = {0.0F, 0.0F, -0.5F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * x = NewTensor(order, dimSize);
+    XTensor * y = NewTensor(order, dimSize);
+    XTensor * g = NewTensor(order, dimSize);
+    XTensor * dedy = NewTensor(order, dimSize);
+    XTensor * dedx = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    x->SetData(xData, unitNum);
+    g->SetData(gData, unitNum);
+    y->SetZeroAll();
+    dedx->SetZeroAll();
+    dedy->SetZeroAll();
+
+    /* call Identity function */
+    Identity(x, y);
+
+    /* call IdentityBackward function */
+    IdentityBackward(g, y, x, dedy, dedx, CROSSENTROPY);
+    
+    /* check result */
+    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+        /* create tensors */
+    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * gGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    xGPU->SetData(xData, unitNum);
+    gGPU->SetData(gData, unitNum);
+    yGPU->SetZeroAll();
+    dedxGPU->SetZeroAll();
+    dedyGPU->SetZeroAll();
+
+    /* call Identity function */
+    Identity(xGPU, yGPU);
+
+    /* call IdentityBackward function */
+    IdentityBackward(gGPU, yGPU, xGPU, dedyGPU, dedxGPU, CROSSENTROPY);
+    
+    /* check result */
+    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete g;
+    delete dedx;
+    delete dedy;
+    delete xGPU;
+    delete yGPU;
+    delete gGPU;
+    delete dedxGPU;
+    delete dedyGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete g;
+    delete dedx;
+    delete dedy;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for Identity Function */
+bool TestIdentity()
+{
+    XPRINT(0, stdout, "[TEST Identity] identity function and its backward computation \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestIdentity1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestIdentity2();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TLoss.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#include "../core/math/ScaleAndShift.h"
+#include "TLoss.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test LossCompute function.
+In this case, Loss function name = SQUAREDERROR.
+loss = sum_{i} 0.5*(t_i - y_i)^2, 
+where t_i is the gold standard and y_i is the model output.
+*/
+bool TestLoss1()
+{
+    /* a tensor of size (10, 1) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 10;
+    dimSize[1] = 1;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    DTYPE answer = 5.0F;
+
+    /* create tensors */
+    XTensor * output = NewTensor(order, dimSize);
+    XTensor * gold = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    output->SetZeroAll();
+    gold->SetZeroAll();
+    _ScaleAndShiftMe(output, 1, 1);
+    _ScaleAndShiftMe(gold, 1, 2);
+
+    DTYPE error;
+    error = LossCompute(gold, output, SQUAREDERROR, false, 0, 0, dimSize[0], 0);
+    
+    /* check results */
+    cpuTest = (error == answer);
+    
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * outputGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    outputGPU->SetZeroAll();
+    goldGPU->SetZeroAll();
+    _ScaleAndShiftMe(outputGPU, 1, 1);
+    _ScaleAndShiftMe(goldGPU, 1, 2);
+
+    /* call LossCompute function */
+    error = LossCompute(goldGPU, outputGPU, SQUAREDERROR, false, 0, 0, dimSize[0], 0);
+    
+    /* check results */
+    gpuTest = (error == answer);
+
+    /* destroy variables */
+    delete output;
+    delete gold;
+    delete outputGPU;
+    delete goldGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete output;
+    delete gold;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: test LossCompute function.
+In this case, Loss function name = CROSSENTROPY.
+loss = sum_{i} (-t_i * log(y_i))
+where t_i is the gold standard and y_i is the model output.
+*/
+bool TestLoss2()
+{
+    /* a tensor of size (10, 1) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 10;
+    dimSize[1] = 1;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    DTYPE answer = 0.0F;
+
+    /* create tensors */
+    XTensor * output = NewTensor(order, dimSize);
+    XTensor * gold = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    output->SetZeroAll();
+    gold->SetZeroAll();
+    _ScaleAndShiftMe(output, 1, 1);
+    _ScaleAndShiftMe(gold, 1, 2);
+
+    DTYPE error;
+    error = LossCompute(gold, output, CROSSENTROPY, false, 0, 0, dimSize[0], 0);
+    
+    /* check results */
+    cpuTest = (error == answer);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * outputGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    outputGPU->SetZeroAll();
+    goldGPU->SetZeroAll();
+    _ScaleAndShiftMe(outputGPU, 1, 1);
+    _ScaleAndShiftMe(goldGPU, 1, 2);
+
+    /* call LossCompute function */
+    error = LossCompute(goldGPU, outputGPU, CROSSENTROPY, false, 0, 0, dimSize[0], 0);
+    
+    /* check results */
+    gpuTest = (error == answer);
+
+    /* destroy variables */
+    delete output;
+    delete gold;
+    delete outputGPU;
+    delete goldGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete output;
+    delete gold;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 3: test LossCompute function.
+In this case, Loss function name = ONEHOTERROR.
+loss = sum_{i} e_i
+where e_i = 0.5*(t_i - y_i)^2 if t_i = 1, e_i = 0 otherwise.
+*/
+bool TestLoss3()
+{
+    /* a tensor of size (10, 1) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 5;
+    dimSize[1] = 1;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+    DTYPE outputData[5][1] = { {0.5F},
+                               {0.5F},
+                               {0.5F},
+                               {0.5F},
+                               {0.5F} };
+    DTYPE goldData[5][1] = { {1.0F},
+                             {1.0F},
+                             {0.0F},
+                             {0.0F},
+                             {0.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    DTYPE answer = 0.25F;
+
+    /* create tensors */
+    XTensor * output = NewTensor(order, dimSize);
+    XTensor * gold = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    output->SetData(outputData, unitNum);
+    gold->SetData(goldData, unitNum);
+
+    DTYPE error;
+    error = LossCompute(gold, output, ONEHOTERROR, false, 0, 0, dimSize[0], 0);
+    
+    /* check results */
+    cpuTest = (error == answer);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * outputGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    outputGPU->SetData(outputData, unitNum);
+    goldGPU->SetData(goldData, unitNum);
+
+    /* call LossCompute function */
+    error = LossCompute(goldGPU, outputGPU, ONEHOTERROR, false, 0, 0, dimSize[0], 0);
+    
+    /* check results */
+    gpuTest = (error == answer);
+
+    /* destroy variables */
+    delete output;
+    delete gold;
+    delete outputGPU;
+    delete goldGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete output;
+    delete gold;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Loss Function */
+bool TestLoss()
+{
+    XPRINT(0, stdout, "[TEST Loss] compute the loss \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestLoss1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestLoss2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+        
+    caseFlag = TestLoss3();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 3 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 3 passed!\n");
+
+    ///* other cases test */
+    ///*
+    //TODO!!
+    //*/
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TMatrixMULBatchedCPU.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
+*/
+
+#include "TMatrixMULBatchedCPU.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: matrix multiplication in batch mode (CPU code). 
+In this case, aList=2*(2, 3), bList=2*(3, 2) -> c=2*(2, 2), transposedA=X_NOTRANS, transposedB=X_NOTRANS.
+*/
+bool TestMatrixMulBatchedCPU1()
+{
+    /* create list */
+    XList * aList = new XList();
+    XList * bList = new XList();
+    XList * cList = new XList();
+
+    /* a source tensor of size (2, 3) */
+    int aOrder = 2;
+    int * aDimSize = new int[aOrder];
+    aDimSize[0] = 2;
+    aDimSize[1] = 3;
+
+    int aUnitNum = 1;
+    for (int i = 0; i < aOrder; i++)
+        aUnitNum *= aDimSize[i];
+
+    /* a source tensor of size (3, 2) */
+    int bOrder = 2;
+    int * bDimSize = new int[bOrder];
+    bDimSize[0] = 3;
+    bDimSize[1] = 2;
+
+    int bUnitNum = 1;
+    for (int i = 0; i < bOrder; i++)
+        bUnitNum *= bDimSize[i];
+
+    /* a target tensor of size (2, 2) */
+    int cOrder = 2;
+    int * cDimSize = new int[cOrder];
+    cDimSize[0] = 2;
+    cDimSize[1] = 2;
+
+    int cUnitNum = 1;
+    for (int i = 0; i < cOrder; i++)
+        cUnitNum *= cDimSize[i];
+
+    DTYPE aData1[2][3] = { {1.0F, 2.0F, 3.0F},
+                           {-4.0F, 5.0F, 6.0F} };
+    DTYPE aData2[2][3] = { {1.0F, -2.0F, -3.0F},
+                           {-4.0F, 3.0F, 2.0F} };
+    DTYPE bData1[3][2] = { {0.0F, -1.0F},
+                           {1.0F, 2.0F}, 
+                           {2.0F, 1.0F} };
+    DTYPE bData2[3][2] = { {0.0F, 1.0F},
+                           {3.0F, 2.0F}, 
+                           {2.0F, 1.0F} };
+    DTYPE answer1[2][2] = { {8.0F, 6.0F}, 
+                            {17.0F, 20.0F} };
+    DTYPE answer2[2][2] = { {-12.0F, -6.0F}, 
+                            {13.0F, 4.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a1 = NewTensor(aOrder, aDimSize);
+    XTensor * a2 = NewTensor(aOrder, aDimSize);
+    XTensor * b1 = NewTensor(bOrder, bDimSize);
+    XTensor * b2 = NewTensor(bOrder, bDimSize);
+    XTensor * c1 = NewTensor(cOrder, cDimSize);
+    XTensor * c2 = NewTensor(cOrder, cDimSize);
+
+    /* initialize variables */
+    a1->SetData(aData1, aUnitNum);
+    a2->SetData(aData2, aUnitNum);
+    b1->SetData(bData1, aUnitNum);
+    b2->SetData(bData2, aUnitNum);
+    c1->SetZeroAll();
+    c2->SetZeroAll();
+
+    /* add tensors to list */
+    aList->Add(a1);
+    aList->Add(a2);
+    bList->Add(b1);
+    bList->Add(b2);
+    cList->Add(c1);
+    cList->Add(c2);
+
+    /* call MatrixMULBatchedCPU function */
+    MatrixMULBatchedCPU(aList, X_NOTRANS, bList, X_NOTRANS, cList);
+
+    /* check results */
+    cpuTest = c1->CheckData(answer1, cUnitNum) && c2->CheckData(answer2, cUnitNum);
+    
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * aGPU1 = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * aGPU2 = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU1 = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU2 = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * cGPU1 = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * cGPU2 = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    aGPU1->SetData(aData1, aUnitNum);
+    aGPU2->SetData(aData2, aUnitNum);
+    bGPU1->SetData(bData1, aUnitNum);
+    bGPU2->SetData(bData2, aUnitNum);
+    cGPU1->SetZeroAll();
+    cGPU2->SetZeroAll();
+    
+    /* clear list */
+    aList->Clear();
+    bList->Clear();
+    cList->Clear();
+
+    /* add tensors to list */
+    aList->Add(aGPU1);
+    aList->Add(aGPU2);
+    bList->Add(bGPU1);
+    bList->Add(bGPU2);
+    cList->Add(cGPU1);
+    cList->Add(cGPU2);
+
+    /* call MatrixMULBatchedCPU function */
+    MatrixMULBatchedCPU(aList, X_NOTRANS, bList, X_NOTRANS, cList);
+
+    /* check results */
+    gpuTest = cGPU1->CheckData(answer1, cUnitNum) && gpuTest;
+    gpuTest = cGPU2->CheckData(answer2, cUnitNum) && gpuTest;
+
+    /* destroy variables */
+    delete a1;
+    delete a2;
+    delete b1;
+    delete b2;
+    delete c1;
+    delete c2;
+    delete aGPU1;
+    delete aGPU2;
+    delete bGPU1;
+    delete bGPU2;
+    delete cGPU1;
+    delete cGPU2;
+    delete[] aDimSize;
+    delete[] bDimSize;
+    delete[] cDimSize;
+    
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a1;
+    delete a2;
+    delete b1;
+    delete b2;
+    delete c1;
+    delete c2;
+    delete[] aDimSize;
+    delete[] bDimSize;
+    delete[] cDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for MatrixMulBatchedCPU Function */
+extern "C"
+bool TestMatrixMulBatchedCPU()
+{
+    XPRINT(0, stdout, "[TEST MATRIXMULBATCHEDCPU] matrix multiplication in batch mode (CPU code) \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestMatrixMulBatchedCPU1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TMatrixMul2D.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
+*/
+
+#include "TMatrixMul2D.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: matrix multiplication (for 2d tensors). 
+In this case, a=(2, 3), b=(3, 2) -> c=(2, 2), 
+transposedA=X_NOTRANS, transposedB=X_NOTRANS.
+*/
+bool TestMatrixMul2D1()
+{
+    /* a source tensor of size (2, 3) */
+    int sOrder1 = 2;
+    int * sDimSize1 = new int[sOrder1];
+    sDimSize1[0] = 2;
+    sDimSize1[1] = 3;
+
+    int sUnitNum1 = 1;
+    for (int i = 0; i < sOrder1; i++)
+        sUnitNum1 *= sDimSize1[i];
+
+    /* a source tensor of size (3, 2) */
+    int sOrder2 = 2;
+    int * sDimSize2 = new int[sOrder2];
+    sDimSize2[0] = 3;
+    sDimSize2[1] = 2;
+
+    int sUnitNum2 = 1;
+    for (int i = 0; i < sOrder2; i++)
+        sUnitNum2 *= sDimSize2[i];
+
+    /* a target tensor of size (2, 2) */
+    int tOrder = 2;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData1[2][3] = { {1.0F, 2.0F, 3.0F},
+                           {-4.0F, 5.0F, 6.0F} };
+    DTYPE sData2[3][2] = { {0.0F, -1.0F},
+                           {1.0F, 2.0F}, 
+                           {2.0F, 1.0F} };
+    DTYPE answer[2][2] = { {8.0F, 6.0F}, 
+                           {17.0F, 20.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
+    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s1->SetData(sData1, sUnitNum1);
+    s2->SetData(sData2, sUnitNum2);
+    t->SetZeroAll();
+
+    /* call MatrixMul2D function */
+    MatrixMul2D(s1, X_NOTRANS, s2, X_NOTRANS, t);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    sGPU1->SetData(sData1, sUnitNum1);
+    sGPU2->SetData(sData2, sUnitNum2);
+    tGPU->SetZeroAll();
+
+    /* call MatrixMul2D function */
+    MatrixMul2D(sGPU1, X_NOTRANS, sGPU2, X_NOTRANS, tGPU);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete sGPU1;
+    delete sGPU2;
+    delete tGPU;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: matrix multiplication (for 2d tensors).
+In this case, a=(3, 2), b=(3, 2) -> c=(2, 2), 
+transposedA=X_TRANS, transposedB=X_NOTRANS.
+*/
+bool TestMatrixMul2D2()
+{
+    /* a source tensor of size (3, 2) */
+    int sOrder1 = 2;
+    int * sDimSize1 = new int[sOrder1];
+    sDimSize1[0] = 3;
+    sDimSize1[1] = 2;
+
+    int sUnitNum1 = 1;
+    for (int i = 0; i < sOrder1; i++)
+        sUnitNum1 *= sDimSize1[i];
+
+    /* a source tensor of size (3, 2) */
+    int sOrder2 = 2;
+    int * sDimSize2 = new int[sOrder2];
+    sDimSize2[0] = 3;
+    sDimSize2[1] = 2;
+
+    int sUnitNum2 = 1;
+    for (int i = 0; i < sOrder2; i++)
+        sUnitNum2 *= sDimSize2[i];
+
+    /* a target tensor of size (2, 2) */
+    int tOrder = 2;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData1[3][2] = { {1.0F, -4.0F},
+                           {2.0F, 5.0F},
+                           {3.0F, 6.0F} };
+    DTYPE sData2[3][2] = { {0.0F, -1.0F},
+                           {1.0F, 2.0F},
+                           {2.0F, 1.0F} };
+    DTYPE answer[2][2] = { {8.0F, 6.0F},
+                           {17.0F, 20.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
+    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s1->SetData(sData1, sUnitNum1);
+    s2->SetData(sData2, sUnitNum2);
+    t->SetZeroAll();
+
+    /* call MatrixMul2D function */
+    MatrixMul2D(s1, X_TRANS, s2, X_NOTRANS, t);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    sGPU1->SetData(sData1, sUnitNum1);
+    sGPU2->SetData(sData2, sUnitNum2);
+    tGPU->SetZeroAll();
+
+    /* call MatrixMul2D function */
+    MatrixMul2D(sGPU1, X_TRANS, sGPU2, X_NOTRANS, tGPU);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete sGPU1;
+    delete sGPU2;
+    delete tGPU;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for MatrixMul2D Function */
+extern "C"
+bool TestMatrixMul2D()
+{
+    XPRINT(0, stdout, "[TEST MATRIXMUL2D] matrix multiplication (for 2d tensors) \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestMatrixMul2D1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestMatrixMul2D2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TMatrixMul2DParallel.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#include "TMatrixMul2DParallel.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/*
+case 1: matrix multiplication (for 2d tensors) with multi-threading. 
+In this case, a=(2, 3), b=(3, 2) -> c=(2, 2), 
+transposedA=X_NOTRANS, transposedB=X_NOTRANS.
+*/
+bool TestMatrixMul2DParallel1()
+{
+    /* a source tensor of size (2, 3) */
+    int sOrder1 = 2;
+    int * sDimSize1 = new int[sOrder1];
+    sDimSize1[0] = 2;
+    sDimSize1[1] = 3;
+
+    int sUnitNum1 = 1;
+    for (int i = 0; i < sOrder1; i++)
+        sUnitNum1 *= sDimSize1[i];
+
+    /* a source tensor of size (3, 2) */
+    int sOrder2 = 2;
+    int * sDimSize2 = new int[sOrder2];
+    sDimSize2[0] = 3;
+    sDimSize2[1] = 2;
+
+    int sUnitNum2 = 1;
+    for (int i = 0; i < sOrder2; i++)
+        sUnitNum2 *= sDimSize2[i];
+
+    /* a target tensor of size (2, 2) */
+    int tOrder = 2;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData1[2][3] = { {1.0F, 2.0F, 3.0F},
+                           {-4.0F, 5.0F, 6.0F} };
+    DTYPE sData2[3][2] = { {0.0F, -1.0F},
+                           {1.0F, 2.0F}, 
+                           {2.0F, 1.0F} };
+    DTYPE answer[2][2] = { {8.0F, 6.0F}, 
+                           {17.0F, 20.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
+    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s1->SetData(sData1, sUnitNum1);
+    s2->SetData(sData2, sUnitNum2);
+    t->SetZeroAll();
+
+    /* call MatrixMul2DParallel function */
+    MatrixMul2DParallel(s1, X_NOTRANS, s2, X_NOTRANS, t);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest;
+}
+
+/* 
+case 2: matrix multiplication (for 2d tensors) with multi-threading.
+In this case, a=(3, 2), b=(3, 2) -> c=(2, 2), 
+transposedA=X_TRANS, transposedB=X_NOTRANS.
+*/
+bool TestMatrixMul2DParallel2()
+{
+    /* a source tensor of size (3, 2) */
+    int sOrder1 = 2;
+    int * sDimSize1 = new int[sOrder1];
+    sDimSize1[0] = 3;
+    sDimSize1[1] = 2;
+
+    int sUnitNum1 = 1;
+    for (int i = 0; i < sOrder1; i++)
+        sUnitNum1 *= sDimSize1[i];
+
+    /* a source tensor of size (3, 2) */
+    int sOrder2 = 2;
+    int * sDimSize2 = new int[sOrder2];
+    sDimSize2[0] = 3;
+    sDimSize2[1] = 2;
+
+    int sUnitNum2 = 1;
+    for (int i = 0; i < sOrder2; i++)
+        sUnitNum2 *= sDimSize2[i];
+
+    /* a target tensor of size (2, 2) */
+    int tOrder = 2;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData1[3][2] = { {1.0F, -4.0F},
+                           {2.0F, 5.0F},
+                           {3.0F, 6.0F} };
+    DTYPE sData2[3][2] = { {0.0F, -1.0F},
+                           {1.0F, 2.0F},
+                           {2.0F, 1.0F} };
+    DTYPE answer[2][2] = { {8.0F, 6.0F},
+                           {17.0F, 20.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
+    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s1->SetData(sData1, sUnitNum1);
+    s2->SetData(sData2, sUnitNum2);
+    t->SetZeroAll();
+
+    /* call MatrixMul2DParallel function */
+    MatrixMul2DParallel(s1, X_TRANS, s2, X_NOTRANS, t);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest;
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for MatrixMul2DParallel Function */
+bool TestMatrixMul2DParallel()
+{
+    XPRINT(0, stdout, "[TEST MatrixMul2DParallel] matrix multiplication (for 2d tensors) with multi-threading \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestMatrixMul2DParallel1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestMatrixMul2DParallel2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TMatrixMul2DParallel.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#ifndef __TEST_MATRIXMUL2DPARALLEL_H__
+#define __TEST_MATRIXMUL2DPARALLEL_H__
+
+#include "../core/arithmetic/MatrixMul2DParallel.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for MatrixMul2DParallel Function */
+extern "C"
+bool TestMatrixMul2DParallel();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_MATRIXMUL2DPARALLEL_H__

source/tensor/test/TMatrixMulBatched.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
+*/
+
+#include "../XTensor.h"
+#include "TMatrixMulBatched.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: matrix multiplication of the two tensors. 
+In this case, a=(2, 3), b=(2, 3) -> c=(2, 2), transposedA=X_NOTRANS, transposedB=X_NOTRANS.
+*/
+bool TestMatrixMulBatched1()
+{
+    /* a source tensor of size (2, 3) */
+    int sOrder1 = 2;
+    int * sDimSize1 = new int[sOrder1];
+    sDimSize1[0] = 2;
+    sDimSize1[1] = 3;
+
+    int sUnitNum1 = 1;
+    for (int i = 0; i < sOrder1; i++)
+        sUnitNum1 *= sDimSize1[i];
+
+    /* a source tensor of size (3, 2) */
+    int sOrder2 = 2;
+    int * sDimSize2 = new int[sOrder2];
+    sDimSize2[0] = 3;
+    sDimSize2[1] = 2;
+
+    int sUnitNum2 = 1;
+    for (int i = 0; i < sOrder2; i++)
+        sUnitNum2 *= sDimSize2[i];
+
+    /* a target tensor of size (2, 2) */
+    int tOrder = 2;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData1[2][3] = { {1.0F, 2.0F, 3.0F},
+                           {-4.0F, 5.0F, 6.0F} };
+    DTYPE sData2[3][2] = { {0.0F, -1.0F},
+                           {1.0F, 2.0F}, 
+                           {2.0F, 1.0F} };
+    DTYPE answer[2][2] = { {8.0F, 6.0F}, 
+                           {17.0F, 20.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
+    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s1->SetData(sData1, sUnitNum1);
+    s2->SetData(sData2, sUnitNum2);
+    t->SetZeroAll();
+
+    /* call MatrixMulBatched function */
+    MatrixMulBatched(s1, X_NOTRANS, s2, X_NOTRANS, t);
+    
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+    
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    sGPU1->SetData(sData1, sUnitNum1);
+    sGPU2->SetData(sData2, sUnitNum2);
+    tGPU->SetZeroAll();
+    
+    /* call MatrixMulBatched function */
+    MatrixMulBatched(sGPU1, X_NOTRANS, sGPU2, X_NOTRANS, tGPU);
+    
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+    
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete sGPU1;
+    delete sGPU2;
+    delete tGPU;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/*
+case 2: matrix multiplication of the two tensors. 
+In this case, a=(2, 2, 3), b=(2, 3, 2) -> c=(2, 2, 2), transposedA=X_NOTRANS, transposedB=X_NOTRANS.
+*/
+bool TestMatrixMulBatched2()
+{
+    /* a source tensor of size (2, 2, 3) */
+    int sOrder1 = 3;
+    int * sDimSize1 = new int[sOrder1];
+    sDimSize1[0] = 2;
+    sDimSize1[1] = 2;
+    sDimSize1[2] = 3;
+
+    int sUnitNum1 = 1;
+    for (int i = 0; i < sOrder1; i++)
+        sUnitNum1 *= sDimSize1[i];
+
+    /* a source tensor of size (2, 3, 2) */
+    int sOrder2 = 3;
+    int * sDimSize2 = new int[sOrder2];
+    sDimSize2[0] = 2;
+    sDimSize2[1] = 3;
+    sDimSize2[2] = 2;
+
+    int sUnitNum2 = 1;
+    for (int i = 0; i < sOrder2; i++)
+        sUnitNum2 *= sDimSize2[i];
+
+    /* a target tensor of size (2, 2, 2) */
+    int tOrder = 3;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+    tDimSize[2] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData1[2][2][3] = { { {0.0F, -1.0F, 2.0F},
+                                {2.0F, 1.0F, 3.0F} },
+                              { {1.0F, 2.0F, 4.0F}, 
+                                {3.0F, 1.0F, 2.0F} } };
+    DTYPE sData2[2][3][2] = { { {1.0F, 2.0F},
+                                {-4.0F, 3.0F},
+                                {2.0F, 6.0F} },
+                              { {1.0F, 2.0F},
+                                {3.0F, 4.0F},
+                                {5.0F, 6.0F} } };
+    DTYPE answer[2][2][2] = { { {8.0F, 9.0F}, 
+                                {4.0F, 25.0F} },
+                              { {27.0F, 34.0F}, 
+                                {16.0F, 22.0F} } };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
+    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s1->SetData(sData1, sUnitNum1);
+    s2->SetData(sData2, sUnitNum2);
+    t->SetZeroAll();
+
+    /* call MatrixMulBatched function */
+    MatrixMulBatched(s1, X_NOTRANS, s2, X_NOTRANS, t);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+    
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    sGPU1->SetData(sData1, sUnitNum1);
+    sGPU2->SetData(sData2, sUnitNum2);
+    tGPU->SetZeroAll();
+
+    /* call MatrixMulBatched function */
+    MatrixMulBatched(sGPU1, X_NOTRANS, sGPU2, X_NOTRANS, tGPU);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete sGPU1;
+    delete sGPU2;
+    delete tGPU;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s1;
+    delete s2;
+    delete t;
+    delete[] sDimSize1;
+    delete[] sDimSize2;
+    delete[] tDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for TestMatrixMulBatched Function */
+bool TestMatrixMulBatched()
+{
+    XPRINT(0, stdout, "[TEST MATRIXMULBATCHED] matrix multiplication of the two tensors \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestMatrixMulBatched1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestMatrixMulBatched2();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TMatrixMulBatched.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
+*/
+
+#ifndef __TEST_MATRIXMULBATCHED_H__
+#define __TEST_MATRIXMULBATCHED_H__
+
+#include "../core/arithmetic/MatrixMulBatched.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for MatrixMulBatched Function */
+extern "C"
+bool TestMatrixMulBatched();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_MATRIXMULBATCHED_H__

source/tensor/test/TMerge.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-13
+*/
+
+#ifndef __TEST_MERGE_H__
+#define __TEST_MERGE_H__
+
+#include "../core/shape/Merge.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Merge Function */
+extern "C"
+bool TestMerge();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_MERGE_H__

source/test/TMultiply.cpp → source/tensor/test/TMultiply.cpp

@@ -81,7 +81,7 @@ bool TestMultiply1()
 	t->SetZeroAll();
 	t->SetZeroAll();
 
 
 	/* call MultiplyElementWise function */
 	/* call MultiplyElementWise function */
-	Multiply(s1, s2, t, 0);
+	_Multiply(s1, s2, t, 0, 0);
 
 
 	/* check results */
 	/* check results */
 	cpuTest = t->CheckData(answer, tUnitNum);
 	cpuTest = t->CheckData(answer, tUnitNum);
@@ -101,7 +101,7 @@ bool TestMultiply1()
 	tGPU->SetZeroAll();
 	tGPU->SetZeroAll();
 
 
 	/* call MultiplyElementWise function */
 	/* call MultiplyElementWise function */
-	Multiply(sGPU1, sGPU2, tGPU, 0);
+	_Multiply(sGPU1, sGPU2, tGPU, 0, 0);
 
 
 	/* check results */
 	/* check results */
 	gpuTest = tGPU->CheckData(answer, tUnitNum);
 	gpuTest = tGPU->CheckData(answer, tUnitNum);
@@ -189,7 +189,7 @@ bool TestMultiply2()
 	t->SetZeroAll();
 	t->SetZeroAll();
 
 
 	/* call MultiplyElementWise function */
 	/* call MultiplyElementWise function */
-	Multiply(s1, s2, t, 0);
+	_Multiply(s1, s2, t, 0, 0);
 
 
 	/* check results */
 	/* check results */
 	cpuTest = t->CheckData(answer, tUnitNum);
 	cpuTest = t->CheckData(answer, tUnitNum);
@@ -209,7 +209,7 @@ bool TestMultiply2()
 	tGPU->SetZeroAll();
 	tGPU->SetZeroAll();
 
 
 	/* call MultiplyElementWise function */
 	/* call MultiplyElementWise function */
-	Multiply(sGPU1, sGPU2, tGPU, 0);
+	_Multiply(sGPU1, sGPU2, tGPU, 0, 0);
 
 
 	/* check results */
 	/* check results */
 	gpuTest = tGPU->CheckData(answer, tUnitNum);
 	gpuTest = tGPU->CheckData(answer, tUnitNum);
@@ -296,7 +296,7 @@ bool TestMultiply3()
 	t->SetZeroAll();
 	t->SetZeroAll();
 
 
 	/* call MultiplyElementWise function */
 	/* call MultiplyElementWise function */
-	Multiply(s1, s2, t, 1);
+	_Multiply(s1, s2, t, 0, 1);
 
 
 	/* check results */
 	/* check results */
 	cpuTest = t->CheckData(answer, tUnitNum);
 	cpuTest = t->CheckData(answer, tUnitNum);
@@ -316,7 +316,7 @@ bool TestMultiply3()
 	tGPU->SetZeroAll();
 	tGPU->SetZeroAll();
 
 
 	/* call MultiplyElementWise function */
 	/* call MultiplyElementWise function */
-	Multiply(sGPU1, sGPU2, tGPU, 1);
+	_Multiply(sGPU1, sGPU2, tGPU, 0, 1);
 
 
 	/* check results */
 	/* check results */
 	gpuTest = tGPU->CheckData(answer, tUnitNum);
 	gpuTest = tGPU->CheckData(answer, tUnitNum);

source/tensor/test/TMultiply.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-15
+*/
+
+#ifndef __TEST_MULTIPLYELEMENTWISE_H__
+#define __TEST_MULTIPLYELEMENTWISE_H__
+
+#include "../core/arithmetic/Multiply.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for MultiplyElementWise Function */
+extern "C"
+bool TestMultiply();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_MULTIPLYELEMENTWISE_H__

source/tensor/test/TNegate.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
+*/
+
+#include "TNegate.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* case 1: set every entry to its minus value */
+bool TestNegate1()
+{
+	/* a tensor of size (3, 2) */
+	int aOrder = 2;
+	int * aDimSize = new int[aOrder];
+	aDimSize[0] = 3;
+	aDimSize[1] = 2;
+
+	int aUnitNum = 1;
+	for (int i = 0; i < aOrder; i++)
+		aUnitNum *= aDimSize[i];
+
+	DTYPE aData[3][2] = { {1.0F, -2.0F}, 
+	                      {-3.0F, 4.0F},
+	                      {5.0F, -6.0F} };
+	DTYPE answer[3][2] = { {-1.0F, 2.0F},
+	                       {3.0F, -4.0F},
+	                       {-5.0F, 6.0F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * a = NewTensor(aOrder, aDimSize);
+
+	/* initialize variables */
+	a->SetData(aData, aUnitNum);
+
+	/* call Negate function */
+	Negate(a);
+
+	/* check results */
+	cpuTest = a->CheckData(answer, aUnitNum);
+    
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensor */
+	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+
+	/* Initialize variables */
+	aGPU->SetData(aData, aUnitNum);
+
+	/* call Negate function */
+	Negate(aGPU);
+
+	/* check results */
+	gpuTest = aGPU->CheckData(answer, aUnitNum);
+
+	/* destroy variables */
+	delete a;
+    delete aGPU;
+	delete[] aDimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete a;
+	delete[] aDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* case 2: set every entry to its minus value */
+bool TestNegate2()
+{
+	/* a tensor of size (3, 2) */
+	int aOrder = 2;
+	int * aDimSize = new int[aOrder];
+	aDimSize[0] = 3;
+	aDimSize[1] = 2;
+
+	int aUnitNum = 1;
+	for (int i = 0; i < aOrder; i++)
+		aUnitNum *= aDimSize[i];
+
+	DTYPE aData[3][2] = { {0.0F, 0.0F},
+	                      {0.0F, 0.0F},
+	                      {0.0F, 0.0F} };
+	DTYPE answer[3][2] = { {-0.0F, -0.0F},
+	                       {-0.0F, -0.0F},
+	                       {-0.0F, -0.0F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * a = NewTensor(aOrder, aDimSize);
+
+	/* initialize variables */
+	a->SetData(aData, aUnitNum);
+
+	/* call Negate function */
+	Negate(a);
+
+	/* check results */
+	cpuTest = a->CheckData(answer, aUnitNum);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensor */
+	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+
+	/* Initialize variables */
+	aGPU->SetData(aData, aUnitNum);
+
+	/* call Negate function */
+	Negate(aGPU);
+
+	/* check results */
+	gpuTest = aGPU->CheckData(answer, aUnitNum);
+
+	/* destroy variables */
+	delete a;
+    delete aGPU;
+	delete[] aDimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete a;
+	delete[] aDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Negate Function */
+bool TestNegate()
+{
+	XPRINT(0, stdout, "[TEST NEGATE] set every entry to its minus value \n");
+	bool returnFlag = true, caseFlag = true;
+
+	/* case 1 test */
+	caseFlag = TestNegate1();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 1 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 1 passed!\n");
+
+	/* case 2 test */
+	caseFlag = TestNegate2();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 2 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 2 passed!\n");
+
+	/* other cases test */
+	/*
+	TODO!!
+	*/
+
+	if (returnFlag) {
+		XPRINT(0, stdout, ">> All Passed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> Failed!\n");
+
+	XPRINT(0, stdout, "\n");
+
+	return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/test/TMatrixMulBatched.h → source/tensor/test/TNegate.h

@@ -16,19 +16,19 @@
 */
 */
 
 
 /*
 /*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
 */
 */
 
 
-#ifndef __TEST_MATRIXMULBATCHED_H__
-#define __TEST_MATRIXMULBATCHED_H__
+#ifndef __TEST_NEGATE_H__
+#define __TEST_NEGATE_H__
 
 
-#include "../core/arithmetic/MatrixMulBatched.h"
+#include "../core/arithmetic/Negate.h"
 
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
-/* test for MatrixMulBatched Function */
+/* test for Negate Function */
 extern "C"
 extern "C"
-bool TestMatrixMulBatched();
+bool TestNegate();
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_MATRIXMULBATCHED_H__
+#endif // __TEST_NEGATE_H__

source/tensor/test/TNormalize.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-20
+*/
+
+#include "TNormalize.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/*
+case 1: normalized the data with normal distribution 
+For an input x, y = a * (x-mean)/sqrt(variance+\epsilon) + b.
+where a and b are the scalar and bias respectively, 
+and \epsilon is the adjustment parameter.
+*/
+bool TestNormalize1()
+{
+	/* a source tensor of size (2, 3) */
+	int sOrder = 2;
+	int * sDimSize = new int[sOrder];
+	sDimSize[0] = 2;
+	sDimSize[1] = 3;
+
+	int sUnitNum = 1;
+	for (int i = 0; i < sOrder; i++)
+		sUnitNum *= sDimSize[i];
+
+	/* a target tensor of size (2, 3) */
+	int tOrder = 2;
+	int * tDimSize = new int[tOrder];
+	tDimSize[0] = 2;
+	tDimSize[1] = 3;
+
+	int tUnitNum = 1;
+	for (int i = 0; i < tOrder; i++)
+		tUnitNum *= tDimSize[i];
+
+	/* a mean tensor of size (3) */
+	int meanOrder = 1;
+	int * meanDimSize = new int[meanOrder];
+	meanDimSize[0] = 3;
+
+	int meanUnitNum = 1;
+	for (int i = 0; i < meanOrder; i++)
+		meanUnitNum *= meanDimSize[i];
+
+	/* a variance tensor of size (3) */
+	int varOrder = 1;
+	int * varDimSize = new int[varOrder];
+	varDimSize[0] = 3;
+
+	int varUnitNum = 1;
+	for (int i = 0; i < varOrder; i++)
+		varUnitNum *= varDimSize[i];
+
+	/* a scalar tensor of size (2, 3) */
+	int aOrder = 2;
+	int * aDimSize = new int[aOrder];
+	aDimSize[0] = 2;
+	aDimSize[1] = 3;
+
+	int aUnitNum = 1;
+	for (int i = 0; i < aOrder; i++)
+		aUnitNum *= aDimSize[i];
+
+	/* a bias tensor of size (2, 3) */
+	int bOrder = 2;
+	int * bDimSize = new int[bOrder];
+	bDimSize[0] = 2;
+	bDimSize[1] = 3;
+
+	int bUnitNum = 1;
+	for (int i = 0; i < bOrder; i++)
+		bUnitNum *= bDimSize[i];
+
+	DTYPE sData[2][3] = { {1.0F, 2.0F, 3.0F},
+	                      {1.5F, 2.5F, 3.5F} };
+	DTYPE meanData[3] = {1.0F, 1.5F, 2.0F};
+	DTYPE varData[3] = {1.0F, 1.0F, 4.0F};
+    DTYPE aData[2][3] = { {1.0F, 1.0F, 1.0F},
+	                      {1.0F, 1.0F, 1.0F} };
+	DTYPE answer[2][3] = { {0.0F, 0.5F, 0.5F},
+	                       {0.5F, 1.0F, 0.75F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * s = NewTensor(sOrder, sDimSize);
+	XTensor * t = NewTensor(tOrder, tDimSize);
+	XTensor * mean = NewTensor(meanOrder, meanDimSize);
+	XTensor * var = NewTensor(varOrder, varDimSize);
+	XTensor * a = NewTensor(aOrder, aDimSize);
+	XTensor * b = NewTensor(bOrder, bDimSize);
+
+	/* initialize variables */
+	s->SetData(sData, sUnitNum);
+	mean->SetData(meanData, meanUnitNum);
+	var->SetData(varData, varUnitNum);
+	a->SetData(aData, aUnitNum);
+	b->SetZeroAll();
+	t->SetZeroAll();
+
+	/* call normalize function */
+	Normalize(s, t, 0, mean, var, a, b, 0.0F);
+    
+	/* check results */
+	cpuTest = t->CheckData(answer, tUnitNum, 1e-4F, 0);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensors */
+	XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * meanGPU = NewTensor(meanOrder, meanDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * varGPU = NewTensor(varOrder, varDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+	/* initialize variables */
+	sGPU->SetData(sData, sUnitNum);
+	meanGPU->SetData(meanData, meanUnitNum);
+	varGPU->SetData(varData, varUnitNum);
+	aGPU->SetData(aData, aUnitNum);
+	bGPU->SetZeroAll();
+	tGPU->SetZeroAll();
+
+	/* call Normalize function */
+	Normalize(sGPU, tGPU, 0, meanGPU, varGPU, aGPU, bGPU, 0.0F);
+
+	/* check results */
+	gpuTest = tGPU->CheckData(answer, tUnitNum, 1e-4F, 0);
+
+	/* destroy variables */
+	delete s;
+	delete t;
+	delete mean;
+	delete var;
+	delete a;
+	delete b;
+	delete sGPU;
+	delete tGPU;
+	delete meanGPU;
+	delete varGPU;
+	delete aGPU;
+	delete bGPU;
+	delete[] sDimSize;
+	delete[] tDimSize;
+	delete[] meanDimSize;
+	delete[] varDimSize;
+	delete[] aDimSize;
+	delete[] bDimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete s;
+	delete t;
+	delete mean;
+	delete var;
+	delete a;
+	delete b;
+	delete[] sDimSize;
+	delete[] tDimSize;
+	delete[] meanDimSize;
+	delete[] varDimSize;
+	delete[] aDimSize;
+	delete[] bDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Normalize Function */
+bool TestNormalize()
+{
+	XPRINT(0, stdout, "[TEST NORMALIZE] normalized the data with normal distribution \n");
+	bool returnFlag = true, caseFlag = true;
+
+	/* case 1 test */
+	caseFlag = TestNormalize1();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 1 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 1 passed!\n");
+
+	/* other cases test */
+	/*
+	TODO!!
+	*/
+
+	if (returnFlag) {
+		XPRINT(0, stdout, ">> All Passed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> Failed!\n");
+
+	XPRINT(0, stdout, "\n");
+
+	return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TNormalize.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-20
+*/
+
+#ifndef __TEST_NORMALIZE_H__
+#define __TEST_NORMALIZE_H__
+
+#include "../core/math/Normalize.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Normalize Function */
+extern "C"
+bool TestNormalize();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_NORMALIZE_H__

source/tensor/test/TPower.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-15
+*/
+
+#include "../XUtility.h"
+#include "TPower.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: get the power(a, p) 
+In this case, p=2.
+*/
+bool TestPower1()
+{
+	/* a tensor of size (3, 2) */
+	int aOrder = 2;
+	int * aDimSize = new int[aOrder];
+	aDimSize[0] = 3;
+	aDimSize[1] = 2;
+
+	int aUnitNum = 1;
+	for (int i = 0; i < aOrder; i++)
+		aUnitNum *= aDimSize[i];
+
+	DTYPE aData[3][2] = { {1.0F, 2.0F},
+	                      {3.0F, 4.0F},
+	                      {5.0F, 6.0F} };
+	DTYPE answer[3][2] = { {1.0F, 4.0F},
+	                       {9.0F, 16.0F},
+	                       {25.0F, 36.0F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * a = NewTensor(aOrder, aDimSize);
+
+	/* initialize variables */
+	a->SetData(aData, aUnitNum);
+
+	/* call Power function */
+	Power(a, 2.0F);
+
+	/* check results */
+	cpuTest = a->CheckData(answer, aUnitNum, 1e-4F);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensor */
+	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+
+	/* Initialize variables */
+	aGPU->SetData(aData, aUnitNum);
+
+	/* call power function */
+	Power(aGPU, 2.0F);
+
+	/* check results */
+	gpuTest = aGPU->CheckData(answer, aUnitNum, 1e-4F);
+
+	/* destroy variables */
+	delete a;
+    delete aGPU;
+	delete[] aDimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete a;
+	delete[] aDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: get the power(a, p)
+In this case, p=1.
+*/
+bool TestPower2()
+{
+	/* a tensor of size (3, 2) */
+	int aOrder = 2;
+	int * aDimSize = new int[aOrder];
+	aDimSize[0] = 3;
+	aDimSize[1] = 2;
+
+	int aUnitNum = 1;
+	for (int i = 0; i < aOrder; i++)
+		aUnitNum *= aDimSize[i];
+
+	DTYPE aData[3][2] = { {0.0F, 1.0F},
+	                      {2.0F, 3.0F},
+	                      {4.0F, 5.0F} };
+	DTYPE answer[3][2] = { {0.0F, 1.0F},
+	                       {2.0F, 3.0F},
+	                       {4.0F, 5.0F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * a = NewTensor(aOrder, aDimSize);
+
+	/* initialize variables */
+	a->SetData(aData, aUnitNum);
+
+	/* call Power function */
+	Power(a, 1.0F);
+
+	/* check results */
+	cpuTest = a->CheckData(answer, aUnitNum, 1e-4F);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensor */
+	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+
+	/* Initialize variables */
+	aGPU->SetData(aData, aUnitNum);
+
+	/* call Power function */
+	Power(aGPU, 1.0F);
+
+	/* check results */
+	gpuTest = aGPU->CheckData(answer, aUnitNum, 1e-4F);
+
+	/* destroy variables */
+	delete a;
+    delete aGPU;
+	delete[] aDimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete a;
+	delete[] aDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 3: get the power(a, p)
+In this case, p=0.
+*/
+bool TestPower3()
+{
+	/* a tensor of size (3, 2) */
+	int aOrder = 2;
+	int * aDimSize = new int[aOrder];
+	aDimSize[0] = 3;
+	aDimSize[1] = 2;
+
+	int aUnitNum = 1;
+	for (int i = 0; i < aOrder; i++)
+		aUnitNum *= aDimSize[i];
+
+	DTYPE aData[3][2] = { {0.0F, 1.0F},
+	                      {2.0F, 3.0F},
+	                      {4.0F, 5.0F} };
+	DTYPE answer[3][2] = { {1.0F, 1.0F},
+	                       {1.0F, 1.0F},
+	                       {1.0F, 1.0F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * a = NewTensor(aOrder, aDimSize);
+
+	/* initialize variables */
+	a->SetData(aData, aUnitNum);
+
+	/* call Power function */
+	Power(a, 0.0F);
+
+	/* check results */
+	cpuTest = a->CheckData(answer, aUnitNum, 1e-4F);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensor */
+	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+
+	/* Initialize variables */
+	aGPU->SetData(aData, aUnitNum);
+
+	/* call Power function */
+	Power(aGPU, 0.0F);
+
+	/* check results */
+	gpuTest = aGPU->CheckData(answer, aUnitNum, 1e-4F);
+
+	/* destroy variables */
+	delete a;
+    delete aGPU;
+	delete[] aDimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete a;
+	delete[] aDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Power Function */
+bool TestPower()
+{
+	XPRINT(0, stdout, "[TEST POWER] get the power(a, p) \n");
+	bool returnFlag = true, caseFlag = true;
+
+	/* case 1 test */
+	caseFlag = TestPower1();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 1 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 1 passed!\n");
+
+	/* case 2 test */
+	caseFlag = TestPower2();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 2 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 2 passed!\n");
+
+	/* case 3 test */
+	caseFlag = TestPower3();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 3 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 3 passed!\n");
+
+	/* other cases test */
+	/*
+	TODO!!
+	*/
+
+	if (returnFlag) {
+		XPRINT(0, stdout, ">> All Passed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> Failed!\n");
+
+	XPRINT(0, stdout, "\n");
+
+	return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TPower.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-15
+*/
+
+#ifndef __TEST_POWER_H__
+#define __TEST_POWER_H__
+
+#include "../core/math/Power.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Power Function */
+extern "C"
+bool TestPower();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_POWER_H__

source/tensor/test/TRectify.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
+*/
+
+#include "TRectify.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test rectify function
+In this case, y = max(0, x) 
+*/
+bool TestRectify1()
+{
+    /* a tensor of size (2, 3) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[2][3] = { {0.0F, -1.0F, 2.0F},
+                          {3.0F, -4.0F, -5.0F} };
+    DTYPE answer[2][3] = { {0.0F, 0.0F, 2.0F},
+                           {3.0F, 0.0F, 0.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * x = NewTensor(order, dimSize);
+    XTensor * y = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    x->SetData(xData, unitNum);
+    y->SetZeroAll();
+
+    /* call Rectify function */
+    Rectify(x, y);
+
+    /* check results */
+    cpuTest = y->CheckData(answer, unitNum);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensor */
+	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+	/* Initialize variables */
+	xGPU->SetData(xData, unitNum);
+	yGPU->SetZeroAll();
+
+	/* call Rectify function */
+	Rectify(xGPU, yGPU);
+
+	/* check results */
+	gpuTest = yGPU->CheckData(answer, unitNum);
+
+	/* destroy variables */
+	delete x;
+    delete y;
+    delete xGPU;
+    delete yGPU;
+	delete[] dimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+	delete x;
+    delete y;
+	delete[] dimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: backward computation 
+dE/dx = dE/dy * dy/dx 
+rectified: y = max(0, x) 
+In this case, lossName=CROSSENTROPY.
+*/
+bool TestRectify2()
+{
+	/* a tensor of size (2, 3) */
+	int order = 2;
+	int * dimSize = new int[order];
+	dimSize[0] = 2;
+	dimSize[1] = 3;
+
+	int unitNum = 1;
+	for (int i = 0; i < order; i++)
+		unitNum *= dimSize[i];
+
+	DTYPE xData[2][3] = { {1.0F, 1.0F, 2.0F},
+	                      {2.0F, 4.0F, 5.0F} };
+	DTYPE goldData[2][3] = { {1.0F, 1.0F, 1.0F},
+	                         {1.0F, 1.0F, 1.0F} };
+    DTYPE yAnswer[2][3] = { {1.0F, 1.0F, 2.0F},
+	                        {2.0F, 4.0F, 5.0F} };
+	DTYPE dedyAnswer[2][3] = { {-1.0F, -1.0F, -0.5F},
+	                           {-0.5F, -0.25F, -0.2F} };
+	DTYPE dedxAnswer[2][3] = { {-1.0F, -1.0F, -0.5F},
+	                           {-0.5F, -0.25F, -0.2F} };
+
+	/* CPU test */
+	bool cpuTest = true;
+
+	/* create tensors */
+	XTensor * x = NewTensor(order, dimSize);
+	XTensor * y = NewTensor(order, dimSize);
+	XTensor * gold = NewTensor(order, dimSize);
+	XTensor * dedy = NewTensor(order, dimSize);
+	XTensor * dedx = NewTensor(order, dimSize);
+
+	/* initialize variables */
+	x->SetData(xData, unitNum);
+	gold->SetData(goldData, unitNum);
+	y->SetZeroAll();
+	dedy->SetZeroAll();
+	dedx->SetZeroAll();
+
+    /* call Rectify function */
+    Rectify(x, y);
+
+	/* call RectifyBackward function */
+	RectifyBackward(gold, y, x, dedy, dedx, CROSSENTROPY);
+
+	/* check results */
+    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+	/* GPU test */
+	bool gpuTest = true;
+
+	/* create tensors */
+	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+	XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+	/* initialize variables */
+	xGPU->SetData(xData, unitNum);
+	goldGPU->SetData(goldData, unitNum);
+	yGPU->SetZeroAll();
+	dedyGPU->SetZeroAll();
+	dedxGPU->SetZeroAll();
+    
+    /* call Rectify function */
+    Rectify(xGPU, yGPU);
+
+	/* call rectifybackward function */
+	RectifyBackward(goldGPU, yGPU, xGPU, dedyGPU, dedxGPU, CROSSENTROPY);
+    
+	/* check results */
+    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+	/* destroy variables */
+    delete x;
+    delete y;
+    delete dedy;
+    delete dedx;
+    delete gold;
+    delete xGPU;
+    delete yGPU;
+    delete dedyGPU;
+    delete dedxGPU;
+    delete goldGPU;
+	delete[] dimSize;
+
+	return cpuTest && gpuTest;
+#else
+	/* destroy variables */
+    delete x;
+    delete y;
+    delete dedy;
+    delete dedx;
+    delete gold;
+	delete[] dimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Rectify Function */
+bool TestRectify()
+{
+    XPRINT(0, stdout, "[TEST RECTIFY] rectify function and its backward computation \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestRectify1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestRectify2();
+
+	if (!caseFlag) {
+		returnFlag = false;
+		XPRINT(0, stdout, ">> case 2 failed!\n");
+	}
+	else
+		XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TRectify.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
+*/
+
+#ifndef __TEST_RECTIFY_H__
+#define __TEST_RECTIFY_H__
+
+#include "../function/Rectify.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Rectify Function */
+extern "C"
+bool TestRectify();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_RECTIFY_H__

source/tensor/test/TReduceMax.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-30
+*/
+
+#include "TReduceMax.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: get the max value of the items along a dimension of the tensor. 
+In this case,
+(2, 4) -> (4), dim = 0
+(2, 4) -> (2), dim = 1
+*/
+bool TestReduceMax1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a output tensor of size (4) */
+    int tOrder1 = 1;
+    int * tDimSize1 = new int[tOrder1];
+    tDimSize1[0] = 4;
+
+    int tUnitNum1 = 1;
+    for (int i = 0; i < tOrder1; i++)
+        tUnitNum1 *= tDimSize1[i];
+
+    /* a output tensor of size (2) */
+    int tOrder2 = 1;
+    int * tDimSize2 = new int[tOrder2];
+    tDimSize2[0] = 2;
+
+    int tUnitNum2 = 1;
+    for (int i = 0; i < tOrder2; i++)
+        tUnitNum2 *= tDimSize2[i];
+
+    DTYPE sData[2][4] = { {0.0F, 5.0F, 2.0F, 3.0F},
+                          {4.0F, 1.0F, 6.0F, 7.0F} };
+    DTYPE answer1[4] = {4.0F, 5.0F, 6.0F, 7.0F};
+    DTYPE answer2[2] = {5.0F, 7.0F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
+    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t1->SetZeroAll();
+    t2->SetZeroAll();
+
+    /* call ReduceMax function */
+    ReduceMax(s, t1, 0);
+    ReduceMax(s, t2, 1);
+
+    /* check results */
+    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU1->SetZeroAll();
+    tGPU2->SetZeroAll();
+
+    /* call ReduceMax function */
+    ReduceMax(sGPU, tGPU1, 0);
+    ReduceMax(sGPU, tGPU2, 1);
+
+    /* check results */
+    gpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
+
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete sGPU;
+    delete tGPU1;
+    delete tGPU2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for ReduceMax Function */
+bool TestReduceMax()
+{
+    XPRINT(0, stdout, "[TEST ReduceMax] get the max value of the items along a dimension of the tensor\n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestReduceMax1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TReduceMax.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-30
+*/
+
+#ifndef __TEST_REDUCEMAX_H__
+#define __TEST_REDUCEMAX_H__
+
+#include "../core/reduce/ReduceMax.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for ReduceMax Function */
+extern "C"
+bool TestReduceMax();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_REDUCEMAX_H__

source/tensor/test/TReduceMean.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#include "TReduceMean.h"
+
+namespace nts { // namespace nt(NiuTrans.Tensor)
+
+/* case 1: get the mean value along a dimension of the tensor */
+bool TestReduceMean1()
+{
+    /* a tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a tensor of size (4) */
+    int tOrder1 = 1;
+    int * tDimSize1 = new int[tOrder1];
+    tDimSize1[0] = 4;
+
+    int tUnitNum1 = 1;
+    for (int i = 0; i < tOrder1; i++)
+        tUnitNum1 *= tDimSize1[i];
+
+    /* a tensor of size (2) */
+    int tOrder2 = 1;
+    int * tDimSize2 = new int[tOrder2];
+    tDimSize2[0] = 2;
+
+    int tUnitNum2 = 1;
+    for (int i = 0; i < tOrder2; i++)
+        tUnitNum2 *= tDimSize2[i];
+
+    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE answer1[4] = {2.0F, 3.0F, 4.0F, 5.0F};
+    DTYPE answer2[2] = {1.5F, 5.5F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
+    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t1->SetZeroAll();
+    t2->SetZeroAll();
+
+    /* call ReduceMean function */
+    ReduceMean(s, t1, 0);
+    ReduceMean(s, t2, 1);
+
+    /* check results */
+    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU1->SetZeroAll();
+    tGPU2->SetZeroAll();
+
+    /* call ReduceMean function */
+    ReduceMean(sGPU, tGPU1, 0);
+    ReduceMean(sGPU, tGPU2, 1);
+
+    /* check results */
+    cpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
+
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete sGPU;
+    delete tGPU1;
+    delete tGPU2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+    
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for ReduceMean Function */
+bool TestReduceMean()
+{
+    XPRINT(0, stdout, "[TEST ReduceMean] get the mean value along a dimension of the tensor \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestReduceMean1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    ///* other cases test */
+    ///*
+    //TODO!!
+    //*/
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TReduceMean.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#ifndef __TEST_REDUCEMEAN_H__
+#define __TEST_REDUCEMEAN_H__
+
+#include "../core/reduce/ReduceMean.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for ReduceMean Function */
+extern "C"
+bool TestReduceMean();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_REDUCEMEAN_H__
+
+

source/tensor/test/TReduceSum.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#include "TReduceSum.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: sum the items along a dimension of the tensor.
+In this case, 
+(2, 4) -> (4), dim = 0
+(2, 4) -> (2), dim = 1
+*/
+bool TestReduceSum1()
+{
+    /* a tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a tensor of size (4) */
+    int tOrder1 = 1;
+    int * tDimSize1 = new int[tOrder1];
+    tDimSize1[0] = 4;
+
+    int tUnitNum1 = 1;
+    for (int i = 0; i < tOrder1; i++)
+        tUnitNum1 *= tDimSize1[i];
+
+    /* a tensor of size (2) */
+    int tOrder2 = 1;
+    int * tDimSize2 = new int[tOrder2];
+    tDimSize2[0] = 2;
+
+    int tUnitNum2 = 1;
+    for (int i = 0; i < tOrder2; i++)
+        tUnitNum2 *= tDimSize2[i];
+
+    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                           {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE answer1[4] = {4.0F, 6.0F, 8.0F, 10.0F};
+    DTYPE answer2[2] = {6.0F, 22.0F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
+    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t1->SetZeroAll();
+    t2->SetZeroAll();
+
+    /* call ReduceSum function */
+    ReduceSum(s, t1, 0);
+    ReduceSum(s, t2, 1);
+
+    /* check results */
+    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU1->SetZeroAll();
+    tGPU2->SetZeroAll();
+
+    /* call ReduceSum function */
+    ReduceSum(sGPU, tGPU1, 0);
+    ReduceSum(sGPU, tGPU2, 1);
+
+    /* check results */
+    cpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
+
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete sGPU;
+    delete tGPU1;
+    delete tGPU2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+    
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for ReduceSum Function */
+bool TestReduceSum()
+{
+    XPRINT(0, stdout, "[TEST ReduceSum] sum the items along a dimension of the tensor.\n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestReduceSum1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TReduceSum.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#ifndef __TEST_REDUCESUM_H__
+#define __TEST_REDUCESUM_H__
+
+#include "../core/reduce/ReduceSum.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for ReduceSum Function */
+extern "C"
+bool TestReduceSum();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_REDUCESUM_H__
+
+
+

source/tensor/test/TReduceSumSquared.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#include "TReduceSumSquared.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: squared sum of the items along a dimension of the tensor. 
+For a 1-dimensional data array a, sum = \sum_i (a_i - shift)^2.
+In this case, (2, 4) -> (4), dim = 0.
+*/
+bool TestReduceSumSquared1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a output tensor of size (4) */
+    int tOrder = 1;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 4;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    /* a shift tensor of size (4) */
+    int shiftOrder = 1;
+    int * shiftDimSize = new int[shiftOrder];
+    shiftDimSize[0] = 4;
+
+    int shiftUnitNum = 1;
+    for (int i = 0; i < shiftOrder; i++)
+        shiftUnitNum *= shiftDimSize[i];
+
+    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE shiftData[4] = {1.0F, -1.0F, -1.0F, 0.0F};
+    DTYPE answer[4] = {10.0F, 40.0F, 58.0F, 58.0F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+    XTensor * shift = NewTensor(shiftOrder, shiftDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    shift->SetData(shiftData, shiftUnitNum);
+    t->SetZeroAll();
+
+    /* call ReduceSumSquared function */
+    ReduceSumSquared(s, t, 0, shift);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * shiftGPU = NewTensor(shiftOrder, shiftDimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    shiftGPU->SetData(shiftData, shiftUnitNum);
+    tGPU->SetZeroAll();
+
+    /* call ReduceSumSquared function */
+    ReduceSumSquared(sGPU, tGPU, 0, shiftGPU);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete shift;
+    delete sGPU;
+    delete tGPU;
+    delete shiftGPU;
+    delete[] sDimSize;
+    delete[] tDimSize;
+    delete[] shiftDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete shift;
+    delete[] sDimSize;
+    delete[] tDimSize;
+    delete[] shiftDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: squared sum of the items along a dimension of the tensor. 
+For a 1-dimensional data array a, sum = \sum_i (a_i - shift)^2.
+In this case, (2, 4) -> (2), dim = 1.
+*/
+bool TestReduceSumSquared2()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a output tensor of size (2) */
+    int tOrder = 1;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    /* a shift tensor of size (2) */
+    int shiftOrder = 1;
+    int * shiftDimSize = new int[shiftOrder];
+    shiftDimSize[0] = 2;
+
+    int shiftUnitNum = 1;
+    for (int i = 0; i < shiftOrder; i++)
+        shiftUnitNum *= shiftDimSize[i];
+
+    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE shiftData[2] = {-1.0F, 1.0F};
+    DTYPE answer[2] = {30.0F, 86.0F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+    XTensor * shift = NewTensor(shiftOrder, shiftDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    shift->SetData(shiftData, shiftUnitNum);
+    t->SetZeroAll();
+
+    /* call ReduceSumSquared function */
+    ReduceSumSquared(s, t, 1, shift);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * shiftGPU = NewTensor(shiftOrder, shiftDimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    shiftGPU->SetData(shiftData, shiftUnitNum);
+    tGPU->SetZeroAll();
+
+    /* call ReduceSumSquared function */
+    ReduceSumSquared(sGPU, tGPU, 1, shiftGPU);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete shift;
+    delete sGPU;
+    delete tGPU;
+    delete shiftGPU;
+    delete[] sDimSize;
+    delete[] tDimSize;
+    delete[] shiftDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete shift;
+    delete[] sDimSize;
+    delete[] tDimSize;
+    delete[] shiftDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for ReduceSumSquared Function */
+bool TestReduceSumSquared()
+{
+    XPRINT(0, stdout, "[TEST ReduceSumSquared] squared sum of the items along a dimension of the tensor\n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestReduceSumSquared1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+    
+    /* case 2 test */
+    caseFlag = TestReduceSumSquared2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TReduceSumSquared.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#ifndef __TEST_REDUCESUMSQUARED_H__
+#define __TEST_REDUCESUMSQUARED_H__
+
+#include "../core/reduce/ReduceSumSquared.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for ReduceSumSquared Function */
+extern "C"
+bool TestReduceSumSquared();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_REDUCESUMSQUARED_H__

source/tensor/test/TReduceVariance.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#include "TReduceVariance.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/*
+case 1: variance of the items along a dimension of the tensor. 
+For a 1-dimensional data array a, variance = 1/n * \sum_i (a_i - mean)^2.
+In this case, (2, 4) -> (4), dim = 0.
+*/
+bool TestReduceVariance1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a output tensor of size (4) */
+    int tOrder = 1;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 4;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    /* a mean tensor of size (4) */
+    int meanOrder = 1;
+    int * meanDimSize = new int[meanOrder];
+    meanDimSize[0] = 4;
+
+    int meanUnitNum = 1;
+    for (int i = 0; i < meanOrder; i++)
+        meanUnitNum *= meanDimSize[i];
+
+    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE meanData[4] = {2.0F, 3.0F, 4.0F, 5.0F};
+    DTYPE answer[4] = {4.0F, 4.0F, 4.0F, 4.0F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+    XTensor * mean = NewTensor(meanOrder, meanDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    mean->SetData(meanData, meanUnitNum);
+    t->SetZeroAll();
+
+    /* call ReduceVariance function */
+    ReduceVariance(s, t, 0, mean);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * meanGPU = NewTensor(meanOrder, meanDimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    meanGPU->SetData(meanData, meanUnitNum);
+    tGPU->SetZeroAll();
+
+    /* call ReduceVariance function */
+    ReduceVariance(sGPU, tGPU, 0, meanGPU);
+
+    /* check results */
+    gpuTest = t->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete mean;
+    delete sGPU;
+    delete tGPU;
+    delete meanGPU;
+    delete[] sDimSize;
+    delete[] tDimSize;
+    delete[] meanDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete mean;
+    delete[] sDimSize;
+    delete[] tDimSize;
+    delete[] meanDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for ReduceVariance Function */
+bool TestReduceVariance()
+{
+    XPRINT(0, stdout, "[TEST ReduceVariance] variance of the items along a dimension of the tensor\n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestReduceVariance1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TReduceVariance.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#ifndef __TEST_REDUCEVARIANCE_H__
+#define __TEST_REDUCEVARIANCE_H__
+
+#include "../core/reduce/ReduceVariance.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for ReduceVariance Function */
+extern "C"
+bool TestReduceVariance();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_REDUCEVARIANCE_H__

source/tensor/test/TScaleAndShift.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#include "TScaleAndShift.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: scale and shift all tensor entires.
+p = p * scale + shift
+*/
+bool TestScaleAndShift1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE answer[2][4] = { {0.5F, 2.5F, 4.5F, 6.5F},
+                           {8.5F, 10.5F, 12.5F, 14.5F} };
+
+    DTYPE scaleFactor = 2.0F;
+    DTYPE shiftFactor = 0.5F;
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+
+    /* call ScaleAndShift function */
+    _ScaleAndShift(s, s, scaleFactor, shiftFactor);
+
+    /* check results */
+    cpuTest = s->CheckData(answer, sUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+
+    /* call ScaleAndShift function */
+    _ScaleAndShift(sGPU, sGPU, scaleFactor, shiftFactor);
+
+    /* check results */
+    gpuTest = sGPU->CheckData(answer, sUnitNum);
+
+    /* destroy variables */
+    delete s;
+    delete sGPU;
+    delete[] sDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete[] sDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for ScaleAndShift Function */
+bool TestScaleAndShift()
+{
+    XPRINT(0, stdout, "[TEST ScaleAndShift] scale and shift all tensor entires\n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestScaleAndShift1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TScaleAndShift.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#ifndef __TEST_SCALEANDSHIFT_H__
+#define __TEST_SCALEANDSHIFT_H__
+
+#include "../core/math/ScaleAndShift.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for ScaleAndShift Function */
+extern "C"
+bool TestScaleAndShift();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SCALEANDSHIFT_H__

source/tensor/test/TSelect.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-04
+*/
+
+#include "TSelect.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test SelectRange function.
+It can generate a tensor with seleccted data in range[low,high] along the given dimension.
+In this case, (2, 2, 4) -> (2, 2, 2), dim = 2, low = 1, high = 3.
+*/
+bool TestSelect1()
+{
+    /* a input tensor of size (2, 2, 4) */
+    int sOrder = 3;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 2;
+    sDimSize[2] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a output tensor of size (2, 2, 2) */
+    int tOrder = 3;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+    tDimSize[2] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData[2][2][4] = { { {0.0F, 1.0F, 2.0F, 3.0F},
+                               {4.0F, 5.0F, 6.0F, 7.0F} },
+                             { {1.0F, 2.0F, 3.0F, 4.0F},
+                               {5.0F, 6.0F, 7.0F, 8.0F} } };
+    DTYPE answer[2][2][2] = { { {1.0F, 2.0F},
+                                {5.0F, 6.0F} },
+                              { {2.0F, 3.0F},
+                                {6.0F, 7.0F} } };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t->SetZeroAll();
+
+    /* call SelectRange function */
+    SelectRange(s, t, 2, 1, 3);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+    
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU->SetZeroAll();
+
+    /* call Select function */
+    SelectRange(sGPU, tGPU, 2, 1, 3);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+    
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete sGPU;
+    delete tGPU;
+    delete[] sDimSize;
+    delete[] tDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete[] sDimSize;
+    delete[] tDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Select Function */
+bool TestSelect()
+{
+    XPRINT(0, stdout, "[TEST Select] generate a tensor with seleccted data in range[low,high] along the given dimension \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSelect1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSelect.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-04
+*/
+
+#ifndef __TEST_SELECT_H__
+#define __TEST_SELECT_H__
+
+#include "../core/getandset/Select.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Select Function */
+extern "C"
+bool TestSelect();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SELECT_H__

source/tensor/test/TSetAscendingOrder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#include "TSetAscendingOrder.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* case 1: set the cell to the ascending order along a given dimension. */
+bool TestSetAscendingOrder1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    int answer[2][4] = { {0, 1, 2, 3},
+                         {0, 1, 2, 3} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize, X_INT);
+
+    /* initialize variables */
+    s->SetZeroAll();
+
+    /* call SetAscendingOrder function */
+    s->SetAscendingOrder(1);
+    
+    /* check results */
+    cpuTest = s->CheckData(answer, sUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_INT, 1.0F, 0);
+
+    /* initialize variables */
+    sGPU->SetZeroAll();
+
+    /* call SetAscendingOrder function */
+    sGPU->SetAscendingOrder(1);
+
+    /* check results */
+    gpuTest = sGPU->CheckData(answer, sUnitNum);
+
+    /* destroy variables */
+    delete s;
+    delete sGPU;
+    delete[] sDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete[] sDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for SetAscendingOrder Function */
+bool TestSetAscendingOrder()
+{
+    XPRINT(0, stdout, "[TEST SetAscendingOrder] set the cell to the ascending order along a given dimension \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSetAscendingOrder1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSetAscendingOrder.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#ifndef __TEST_SETASCENDINGORDER_H__
+#define __TEST_SETASCENDINGORDER_H__
+
+#include "../XTensor.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for SetAscendingOrder Function */
+extern "C"
+bool TestSetAscendingOrder();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SETASCENDINGORDER_H__

source/tensor/test/TSetData.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#include "TSetData.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test SetDataRand function.
+set the tensor items by a uniform distribution in range [lower, upper]. 
+*/
+bool TestSetData1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    DTYPE answer[2][4] = {0};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+
+    /* call SetDataRand function */
+    s->SetDataRand(0.0, 1.0);
+    
+    /* check results */
+    cpuTest = s->CheckData(answer, sUnitNum, 1.0F);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+
+    /* call SetDataRand function */
+    sGPU->SetDataRand(0.0, 1.0);
+    
+    gpuTest = sGPU->CheckData(answer, sUnitNum, 1.0F);
+
+    /* destroy variables */
+    delete s;
+    delete sGPU;
+    delete[] sDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete[] sDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for SetData Function */
+bool TestSetData()
+{
+    XPRINT(0, stdout, "[TEST SetData] set the data of tensor \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSetData1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/test/TMatrixMul2DParallel.h → source/tensor/test/TSetData.h

@@ -19,16 +19,16 @@
 * $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
 * $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
 */
 */
 
 
-#ifndef __TEST_MATRIXMUL2DPARALLEL_H__
-#define __TEST_MATRIXMUL2DPARALLEL_H__
+#ifndef __TEST_SETDATA_H__
+#define __TEST_SETDATA_H__
 
 
-#include "../core/arithmetic/MatrixMul2DParallel.h"
+#include "../XTensor.h"
 
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 
-/* test for MatrixMul2DParallel Function */
+/* test for SetData Function */
 extern "C"
 extern "C"
-bool TestMatrixMul2DParallel();
+bool TestSetData();
 
 
 } // namespace nts(NiuTrans.Tensor)
 } // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_MATRIXMUL2DPARALLEL_H__
+#endif // __TEST_SETDATA_H__

source/tensor/test/TSigmoid.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
+*/
+
+#include "../XUtility.h"
+#include "TSigmoid.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test Sigmoid function.
+sigmoid function: y = 1/(1+exp(-x))
+*/
+bool TestSigmoid1()
+{
+    /* a input tensor of size (3) */
+    int order = 1;
+    int * dimSize = new int[order];
+    dimSize[0] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[3] = {0.0F, 1.0F, 2.0F};
+    DTYPE answer[3] = {0.5F, 0.7311F, 0.8808F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * x = NewTensor(order, dimSize);
+    XTensor * y = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    x->SetData(xData, unitNum);
+    y->SetZeroAll();
+
+    /* call Sigmoid function */
+    Sigmoid(x, y);
+
+    /* check result */
+    cpuTest = y->CheckData(answer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+        /* create tensors */
+    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    xGPU->SetData(xData, unitNum);
+    yGPU->SetZeroAll();
+
+    /* call Sigmoid function */
+    Sigmoid(xGPU, yGPU);
+
+    /* check result */
+    gpuTest = yGPU->CheckData(answer, unitNum, 1e-4F);
+
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete xGPU;
+    delete yGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: test Sigmoid function and SigmoidBackward function.
+sigmoid function: y = 1/(1+exp(-x))
+backward computation: 
+dE/ds = dE/dy * dy/dx
+dy/dx = y * (1 -y)
+In this case, LossName=CROSSENTROPY.
+*/
+bool TestSigmoid2()
+{
+    /* a input tensor of size (3) */
+    int order = 1;
+    int * dimSize = new int[order];
+    dimSize[0] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[3] = {0.0F, 1.0F, 2.0F};
+    DTYPE gData[3] = {0.4F, 0.8F, 1.0F};
+    DTYPE yAnswer[3] = {0.5F, 0.7311F, 0.8808F};
+    DTYPE dedyAnswer[3] = {-0.8F, -1.0943F, -1.1353F};
+    DTYPE dedxAnswer[3] = {-0.2F, -0.2151F, -0.1192F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * x = NewTensor(order, dimSize);
+    XTensor * y = NewTensor(order, dimSize);
+    XTensor * g = NewTensor(order, dimSize);
+    XTensor * dedy = NewTensor(order, dimSize);
+    XTensor * dedx = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    x->SetData(xData, unitNum);
+    g->SetData(gData, unitNum);
+    y->SetZeroAll();
+    dedy->SetZeroAll();
+    dedx->SetZeroAll();
+
+    /* call Sigmoid function */
+    Sigmoid(x, y);
+
+    /* call SigmoidBackward function */
+    SigmoidBackward(g, y, x, dedy, dedx, CROSSENTROPY);
+    
+    /* check result */
+    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+        /* create tensors */
+    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * gGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    xGPU->SetData(xData, unitNum);
+    gGPU->SetData(gData, unitNum);
+    yGPU->SetZeroAll();
+    dedyGPU->SetZeroAll();
+    dedxGPU->SetZeroAll();
+
+    /* call Sigmoid function */
+    Sigmoid(xGPU, yGPU);
+
+    /* call SigmoidBackward function */
+    SigmoidBackward(gGPU, yGPU, xGPU, dedyGPU, dedxGPU, CROSSENTROPY);
+    
+    /* check result */
+    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
+              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete g;
+    delete dedx;
+    delete dedy;
+    delete xGPU;
+    delete yGPU;
+    delete gGPU;
+    delete dedxGPU;
+    delete dedyGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete g;
+    delete dedx;
+    delete dedy;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for Sigmoid Function */
+bool TestSigmoid()
+{
+    XPRINT(0, stdout, "[TEST SIGMOID] sigmoid function and its backward computation \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSigmoid1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+    
+    /* case 2 test */
+    caseFlag = TestSigmoid2();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSigmoid.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
+*/
+
+#ifndef __TEST_SIGMOID_H__
+#define __TEST_SIGMOID_H__
+
+#include "../function/Sigmoid.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Sigmoid Function */
+extern "C"
+bool TestSigmoid();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SIGMOID_H__

source/tensor/test/TSoftmax.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
+*/
+
+#include "../XTensor.h"
+#include "../XUtility.h"
+#include "TSoftmax.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test Softmax function.
+softmax function: y = e^x / \sum_{i} e^{x_i}
+*/
+bool TestSoftmax1()
+{
+    /* a tensor of size (2, 3) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[2][3] = { {0.0F, 1.0F, 2.0F}, 
+                          {0.5F, 0.7F, 1.4F} };
+    DTYPE answer[2][3] = { {0.0900F, 0.2447F, 0.6652F}, 
+                           {0.2136F, 0.2609F, 0.5254F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * x = NewTensor(order, dimSize);
+    XTensor * y = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    x->SetData(xData, unitNum);
+    y->SetZeroAll();
+
+    /* call Softmax function */
+    Softmax(x, y, 1);
+    
+    /* check result */
+    cpuTest = y->CheckData(answer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    xGPU->SetData(xData, unitNum);
+    yGPU->SetZeroAll();
+
+    /* call Softmax function */
+    Softmax(xGPU, yGPU, 1);
+    
+    /* check result */
+    gpuTest = yGPU->CheckData(answer, unitNum, 1e-4F);
+
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete xGPU;
+    delete yGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: test SoftmaxBackward function.
+SoftmaxBackward function: dE/dx_j = -gold_j + y_j
+In this case, LossName=CROSSENTROPY.
+*/
+bool TestSoftmax2()
+{
+    /* a input tensor of size (2, 3) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 1;
+    dimSize[1] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[1][3] = { {0.0F, 1.0F, 2.0F} };
+    DTYPE gData[1][3] = { {0.0F, 0.0F, 1.0F} };
+    DTYPE yAnswer[1][3] = { {0.0900F, 0.2447F, 0.6652F} };
+    DTYPE dedxAnswer[1][3] = {0.0900F, 0.2447F, -0.3347F};
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * x = NewTensor(order, dimSize);
+    XTensor * y = NewTensor(order, dimSize);
+    XTensor * g = NewTensor(order, dimSize);
+    XTensor * dedy = NewTensor(order, dimSize);
+    XTensor * dedx = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    x->SetData(xData, unitNum);
+    g->SetData(gData, unitNum);
+    y->SetZeroAll();
+    dedx->SetZeroAll();
+    dedy->SetZeroAll();
+
+    /* call Softmax function */
+    Softmax(x, y, 1);
+    
+    /* call SoftmaxBackward function */
+    SoftmaxBackward(g, y, x, dedy, dedx, 1, CROSSENTROPY);
+    
+    /* check result */
+    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+        /* create tensors */
+    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * gGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* initialize variables */
+    xGPU->SetData(xData, unitNum);
+    gGPU->SetData(gData, unitNum);
+    yGPU->SetZeroAll();
+    dedxGPU->SetZeroAll();
+    dedyGPU->SetZeroAll();
+
+    /* call Softmax function */
+    Softmax(xGPU, yGPU, 1);
+
+    /* call SoftmaxBackward function */
+    SoftmaxBackward(gGPU, yGPU, xGPU, dedyGPU, dedxGPU, 1, CROSSENTROPY);
+    
+    /* check result */
+    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
+              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F);
+
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete g;
+    delete dedx;
+    delete dedy;
+    delete xGPU;
+    delete yGPU;
+    delete gGPU;
+    delete dedxGPU;
+    delete dedyGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete x;
+    delete y;
+    delete g;
+    delete dedx;
+    delete dedy;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for Softmax Function */
+bool TestSoftmax()
+{
+    XPRINT(0, stdout, "[TEST SOFTMAX] softmax function and its backward computation \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSoftmax1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestSoftmax2();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSoftmax.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
+*/
+
+#ifndef __TEST_SOFTMAX_H__
+#define __TEST_SOFTMAX_H__
+
+#include "../function/Softmax.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Softmax Function */
+extern "C"
+bool TestSoftmax();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SOFTMAX_H__

source/tensor/test/TSort.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#include "TSort.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* case 1: sort the tensor along a given dimension */
+bool TestSort1()
+{
+    /* a tensor of size (2, 4) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 4;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE answer[2][4] = { {4.0F, 5.0F, 6.0F, 7.0F},
+                           {0.0F, 1.0F, 2.0F, 3.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a = NewTensor(order, dimSize);
+    XTensor * b = NewTensor(order, dimSize, X_INT);
+
+    /* initialize variables */
+    a->SetData(aData, unitNum);
+    b->SetZeroAll();
+
+    /* call Sort function */
+    Sort(a, b, 0);
+
+    cpuTest = a->CheckData(answer, unitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(order, dimSize, X_INT, 1.0F, 0);
+    
+    /* Initialize variables */
+    aGPU->SetData(aData, unitNum);
+    bGPU->SetZeroAll();
+
+    /* call sum function */
+    Sort(aGPU, bGPU, 0);
+
+    /* check results */
+    gpuTest = aGPU->CheckData(answer, unitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete aGPU;
+    delete bGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+bool TestSort2()
+{
+    /* a tensor of size (2, 4) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 4;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE answer[2][4] = { {3.0F, 2.0F, 1.0F, 0.0F},
+                           {7.0F, 6.0F, 5.0F, 4.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a = NewTensor(order, dimSize);
+    XTensor * b = NewTensor(order, dimSize, X_INT);
+
+    /* initialize variables */
+    a->SetData(aData, unitNum);
+
+    /* call Sort function */
+    Sort(a, b, 1);
+
+    /* check results */
+    cpuTest = a->CheckData(answer, unitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(order, dimSize, X_INT, 1.0F, 0);
+
+    /* Initialize variables */
+    aGPU->SetData(aData, unitNum);
+
+    /* call sum function */
+    Sort(aGPU, bGPU, 1);
+
+    /* check results */
+    gpuTest = aGPU->CheckData(answer, unitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete aGPU;
+    delete bGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Sort Function */
+bool TestSort()
+{
+    XPRINT(0, stdout, "[TEST SORT] sort the tensor along a given dimension \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSort1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestSort2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSort.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#ifndef __TEST_SORT_H__
+#define __TEST_SORT_H__
+
+#include "../core/sort/Sort.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Sort Function */
+extern "C"
+bool TestSort();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SORT_H__

source/tensor/test/TSplit.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-13
+*/
+
+#include "TSplit.h"
+
+namespace nts { // namespace nt(NiuTrans.Tensor)
+
+/* 
+case 1: transform a tensor by splitting it, e.g., (N, M) -> (N/3, M, 3)
+In this case, (4, 3) -> (2, 2, 3), whereToSplit=0, splitNum=2.
+*/
+bool TestSplit1()
+{
+    /* a source tensor of size (4, 3) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 4;
+    sDimSize[1] = 3;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a target tensor of size (2, 2, 3) */
+    int tOrder = 3;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+    tDimSize[2] = 3;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData[4][3] = { {0.0F, 1.0F, 2.0F},
+                          {3.0F, 4.0F, 5.0F},
+                          {0.1F, 1.1F, 2.1F},
+                          {3.1F, 4.1F, 5.1F} };
+	DTYPE answer[2][2][3] = { { {0.0F, 1.0F, 2.0F},
+	                            {3.0F, 4.0F, 5.0F} },
+							  { {0.1F, 1.1F, 2.1F},
+	                            {3.1F, 4.1F, 5.1F} } };
+  
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t->SetZeroAll();
+
+    /* call split function */
+    Split(s, t, 0, 2);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+	sGPU->SetData(sData, sUnitNum);
+	tGPU->SetZeroAll();
+
+    /* call sum function */
+	Split(sGPU, tGPU, 0, 2);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+	delete s, t, sGPU, tGPU;
+	delete[] sDimSize, tDimSize;
+
+	return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+	delete s, t;
+	delete[] sDimSize, tDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: transform a tensor by splitting it, e.g., (N, M) -> (N/3, M, 3)
+In this case, (3, 4) -> (2, 3, 2), whereToSplit=1, splitNum=2.
+*/
+bool TestSplit2()
+{
+    /* a source tensor of size (3, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 3;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a target tensor of size (2, 3, 2) */
+    int tOrder = 3;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 3;
+    tDimSize[2] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData[3][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 0.1F, 1.1F},
+                          {2.1F, 3.1F, 4.1F, 5.1F} };
+	DTYPE answer[2][3][2] = { { {0.0F, 1.0F},
+								{4.0F, 5.0F},
+								{2.1F, 3.1F} },
+	                          { {2.0F, 3.0F},
+							    {0.1F, 1.1F},
+								{4.1F, 5.1F} } };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t->SetZeroAll();;
+
+    /* call split function */
+    Split(s, t, 1, 2);
+
+    /* check results */
+    cpuTest = t->CheckData(answer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+	XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+	sGPU->SetData(sData, sUnitNum);
+	tGPU->SetZeroAll();
+
+    /* call sum function */
+	Split(sGPU, tGPU, 1, 2);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(answer, tUnitNum);
+
+    /* destroy variables */
+	delete s;
+    delete t;
+    delete sGPU;
+    delete tGPU;
+	delete[] sDimSize;
+	delete[] tDimSize;
+
+	return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+	delete s;
+    delete t;
+	delete[] sDimSize;
+	delete[] tDimSize;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 3: split a big tensor into small tensors
+In this case, (3, 4) -> 2 * (3, 2) , whereToSplit=1, splitNum=2.
+*/
+bool TestSplit3()
+{
+	/* create list */
+    XList tList;
+    tList = XList();
+
+    /* a source tensor of size (3, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 3;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a target tensor of size (3, 2) */
+    int tOrder1 = 2;
+    int * tDimSize1 = new int[tOrder1];
+    tDimSize1[0] = 3;
+    tDimSize1[1] = 2;
+
+    int tUnitNum1 = 1;
+    for (int i = 0; i < tOrder1; i++)
+        tUnitNum1 *= tDimSize1[i];
+
+    /* a target tensor of size (3 * 2) */
+    int tOrder2 = 2;
+    int * tDimSize2 = new int[tOrder2];
+    tDimSize2[0] = 3;
+    tDimSize2[1] = 2;
+
+    int tUnitNum2 = 1;
+    for (int i = 0; i < tOrder2; i++)
+        tUnitNum2 *= tDimSize2[i];
+
+    DTYPE sData[3][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 0.1F, 1.1F},
+                          {2.1F, 3.1F, 4.1F, 5.1F} };
+    DTYPE answer1[3][2] = { {0.0F, 1.0F},
+                            {4.0F, 5.0F},
+                            {2.1F, 3.1F} };
+    DTYPE answer2[3][2] = { {2.0F, 3.0F},
+                            {0.1F, 1.1F},
+                            {4.1F, 5.1F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
+    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t1->SetZeroAll();
+    t2->SetZeroAll();
+
+	/* add tensors to list */
+    tList.Add(t1);
+    tList.Add(t2);
+
+    /* call split function */
+    Split(s, &tList, 1, 2);
+
+    /* check results */
+    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+	/* clear list */
+	tList.Clear();
+
+    /* create tensor */
+	XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+	XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
+	XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+	sGPU->SetData(sData, sUnitNum);
+	tGPU1->SetZeroAll();
+	tGPU2->SetZeroAll();
+
+	/* add tensors to list */
+	tList.Add(tGPU1);
+	tList.Add(tGPU2);
+
+	/* call split function */
+	Split(s, &tList, 1, 2);
+
+    /* check results */
+	gpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
+
+    /* destroy variables */
+	delete s;
+    delete t1;
+    delete t2;
+    delete sGPU;
+    delete tGPU1;
+    delete tGPU2;
+	delete[] sDimSize;
+	delete[] tDimSize1;
+	delete[] tDimSize2;
+
+	return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+	delete s;
+    delete t1;
+    delete t2;
+	delete[] sDimSize;
+	delete[] tDimSize1;
+	delete[] tDimSize2;
+
+	return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for Split Function */
+bool TestSplit()
+{
+    XPRINT(0, stdout, "[TEST SPLIT] split a big tensor into small tensors \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSplit1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestSplit2();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* case 3 test */
+    caseFlag = TestSplit3();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 3 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 3 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSplit.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-13
+*/
+
+#ifndef __TEST_SPLIT_H__
+#define __TEST_SPLIT_H__
+
+#include "../core/shape/Split.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Split Function */
+extern "C"
+bool TestSplit();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SPLIT_H__

source/tensor/test/TSum.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#include "TSum.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* case 1: tensor summation c = a + b * \beta */
+bool TestSum1()
+{
+    /* a tensor of size (2, 4) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 4;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE bData[2][4] = { {1.0F, -1.0F, -3.0F, -5.0F}, 
+                          {-7.0F, -9.0F, -11.0F, -13.0F} };
+    DTYPE answer[2][4] = { {1.0F, 0.0F, -1.0F, -2.0F},
+                           {-3.0F, -4.0F, -5.0F, -6.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a = NewTensor(order, dimSize);
+    XTensor * b = NewTensor(order, dimSize);
+
+    /* initialize variables */
+    a->SetData(aData, unitNum);
+    b->SetData(bData, unitNum);
+
+    /* call sum function */
+    _Sum(a, b, a);
+
+    /* check results */
+    cpuTest = a->CheckData(answer, unitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    aGPU->SetData(aData, unitNum);
+    bGPU->SetData(bData, unitNum);
+
+    /* call sum function */
+    _Sum(aGPU, bGPU, aGPU);
+
+    /* check results */
+    gpuTest = aGPU->CheckData(answer, unitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete aGPU;
+    delete bGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+	delete b;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* case 2: tensor summation c = a + b * \beta */
+bool TestSum2()
+{
+    /* a tensor of size (2, 4) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 4;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++) {
+        unitNum *= dimSize[i];
+    }
+    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE bData[2][4] = { {1.0F, -1.0F, -3.0F, -5.0F}, 
+                          {-7.0F, -9.0F, -11.0F, -13.0F} };
+    DTYPE answer[2][4] = { {0.5F, 0.5F, 0.5F, 0.5F},
+                           {0.5F, 0.5F, 0.5F, 0.5F} };
+    float beta = 0.5F;
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensor */
+    XTensor * a = NewTensor(order, dimSize);
+    XTensor * b = NewTensor(order, dimSize);
+    XTensor * c = NewTensor(order, dimSize);
+
+    /* Initalize variables */
+    a->SetData(aData, unitNum);
+    b->SetData(bData, unitNum);
+    c->SetZeroAll();
+
+    /* call Sum function */
+    _Sum(a, b, c, beta);
+
+    /* check results */
+    cpuTest = c->CheckData(answer, unitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * cGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initalize variables */
+    aGPU->SetData(aData, unitNum);
+    bGPU->SetData(bData, unitNum);
+    cGPU->SetZeroAll();
+
+    /* call Sum function */
+    _Sum(aGPU, bGPU, cGPU, beta);
+
+    /* check results */
+    gpuTest = cGPU->CheckData(answer, unitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete c;
+    delete aGPU;
+    delete bGPU;
+    delete cGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete c;
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for Sum Function */
+bool TestSum()
+{
+    XPRINT(0, stdout, "[TEST SUM] tensor summation c = a + b * beta\n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSum1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestSum2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+        TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSum.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
+*/
+
+#ifndef __TEST_SUM_H__
+#define __TEST_SUM_H__
+
+#include "../core/arithmetic/Sum.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Sum Function */
+extern "C"
+bool TestSum();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SUM_H__

source/tensor/test/TSumByColumnTV.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#include "TSumByColumnTV.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test SumByColumnTV function
+sum of a tensor and a vector (column vector) in a column by column manner
+*/
+bool TestSumByColumnTV1()
+{
+    /* a tensor of size (2, 4) */
+    int aOrder = 2;
+    int * aDimSize = new int[aOrder];
+    aDimSize[0] = 2;
+    aDimSize[1] = 4;
+
+    int aUnitNum = 1;
+    for (int i = 0; i < aOrder; i++)
+        aUnitNum *= aDimSize[i];
+
+    /* a tensor of size (2, 1) */
+    int bOrder = 2;
+    int * bDimSize = new int[bOrder];
+    bDimSize[0] = 2;
+    bDimSize[1] = 1;
+
+    int bUnitNum = 1;
+    for (int i = 0; i < bOrder; i++)
+        bUnitNum *= bDimSize[i];
+
+    /* a tensor of size (2, 4) */
+    int cOrder = 2;
+    int * cDimSize = new int[cOrder];
+    cDimSize[0] = 2;
+    cDimSize[1] = 4;
+
+    int cUnitNum = 1;
+    for (int i = 0; i < cOrder; i++)
+        cUnitNum *= cDimSize[i];
+
+    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE bData[2][1] = { {1.0F},
+                          {0.0F} };
+    DTYPE answer[2][4] = { {1.0F, 2.0F, 3.0F, 4.0F},
+                           {4.0F, 5.0F, 6.0F, 7.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a = NewTensor(aOrder, aDimSize);
+    XTensor * b = NewTensor(bOrder, bDimSize);
+    XTensor * c = NewTensor(cOrder, cDimSize);
+
+    /* initialize variables */
+    a->SetData(aData, aUnitNum);
+    b->SetData(bData, bUnitNum);
+
+    /* call SumByColumnTV function */
+    SumByColumnTV(a, b, c);
+
+    /* check results */
+    cpuTest = c->CheckData(answer, cUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * cGPU = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    aGPU->SetData(aData, aUnitNum);
+    bGPU->SetData(bData, bUnitNum);
+    cGPU->SetZeroAll();
+
+    /* call SumByColumnTV function */
+    SumByColumnTV(aGPU, bGPU, cGPU);
+
+    /* check results */
+    gpuTest = cGPU->CheckData(answer, cUnitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete c;
+    delete aGPU;
+    delete bGPU;
+    delete cGPU;
+    delete[] aDimSize;
+    delete[] bDimSize;
+    delete[] cDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete c;
+    delete[] aDimSize;
+    delete[] bDimSize;
+    delete[] cDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: test SumByColumnTV function
+sum of a tensor and a vector (column vector) in a column by column manner
+*/
+bool TestSumByColumnTV2()
+{
+    /* a tensor of size (2, 4) */
+    int aOrder = 2;
+    int * aDimSize = new int[aOrder];
+    aDimSize[0] = 2;
+    aDimSize[1] = 4;
+
+    int aUnitNum = 1;
+    for (int i = 0; i < aOrder; i++)
+        aUnitNum *= aDimSize[i];
+
+    /* a tensor of size (2, 1) */
+    int bOrder = 2;
+    int * bDimSize = new int[bOrder];
+    bDimSize[0] = 2;
+    bDimSize[1] = 1;
+
+    int bUnitNum = 1;
+    for (int i = 0; i < bOrder; i++)
+        bUnitNum *= bDimSize[i];
+
+    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE bData[2][1] = { {1.0F},
+                          {0.0F} };
+    DTYPE answer[2][4] = { {1.0F, 2.0F, 3.0F, 4.0F},
+                           {4.0F, 5.0F, 6.0F, 7.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a = NewTensor(aOrder, aDimSize);
+    XTensor * b = NewTensor(bOrder, bDimSize);
+
+    /* initialize variables */
+    a->SetData(aData, aUnitNum);
+    b->SetData(bData, bUnitNum);
+
+    /* call SumByColumnTV function */
+    SumByColumnTV(a, b);
+
+    /* check results */
+    cpuTest = a->CheckData(answer, aUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    aGPU->SetData(aData, aUnitNum);
+    bGPU->SetData(bData, bUnitNum);
+
+    /* call SumByColumnTV function */
+    SumByColumnTV(aGPU, bGPU);
+
+    /* check results */
+    gpuTest = aGPU->CheckData(answer, aUnitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete aGPU;
+    delete bGPU;
+    delete[] aDimSize;
+    delete[] bDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete[] aDimSize;
+    delete[] bDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for SumByColumnTV Function */
+bool TestSumByColumnTV() 
+{
+    XPRINT(0, stdout, "[TEST SumByColumnTV] sum of a tensor and a vector (column vector) in a column by column manner \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSumByColumnTV1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestSumByColumnTV2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+        TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSumByColumnTV.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#ifndef __TEST_SUMBYCOLUMNTV_H__
+#define __TEST_SUMBYCOLUMNTV_H__
+
+#include "../core/arithmetic/SumByColumnTV.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for SumByColumnTV Function */
+extern "C"
+bool TestSumByColumnTV();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SUMBYCOLUMNTV_H__

source/tensor/test/TSumByColumnVT.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#include "TSumByColumnVT.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: test SumByColumnVT function
+sum of a vector (column vector) and a tensor in a column by column manner
+*/
+bool TestSumByColumnVT1()
+{
+    /* a tensor of size (2, 1) */
+    int aOrder = 2;
+    int * aDimSize = new int[aOrder];
+    aDimSize[0] = 2;
+    aDimSize[1] = 1;
+
+    int aUnitNum = 1;
+    for (int i = 0; i < aOrder; i++)
+        aUnitNum *= aDimSize[i];
+
+    /* a tensor of size (2, 4) */
+    int bOrder = 2;
+    int * bDimSize = new int[bOrder];
+    bDimSize[0] = 2;
+    bDimSize[1] = 4;
+
+    int bUnitNum = 1;
+    for (int i = 0; i < bOrder; i++)
+        bUnitNum *= bDimSize[i];
+
+    /* a tensor of size (2, 1) */
+    int cOrder = 2;
+    int * cDimSize = new int[cOrder];
+    cDimSize[0] = 2;
+    cDimSize[1] = 1;
+
+    int cUnitNum = 1;
+    for (int i = 0; i < cOrder; i++)
+        cUnitNum *= cDimSize[i];
+
+    DTYPE aData[2][1] = { {1.0F},
+                          {0.0F} };
+    DTYPE bData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE answer[2][1] = { {7.0F},
+                           {22.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a = NewTensor(aOrder, aDimSize);
+    XTensor * b = NewTensor(bOrder, bDimSize);
+    XTensor * c = NewTensor(cOrder, cDimSize);
+
+    /* initialize variables */
+    a->SetData(aData, aUnitNum);
+    b->SetData(bData, bUnitNum);
+    c->SetZeroAll();
+
+    /* call SumByColumnVT function */
+    SumByColumnVT(a, b, c);
+    
+    /* check results */
+    cpuTest = c->CheckData(answer, cUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * cGPU = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    aGPU->SetData(aData, aUnitNum);
+    bGPU->SetData(bData, bUnitNum);
+    cGPU->SetZeroAll();
+
+    /* call SumByColumnVT function */
+    SumByColumnVT(aGPU, bGPU, cGPU);
+    
+    /* check results */
+    gpuTest = cGPU->CheckData(answer, cUnitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete c;
+    delete aGPU;
+    delete bGPU;
+    delete cGPU;
+    delete[] aDimSize;
+    delete[] bDimSize;
+    delete[] cDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete c;
+    delete[] aDimSize;
+    delete[] bDimSize;
+    delete[] cDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* 
+case 2: test SumByColumnVT function
+sum of a vector (column vector) and a tensor in a column by column manner
+*/
+bool TestSumByColumnVT2()
+{
+    /* a tensor of size (2, 1) */
+    int aOrder = 2;
+    int * aDimSize = new int[aOrder];
+    aDimSize[0] = 2;
+    aDimSize[1] = 1;
+
+    int aUnitNum = 1;
+    for (int i = 0; i < aOrder; i++)
+        aUnitNum *= aDimSize[i];
+
+    /* a tensor of size (2, 4) */
+    int bOrder = 2;
+    int * bDimSize = new int[bOrder];
+    bDimSize[0] = 2;
+    bDimSize[1] = 4;
+
+    int bUnitNum = 1;
+    for (int i = 0; i < bOrder; i++)
+        bUnitNum *= bDimSize[i];
+
+    DTYPE aData[2][1] = { {1.0F},
+                          {0.0F} };
+    DTYPE bData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
+                          {4.0F, 5.0F, 6.0F, 7.0F} };
+    DTYPE answer[2][1] = { {7.0F},
+                           {22.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * a = NewTensor(aOrder, aDimSize);
+    XTensor * b = NewTensor(bOrder, bDimSize);
+
+    /* initialize variables */
+    a->SetData(aData, aUnitNum);
+    b->SetData(bData, bUnitNum);
+
+    /* call SumByColumnVT function */
+    SumByColumnVT(a, b);
+
+    /* check results */
+    cpuTest = a->CheckData(answer, aUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    aGPU->SetData(aData, aUnitNum);
+    bGPU->SetData(bData, bUnitNum);
+
+    /* call SumByColumnVT function */
+    SumByColumnVT(aGPU, bGPU);
+
+    /* check results */
+    gpuTest = aGPU->CheckData(answer, aUnitNum);
+
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete aGPU;
+    delete bGPU;
+    delete[] aDimSize;
+    delete[] bDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete a;
+    delete b;
+    delete[] aDimSize;
+    delete[] bDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for SumByColumnVT Function */
+bool TestSumByColumnVT() 
+{
+    XPRINT(0, stdout, "[TEST SumByColumnVT] sum of a vector (column vector) and a tensor in a column by column manner \n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestSumByColumnVT1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* case 2 test */
+    caseFlag = TestSumByColumnVT2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+        TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TSumByColumnVT.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
+*/
+
+#ifndef __TEST_SUMBYCOLUMNVT_H__
+#define __TEST_SUMBYCOLUMNVT_H__
+
+#include "../core/arithmetic/SumByColumnVT.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for SumByColumnVT Function */
+extern "C"
+bool TestSumByColumnVT();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_SUMBYCOLUMNVT_H__

source/tensor/test/TTopK.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#include "TTopK.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: get the top-k items along a given dimension.
+In this case, 
+(2, 4) -> (2, 4), dim = 0, k = 2
+(2, 4) -> (2, 4), dim = 1, k = 4
+*/
+bool TestTopK1()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a output tensor of size (2, 4) */
+    int tOrder = 2;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 4;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData[2][4] = { {5.0F, 1.0F, 2.0F, 8.0F},
+                          {4.0F, 3.0F, 7.0F, 6.0F} };
+
+    DTYPE tAnswer1[2][4] = { {5.0F, 3.0F, 7.0F, 8.0F},
+                             {4.0F, 1.0F, 2.0F, 6.0F} };
+    int indexAnswer1[2][4] = { {0, 1, 1, 0},
+                               {1, 0, 0, 1} };
+
+    DTYPE tAnswer2[2][4] = { {8.0F, 5.0F, 2.0F, 1.0F},
+                             {7.0F, 6.0F, 4.0F, 3.0F} };
+    int indexAnswer2[2][4] = { {3, 0, 2, 1},
+                               {2, 3, 0, 1} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t1 = NewTensor(tOrder, tDimSize);
+    XTensor * t2 = NewTensor(tOrder, tDimSize);
+    XTensor * index1 = NewTensor(tOrder, tDimSize, X_INT);
+    XTensor * index2 = NewTensor(tOrder, tDimSize, X_INT);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t1->SetZeroAll();
+    t2->SetZeroAll();
+    index1->SetZeroAll();
+    index2->SetZeroAll();
+
+    /* call TopK function */
+    int dim = 0;
+    int k = sDimSize[dim];
+    TopK(s, t1, index1, dim, k);
+
+    dim = 1;
+    k = sDimSize[dim];
+    TopK(s, t2, index2, dim, k);
+
+    /* check results */
+    cpuTest = t1->CheckData(tAnswer1, tUnitNum) && 
+              t2->CheckData(tAnswer2, tUnitNum) &&
+              index1->CheckData(indexAnswer1, tUnitNum) &&
+              index2->CheckData(indexAnswer2, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU1 = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU2 = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * indexGPU1 = NewTensor(tOrder, tDimSize, X_INT, 1.0F, 0);
+    XTensor * indexGPU2 = NewTensor(tOrder, tDimSize, X_INT, 1.0F, 0);
+    
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU1->SetZeroAll();
+    tGPU2->SetZeroAll();
+    indexGPU1->SetZeroAll();
+    indexGPU2->SetZeroAll();
+
+    /* call TopK function */
+    dim = 0;
+    k = sDimSize[dim];
+    TopK(sGPU, tGPU1, indexGPU1, dim, k);
+    
+    dim = 1;
+    k = sDimSize[dim];
+    TopK(sGPU, tGPU2, indexGPU2, dim, k);
+    
+    /* check results */
+    gpuTest = tGPU1->CheckData(tAnswer1, tUnitNum) && 
+              tGPU2->CheckData(tAnswer2, tUnitNum) &&
+              indexGPU1->CheckData(indexAnswer1, tUnitNum) &&
+              indexGPU2->CheckData(indexAnswer2, tUnitNum);
+
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete index1;
+    delete index2;
+    delete sGPU;
+    delete tGPU1;
+    delete tGPU2;
+    delete indexGPU1;
+    delete indexGPU2;
+    delete[] sDimSize;
+    delete[] tDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete index1;
+    delete index2;
+    delete[] sDimSize;
+    delete[] tDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/*
+case 2: get the top-k items along a given dimension.
+In this case, (2, 4) -> (2, 2), dim = 1, k = 2.
+*/
+bool TestTopK2()
+{
+    /* a input tensor of size (2, 4) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 4;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a output tensor of size (2, 2) */
+    int tOrder = 2;
+    int * tDimSize = new int[tOrder];
+    tDimSize[0] = 2;
+    tDimSize[1] = 2;
+
+    int tUnitNum = 1;
+    for (int i = 0; i < tOrder; i++)
+        tUnitNum *= tDimSize[i];
+
+    DTYPE sData[2][4] = { {5.0F, 1.0F, 2.0F, 8.0F},
+                          {4.0F, 3.0F, 7.0F, 6.0F} };
+    DTYPE tAnswer[2][2] = { {8.0F, 5.0F},
+                            {7.0F, 6.0F} };
+    int indexAnswer[2][2] = { {3, 0},
+                              {2, 3} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t = NewTensor(tOrder, tDimSize);
+    XTensor * index = NewTensor(tOrder, tDimSize, X_INT);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t->SetZeroAll();
+    index->SetZeroAll();
+
+    /* call TopK function */
+    int dim = 1;
+    int k = tDimSize[dim];
+    TopK(s, t, index, dim, k);
+
+    /* check results */
+    cpuTest = t->CheckData(tAnswer, tUnitNum) && index->CheckData(indexAnswer, tUnitNum);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * indexGPU = NewTensor(tOrder, tDimSize, X_INT, 1.0F, 0);
+    
+    /* initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU->SetZeroAll();
+    indexGPU->SetZeroAll();
+
+    /* call TopK function */
+    dim = 1;
+    k = tDimSize[dim];
+    TopK(sGPU, tGPU, indexGPU, dim, k);
+
+    /* check results */
+    gpuTest = tGPU->CheckData(tAnswer, tUnitNum) && indexGPU->CheckData(indexAnswer, tUnitNum);
+    
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete index;
+    delete sGPU;
+    delete tGPU;
+    delete indexGPU;
+    delete[] sDimSize;
+    delete[] tDimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t;
+    delete index;
+    delete[] sDimSize;
+    delete[] tDimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+TODO!!
+*/
+
+/* test for TopK Function */
+bool TestTopK()
+{
+    XPRINT(0, stdout, "[TEST TopK] get the top-k items along a given dimension\n");
+    bool returnFlag = true, caseFlag = true;
+    
+    /* case 1 test */
+    caseFlag = TestTopK1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+    
+    /* case 2 test */
+    caseFlag = TestTopK2();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 2 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 2 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+    }
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TTopK.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#ifndef __TEST_TOPK_H__
+#define __TEST_TOPK_H__
+
+#include "../core/sort/TopK.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for TopK Function */
+extern "C"
+bool TestTopK();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_TOPK_H__

source/tensor/test/TUnsqueeze.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-13
+*/
+
+#include "../XList.h"
+#include "TUnsqueeze.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* 
+case 1: insert a dimension by copying the blocks for x times (where x is the size of the inerted dimension)
+In this case, 
+(2, 3) -> (2, 2, 3), dim=1, dSize=2
+(2, 3) -> (2, 3, 2), dim=2, dSize=2
+*/
+bool TestUnsqueeze1()
+{
+    /* a source tensor of size (2, 3) */
+    int sOrder = 2;
+    int * sDimSize = new int[sOrder];
+    sDimSize[0] = 2;
+    sDimSize[1] = 3;
+
+    int sUnitNum = 1;
+    for (int i = 0; i < sOrder; i++)
+        sUnitNum *= sDimSize[i];
+
+    /* a target tensor of size (2, 2, 3) */
+    int tOrder1 = 3;
+    int * tDimSize1 = new int[tOrder1];
+    tDimSize1[0] = 2;
+    tDimSize1[1] = 2;
+    tDimSize1[2] = 3;
+
+    int tUnitNum1 = 1;
+    for (int i = 0; i < tOrder1; i++)
+        tUnitNum1 *= tDimSize1[i];
+
+    /* a target tensor of size (2, 3, 2) */
+    int tOrder2 = 3;
+    int * tDimSize2 = new int[tOrder2];
+    tDimSize2[0] = 2;
+    tDimSize2[1] = 3;
+    tDimSize2[2] = 2;
+
+    int tUnitNum2 = 1;
+    for (int i = 0; i < tOrder2; i++)
+        tUnitNum2 *= tDimSize2[i];
+
+    DTYPE sData[2][3] = { {0.0F, 1.0F, 2.0F},
+                          {3.0F, 4.0F, 5.0F} };
+    DTYPE answer1[2][2][3] = { { {0.0F, 1.0F, 2.0F},
+                                 {0.0F, 1.0F, 2.0F} },
+                               { {3.0F, 4.0F, 5.0F},
+                                 {3.0F, 4.0F, 5.0F} } };
+    DTYPE answer2[2][3][2] = { { {0.0F, 0.0F}, 
+                                 {1.0F, 1.0F}, 
+                                 {2.0F, 2.0F} },
+                               { {3.0F, 3.0F}, 
+                                 {4.0F, 4.0F}, 
+                                 {5.0F, 5.0F} } };
+    
+    /* CPU test */
+    bool cpuTest = true;
+
+    /* create tensors */
+    XTensor * s = NewTensor(sOrder, sDimSize);
+    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
+    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
+
+    /* initialize variables */
+    s->SetData(sData, sUnitNum);
+    t1->SetZeroAll();
+    t2->SetZeroAll();
+
+    /* call Unsqueeze function */
+    Unsqueeze(s, t1, 1, 2);
+    Unsqueeze(s, t2, 2, 2);
+
+    /* check results */
+    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensor */
+    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
+    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
+
+    /* Initialize variables */
+    sGPU->SetData(sData, sUnitNum);
+    tGPU1->SetZeroAll();
+    tGPU2->SetZeroAll();
+
+    /* call Unsqueeze function */
+    Unsqueeze(sGPU, tGPU1, 1, 2);
+    Unsqueeze(sGPU, tGPU2, 2, 2);
+
+    /* check results */
+    gpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
+
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete sGPU;
+    delete tGPU1;
+    delete tGPU2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete s;
+    delete t1;
+    delete t2;
+    delete[] sDimSize;
+    delete[] tDimSize1;
+    delete[] tDimSize2;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
+/* other cases */
+/*
+    TODO!!
+*/
+
+/* test for Unsqueeze Function */
+bool TestUnsqueeze()
+{
+    XPRINT(0, stdout, "[TEST Unsqueeze] insert a dimension by copying the blocks for x times\n");
+    bool returnFlag = true, caseFlag = true;
+
+    /* case 1 test */
+    caseFlag = TestUnsqueeze1();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    /* other cases test */
+    /*
+    TODO!!
+    */
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    XPRINT(0, stdout, "\n");
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/tensor/test/TUnsqueeze.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
+*/
+
+#ifndef __TEST_UNSQUEEZE_H__
+#define __TEST_UNSQUEEZE_H__
+
+#include "../core/shape/Unsqueeze.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for Unsqueeze Function */
+extern "C"
+bool TestUnsqueeze();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_UNSQUEEZE_H__

source/tensor/test/TXMem.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-6-24
+ */
+
+#include "../XGlobal.h"
+#include "../XUtility.h"
+#include "TXMem.h"
+
+namespace nts{ // namespace nts(NiuTrans.Tensor)
+
+/* case 1: test memory pool class */
+bool TestXMemCase1()
+{
+    bool ok = true;
+    int caseNum = 1000;
+    int blcokSize = 16;
+    int testNum = caseNum * 10;
+
+    for(int i = 0, scalar = 1; i < 3; i++){
+        XMem mem;
+        mem.Initialize(-1, FREE_ON_THE_FLY, blcokSize * sizeof(int) * scalar * scalar, 1000, 0);
+        mem.SetIndex(10000, blcokSize * sizeof(int) / 2);
+
+        srand(907);
+
+        int ** p = new int*[caseNum];
+        int * size = new int[caseNum];
+
+        for(int i = 0; i < caseNum; i++){
+            p[i] = NULL;
+            size[i] = rand() % (2*blcokSize);
+        }
+
+        for(int i = 0; i < testNum * scalar; i++){
+            int j = rand() % caseNum;
+
+            if(p[j] == NULL){
+                p[j] = (int*)mem.AllocStandard(mem.devID, size[j] * sizeof(int));
+                for(int k = 0; k < size[j]; k++)
+                    p[j][k] = j;
+            }
+            else{
+                mem.ReleaseStandard(mem.devID, p[j]);
+                for(int k = 0; k < size[j]; k++)
+                    p[j][k] = -1;
+                p[j] = NULL;
+            }
+
+            for(int k = 0; k < caseNum; k++){
+                if(p[k] != NULL){
+                    for(int o = 0; o < size[k]; o++){
+                        if(p[k][o] != k){
+                            ok = false;
+                        }
+                    }
+                }
+            }
+        }
+
+        delete[] p;
+        delete[] size;
+        scalar *= 2;
+    }
+
+    return ok;
+}
+
+/* test for memory pool class */
+bool TestXMem()
+{
+    XPRINT(0, stdout, "[Test] Memory pool ... Began\n");
+    bool returnFlag = true;
+    bool caseFlag = true;
+
+    double startT = GetClock();
+
+    /* case 1 test */
+    caseFlag = TestXMemCase1();
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 1 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 1 passed!\n");
+
+    if (returnFlag) {
+        XPRINT(0, stdout, ">> All Passed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> Failed!\n");
+
+    double endT = GetClock();
+
+    XPRINT1(0, stdout, "[Test] Finished (took %.3lfms)\n\n", endT - startT);
+
+    return returnFlag;
+}
+
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/tensor/test/TXMem.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-6-24
+ */
+
+#ifndef __TXMEM_H__
+#define __TXMEM_H__
+
+#include "../XMem.h"
+
+namespace nts{ // namespace nts(NiuTrans.Tensor)
+
+/* test for memory pool class */
+extern "C"
+bool TestXMem();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TXMEM_H__

source/tensor/test/Test.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-05-01
+*/
+
+#include "Test.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for all Function */
+bool Test()
+{
+    bool wrong = false;
+    XPRINT(0, stdout, "Testing the XTensor utilites ... \n\n");
+
+    wrong = !TestConcatenate() || wrong;
+    wrong = !TestConcatenateSolely() || wrong;
+    wrong = !TestCopyIndexed() || wrong;
+    wrong = !TestCopyValues() || wrong;
+    wrong = !TestMatrixMul() || wrong;
+    wrong = !TestMatrixMul2D() || wrong;
+    wrong = !TestMatrixMul2DParallel() || wrong;
+    wrong = !TestMatrixMulBatched() || wrong;
+    wrong = !TestMatrixMulBatchedCPU() || wrong;
+    wrong = !TestMerge() || wrong;
+    wrong = !TestMultiply() || wrong;
+    wrong = !TestNegate() || wrong;
+    wrong = !TestNormalize() || wrong;
+    wrong = !TestPower() || wrong;
+    wrong = !TestReduceMax() || wrong;
+    wrong = !TestReduceMean() || wrong;
+    wrong = !TestReduceSum() || wrong;
+    wrong = !TestReduceSumSquared() || wrong;
+    wrong = !TestReduceVariance() || wrong;
+    wrong = !TestScaleAndShift() || wrong;
+    wrong = !TestSelect() || wrong;
+    wrong = !TestSetAscendingOrder() || wrong;
+    wrong = !TestSetData() || wrong;
+    wrong = !TestSort() || wrong;
+    wrong = !TestSplit() || wrong;
+    wrong = !TestSum() || wrong;
+    wrong = !TestSumByColumnTV() || wrong;
+    wrong = !TestSumByColumnVT() || wrong;
+    wrong = !TestTopK() || wrong;
+    wrong = !TestUnsqueeze() || wrong;
+    wrong = !TestXMem() || wrong;
+
+    wrong = !TestHardTanH() || wrong;
+    wrong = !TestIdentity() || wrong;
+    wrong = !TestLogSoftmax() || wrong;
+    wrong = !TestLoss() || wrong;
+    wrong = !TestRectify() || wrong;
+    wrong = !TestSigmoid() || wrong;
+    wrong = !TestSoftmax() || wrong;
+
+    /* other test */
+    /*
+    TODO!!
+    */
+
+    if (wrong) {
+        XPRINT(0, stdout, "Something goes wrong! Please check the code!\n");
+        return false;
+    }
+    else {
+        XPRINT(0, stdout, "OK! Everything is good!\n");
+        return true;
+    }
+}
+
+} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/tensor/test/Test.h

+/* NiuTrans.Tensor - an open-source tensor library
+* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+* All rights reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-05-01
+*/
+
+#ifndef __TEST_H__
+#define __TEST_H__
+
+#include "TConcatenate.h"
+#include "TConcatenateSolely.h"
+#include "TCopyIndexed.h"
+#include "TCopyValues.h"
+#include "TMatrixMul.h"
+#include "TMatrixMul2D.h"
+#include "TMatrixMul2DParallel.h"
+#include "TMatrixMulBatched.h"
+#include "TMatrixMULBatchedCPU.h"
+#include "TMerge.h"
+#include "TMultiply.h"
+#include "TNegate.h"
+#include "TNormalize.h"
+#include "TPower.h"
+#include "TReduceMax.h"
+#include "TReduceMean.h"
+#include "TReduceSum.h"
+#include "TReduceSumSquared.h"
+#include "TReduceVariance.h"
+#include "TScaleAndShift.h"
+#include "TSelect.h"
+#include "TSetAscendingOrder.h"
+#include "TSetData.h"
+#include "TSort.h"
+#include "TSplit.h"
+#include "TSum.h"
+#include "TSumByColumnTV.h"
+#include "TSumByColumnVT.h"
+#include "TTopK.h"
+#include "TUnsqueeze.h"
+#include "TXMem.h"
+
+#include "THardTanH.h"
+#include "TIdentity.h"
+#include "TLogSoftmax.h"
+#include "TLoss.h"
+#include "TRectify.h"
+#include "TSigmoid.h"
+#include "TSoftmax.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* test for all Function */
+extern "C"
+bool Test();
+
+} // namespace nts(NiuTrans.Tensor)
+#endif // __TEST_H__

source/test/TCopyValues.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#include "../XUtility.h"
-#include "TCopyValues.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* case 1: copy tensor s to tensor t */
-bool TestCopyValues1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-
-    DTYPE scaleFactor = 2.0F;
-    DTYPE shiftFactor = 0.5F;
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(sOrder, sDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t->SetZeroAll();
-
-    /* call CopyValues function */
-    CopyValues(s, t);
-
-    /* check results */
-    cpuTest = t->CheckData(s->data, sUnitNum);
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU->SetData(sData, sUnitNum);
-
-    /* call CopyValues function */
-    CopyValues(sGPU, tGPU);
-
-    /* check results */
-    DTYPE * dataGPU = (DTYPE*)sGPU->data;
-    int size = sUnitNum * sGPU->unitSize;
-    char * dataCPU = new char[size];
-    XMemCopy(dataCPU, -1, dataGPU, sGPU->devID, size);
-
-    gpuTest = tGPU->CheckData(dataCPU, sUnitNum);
-
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete sGPU;
-    delete tGPU;
-    delete[] sDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete[] sDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for CopyValues Function */
-bool TestCopyValues()
-{
-    XPRINT(0, stdout, "[TEST CopyValues] copy tensor s to tensor t \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestCopyValues1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/THardTanH.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-20
-*/
-
-#include "../XTensor.h"
-#include "THardTanH.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test HardTanH function 
-y =  1    if x > 1
-     x    if -1 <= x <= 1
-    -1    if x < -1
-*/
-bool TestHardTanH1()
-{
-	/* a tensor of size (2, 3) */
-	int order = 2;
-	int * dimSize = new int[order];
-	dimSize[0] = 2;
-	dimSize[1] = 3;
-
-	int unitNum = 1;
-	for (int i = 0; i < order; i++)
-		unitNum *= dimSize[i];
-
-	DTYPE xData[2][3] = { {0.5F, -1.0F, 2.0F},
-	                      {3.5F, -4.5F, 1.0F} };
-	DTYPE answer[2][3] = { {0.5F, -1.0F, 1.0F},
-	                       {1.0F, -1.0F, 1.0F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * x = NewTensor(order, dimSize);
-	XTensor * y = NewTensor(order, dimSize);
-
-	/* initialize variables */
-	x->SetData(xData, unitNum);
-	y->SetZeroAll();
-
-	/* call hardtanh function */
-	HardTanH(x, y);
-
-	/* check results */
-	cpuTest = y->CheckData(answer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensor */
-	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-	/* Initialize variables */
-	xGPU->SetData(xData, unitNum);
-	yGPU->SetZeroAll();
-
-	/* call hardtanh function */
-	HardTanH(xGPU, yGPU);
-
-	/* check results */
-	gpuTest = yGPU->CheckData(answer, unitNum, 1e-4F);
-
-	/* destroy variables */
-	delete x;
-    delete y;
-    delete xGPU;
-    delete yGPU;
-	delete[] dimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete x;
-    delete y;
-	delete[] dimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/*
-case 2: test backward computation of HardTanH function.
-dE/dx = dE/dy * dy/dx
-hard tanh: y =  1    if x > 1
-                x    if -1 <= x <= 1
-               -1    if x< -1
-
-   and dy/dx =  1    if -1 <= x <= 1
-                0    otherwise
-In this case, lossName=SQUAREDERROR.
-*/
-bool TestHardTanH2()
-{
-	/* a tensor of size (2, 3) */
-	int order = 2;
-	int * dimSize = new int[order];
-	dimSize[0] = 2;
-	dimSize[1] = 3;
-
-	int unitNum = 1;
-	for (int i = 0; i < order; i++)
-		unitNum *= dimSize[i];
-
-	DTYPE xData[2][3] = { {0.5F, -1.0F, 2.0F},
-	                      {3.5F, -4.5F, 1.0F} };
-	DTYPE goldData[2][3] = { {1.0F, 1.0F, 1.0F},
-	                         {1.0F, 1.0F, 1.0F} };
-	DTYPE yAnswer[2][3] = { {0.5F, -1.0F, 1.0F},
-	                        {1.0F, -1.0F, 1.0F} };
-    DTYPE dedyAnswer[2][3] = { {-0.5F, -2.0F, 0.0F},
-	                           {0.0F, -2.0F, 0.0F} };
-	DTYPE dedxAnswer[2][3] = { {-0.5F, -2.0F, 0.0F},
-	                           {0.0F, 0.0F, -0.0F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * x = NewTensor(order, dimSize);
-	XTensor * y = NewTensor(order, dimSize);
-	XTensor * gold = NewTensor(order, dimSize);
-	XTensor * dedy = NewTensor(order, dimSize);
-	XTensor * dedx = NewTensor(order, dimSize);
-
-	/* initialize variables */
-	x->SetData(xData, unitNum);
-	gold->SetData(goldData, unitNum);
-    y->SetZeroAll();
-    dedy->SetZeroAll();
-	dedx->SetZeroAll();
-
-    /* call HardTanH function */
-    HardTanH(x, y);
-
-	/* call HardTanHBackward function */
-	HardTanHBackward(gold, y, x, dedy, dedx, SQUAREDERROR);
-
-	/* check results */
-	cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F) 
-              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensors */
-	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-	/* initialize variables */
-	xGPU->SetData(xData, unitNum);
-	goldGPU->SetData(goldData, unitNum);
-    yGPU->SetZeroAll();
-    dedyGPU->SetZeroAll();
-	dedxGPU->SetZeroAll();
-
-    /* call HardTanH function */
-    HardTanH(xGPU, yGPU);
-
-	/* call hardtanhbackward function */
-	HardTanHBackward(goldGPU, yGPU, xGPU, dedyGPU, dedxGPU, SQUAREDERROR);
-
-	/* check results */
-	gpuTest = y->CheckData(yAnswer, unitNum, 1e-4F) 
-              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
-
-	/* destroy variables */
-    delete x;
-    delete y;
-    delete gold;
-    delete dedx;
-    delete dedy;
-    delete xGPU;
-    delete yGPU;
-    delete goldGPU;
-    delete dedxGPU;
-    delete dedyGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete gold;
-    delete dedx;
-    delete dedy;
-    delete[] dimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for HardTanH Function */
-bool TestHardTanH()
-{
-	XPRINT(0, stdout, "[TEST HARDTANH] test hardtanh and its backward computation \n");
-	bool returnFlag = true, caseFlag = true;
-
-	/* case 1 test */
-	caseFlag = TestHardTanH1();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 1 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 1 passed!\n");
-
-	/* case 2 test */
-	caseFlag = TestHardTanH2();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 2 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 2 passed!\n");
-
-	/* other cases test */
-	/*
-	TODO!!
-	*/
-
-	if (returnFlag) {
-		XPRINT(0, stdout, ">> All Passed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> Failed!\n");
-
-	XPRINT(0, stdout, "\n");
-
-	return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TIdentity.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-29
-*/
-
-#include "../XUtility.h"
-#include "TIdentity.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/*
-case 1: test Identity function.
-Identity function: y = x 
-*/
-bool TestIdentity1()
-{
-    /* a tensor of size (2, 3) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 2;
-    dimSize[1] = 3;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE xData[2][3] = { {0.0F, 1.0F, 2.0F}, 
-                          {0.5F, 0.7F, 1.4F} };
-    DTYPE answer[2][3] = { {0.0F, 1.0F, 2.0F}, 
-                           {0.5F, 0.7F, 1.4F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * x = NewTensor(order, dimSize);
-    XTensor * y = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    x->SetData(xData, unitNum);
-    y->SetZeroAll();
-
-    /* call Identity function */
-    Identity(x, y);
-    
-    /* check result */
-    cpuTest = y->CheckData(answer, unitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    xGPU->SetData(xData, unitNum);
-    yGPU->SetZeroAll();
-
-    /* call Identity function */
-    Identity(xGPU, yGPU);
-    
-    /* check result */
-    gpuTest = yGPU->CheckData(answer, unitNum);
-
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete xGPU;
-    delete yGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: test IdentityBackward function.
-IdentityBackward function: dE/dx = dE/dy * dy/dx = dE/dy
-In this case, lossName=CROSSENTROPY.
-*/
-bool TestIdentity2()
-{
-    /* a tensor of size (2, 3) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 1;
-    dimSize[1] = 3;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE xData[3] = {1.0F, 1.0F, 2.0F};
-    DTYPE gData[3] = {0.0F, 0.0F, 1.0F};
-    DTYPE yAnswer[3] = {1.0F, 1.0F, 2.0F};
-    DTYPE dedyAnswer[3] = {0.0F, 0.0F, -0.5F};
-    DTYPE dedxAnswer[3] = {0.0F, 0.0F, -0.5F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * x = NewTensor(order, dimSize);
-    XTensor * y = NewTensor(order, dimSize);
-    XTensor * g = NewTensor(order, dimSize);
-    XTensor * dedy = NewTensor(order, dimSize);
-    XTensor * dedx = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    x->SetData(xData, unitNum);
-    g->SetData(gData, unitNum);
-    y->SetZeroAll();
-    dedx->SetZeroAll();
-    dedy->SetZeroAll();
-
-    /* call Identity function */
-    Identity(x, y);
-
-    /* call IdentityBackward function */
-    IdentityBackward(g, y, x, dedy, dedx, CROSSENTROPY);
-    
-    /* check result */
-    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-        /* create tensors */
-    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * gGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    xGPU->SetData(xData, unitNum);
-    gGPU->SetData(gData, unitNum);
-    yGPU->SetZeroAll();
-    dedxGPU->SetZeroAll();
-    dedyGPU->SetZeroAll();
-
-    /* call Identity function */
-    Identity(xGPU, yGPU);
-
-    /* call IdentityBackward function */
-    IdentityBackward(gGPU, yGPU, xGPU, dedyGPU, dedxGPU, CROSSENTROPY);
-    
-    /* check result */
-    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
-
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete g;
-    delete dedx;
-    delete dedy;
-    delete xGPU;
-    delete yGPU;
-    delete gGPU;
-    delete dedxGPU;
-    delete dedyGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete g;
-    delete dedx;
-    delete dedy;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for Identity Function */
-bool TestIdentity()
-{
-    XPRINT(0, stdout, "[TEST Identity] identity function and its backward computation \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestIdentity1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestIdentity2();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TLoss.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#include "../core/math/ScaleAndShift.h"
-#include "TLoss.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test LossCompute function.
-In this case, Loss function name = SQUAREDERROR.
-loss = sum_{i} 0.5*(t_i - y_i)^2, 
-where t_i is the gold standard and y_i is the model output.
-*/
-bool TestLoss1()
-{
-    /* a tensor of size (10, 1) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 10;
-    dimSize[1] = 1;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    DTYPE answer = 5.0F;
-
-    /* create tensors */
-    XTensor * output = NewTensor(order, dimSize);
-    XTensor * gold = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    output->SetZeroAll();
-    gold->SetZeroAll();
-    ScaleAndShift(output, 1, 1);
-    ScaleAndShift(gold, 1, 2);
-
-    DTYPE error;
-    error = LossCompute(gold, output, SQUAREDERROR, false, 0, 0, dimSize[0], 0);
-    
-    /* check results */
-    cpuTest = (error == answer);
-    
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * outputGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    outputGPU->SetZeroAll();
-    goldGPU->SetZeroAll();
-    ScaleAndShift(outputGPU, 1, 1);
-    ScaleAndShift(goldGPU, 1, 2);
-
-    /* call LossCompute function */
-    error = LossCompute(goldGPU, outputGPU, SQUAREDERROR, false, 0, 0, dimSize[0], 0);
-    
-    /* check results */
-    gpuTest = (error == answer);
-
-    /* destroy variables */
-    delete output;
-    delete gold;
-    delete outputGPU;
-    delete goldGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete output;
-    delete gold;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: test LossCompute function.
-In this case, Loss function name = CROSSENTROPY.
-loss = sum_{i} (-t_i * log(y_i))
-where t_i is the gold standard and y_i is the model output.
-*/
-bool TestLoss2()
-{
-    /* a tensor of size (10, 1) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 10;
-    dimSize[1] = 1;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    DTYPE answer = 0.0F;
-
-    /* create tensors */
-    XTensor * output = NewTensor(order, dimSize);
-    XTensor * gold = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    output->SetZeroAll();
-    gold->SetZeroAll();
-    ScaleAndShift(output, 1, 1);
-    ScaleAndShift(gold, 1, 2);
-
-    DTYPE error;
-    error = LossCompute(gold, output, CROSSENTROPY, false, 0, 0, dimSize[0], 0);
-    
-    /* check results */
-    cpuTest = (error == answer);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * outputGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    outputGPU->SetZeroAll();
-    goldGPU->SetZeroAll();
-    ScaleAndShift(outputGPU, 1, 1);
-    ScaleAndShift(goldGPU, 1, 2);
-
-    /* call LossCompute function */
-    error = LossCompute(goldGPU, outputGPU, CROSSENTROPY, false, 0, 0, dimSize[0], 0);
-    
-    /* check results */
-    gpuTest = (error == answer);
-
-    /* destroy variables */
-    delete output;
-    delete gold;
-    delete outputGPU;
-    delete goldGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete output;
-    delete gold;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 3: test LossCompute function.
-In this case, Loss function name = ONEHOTERROR.
-loss = sum_{i} e_i
-where e_i = 0.5*(t_i - y_i)^2 if t_i = 1, e_i = 0 otherwise.
-*/
-bool TestLoss3()
-{
-    /* a tensor of size (10, 1) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 5;
-    dimSize[1] = 1;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-    DTYPE outputData[5][1] = { {0.5F},
-                               {0.5F},
-                               {0.5F},
-                               {0.5F},
-                               {0.5F} };
-    DTYPE goldData[5][1] = { {1.0F},
-                             {1.0F},
-                             {0.0F},
-                             {0.0F},
-                             {0.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    DTYPE answer = 0.25F;
-
-    /* create tensors */
-    XTensor * output = NewTensor(order, dimSize);
-    XTensor * gold = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    output->SetData(outputData, unitNum);
-    gold->SetData(goldData, unitNum);
-
-    DTYPE error;
-    error = LossCompute(gold, output, ONEHOTERROR, false, 0, 0, dimSize[0], 0);
-    
-    /* check results */
-    cpuTest = (error == answer);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * outputGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    outputGPU->SetData(outputData, unitNum);
-    goldGPU->SetData(goldData, unitNum);
-
-    /* call LossCompute function */
-    error = LossCompute(goldGPU, outputGPU, ONEHOTERROR, false, 0, 0, dimSize[0], 0);
-    
-    /* check results */
-    gpuTest = (error == answer);
-
-    /* destroy variables */
-    delete output;
-    delete gold;
-    delete outputGPU;
-    delete goldGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete output;
-    delete gold;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Loss Function */
-bool TestLoss()
-{
-    XPRINT(0, stdout, "[TEST Loss] compute the loss \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestLoss1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestLoss2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-        
-    caseFlag = TestLoss3();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 3 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 3 passed!\n");
-
-    ///* other cases test */
-    ///*
-    //TODO!!
-    //*/
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TMatrixMULBatchedCPU.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
-*/
-
-#include "TMatrixMULBatchedCPU.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: matrix multiplication in batch mode (CPU code). 
-In this case, aList=2*(2, 3), bList=2*(3, 2) -> c=2*(2, 2), transposedA=X_NOTRANS, transposedB=X_NOTRANS.
-*/
-bool TestMatrixMulBatchedCPU1()
-{
-    /* create list */
-    XList * aList = new XList();
-    XList * bList = new XList();
-    XList * cList = new XList();
-
-    /* a source tensor of size (2, 3) */
-    int aOrder = 2;
-    int * aDimSize = new int[aOrder];
-    aDimSize[0] = 2;
-    aDimSize[1] = 3;
-
-    int aUnitNum = 1;
-    for (int i = 0; i < aOrder; i++)
-        aUnitNum *= aDimSize[i];
-
-    /* a source tensor of size (3, 2) */
-    int bOrder = 2;
-    int * bDimSize = new int[bOrder];
-    bDimSize[0] = 3;
-    bDimSize[1] = 2;
-
-    int bUnitNum = 1;
-    for (int i = 0; i < bOrder; i++)
-        bUnitNum *= bDimSize[i];
-
-    /* a target tensor of size (2, 2) */
-    int cOrder = 2;
-    int * cDimSize = new int[cOrder];
-    cDimSize[0] = 2;
-    cDimSize[1] = 2;
-
-    int cUnitNum = 1;
-    for (int i = 0; i < cOrder; i++)
-        cUnitNum *= cDimSize[i];
-
-    DTYPE aData1[2][3] = { {1.0F, 2.0F, 3.0F},
-                           {-4.0F, 5.0F, 6.0F} };
-    DTYPE aData2[2][3] = { {1.0F, -2.0F, -3.0F},
-                           {-4.0F, 3.0F, 2.0F} };
-    DTYPE bData1[3][2] = { {0.0F, -1.0F},
-                           {1.0F, 2.0F}, 
-                           {2.0F, 1.0F} };
-    DTYPE bData2[3][2] = { {0.0F, 1.0F},
-                           {3.0F, 2.0F}, 
-                           {2.0F, 1.0F} };
-    DTYPE answer1[2][2] = { {8.0F, 6.0F}, 
-                            {17.0F, 20.0F} };
-    DTYPE answer2[2][2] = { {-12.0F, -6.0F}, 
-                            {13.0F, 4.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a1 = NewTensor(aOrder, aDimSize);
-    XTensor * a2 = NewTensor(aOrder, aDimSize);
-    XTensor * b1 = NewTensor(bOrder, bDimSize);
-    XTensor * b2 = NewTensor(bOrder, bDimSize);
-    XTensor * c1 = NewTensor(cOrder, cDimSize);
-    XTensor * c2 = NewTensor(cOrder, cDimSize);
-
-    /* initialize variables */
-    a1->SetData(aData1, aUnitNum);
-    a2->SetData(aData2, aUnitNum);
-    b1->SetData(bData1, aUnitNum);
-    b2->SetData(bData2, aUnitNum);
-    c1->SetZeroAll();
-    c2->SetZeroAll();
-
-    /* add tensors to list */
-    aList->Add(a1);
-    aList->Add(a2);
-    bList->Add(b1);
-    bList->Add(b2);
-    cList->Add(c1);
-    cList->Add(c2);
-
-    /* call MatrixMULBatchedCPU function */
-    MatrixMULBatchedCPU(aList, X_NOTRANS, bList, X_NOTRANS, cList);
-
-    /* check results */
-    cpuTest = c1->CheckData(answer1, cUnitNum) && c2->CheckData(answer2, cUnitNum);
-    
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * aGPU1 = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * aGPU2 = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU1 = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU2 = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * cGPU1 = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * cGPU2 = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    aGPU1->SetData(aData1, aUnitNum);
-    aGPU2->SetData(aData2, aUnitNum);
-    bGPU1->SetData(bData1, aUnitNum);
-    bGPU2->SetData(bData2, aUnitNum);
-    cGPU1->SetZeroAll();
-    cGPU2->SetZeroAll();
-    
-    /* clear list */
-    aList->Clear();
-    bList->Clear();
-    cList->Clear();
-
-    /* add tensors to list */
-    aList->Add(aGPU1);
-    aList->Add(aGPU2);
-    bList->Add(bGPU1);
-    bList->Add(bGPU2);
-    cList->Add(cGPU1);
-    cList->Add(cGPU2);
-
-    /* call MatrixMULBatchedCPU function */
-    MatrixMULBatchedCPU(aList, X_NOTRANS, bList, X_NOTRANS, cList);
-
-    /* check results */
-    gpuTest = cGPU1->CheckData(answer1, cUnitNum) && gpuTest;
-    gpuTest = cGPU2->CheckData(answer2, cUnitNum) && gpuTest;
-
-    /* destroy variables */
-    delete a1;
-    delete a2;
-    delete b1;
-    delete b2;
-    delete c1;
-    delete c2;
-    delete aGPU1;
-    delete aGPU2;
-    delete bGPU1;
-    delete bGPU2;
-    delete cGPU1;
-    delete cGPU2;
-    delete[] aDimSize;
-    delete[] bDimSize;
-    delete[] cDimSize;
-    
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a1;
-    delete a2;
-    delete b1;
-    delete b2;
-    delete c1;
-    delete c2;
-    delete[] aDimSize;
-    delete[] bDimSize;
-    delete[] cDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for MatrixMulBatchedCPU Function */
-extern "C"
-bool TestMatrixMulBatchedCPU()
-{
-    XPRINT(0, stdout, "[TEST MATRIXMULBATCHEDCPU] matrix multiplication in batch mode (CPU code) \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestMatrixMulBatchedCPU1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TMatrixMul2D.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
-*/
-
-#include "TMatrixMul2D.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: matrix multiplication (for 2d tensors). 
-In this case, a=(2, 3), b=(3, 2) -> c=(2, 2), 
-transposedA=X_NOTRANS, transposedB=X_NOTRANS.
-*/
-bool TestMatrixMul2D1()
-{
-    /* a source tensor of size (2, 3) */
-    int sOrder1 = 2;
-    int * sDimSize1 = new int[sOrder1];
-    sDimSize1[0] = 2;
-    sDimSize1[1] = 3;
-
-    int sUnitNum1 = 1;
-    for (int i = 0; i < sOrder1; i++)
-        sUnitNum1 *= sDimSize1[i];
-
-    /* a source tensor of size (3, 2) */
-    int sOrder2 = 2;
-    int * sDimSize2 = new int[sOrder2];
-    sDimSize2[0] = 3;
-    sDimSize2[1] = 2;
-
-    int sUnitNum2 = 1;
-    for (int i = 0; i < sOrder2; i++)
-        sUnitNum2 *= sDimSize2[i];
-
-    /* a target tensor of size (2, 2) */
-    int tOrder = 2;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData1[2][3] = { {1.0F, 2.0F, 3.0F},
-                           {-4.0F, 5.0F, 6.0F} };
-    DTYPE sData2[3][2] = { {0.0F, -1.0F},
-                           {1.0F, 2.0F}, 
-                           {2.0F, 1.0F} };
-    DTYPE answer[2][2] = { {8.0F, 6.0F}, 
-                           {17.0F, 20.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
-    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s1->SetData(sData1, sUnitNum1);
-    s2->SetData(sData2, sUnitNum2);
-    t->SetZeroAll();
-
-    /* call MatrixMul2D function */
-    MatrixMul2D(s1, X_NOTRANS, s2, X_NOTRANS, t);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    sGPU1->SetData(sData1, sUnitNum1);
-    sGPU2->SetData(sData2, sUnitNum2);
-    tGPU->SetZeroAll();
-
-    /* call MatrixMul2D function */
-    MatrixMul2D(sGPU1, X_NOTRANS, sGPU2, X_NOTRANS, tGPU);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete sGPU1;
-    delete sGPU2;
-    delete tGPU;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: matrix multiplication (for 2d tensors).
-In this case, a=(3, 2), b=(3, 2) -> c=(2, 2), 
-transposedA=X_TRANS, transposedB=X_NOTRANS.
-*/
-bool TestMatrixMul2D2()
-{
-    /* a source tensor of size (3, 2) */
-    int sOrder1 = 2;
-    int * sDimSize1 = new int[sOrder1];
-    sDimSize1[0] = 3;
-    sDimSize1[1] = 2;
-
-    int sUnitNum1 = 1;
-    for (int i = 0; i < sOrder1; i++)
-        sUnitNum1 *= sDimSize1[i];
-
-    /* a source tensor of size (3, 2) */
-    int sOrder2 = 2;
-    int * sDimSize2 = new int[sOrder2];
-    sDimSize2[0] = 3;
-    sDimSize2[1] = 2;
-
-    int sUnitNum2 = 1;
-    for (int i = 0; i < sOrder2; i++)
-        sUnitNum2 *= sDimSize2[i];
-
-    /* a target tensor of size (2, 2) */
-    int tOrder = 2;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData1[3][2] = { {1.0F, -4.0F},
-                           {2.0F, 5.0F},
-                           {3.0F, 6.0F} };
-    DTYPE sData2[3][2] = { {0.0F, -1.0F},
-                           {1.0F, 2.0F},
-                           {2.0F, 1.0F} };
-    DTYPE answer[2][2] = { {8.0F, 6.0F},
-                           {17.0F, 20.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
-    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s1->SetData(sData1, sUnitNum1);
-    s2->SetData(sData2, sUnitNum2);
-    t->SetZeroAll();
-
-    /* call MatrixMul2D function */
-    MatrixMul2D(s1, X_TRANS, s2, X_NOTRANS, t);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    sGPU1->SetData(sData1, sUnitNum1);
-    sGPU2->SetData(sData2, sUnitNum2);
-    tGPU->SetZeroAll();
-
-    /* call MatrixMul2D function */
-    MatrixMul2D(sGPU1, X_TRANS, sGPU2, X_NOTRANS, tGPU);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete sGPU1;
-    delete sGPU2;
-    delete tGPU;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for MatrixMul2D Function */
-extern "C"
-bool TestMatrixMul2D()
-{
-    XPRINT(0, stdout, "[TEST MATRIXMUL2D] matrix multiplication (for 2d tensors) \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestMatrixMul2D1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestMatrixMul2D2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TMatrixMul2DParallel.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#include "TMatrixMul2DParallel.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/*
-case 1: matrix multiplication (for 2d tensors) with multi-threading. 
-In this case, a=(2, 3), b=(3, 2) -> c=(2, 2), 
-transposedA=X_NOTRANS, transposedB=X_NOTRANS.
-*/
-bool TestMatrixMul2DParallel1()
-{
-    /* a source tensor of size (2, 3) */
-    int sOrder1 = 2;
-    int * sDimSize1 = new int[sOrder1];
-    sDimSize1[0] = 2;
-    sDimSize1[1] = 3;
-
-    int sUnitNum1 = 1;
-    for (int i = 0; i < sOrder1; i++)
-        sUnitNum1 *= sDimSize1[i];
-
-    /* a source tensor of size (3, 2) */
-    int sOrder2 = 2;
-    int * sDimSize2 = new int[sOrder2];
-    sDimSize2[0] = 3;
-    sDimSize2[1] = 2;
-
-    int sUnitNum2 = 1;
-    for (int i = 0; i < sOrder2; i++)
-        sUnitNum2 *= sDimSize2[i];
-
-    /* a target tensor of size (2, 2) */
-    int tOrder = 2;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData1[2][3] = { {1.0F, 2.0F, 3.0F},
-                           {-4.0F, 5.0F, 6.0F} };
-    DTYPE sData2[3][2] = { {0.0F, -1.0F},
-                           {1.0F, 2.0F}, 
-                           {2.0F, 1.0F} };
-    DTYPE answer[2][2] = { {8.0F, 6.0F}, 
-                           {17.0F, 20.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
-    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s1->SetData(sData1, sUnitNum1);
-    s2->SetData(sData2, sUnitNum2);
-    t->SetZeroAll();
-
-    /* call MatrixMul2DParallel function */
-    MatrixMul2DParallel(s1, X_NOTRANS, s2, X_NOTRANS, t);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest;
-}
-
-/* 
-case 2: matrix multiplication (for 2d tensors) with multi-threading.
-In this case, a=(3, 2), b=(3, 2) -> c=(2, 2), 
-transposedA=X_TRANS, transposedB=X_NOTRANS.
-*/
-bool TestMatrixMul2DParallel2()
-{
-    /* a source tensor of size (3, 2) */
-    int sOrder1 = 2;
-    int * sDimSize1 = new int[sOrder1];
-    sDimSize1[0] = 3;
-    sDimSize1[1] = 2;
-
-    int sUnitNum1 = 1;
-    for (int i = 0; i < sOrder1; i++)
-        sUnitNum1 *= sDimSize1[i];
-
-    /* a source tensor of size (3, 2) */
-    int sOrder2 = 2;
-    int * sDimSize2 = new int[sOrder2];
-    sDimSize2[0] = 3;
-    sDimSize2[1] = 2;
-
-    int sUnitNum2 = 1;
-    for (int i = 0; i < sOrder2; i++)
-        sUnitNum2 *= sDimSize2[i];
-
-    /* a target tensor of size (2, 2) */
-    int tOrder = 2;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData1[3][2] = { {1.0F, -4.0F},
-                           {2.0F, 5.0F},
-                           {3.0F, 6.0F} };
-    DTYPE sData2[3][2] = { {0.0F, -1.0F},
-                           {1.0F, 2.0F},
-                           {2.0F, 1.0F} };
-    DTYPE answer[2][2] = { {8.0F, 6.0F},
-                           {17.0F, 20.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
-    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s1->SetData(sData1, sUnitNum1);
-    s2->SetData(sData2, sUnitNum2);
-    t->SetZeroAll();
-
-    /* call MatrixMul2DParallel function */
-    MatrixMul2DParallel(s1, X_TRANS, s2, X_NOTRANS, t);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest;
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for MatrixMul2DParallel Function */
-bool TestMatrixMul2DParallel()
-{
-    XPRINT(0, stdout, "[TEST MatrixMul2DParallel] matrix multiplication (for 2d tensors) with multi-threading \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestMatrixMul2DParallel1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestMatrixMul2DParallel2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TMatrixMulBatched.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-15
-*/
-
-#include "../XTensor.h"
-#include "TMatrixMulBatched.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: matrix multiplication of the two tensors. 
-In this case, a=(2, 3), b=(2, 3) -> c=(2, 2), transposedA=X_NOTRANS, transposedB=X_NOTRANS.
-*/
-bool TestMatrixMulBatched1()
-{
-    /* a source tensor of size (2, 3) */
-    int sOrder1 = 2;
-    int * sDimSize1 = new int[sOrder1];
-    sDimSize1[0] = 2;
-    sDimSize1[1] = 3;
-
-    int sUnitNum1 = 1;
-    for (int i = 0; i < sOrder1; i++)
-        sUnitNum1 *= sDimSize1[i];
-
-    /* a source tensor of size (3, 2) */
-    int sOrder2 = 2;
-    int * sDimSize2 = new int[sOrder2];
-    sDimSize2[0] = 3;
-    sDimSize2[1] = 2;
-
-    int sUnitNum2 = 1;
-    for (int i = 0; i < sOrder2; i++)
-        sUnitNum2 *= sDimSize2[i];
-
-    /* a target tensor of size (2, 2) */
-    int tOrder = 2;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData1[2][3] = { {1.0F, 2.0F, 3.0F},
-                           {-4.0F, 5.0F, 6.0F} };
-    DTYPE sData2[3][2] = { {0.0F, -1.0F},
-                           {1.0F, 2.0F}, 
-                           {2.0F, 1.0F} };
-    DTYPE answer[2][2] = { {8.0F, 6.0F}, 
-                           {17.0F, 20.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
-    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s1->SetData(sData1, sUnitNum1);
-    s2->SetData(sData2, sUnitNum2);
-    t->SetZeroAll();
-
-    /* call MatrixMulBatched function */
-    MatrixMulBatched(s1, X_NOTRANS, s2, X_NOTRANS, t);
-    
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-    
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    sGPU1->SetData(sData1, sUnitNum1);
-    sGPU2->SetData(sData2, sUnitNum2);
-    tGPU->SetZeroAll();
-    
-    /* call MatrixMulBatched function */
-    MatrixMulBatched(sGPU1, X_NOTRANS, sGPU2, X_NOTRANS, tGPU);
-    
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-    
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete sGPU1;
-    delete sGPU2;
-    delete tGPU;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/*
-case 2: matrix multiplication of the two tensors. 
-In this case, a=(2, 2, 3), b=(2, 3, 2) -> c=(2, 2, 2), transposedA=X_NOTRANS, transposedB=X_NOTRANS.
-*/
-bool TestMatrixMulBatched2()
-{
-    /* a source tensor of size (2, 2, 3) */
-    int sOrder1 = 3;
-    int * sDimSize1 = new int[sOrder1];
-    sDimSize1[0] = 2;
-    sDimSize1[1] = 2;
-    sDimSize1[2] = 3;
-
-    int sUnitNum1 = 1;
-    for (int i = 0; i < sOrder1; i++)
-        sUnitNum1 *= sDimSize1[i];
-
-    /* a source tensor of size (2, 3, 2) */
-    int sOrder2 = 3;
-    int * sDimSize2 = new int[sOrder2];
-    sDimSize2[0] = 2;
-    sDimSize2[1] = 3;
-    sDimSize2[2] = 2;
-
-    int sUnitNum2 = 1;
-    for (int i = 0; i < sOrder2; i++)
-        sUnitNum2 *= sDimSize2[i];
-
-    /* a target tensor of size (2, 2, 2) */
-    int tOrder = 3;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-    tDimSize[2] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData1[2][2][3] = { { {0.0F, -1.0F, 2.0F},
-                                {2.0F, 1.0F, 3.0F} },
-                              { {1.0F, 2.0F, 4.0F}, 
-                                {3.0F, 1.0F, 2.0F} } };
-    DTYPE sData2[2][3][2] = { { {1.0F, 2.0F},
-                                {-4.0F, 3.0F},
-                                {2.0F, 6.0F} },
-                              { {1.0F, 2.0F},
-                                {3.0F, 4.0F},
-                                {5.0F, 6.0F} } };
-    DTYPE answer[2][2][2] = { { {8.0F, 9.0F}, 
-                                {4.0F, 25.0F} },
-                              { {27.0F, 34.0F}, 
-                                {16.0F, 22.0F} } };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s1 = NewTensor(sOrder1, sDimSize1);
-    XTensor * s2 = NewTensor(sOrder2, sDimSize2);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s1->SetData(sData1, sUnitNum1);
-    s2->SetData(sData2, sUnitNum2);
-    t->SetZeroAll();
-
-    /* call MatrixMulBatched function */
-    MatrixMulBatched(s1, X_NOTRANS, s2, X_NOTRANS, t);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-    
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * sGPU1 = NewTensor(sOrder1, sDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * sGPU2 = NewTensor(sOrder2, sDimSize2, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    sGPU1->SetData(sData1, sUnitNum1);
-    sGPU2->SetData(sData2, sUnitNum2);
-    tGPU->SetZeroAll();
-
-    /* call MatrixMulBatched function */
-    MatrixMulBatched(sGPU1, X_NOTRANS, sGPU2, X_NOTRANS, tGPU);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete sGPU1;
-    delete sGPU2;
-    delete tGPU;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s1;
-    delete s2;
-    delete t;
-    delete[] sDimSize1;
-    delete[] sDimSize2;
-    delete[] tDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for TestMatrixMulBatched Function */
-bool TestMatrixMulBatched()
-{
-    XPRINT(0, stdout, "[TEST MATRIXMULBATCHED] matrix multiplication of the two tensors \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestMatrixMulBatched1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestMatrixMulBatched2();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TMerge.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-13
-*/
-
-#ifndef __TEST_MERGE_H__
-#define __TEST_MERGE_H__
-
-#include "../core/shape/Merge.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Merge Function */
-extern "C"
-bool TestMerge();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_MERGE_H__

source/test/TMultiply.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-15
-*/
-
-#ifndef __TEST_MULTIPLYELEMENTWISE_H__
-#define __TEST_MULTIPLYELEMENTWISE_H__
-
-#include "../core/arithmetic/Multiply.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for MultiplyElementWise Function */
-extern "C"
-bool TestMultiply();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_MULTIPLYELEMENTWISE_H__

source/test/TNegate.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
-*/
-
-#include "TNegate.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* case 1: set every entry to its minus value */
-bool TestNegate1()
-{
-	/* a tensor of size (3, 2) */
-	int aOrder = 2;
-	int * aDimSize = new int[aOrder];
-	aDimSize[0] = 3;
-	aDimSize[1] = 2;
-
-	int aUnitNum = 1;
-	for (int i = 0; i < aOrder; i++)
-		aUnitNum *= aDimSize[i];
-
-	DTYPE aData[3][2] = { {1.0F, -2.0F}, 
-	                      {-3.0F, 4.0F},
-	                      {5.0F, -6.0F} };
-	DTYPE answer[3][2] = { {-1.0F, 2.0F},
-	                       {3.0F, -4.0F},
-	                       {-5.0F, 6.0F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * a = NewTensor(aOrder, aDimSize);
-
-	/* initialize variables */
-	a->SetData(aData, aUnitNum);
-
-	/* call Negate function */
-	Negate(a);
-
-	/* check results */
-	cpuTest = a->CheckData(answer, aUnitNum);
-    
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensor */
-	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-
-	/* Initialize variables */
-	aGPU->SetData(aData, aUnitNum);
-
-	/* call Negate function */
-	Negate(aGPU);
-
-	/* check results */
-	gpuTest = aGPU->CheckData(answer, aUnitNum);
-
-	/* destroy variables */
-	delete a;
-    delete aGPU;
-	delete[] aDimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete a;
-	delete[] aDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* case 2: set every entry to its minus value */
-bool TestNegate2()
-{
-	/* a tensor of size (3, 2) */
-	int aOrder = 2;
-	int * aDimSize = new int[aOrder];
-	aDimSize[0] = 3;
-	aDimSize[1] = 2;
-
-	int aUnitNum = 1;
-	for (int i = 0; i < aOrder; i++)
-		aUnitNum *= aDimSize[i];
-
-	DTYPE aData[3][2] = { {0.0F, 0.0F},
-	                      {0.0F, 0.0F},
-	                      {0.0F, 0.0F} };
-	DTYPE answer[3][2] = { {-0.0F, -0.0F},
-	                       {-0.0F, -0.0F},
-	                       {-0.0F, -0.0F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * a = NewTensor(aOrder, aDimSize);
-
-	/* initialize variables */
-	a->SetData(aData, aUnitNum);
-
-	/* call Negate function */
-	Negate(a);
-
-	/* check results */
-	cpuTest = a->CheckData(answer, aUnitNum);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensor */
-	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-
-	/* Initialize variables */
-	aGPU->SetData(aData, aUnitNum);
-
-	/* call Negate function */
-	Negate(aGPU);
-
-	/* check results */
-	gpuTest = aGPU->CheckData(answer, aUnitNum);
-
-	/* destroy variables */
-	delete a;
-    delete aGPU;
-	delete[] aDimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete a;
-	delete[] aDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Negate Function */
-bool TestNegate()
-{
-	XPRINT(0, stdout, "[TEST NEGATE] set every entry to its minus value \n");
-	bool returnFlag = true, caseFlag = true;
-
-	/* case 1 test */
-	caseFlag = TestNegate1();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 1 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 1 passed!\n");
-
-	/* case 2 test */
-	caseFlag = TestNegate2();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 2 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 2 passed!\n");
-
-	/* other cases test */
-	/*
-	TODO!!
-	*/
-
-	if (returnFlag) {
-		XPRINT(0, stdout, ">> All Passed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> Failed!\n");
-
-	XPRINT(0, stdout, "\n");
-
-	return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TNegate.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
-*/
-
-#ifndef __TEST_NEGATE_H__
-#define __TEST_NEGATE_H__
-
-#include "../core/arithmetic/Negate.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Negate Function */
-extern "C"
-bool TestNegate();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_NEGATE_H__

source/test/TNormalize.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-20
-*/
-
-#include "TNormalize.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/*
-case 1: normalized the data with normal distribution 
-For an input x, y = a * (x-mean)/sqrt(variance+\epsilon) + b.
-where a and b are the scalar and bias respectively, 
-and \epsilon is the adjustment parameter.
-*/
-bool TestNormalize1()
-{
-	/* a source tensor of size (2, 3) */
-	int sOrder = 2;
-	int * sDimSize = new int[sOrder];
-	sDimSize[0] = 2;
-	sDimSize[1] = 3;
-
-	int sUnitNum = 1;
-	for (int i = 0; i < sOrder; i++)
-		sUnitNum *= sDimSize[i];
-
-	/* a target tensor of size (2, 3) */
-	int tOrder = 2;
-	int * tDimSize = new int[tOrder];
-	tDimSize[0] = 2;
-	tDimSize[1] = 3;
-
-	int tUnitNum = 1;
-	for (int i = 0; i < tOrder; i++)
-		tUnitNum *= tDimSize[i];
-
-	/* a mean tensor of size (3) */
-	int meanOrder = 1;
-	int * meanDimSize = new int[meanOrder];
-	meanDimSize[0] = 3;
-
-	int meanUnitNum = 1;
-	for (int i = 0; i < meanOrder; i++)
-		meanUnitNum *= meanDimSize[i];
-
-	/* a variance tensor of size (3) */
-	int varOrder = 1;
-	int * varDimSize = new int[varOrder];
-	varDimSize[0] = 3;
-
-	int varUnitNum = 1;
-	for (int i = 0; i < varOrder; i++)
-		varUnitNum *= varDimSize[i];
-
-	/* a scalar tensor of size (2, 3) */
-	int aOrder = 2;
-	int * aDimSize = new int[aOrder];
-	aDimSize[0] = 2;
-	aDimSize[1] = 3;
-
-	int aUnitNum = 1;
-	for (int i = 0; i < aOrder; i++)
-		aUnitNum *= aDimSize[i];
-
-	/* a bias tensor of size (2, 3) */
-	int bOrder = 2;
-	int * bDimSize = new int[bOrder];
-	bDimSize[0] = 2;
-	bDimSize[1] = 3;
-
-	int bUnitNum = 1;
-	for (int i = 0; i < bOrder; i++)
-		bUnitNum *= bDimSize[i];
-
-	DTYPE sData[2][3] = { {1.0F, 2.0F, 3.0F},
-	                      {1.5F, 2.5F, 3.5F} };
-	DTYPE meanData[3] = {1.0F, 1.5F, 2.0F};
-	DTYPE varData[3] = {1.0F, 1.0F, 4.0F};
-    DTYPE aData[2][3] = { {1.0F, 1.0F, 1.0F},
-	                      {1.0F, 1.0F, 1.0F} };
-	DTYPE answer[2][3] = { {0.0F, 0.5F, 0.5F},
-	                       {0.5F, 1.0F, 0.75F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * s = NewTensor(sOrder, sDimSize);
-	XTensor * t = NewTensor(tOrder, tDimSize);
-	XTensor * mean = NewTensor(meanOrder, meanDimSize);
-	XTensor * var = NewTensor(varOrder, varDimSize);
-	XTensor * a = NewTensor(aOrder, aDimSize);
-	XTensor * b = NewTensor(bOrder, bDimSize);
-
-	/* initialize variables */
-	s->SetData(sData, sUnitNum);
-	mean->SetData(meanData, meanUnitNum);
-	var->SetData(varData, varUnitNum);
-	a->SetData(aData, aUnitNum);
-	b->SetZeroAll();
-	t->SetZeroAll();
-
-	/* call normalize function */
-	Normalize(s, t, 0, mean, var, a, b, 0.0F);
-    
-	/* check results */
-	cpuTest = t->CheckData(answer, tUnitNum, 1e-4F, 0);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensors */
-	XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * meanGPU = NewTensor(meanOrder, meanDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * varGPU = NewTensor(varOrder, varDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-	/* initialize variables */
-	sGPU->SetData(sData, sUnitNum);
-	meanGPU->SetData(meanData, meanUnitNum);
-	varGPU->SetData(varData, varUnitNum);
-	aGPU->SetData(aData, aUnitNum);
-	bGPU->SetZeroAll();
-	tGPU->SetZeroAll();
-
-	/* call Normalize function */
-	Normalize(sGPU, tGPU, 0, meanGPU, varGPU, aGPU, bGPU, 0.0F);
-
-	/* check results */
-	gpuTest = tGPU->CheckData(answer, tUnitNum, 1e-4F, 0);
-
-	/* destroy variables */
-	delete s;
-	delete t;
-	delete mean;
-	delete var;
-	delete a;
-	delete b;
-	delete sGPU;
-	delete tGPU;
-	delete meanGPU;
-	delete varGPU;
-	delete aGPU;
-	delete bGPU;
-	delete[] sDimSize;
-	delete[] tDimSize;
-	delete[] meanDimSize;
-	delete[] varDimSize;
-	delete[] aDimSize;
-	delete[] bDimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete s;
-	delete t;
-	delete mean;
-	delete var;
-	delete a;
-	delete b;
-	delete[] sDimSize;
-	delete[] tDimSize;
-	delete[] meanDimSize;
-	delete[] varDimSize;
-	delete[] aDimSize;
-	delete[] bDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Normalize Function */
-bool TestNormalize()
-{
-	XPRINT(0, stdout, "[TEST NORMALIZE] normalized the data with normal distribution \n");
-	bool returnFlag = true, caseFlag = true;
-
-	/* case 1 test */
-	caseFlag = TestNormalize1();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 1 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 1 passed!\n");
-
-	/* other cases test */
-	/*
-	TODO!!
-	*/
-
-	if (returnFlag) {
-		XPRINT(0, stdout, ">> All Passed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> Failed!\n");
-
-	XPRINT(0, stdout, "\n");
-
-	return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TNormalize.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-20
-*/
-
-#ifndef __TEST_NORMALIZE_H__
-#define __TEST_NORMALIZE_H__
-
-#include "../core/math/Normalize.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Normalize Function */
-extern "C"
-bool TestNormalize();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_NORMALIZE_H__

source/test/TPower.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-15
-*/
-
-#include "../XUtility.h"
-#include "TPower.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: get the power(a, p) 
-In this case, p=2.
-*/
-bool TestPower1()
-{
-	/* a tensor of size (3, 2) */
-	int aOrder = 2;
-	int * aDimSize = new int[aOrder];
-	aDimSize[0] = 3;
-	aDimSize[1] = 2;
-
-	int aUnitNum = 1;
-	for (int i = 0; i < aOrder; i++)
-		aUnitNum *= aDimSize[i];
-
-	DTYPE aData[3][2] = { {1.0F, 2.0F},
-	                      {3.0F, 4.0F},
-	                      {5.0F, 6.0F} };
-	DTYPE answer[3][2] = { {1.0F, 4.0F},
-	                       {9.0F, 16.0F},
-	                       {25.0F, 36.0F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * a = NewTensor(aOrder, aDimSize);
-
-	/* initialize variables */
-	a->SetData(aData, aUnitNum);
-
-	/* call Power function */
-	Power(a, 2.0F);
-
-	/* check results */
-	cpuTest = a->CheckData(answer, aUnitNum, 1e-4F);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensor */
-	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-
-	/* Initialize variables */
-	aGPU->SetData(aData, aUnitNum);
-
-	/* call power function */
-	Power(aGPU, 2.0F);
-
-	/* check results */
-	gpuTest = aGPU->CheckData(answer, aUnitNum, 1e-4F);
-
-	/* destroy variables */
-	delete a;
-    delete aGPU;
-	delete[] aDimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete a;
-	delete[] aDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: get the power(a, p)
-In this case, p=1.
-*/
-bool TestPower2()
-{
-	/* a tensor of size (3, 2) */
-	int aOrder = 2;
-	int * aDimSize = new int[aOrder];
-	aDimSize[0] = 3;
-	aDimSize[1] = 2;
-
-	int aUnitNum = 1;
-	for (int i = 0; i < aOrder; i++)
-		aUnitNum *= aDimSize[i];
-
-	DTYPE aData[3][2] = { {0.0F, 1.0F},
-	                      {2.0F, 3.0F},
-	                      {4.0F, 5.0F} };
-	DTYPE answer[3][2] = { {0.0F, 1.0F},
-	                       {2.0F, 3.0F},
-	                       {4.0F, 5.0F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * a = NewTensor(aOrder, aDimSize);
-
-	/* initialize variables */
-	a->SetData(aData, aUnitNum);
-
-	/* call Power function */
-	Power(a, 1.0F);
-
-	/* check results */
-	cpuTest = a->CheckData(answer, aUnitNum, 1e-4F);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensor */
-	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-
-	/* Initialize variables */
-	aGPU->SetData(aData, aUnitNum);
-
-	/* call Power function */
-	Power(aGPU, 1.0F);
-
-	/* check results */
-	gpuTest = aGPU->CheckData(answer, aUnitNum, 1e-4F);
-
-	/* destroy variables */
-	delete a;
-    delete aGPU;
-	delete[] aDimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete a;
-	delete[] aDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 3: get the power(a, p)
-In this case, p=0.
-*/
-bool TestPower3()
-{
-	/* a tensor of size (3, 2) */
-	int aOrder = 2;
-	int * aDimSize = new int[aOrder];
-	aDimSize[0] = 3;
-	aDimSize[1] = 2;
-
-	int aUnitNum = 1;
-	for (int i = 0; i < aOrder; i++)
-		aUnitNum *= aDimSize[i];
-
-	DTYPE aData[3][2] = { {0.0F, 1.0F},
-	                      {2.0F, 3.0F},
-	                      {4.0F, 5.0F} };
-	DTYPE answer[3][2] = { {1.0F, 1.0F},
-	                       {1.0F, 1.0F},
-	                       {1.0F, 1.0F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * a = NewTensor(aOrder, aDimSize);
-
-	/* initialize variables */
-	a->SetData(aData, aUnitNum);
-
-	/* call Power function */
-	Power(a, 0.0F);
-
-	/* check results */
-	cpuTest = a->CheckData(answer, aUnitNum, 1e-4F);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensor */
-	XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-
-	/* Initialize variables */
-	aGPU->SetData(aData, aUnitNum);
-
-	/* call Power function */
-	Power(aGPU, 0.0F);
-
-	/* check results */
-	gpuTest = aGPU->CheckData(answer, aUnitNum, 1e-4F);
-
-	/* destroy variables */
-	delete a;
-    delete aGPU;
-	delete[] aDimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete a;
-	delete[] aDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Power Function */
-bool TestPower()
-{
-	XPRINT(0, stdout, "[TEST POWER] get the power(a, p) \n");
-	bool returnFlag = true, caseFlag = true;
-
-	/* case 1 test */
-	caseFlag = TestPower1();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 1 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 1 passed!\n");
-
-	/* case 2 test */
-	caseFlag = TestPower2();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 2 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 2 passed!\n");
-
-	/* case 3 test */
-	caseFlag = TestPower3();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 3 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 3 passed!\n");
-
-	/* other cases test */
-	/*
-	TODO!!
-	*/
-
-	if (returnFlag) {
-		XPRINT(0, stdout, ">> All Passed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> Failed!\n");
-
-	XPRINT(0, stdout, "\n");
-
-	return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TPower.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-15
-*/
-
-#ifndef __TEST_POWER_H__
-#define __TEST_POWER_H__
-
-#include "../core/math/Power.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Power Function */
-extern "C"
-bool TestPower();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_POWER_H__

source/test/TRectify.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
-*/
-
-#include "TRectify.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test rectify function
-In this case, y = max(0, x) 
-*/
-bool TestRectify1()
-{
-    /* a tensor of size (2, 3) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 2;
-    dimSize[1] = 3;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE xData[2][3] = { {0.0F, -1.0F, 2.0F},
-                          {3.0F, -4.0F, -5.0F} };
-    DTYPE answer[2][3] = { {0.0F, 0.0F, 2.0F},
-                           {3.0F, 0.0F, 0.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * x = NewTensor(order, dimSize);
-    XTensor * y = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    x->SetData(xData, unitNum);
-    y->SetZeroAll();
-
-    /* call Rectify function */
-    Rectify(x, y);
-
-    /* check results */
-    cpuTest = y->CheckData(answer, unitNum);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensor */
-	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-	/* Initialize variables */
-	xGPU->SetData(xData, unitNum);
-	yGPU->SetZeroAll();
-
-	/* call Rectify function */
-	Rectify(xGPU, yGPU);
-
-	/* check results */
-	gpuTest = yGPU->CheckData(answer, unitNum);
-
-	/* destroy variables */
-	delete x;
-    delete y;
-    delete xGPU;
-    delete yGPU;
-	delete[] dimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-	delete x;
-    delete y;
-	delete[] dimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: backward computation 
-dE/dx = dE/dy * dy/dx 
-rectified: y = max(0, x) 
-In this case, lossName=CROSSENTROPY.
-*/
-bool TestRectify2()
-{
-	/* a tensor of size (2, 3) */
-	int order = 2;
-	int * dimSize = new int[order];
-	dimSize[0] = 2;
-	dimSize[1] = 3;
-
-	int unitNum = 1;
-	for (int i = 0; i < order; i++)
-		unitNum *= dimSize[i];
-
-	DTYPE xData[2][3] = { {1.0F, 1.0F, 2.0F},
-	                      {2.0F, 4.0F, 5.0F} };
-	DTYPE goldData[2][3] = { {1.0F, 1.0F, 1.0F},
-	                         {1.0F, 1.0F, 1.0F} };
-    DTYPE yAnswer[2][3] = { {1.0F, 1.0F, 2.0F},
-	                        {2.0F, 4.0F, 5.0F} };
-	DTYPE dedyAnswer[2][3] = { {-1.0F, -1.0F, -0.5F},
-	                           {-0.5F, -0.25F, -0.2F} };
-	DTYPE dedxAnswer[2][3] = { {-1.0F, -1.0F, -0.5F},
-	                           {-0.5F, -0.25F, -0.2F} };
-
-	/* CPU test */
-	bool cpuTest = true;
-
-	/* create tensors */
-	XTensor * x = NewTensor(order, dimSize);
-	XTensor * y = NewTensor(order, dimSize);
-	XTensor * gold = NewTensor(order, dimSize);
-	XTensor * dedy = NewTensor(order, dimSize);
-	XTensor * dedx = NewTensor(order, dimSize);
-
-	/* initialize variables */
-	x->SetData(xData, unitNum);
-	gold->SetData(goldData, unitNum);
-	y->SetZeroAll();
-	dedy->SetZeroAll();
-	dedx->SetZeroAll();
-
-    /* call Rectify function */
-    Rectify(x, y);
-
-	/* call RectifyBackward function */
-	RectifyBackward(gold, y, x, dedy, dedx, CROSSENTROPY);
-
-	/* check results */
-    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-	/* GPU test */
-	bool gpuTest = true;
-
-	/* create tensors */
-	XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-	XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-	/* initialize variables */
-	xGPU->SetData(xData, unitNum);
-	goldGPU->SetData(goldData, unitNum);
-	yGPU->SetZeroAll();
-	dedyGPU->SetZeroAll();
-	dedxGPU->SetZeroAll();
-    
-    /* call Rectify function */
-    Rectify(xGPU, yGPU);
-
-	/* call rectifybackward function */
-	RectifyBackward(goldGPU, yGPU, xGPU, dedyGPU, dedxGPU, CROSSENTROPY);
-    
-	/* check results */
-    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
-
-	/* destroy variables */
-    delete x;
-    delete y;
-    delete dedy;
-    delete dedx;
-    delete gold;
-    delete xGPU;
-    delete yGPU;
-    delete dedyGPU;
-    delete dedxGPU;
-    delete goldGPU;
-	delete[] dimSize;
-
-	return cpuTest && gpuTest;
-#else
-	/* destroy variables */
-    delete x;
-    delete y;
-    delete dedy;
-    delete dedx;
-    delete gold;
-	delete[] dimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Rectify Function */
-bool TestRectify()
-{
-    XPRINT(0, stdout, "[TEST RECTIFY] rectify function and its backward computation \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestRectify1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestRectify2();
-
-	if (!caseFlag) {
-		returnFlag = false;
-		XPRINT(0, stdout, ">> case 2 failed!\n");
-	}
-	else
-		XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TRectify.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-14
-*/
-
-#ifndef __TEST_RECTIFY_H__
-#define __TEST_RECTIFY_H__
-
-#include "../function/Rectify.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Rectify Function */
-extern "C"
-bool TestRectify();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_RECTIFY_H__

source/test/TReduceMax.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-30
-*/
-
-#include "TReduceMax.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: get the max value of the items along a dimension of the tensor. 
-In this case,
-(2, 4) -> (4), dim = 0
-(2, 4) -> (2), dim = 1
-*/
-bool TestReduceMax1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a output tensor of size (4) */
-    int tOrder1 = 1;
-    int * tDimSize1 = new int[tOrder1];
-    tDimSize1[0] = 4;
-
-    int tUnitNum1 = 1;
-    for (int i = 0; i < tOrder1; i++)
-        tUnitNum1 *= tDimSize1[i];
-
-    /* a output tensor of size (2) */
-    int tOrder2 = 1;
-    int * tDimSize2 = new int[tOrder2];
-    tDimSize2[0] = 2;
-
-    int tUnitNum2 = 1;
-    for (int i = 0; i < tOrder2; i++)
-        tUnitNum2 *= tDimSize2[i];
-
-    DTYPE sData[2][4] = { {0.0F, 5.0F, 2.0F, 3.0F},
-                          {4.0F, 1.0F, 6.0F, 7.0F} };
-    DTYPE answer1[4] = {4.0F, 5.0F, 6.0F, 7.0F};
-    DTYPE answer2[2] = {5.0F, 7.0F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
-    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t1->SetZeroAll();
-    t2->SetZeroAll();
-
-    /* call ReduceMax function */
-    ReduceMax(s, t1, 0);
-    ReduceMax(s, t2, 1);
-
-    /* check results */
-    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU1->SetZeroAll();
-    tGPU2->SetZeroAll();
-
-    /* call ReduceMax function */
-    ReduceMax(sGPU, tGPU1, 0);
-    ReduceMax(sGPU, tGPU2, 1);
-
-    /* check results */
-    gpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
-
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete sGPU;
-    delete tGPU1;
-    delete tGPU2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for ReduceMax Function */
-bool TestReduceMax()
-{
-    XPRINT(0, stdout, "[TEST ReduceMax] get the max value of the items along a dimension of the tensor\n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestReduceMax1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TReduceMax.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-30
-*/
-
-#ifndef __TEST_REDUCEMAX_H__
-#define __TEST_REDUCEMAX_H__
-
-#include "../core/reduce/ReduceMax.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for ReduceMax Function */
-extern "C"
-bool TestReduceMax();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_REDUCEMAX_H__

source/test/TReduceMean.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#include "TReduceMean.h"
-
-namespace nts { // namespace nt(NiuTrans.Tensor)
-
-/* case 1: get the mean value along a dimension of the tensor */
-bool TestReduceMean1()
-{
-    /* a tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a tensor of size (4) */
-    int tOrder1 = 1;
-    int * tDimSize1 = new int[tOrder1];
-    tDimSize1[0] = 4;
-
-    int tUnitNum1 = 1;
-    for (int i = 0; i < tOrder1; i++)
-        tUnitNum1 *= tDimSize1[i];
-
-    /* a tensor of size (2) */
-    int tOrder2 = 1;
-    int * tDimSize2 = new int[tOrder2];
-    tDimSize2[0] = 2;
-
-    int tUnitNum2 = 1;
-    for (int i = 0; i < tOrder2; i++)
-        tUnitNum2 *= tDimSize2[i];
-
-    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE answer1[4] = {2.0F, 3.0F, 4.0F, 5.0F};
-    DTYPE answer2[2] = {1.5F, 5.5F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
-    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t1->SetZeroAll();
-    t2->SetZeroAll();
-
-    /* call ReduceMean function */
-    ReduceMean(s, t1, 0);
-    ReduceMean(s, t2, 1);
-
-    /* check results */
-    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU1->SetZeroAll();
-    tGPU2->SetZeroAll();
-
-    /* call ReduceMean function */
-    ReduceMean(sGPU, tGPU1, 0);
-    ReduceMean(sGPU, tGPU2, 1);
-
-    /* check results */
-    cpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
-
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete sGPU;
-    delete tGPU1;
-    delete tGPU2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-    
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for ReduceMean Function */
-bool TestReduceMean()
-{
-    XPRINT(0, stdout, "[TEST ReduceMean] get the mean value along a dimension of the tensor \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestReduceMean1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    ///* other cases test */
-    ///*
-    //TODO!!
-    //*/
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TReduceMean.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#ifndef __TEST_REDUCEMEAN_H__
-#define __TEST_REDUCEMEAN_H__
-
-#include "../core/reduce/ReduceMean.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for ReduceMean Function */
-extern "C"
-bool TestReduceMean();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_REDUCEMEAN_H__
-
-

source/test/TReduceSum.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#include "TReduceSum.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: sum the items along a dimension of the tensor.
-In this case, 
-(2, 4) -> (4), dim = 0
-(2, 4) -> (2), dim = 1
-*/
-bool TestReduceSum1()
-{
-    /* a tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a tensor of size (4) */
-    int tOrder1 = 1;
-    int * tDimSize1 = new int[tOrder1];
-    tDimSize1[0] = 4;
-
-    int tUnitNum1 = 1;
-    for (int i = 0; i < tOrder1; i++)
-        tUnitNum1 *= tDimSize1[i];
-
-    /* a tensor of size (2) */
-    int tOrder2 = 1;
-    int * tDimSize2 = new int[tOrder2];
-    tDimSize2[0] = 2;
-
-    int tUnitNum2 = 1;
-    for (int i = 0; i < tOrder2; i++)
-        tUnitNum2 *= tDimSize2[i];
-
-    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                           {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE answer1[4] = {4.0F, 6.0F, 8.0F, 10.0F};
-    DTYPE answer2[2] = {6.0F, 22.0F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
-    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t1->SetZeroAll();
-    t2->SetZeroAll();
-
-    /* call ReduceSum function */
-    ReduceSum(s, t1, 0);
-    ReduceSum(s, t2, 1);
-
-    /* check results */
-    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU1->SetZeroAll();
-    tGPU2->SetZeroAll();
-
-    /* call ReduceSum function */
-    ReduceSum(sGPU, tGPU1, 0);
-    ReduceSum(sGPU, tGPU2, 1);
-
-    /* check results */
-    cpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
-
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete sGPU;
-    delete tGPU1;
-    delete tGPU2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-    
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for ReduceSum Function */
-bool TestReduceSum()
-{
-    XPRINT(0, stdout, "[TEST ReduceSum] sum the items along a dimension of the tensor.\n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestReduceSum1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TReduceSum.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (email: li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#ifndef __TEST_REDUCESUM_H__
-#define __TEST_REDUCESUM_H__
-
-#include "../core/reduce/ReduceSum.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for ReduceSum Function */
-extern "C"
-bool TestReduceSum();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_REDUCESUM_H__
-
-
-

source/test/TReduceSumSquared.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#include "TReduceSumSquared.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: squared sum of the items along a dimension of the tensor. 
-For a 1-dimensional data array a, sum = \sum_i (a_i - shift)^2.
-In this case, (2, 4) -> (4), dim = 0.
-*/
-bool TestReduceSumSquared1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a output tensor of size (4) */
-    int tOrder = 1;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 4;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    /* a shift tensor of size (4) */
-    int shiftOrder = 1;
-    int * shiftDimSize = new int[shiftOrder];
-    shiftDimSize[0] = 4;
-
-    int shiftUnitNum = 1;
-    for (int i = 0; i < shiftOrder; i++)
-        shiftUnitNum *= shiftDimSize[i];
-
-    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE shiftData[4] = {1.0F, -1.0F, -1.0F, 0.0F};
-    DTYPE answer[4] = {10.0F, 40.0F, 58.0F, 58.0F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-    XTensor * shift = NewTensor(shiftOrder, shiftDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    shift->SetData(shiftData, shiftUnitNum);
-    t->SetZeroAll();
-
-    /* call ReduceSumSquared function */
-    ReduceSumSquared(s, t, 0, shift);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * shiftGPU = NewTensor(shiftOrder, shiftDimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    shiftGPU->SetData(shiftData, shiftUnitNum);
-    tGPU->SetZeroAll();
-
-    /* call ReduceSumSquared function */
-    ReduceSumSquared(sGPU, tGPU, 0, shiftGPU);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete shift;
-    delete sGPU;
-    delete tGPU;
-    delete shiftGPU;
-    delete[] sDimSize;
-    delete[] tDimSize;
-    delete[] shiftDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete shift;
-    delete[] sDimSize;
-    delete[] tDimSize;
-    delete[] shiftDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: squared sum of the items along a dimension of the tensor. 
-For a 1-dimensional data array a, sum = \sum_i (a_i - shift)^2.
-In this case, (2, 4) -> (2), dim = 1.
-*/
-bool TestReduceSumSquared2()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a output tensor of size (2) */
-    int tOrder = 1;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    /* a shift tensor of size (2) */
-    int shiftOrder = 1;
-    int * shiftDimSize = new int[shiftOrder];
-    shiftDimSize[0] = 2;
-
-    int shiftUnitNum = 1;
-    for (int i = 0; i < shiftOrder; i++)
-        shiftUnitNum *= shiftDimSize[i];
-
-    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE shiftData[2] = {-1.0F, 1.0F};
-    DTYPE answer[2] = {30.0F, 86.0F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-    XTensor * shift = NewTensor(shiftOrder, shiftDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    shift->SetData(shiftData, shiftUnitNum);
-    t->SetZeroAll();
-
-    /* call ReduceSumSquared function */
-    ReduceSumSquared(s, t, 1, shift);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * shiftGPU = NewTensor(shiftOrder, shiftDimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    shiftGPU->SetData(shiftData, shiftUnitNum);
-    tGPU->SetZeroAll();
-
-    /* call ReduceSumSquared function */
-    ReduceSumSquared(sGPU, tGPU, 1, shiftGPU);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete shift;
-    delete sGPU;
-    delete tGPU;
-    delete shiftGPU;
-    delete[] sDimSize;
-    delete[] tDimSize;
-    delete[] shiftDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete shift;
-    delete[] sDimSize;
-    delete[] tDimSize;
-    delete[] shiftDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for ReduceSumSquared Function */
-bool TestReduceSumSquared()
-{
-    XPRINT(0, stdout, "[TEST ReduceSumSquared] squared sum of the items along a dimension of the tensor\n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestReduceSumSquared1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-    
-    /* case 2 test */
-    caseFlag = TestReduceSumSquared2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TReduceSumSquared.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#ifndef __TEST_REDUCESUMSQUARED_H__
-#define __TEST_REDUCESUMSQUARED_H__
-
-#include "../core/reduce/ReduceSumSquared.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for ReduceSumSquared Function */
-extern "C"
-bool TestReduceSumSquared();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_REDUCESUMSQUARED_H__

source/test/TReduceVariance.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#include "TReduceVariance.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/*
-case 1: variance of the items along a dimension of the tensor. 
-For a 1-dimensional data array a, variance = 1/n * \sum_i (a_i - mean)^2.
-In this case, (2, 4) -> (4), dim = 0.
-*/
-bool TestReduceVariance1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a output tensor of size (4) */
-    int tOrder = 1;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 4;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    /* a mean tensor of size (4) */
-    int meanOrder = 1;
-    int * meanDimSize = new int[meanOrder];
-    meanDimSize[0] = 4;
-
-    int meanUnitNum = 1;
-    for (int i = 0; i < meanOrder; i++)
-        meanUnitNum *= meanDimSize[i];
-
-    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE meanData[4] = {2.0F, 3.0F, 4.0F, 5.0F};
-    DTYPE answer[4] = {4.0F, 4.0F, 4.0F, 4.0F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-    XTensor * mean = NewTensor(meanOrder, meanDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    mean->SetData(meanData, meanUnitNum);
-    t->SetZeroAll();
-
-    /* call ReduceVariance function */
-    ReduceVariance(s, t, 0, mean);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * meanGPU = NewTensor(meanOrder, meanDimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    meanGPU->SetData(meanData, meanUnitNum);
-    tGPU->SetZeroAll();
-
-    /* call ReduceVariance function */
-    ReduceVariance(sGPU, tGPU, 0, meanGPU);
-
-    /* check results */
-    gpuTest = t->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete mean;
-    delete sGPU;
-    delete tGPU;
-    delete meanGPU;
-    delete[] sDimSize;
-    delete[] tDimSize;
-    delete[] meanDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete mean;
-    delete[] sDimSize;
-    delete[] tDimSize;
-    delete[] meanDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for ReduceVariance Function */
-bool TestReduceVariance()
-{
-    XPRINT(0, stdout, "[TEST ReduceVariance] variance of the items along a dimension of the tensor\n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestReduceVariance1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TReduceVariance.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#ifndef __TEST_REDUCEVARIANCE_H__
-#define __TEST_REDUCEVARIANCE_H__
-
-#include "../core/reduce/ReduceVariance.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for ReduceVariance Function */
-extern "C"
-bool TestReduceVariance();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_REDUCEVARIANCE_H__

source/test/TScaleAndShift.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#include "TScaleAndShift.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: scale and shift all tensor entires.
-p = p * scale + shift
-*/
-bool TestScaleAndShift1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    DTYPE sData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE answer[2][4] = { {0.5F, 2.5F, 4.5F, 6.5F},
-                           {8.5F, 10.5F, 12.5F, 14.5F} };
-
-    DTYPE scaleFactor = 2.0F;
-    DTYPE shiftFactor = 0.5F;
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-
-    /* call ScaleAndShift function */
-    ScaleAndShift(s, scaleFactor, shiftFactor);
-
-    /* check results */
-    cpuTest = s->CheckData(answer, sUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-
-    /* call ScaleAndShift function */
-    ScaleAndShift(sGPU, scaleFactor, shiftFactor);
-
-    /* check results */
-    gpuTest = sGPU->CheckData(answer, sUnitNum);
-
-    /* destroy variables */
-    delete s;
-    delete sGPU;
-    delete[] sDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete[] sDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for ScaleAndShift Function */
-bool TestScaleAndShift()
-{
-    XPRINT(0, stdout, "[TEST ScaleAndShift] scale and shift all tensor entires\n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestScaleAndShift1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TScaleAndShift.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#ifndef __TEST_SCALEANDSHIFT_H__
-#define __TEST_SCALEANDSHIFT_H__
-
-#include "../core/math/ScaleAndShift.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for ScaleAndShift Function */
-extern "C"
-bool TestScaleAndShift();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SCALEANDSHIFT_H__

source/test/TSelect.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-04
-*/
-
-#include "TSelect.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test SelectRange function.
-It can generate a tensor with seleccted data in range[low,high] along the given dimension.
-In this case, (2, 2, 4) -> (2, 2, 2), dim = 2, low = 1, high = 3.
-*/
-bool TestSelect1()
-{
-    /* a input tensor of size (2, 2, 4) */
-    int sOrder = 3;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 2;
-    sDimSize[2] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a output tensor of size (2, 2, 2) */
-    int tOrder = 3;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-    tDimSize[2] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData[2][2][4] = { { {0.0F, 1.0F, 2.0F, 3.0F},
-                               {4.0F, 5.0F, 6.0F, 7.0F} },
-                             { {1.0F, 2.0F, 3.0F, 4.0F},
-                               {5.0F, 6.0F, 7.0F, 8.0F} } };
-    DTYPE answer[2][2][2] = { { {1.0F, 2.0F},
-                                {5.0F, 6.0F} },
-                              { {2.0F, 3.0F},
-                                {6.0F, 7.0F} } };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t->SetZeroAll();
-
-    /* call SelectRange function */
-    SelectRange(s, t, 2, 1, 3);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-    
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU->SetZeroAll();
-
-    /* call Select function */
-    SelectRange(sGPU, tGPU, 2, 1, 3);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-    
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete sGPU;
-    delete tGPU;
-    delete[] sDimSize;
-    delete[] tDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete[] sDimSize;
-    delete[] tDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Select Function */
-bool TestSelect()
-{
-    XPRINT(0, stdout, "[TEST Select] generate a tensor with seleccted data in range[low,high] along the given dimension \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSelect1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSelect.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-04
-*/
-
-#ifndef __TEST_SELECT_H__
-#define __TEST_SELECT_H__
-
-#include "../core/getandset/Select.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Select Function */
-extern "C"
-bool TestSelect();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SELECT_H__

source/test/TSetAscendingOrder.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#include "TSetAscendingOrder.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* case 1: set the cell to the ascending order along a given dimension. */
-bool TestSetAscendingOrder1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    int answer[2][4] = { {0, 1, 2, 3},
-                         {0, 1, 2, 3} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize, X_INT);
-
-    /* initialize variables */
-    s->SetZeroAll();
-
-    /* call SetAscendingOrder function */
-    s->SetAscendingOrder(1);
-    
-    /* check results */
-    cpuTest = s->CheckData(answer, sUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_INT, 1.0F, 0);
-
-    /* initialize variables */
-    sGPU->SetZeroAll();
-
-    /* call SetAscendingOrder function */
-    sGPU->SetAscendingOrder(1);
-
-    /* check results */
-    gpuTest = sGPU->CheckData(answer, sUnitNum);
-
-    /* destroy variables */
-    delete s;
-    delete sGPU;
-    delete[] sDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete[] sDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for SetAscendingOrder Function */
-bool TestSetAscendingOrder()
-{
-    XPRINT(0, stdout, "[TEST SetAscendingOrder] set the cell to the ascending order along a given dimension \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSetAscendingOrder1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSetAscendingOrder.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#ifndef __TEST_SETASCENDINGORDER_H__
-#define __TEST_SETASCENDINGORDER_H__
-
-#include "../XTensor.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for SetAscendingOrder Function */
-extern "C"
-bool TestSetAscendingOrder();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SETASCENDINGORDER_H__

source/test/TSetData.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#include "TSetData.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test SetDataRand function.
-set the tensor items by a uniform distribution in range [lower, upper]. 
-*/
-bool TestSetData1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    DTYPE answer[2][4] = {0};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-
-    /* call SetDataRand function */
-    s->SetDataRand(0.0, 1.0);
-    
-    /* check results */
-    cpuTest = s->CheckData(answer, sUnitNum, 1.0F);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-
-    /* call SetDataRand function */
-    sGPU->SetDataRand(0.0, 1.0);
-    
-    gpuTest = sGPU->CheckData(answer, sUnitNum, 1.0F);
-
-    /* destroy variables */
-    delete s;
-    delete sGPU;
-    delete[] sDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete[] sDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for SetData Function */
-bool TestSetData()
-{
-    XPRINT(0, stdout, "[TEST SetData] set the data of tensor \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSetData1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSetData.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#ifndef __TEST_SETDATA_H__
-#define __TEST_SETDATA_H__
-
-#include "../XTensor.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for SetData Function */
-extern "C"
-bool TestSetData();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SETDATA_H__

source/test/TSigmoid.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
-*/
-
-#include "../XUtility.h"
-#include "TSigmoid.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test Sigmoid function.
-sigmoid function: y = 1/(1+exp(-x))
-*/
-bool TestSigmoid1()
-{
-    /* a input tensor of size (3) */
-    int order = 1;
-    int * dimSize = new int[order];
-    dimSize[0] = 3;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE xData[3] = {0.0F, 1.0F, 2.0F};
-    DTYPE answer[3] = {0.5F, 0.7311F, 0.8808F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * x = NewTensor(order, dimSize);
-    XTensor * y = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    x->SetData(xData, unitNum);
-    y->SetZeroAll();
-
-    /* call Sigmoid function */
-    Sigmoid(x, y);
-
-    /* check result */
-    cpuTest = y->CheckData(answer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-        /* create tensors */
-    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    xGPU->SetData(xData, unitNum);
-    yGPU->SetZeroAll();
-
-    /* call Sigmoid function */
-    Sigmoid(xGPU, yGPU);
-
-    /* check result */
-    gpuTest = yGPU->CheckData(answer, unitNum, 1e-4F);
-
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete xGPU;
-    delete yGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: test Sigmoid function and SigmoidBackward function.
-sigmoid function: y = 1/(1+exp(-x))
-backward computation: 
-dE/ds = dE/dy * dy/dx
-dy/dx = y * (1 -y)
-In this case, LossName=CROSSENTROPY.
-*/
-bool TestSigmoid2()
-{
-    /* a input tensor of size (3) */
-    int order = 1;
-    int * dimSize = new int[order];
-    dimSize[0] = 3;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE xData[3] = {0.0F, 1.0F, 2.0F};
-    DTYPE gData[3] = {0.4F, 0.8F, 1.0F};
-    DTYPE yAnswer[3] = {0.5F, 0.7311F, 0.8808F};
-    DTYPE dedyAnswer[3] = {-0.8F, -1.0943F, -1.1353F};
-    DTYPE dedxAnswer[3] = {-0.2F, -0.2151F, -0.1192F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * x = NewTensor(order, dimSize);
-    XTensor * y = NewTensor(order, dimSize);
-    XTensor * g = NewTensor(order, dimSize);
-    XTensor * dedy = NewTensor(order, dimSize);
-    XTensor * dedx = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    x->SetData(xData, unitNum);
-    g->SetData(gData, unitNum);
-    y->SetZeroAll();
-    dedy->SetZeroAll();
-    dedx->SetZeroAll();
-
-    /* call Sigmoid function */
-    Sigmoid(x, y);
-
-    /* call SigmoidBackward function */
-    SigmoidBackward(g, y, x, dedy, dedx, CROSSENTROPY);
-    
-    /* check result */
-    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedy->CheckData(dedyAnswer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-        /* create tensors */
-    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * gGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    xGPU->SetData(xData, unitNum);
-    gGPU->SetData(gData, unitNum);
-    yGPU->SetZeroAll();
-    dedyGPU->SetZeroAll();
-    dedxGPU->SetZeroAll();
-
-    /* call Sigmoid function */
-    Sigmoid(xGPU, yGPU);
-
-    /* call SigmoidBackward function */
-    SigmoidBackward(gGPU, yGPU, xGPU, dedyGPU, dedxGPU, CROSSENTROPY);
-    
-    /* check result */
-    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F)
-              && dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete g;
-    delete dedx;
-    delete dedy;
-    delete xGPU;
-    delete yGPU;
-    delete gGPU;
-    delete dedxGPU;
-    delete dedyGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete g;
-    delete dedx;
-    delete dedy;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for Sigmoid Function */
-bool TestSigmoid()
-{
-    XPRINT(0, stdout, "[TEST SIGMOID] sigmoid function and its backward computation \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSigmoid1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-    
-    /* case 2 test */
-    caseFlag = TestSigmoid2();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSigmoid.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
-*/
-
-#ifndef __TEST_SIGMOID_H__
-#define __TEST_SIGMOID_H__
-
-#include "../function/Sigmoid.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Sigmoid Function */
-extern "C"
-bool TestSigmoid();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SIGMOID_H__

source/test/TSoftmax.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
-*/
-
-#include "../XTensor.h"
-#include "../XUtility.h"
-#include "TSoftmax.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test Softmax function.
-softmax function: y = e^x / \sum_{i} e^{x_i}
-*/
-bool TestSoftmax1()
-{
-    /* a tensor of size (2, 3) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 2;
-    dimSize[1] = 3;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE xData[2][3] = { {0.0F, 1.0F, 2.0F}, 
-                          {0.5F, 0.7F, 1.4F} };
-    DTYPE answer[2][3] = { {0.0900F, 0.2447F, 0.6652F}, 
-                           {0.2136F, 0.2609F, 0.5254F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * x = NewTensor(order, dimSize);
-    XTensor * y = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    x->SetData(xData, unitNum);
-    y->SetZeroAll();
-
-    /* call Softmax function */
-    Softmax(x, y, 1);
-    
-    /* check result */
-    cpuTest = y->CheckData(answer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    xGPU->SetData(xData, unitNum);
-    yGPU->SetZeroAll();
-
-    /* call Softmax function */
-    Softmax(xGPU, yGPU, 1);
-    
-    /* check result */
-    gpuTest = yGPU->CheckData(answer, unitNum, 1e-4F);
-
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete xGPU;
-    delete yGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: test SoftmaxBackward function.
-SoftmaxBackward function: dE/dx_j = -gold_j + y_j
-In this case, LossName=CROSSENTROPY.
-*/
-bool TestSoftmax2()
-{
-    /* a input tensor of size (2, 3) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 1;
-    dimSize[1] = 3;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE xData[1][3] = { {0.0F, 1.0F, 2.0F} };
-    DTYPE gData[1][3] = { {0.0F, 0.0F, 1.0F} };
-    DTYPE yAnswer[1][3] = { {0.0900F, 0.2447F, 0.6652F} };
-    DTYPE dedxAnswer[1][3] = {0.0900F, 0.2447F, -0.3347F};
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * x = NewTensor(order, dimSize);
-    XTensor * y = NewTensor(order, dimSize);
-    XTensor * g = NewTensor(order, dimSize);
-    XTensor * dedy = NewTensor(order, dimSize);
-    XTensor * dedx = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    x->SetData(xData, unitNum);
-    g->SetData(gData, unitNum);
-    y->SetZeroAll();
-    dedx->SetZeroAll();
-    dedy->SetZeroAll();
-
-    /* call Softmax function */
-    Softmax(x, y, 1);
-    
-    /* call SoftmaxBackward function */
-    SoftmaxBackward(g, y, x, dedy, dedx, 1, CROSSENTROPY);
-    
-    /* check result */
-    cpuTest = y->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedx->CheckData(dedxAnswer, unitNum, 1e-4F);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-        /* create tensors */
-    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * gGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* initialize variables */
-    xGPU->SetData(xData, unitNum);
-    gGPU->SetData(gData, unitNum);
-    yGPU->SetZeroAll();
-    dedxGPU->SetZeroAll();
-    dedyGPU->SetZeroAll();
-
-    /* call Softmax function */
-    Softmax(xGPU, yGPU, 1);
-
-    /* call SoftmaxBackward function */
-    SoftmaxBackward(gGPU, yGPU, xGPU, dedyGPU, dedxGPU, 1, CROSSENTROPY);
-    
-    /* check result */
-    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F)
-              && dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F);
-
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete g;
-    delete dedx;
-    delete dedy;
-    delete xGPU;
-    delete yGPU;
-    delete gGPU;
-    delete dedxGPU;
-    delete dedyGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete x;
-    delete y;
-    delete g;
-    delete dedx;
-    delete dedy;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for Softmax Function */
-bool TestSoftmax()
-{
-    XPRINT(0, stdout, "[TEST SOFTMAX] softmax function and its backward computation \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSoftmax1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestSoftmax2();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSoftmax.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-19
-*/
-
-#ifndef __TEST_SOFTMAX_H__
-#define __TEST_SOFTMAX_H__
-
-#include "../function/Softmax.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Softmax Function */
-extern "C"
-bool TestSoftmax();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SOFTMAX_H__

source/test/TSort.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#include "TSort.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* case 1: sort the tensor along a given dimension */
-bool TestSort1()
-{
-    /* a tensor of size (2, 4) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 2;
-    dimSize[1] = 4;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE answer[2][4] = { {4.0F, 5.0F, 6.0F, 7.0F},
-                           {0.0F, 1.0F, 2.0F, 3.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a = NewTensor(order, dimSize);
-    XTensor * b = NewTensor(order, dimSize, X_INT);
-
-    /* initialize variables */
-    a->SetData(aData, unitNum);
-    b->SetZeroAll();
-
-    /* call Sort function */
-    Sort(a, b, 0);
-
-    cpuTest = a->CheckData(answer, unitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(order, dimSize, X_INT, 1.0F, 0);
-    
-    /* Initialize variables */
-    aGPU->SetData(aData, unitNum);
-    bGPU->SetZeroAll();
-
-    /* call sum function */
-    Sort(aGPU, bGPU, 0);
-
-    /* check results */
-    gpuTest = aGPU->CheckData(answer, unitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete aGPU;
-    delete bGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-bool TestSort2()
-{
-    /* a tensor of size (2, 4) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 2;
-    dimSize[1] = 4;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE answer[2][4] = { {3.0F, 2.0F, 1.0F, 0.0F},
-                           {7.0F, 6.0F, 5.0F, 4.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a = NewTensor(order, dimSize);
-    XTensor * b = NewTensor(order, dimSize, X_INT);
-
-    /* initialize variables */
-    a->SetData(aData, unitNum);
-
-    /* call Sort function */
-    Sort(a, b, 1);
-
-    /* check results */
-    cpuTest = a->CheckData(answer, unitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(order, dimSize, X_INT, 1.0F, 0);
-
-    /* Initialize variables */
-    aGPU->SetData(aData, unitNum);
-
-    /* call sum function */
-    Sort(aGPU, bGPU, 1);
-
-    /* check results */
-    gpuTest = aGPU->CheckData(answer, unitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete aGPU;
-    delete bGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Sort Function */
-bool TestSort()
-{
-    XPRINT(0, stdout, "[TEST SORT] sort the tensor along a given dimension \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSort1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestSort2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSort.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#ifndef __TEST_SORT_H__
-#define __TEST_SORT_H__
-
-#include "../core/sort/Sort.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Sort Function */
-extern "C"
-bool TestSort();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SORT_H__

source/test/TSplit.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-13
-*/
-
-#include "TSplit.h"
-
-namespace nts { // namespace nt(NiuTrans.Tensor)
-
-/* 
-case 1: transform a tensor by splitting it, e.g., (N, M) -> (N/3, M, 3)
-In this case, (4, 3) -> (2, 2, 3), whereToSplit=0, splitNum=2.
-*/
-bool TestSplit1()
-{
-    /* a source tensor of size (4, 3) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 4;
-    sDimSize[1] = 3;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a target tensor of size (2, 2, 3) */
-    int tOrder = 3;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-    tDimSize[2] = 3;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData[4][3] = { {0.0F, 1.0F, 2.0F},
-                          {3.0F, 4.0F, 5.0F},
-                          {0.1F, 1.1F, 2.1F},
-                          {3.1F, 4.1F, 5.1F} };
-	DTYPE answer[2][2][3] = { { {0.0F, 1.0F, 2.0F},
-	                            {3.0F, 4.0F, 5.0F} },
-							  { {0.1F, 1.1F, 2.1F},
-	                            {3.1F, 4.1F, 5.1F} } };
-  
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t->SetZeroAll();
-
-    /* call split function */
-    Split(s, t, 0, 2);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-	sGPU->SetData(sData, sUnitNum);
-	tGPU->SetZeroAll();
-
-    /* call sum function */
-	Split(sGPU, tGPU, 0, 2);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-	delete s, t, sGPU, tGPU;
-	delete[] sDimSize, tDimSize;
-
-	return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-	delete s, t;
-	delete[] sDimSize, tDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: transform a tensor by splitting it, e.g., (N, M) -> (N/3, M, 3)
-In this case, (3, 4) -> (2, 3, 2), whereToSplit=1, splitNum=2.
-*/
-bool TestSplit2()
-{
-    /* a source tensor of size (3, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 3;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a target tensor of size (2, 3, 2) */
-    int tOrder = 3;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 3;
-    tDimSize[2] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData[3][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 0.1F, 1.1F},
-                          {2.1F, 3.1F, 4.1F, 5.1F} };
-	DTYPE answer[2][3][2] = { { {0.0F, 1.0F},
-								{4.0F, 5.0F},
-								{2.1F, 3.1F} },
-	                          { {2.0F, 3.0F},
-							    {0.1F, 1.1F},
-								{4.1F, 5.1F} } };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t->SetZeroAll();;
-
-    /* call split function */
-    Split(s, t, 1, 2);
-
-    /* check results */
-    cpuTest = t->CheckData(answer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-	XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-	sGPU->SetData(sData, sUnitNum);
-	tGPU->SetZeroAll();
-
-    /* call sum function */
-	Split(sGPU, tGPU, 1, 2);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(answer, tUnitNum);
-
-    /* destroy variables */
-	delete s;
-    delete t;
-    delete sGPU;
-    delete tGPU;
-	delete[] sDimSize;
-	delete[] tDimSize;
-
-	return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-	delete s;
-    delete t;
-	delete[] sDimSize;
-	delete[] tDimSize;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 3: split a big tensor into small tensors
-In this case, (3, 4) -> 2 * (3, 2) , whereToSplit=1, splitNum=2.
-*/
-bool TestSplit3()
-{
-	/* create list */
-    XList tList;
-    tList = XList();
-
-    /* a source tensor of size (3, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 3;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a target tensor of size (3, 2) */
-    int tOrder1 = 2;
-    int * tDimSize1 = new int[tOrder1];
-    tDimSize1[0] = 3;
-    tDimSize1[1] = 2;
-
-    int tUnitNum1 = 1;
-    for (int i = 0; i < tOrder1; i++)
-        tUnitNum1 *= tDimSize1[i];
-
-    /* a target tensor of size (3 * 2) */
-    int tOrder2 = 2;
-    int * tDimSize2 = new int[tOrder2];
-    tDimSize2[0] = 3;
-    tDimSize2[1] = 2;
-
-    int tUnitNum2 = 1;
-    for (int i = 0; i < tOrder2; i++)
-        tUnitNum2 *= tDimSize2[i];
-
-    DTYPE sData[3][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 0.1F, 1.1F},
-                          {2.1F, 3.1F, 4.1F, 5.1F} };
-    DTYPE answer1[3][2] = { {0.0F, 1.0F},
-                            {4.0F, 5.0F},
-                            {2.1F, 3.1F} };
-    DTYPE answer2[3][2] = { {2.0F, 3.0F},
-                            {0.1F, 1.1F},
-                            {4.1F, 5.1F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
-    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t1->SetZeroAll();
-    t2->SetZeroAll();
-
-	/* add tensors to list */
-    tList.Add(t1);
-    tList.Add(t2);
-
-    /* call split function */
-    Split(s, &tList, 1, 2);
-
-    /* check results */
-    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-	/* clear list */
-	tList.Clear();
-
-    /* create tensor */
-	XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-	XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
-	XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-	sGPU->SetData(sData, sUnitNum);
-	tGPU1->SetZeroAll();
-	tGPU2->SetZeroAll();
-
-	/* add tensors to list */
-	tList.Add(tGPU1);
-	tList.Add(tGPU2);
-
-	/* call split function */
-	Split(s, &tList, 1, 2);
-
-    /* check results */
-	gpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
-
-    /* destroy variables */
-	delete s;
-    delete t1;
-    delete t2;
-    delete sGPU;
-    delete tGPU1;
-    delete tGPU2;
-	delete[] sDimSize;
-	delete[] tDimSize1;
-	delete[] tDimSize2;
-
-	return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-	delete s;
-    delete t1;
-    delete t2;
-	delete[] sDimSize;
-	delete[] tDimSize1;
-	delete[] tDimSize2;
-
-	return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for Split Function */
-bool TestSplit()
-{
-    XPRINT(0, stdout, "[TEST SPLIT] split a big tensor into small tensors \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSplit1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestSplit2();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* case 3 test */
-    caseFlag = TestSplit3();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 3 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 3 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSplit.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-13
-*/
-
-#ifndef __TEST_SPLIT_H__
-#define __TEST_SPLIT_H__
-
-#include "../core/shape/Split.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Split Function */
-extern "C"
-bool TestSplit();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SPLIT_H__

source/test/TSum.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#include "TSum.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* case 1: tensor summation c = a + b * \beta */
-bool TestSum1()
-{
-    /* a tensor of size (2, 4) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 2;
-    dimSize[1] = 4;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++)
-        unitNum *= dimSize[i];
-
-    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE bData[2][4] = { {1.0F, -1.0F, -3.0F, -5.0F}, 
-                          {-7.0F, -9.0F, -11.0F, -13.0F} };
-    DTYPE answer[2][4] = { {1.0F, 0.0F, -1.0F, -2.0F},
-                           {-3.0F, -4.0F, -5.0F, -6.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a = NewTensor(order, dimSize);
-    XTensor * b = NewTensor(order, dimSize);
-
-    /* initialize variables */
-    a->SetData(aData, unitNum);
-    b->SetData(bData, unitNum);
-
-    /* call sum function */
-    _Sum(a, b);
-
-    /* check results */
-    cpuTest = a->CheckData(answer, unitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    aGPU->SetData(aData, unitNum);
-    bGPU->SetData(bData, unitNum);
-
-    /* call sum function */
-    _Sum(aGPU, bGPU);
-
-    /* check results */
-    gpuTest = aGPU->CheckData(answer, unitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete aGPU;
-    delete bGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-	delete b;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* case 2: tensor summation c = a + b * \beta */
-bool TestSum2()
-{
-    /* a tensor of size (2, 4) */
-    int order = 2;
-    int * dimSize = new int[order];
-    dimSize[0] = 2;
-    dimSize[1] = 4;
-
-    int unitNum = 1;
-    for (int i = 0; i < order; i++) {
-        unitNum *= dimSize[i];
-    }
-    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE bData[2][4] = { {1.0F, -1.0F, -3.0F, -5.0F}, 
-                          {-7.0F, -9.0F, -11.0F, -13.0F} };
-    DTYPE answer[2][4] = { {0.5F, 0.5F, 0.5F, 0.5F},
-                           {0.5F, 0.5F, 0.5F, 0.5F} };
-    float beta = 0.5F;
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensor */
-    XTensor * a = NewTensor(order, dimSize);
-    XTensor * b = NewTensor(order, dimSize);
-    XTensor * c = NewTensor(order, dimSize);
-
-    /* Initalize variables */
-    a->SetData(aData, unitNum);
-    b->SetData(bData, unitNum);
-    c->SetZeroAll();
-
-    /* call Sum function */
-    _Sum(a, b, c, beta);
-
-    /* check results */
-    cpuTest = c->CheckData(answer, unitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-    XTensor * cGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initalize variables */
-    aGPU->SetData(aData, unitNum);
-    bGPU->SetData(bData, unitNum);
-    cGPU->SetZeroAll();
-
-    /* call Sum function */
-    _Sum(aGPU, bGPU, cGPU, beta);
-
-    /* check results */
-    gpuTest = cGPU->CheckData(answer, unitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete c;
-    delete aGPU;
-    delete bGPU;
-    delete cGPU;
-    delete[] dimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete c;
-    delete[] dimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for Sum Function */
-bool TestSum()
-{
-    XPRINT(0, stdout, "[TEST SUM] tensor summation c = a + b * beta\n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSum1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestSum2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-        TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSum.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-04-30
-*/
-
-#ifndef __TEST_SUM_H__
-#define __TEST_SUM_H__
-
-#include "../core/arithmetic/Sum.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Sum Function */
-extern "C"
-bool TestSum();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SUM_H__

source/test/TSumByColumnTV.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#include "TSumByColumnTV.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test SumByColumnTV function
-sum of a tensor and a vector (column vector) in a column by column manner
-*/
-bool TestSumByColumnTV1()
-{
-    /* a tensor of size (2, 4) */
-    int aOrder = 2;
-    int * aDimSize = new int[aOrder];
-    aDimSize[0] = 2;
-    aDimSize[1] = 4;
-
-    int aUnitNum = 1;
-    for (int i = 0; i < aOrder; i++)
-        aUnitNum *= aDimSize[i];
-
-    /* a tensor of size (2, 1) */
-    int bOrder = 2;
-    int * bDimSize = new int[bOrder];
-    bDimSize[0] = 2;
-    bDimSize[1] = 1;
-
-    int bUnitNum = 1;
-    for (int i = 0; i < bOrder; i++)
-        bUnitNum *= bDimSize[i];
-
-    /* a tensor of size (2, 4) */
-    int cOrder = 2;
-    int * cDimSize = new int[cOrder];
-    cDimSize[0] = 2;
-    cDimSize[1] = 4;
-
-    int cUnitNum = 1;
-    for (int i = 0; i < cOrder; i++)
-        cUnitNum *= cDimSize[i];
-
-    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE bData[2][1] = { {1.0F},
-                          {0.0F} };
-    DTYPE answer[2][4] = { {1.0F, 2.0F, 3.0F, 4.0F},
-                           {4.0F, 5.0F, 6.0F, 7.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a = NewTensor(aOrder, aDimSize);
-    XTensor * b = NewTensor(bOrder, bDimSize);
-    XTensor * c = NewTensor(cOrder, cDimSize);
-
-    /* initialize variables */
-    a->SetData(aData, aUnitNum);
-    b->SetData(bData, bUnitNum);
-
-    /* call SumByColumnTV function */
-    SumByColumnTV(a, b, c);
-
-    /* check results */
-    cpuTest = c->CheckData(answer, cUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * cGPU = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    aGPU->SetData(aData, aUnitNum);
-    bGPU->SetData(bData, bUnitNum);
-    cGPU->SetZeroAll();
-
-    /* call SumByColumnTV function */
-    SumByColumnTV(aGPU, bGPU, cGPU);
-
-    /* check results */
-    gpuTest = cGPU->CheckData(answer, cUnitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete c;
-    delete aGPU;
-    delete bGPU;
-    delete cGPU;
-    delete[] aDimSize;
-    delete[] bDimSize;
-    delete[] cDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete c;
-    delete[] aDimSize;
-    delete[] bDimSize;
-    delete[] cDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: test SumByColumnTV function
-sum of a tensor and a vector (column vector) in a column by column manner
-*/
-bool TestSumByColumnTV2()
-{
-    /* a tensor of size (2, 4) */
-    int aOrder = 2;
-    int * aDimSize = new int[aOrder];
-    aDimSize[0] = 2;
-    aDimSize[1] = 4;
-
-    int aUnitNum = 1;
-    for (int i = 0; i < aOrder; i++)
-        aUnitNum *= aDimSize[i];
-
-    /* a tensor of size (2, 1) */
-    int bOrder = 2;
-    int * bDimSize = new int[bOrder];
-    bDimSize[0] = 2;
-    bDimSize[1] = 1;
-
-    int bUnitNum = 1;
-    for (int i = 0; i < bOrder; i++)
-        bUnitNum *= bDimSize[i];
-
-    DTYPE aData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE bData[2][1] = { {1.0F},
-                          {0.0F} };
-    DTYPE answer[2][4] = { {1.0F, 2.0F, 3.0F, 4.0F},
-                           {4.0F, 5.0F, 6.0F, 7.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a = NewTensor(aOrder, aDimSize);
-    XTensor * b = NewTensor(bOrder, bDimSize);
-
-    /* initialize variables */
-    a->SetData(aData, aUnitNum);
-    b->SetData(bData, bUnitNum);
-
-    /* call SumByColumnTV function */
-    SumByColumnTV(a, b);
-
-    /* check results */
-    cpuTest = a->CheckData(answer, aUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    aGPU->SetData(aData, aUnitNum);
-    bGPU->SetData(bData, bUnitNum);
-
-    /* call SumByColumnTV function */
-    SumByColumnTV(aGPU, bGPU);
-
-    /* check results */
-    gpuTest = aGPU->CheckData(answer, aUnitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete aGPU;
-    delete bGPU;
-    delete[] aDimSize;
-    delete[] bDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete[] aDimSize;
-    delete[] bDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for SumByColumnTV Function */
-bool TestSumByColumnTV() 
-{
-    XPRINT(0, stdout, "[TEST SumByColumnTV] sum of a tensor and a vector (column vector) in a column by column manner \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSumByColumnTV1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestSumByColumnTV2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-        TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSumByColumnTV.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#ifndef __TEST_SUMBYCOLUMNTV_H__
-#define __TEST_SUMBYCOLUMNTV_H__
-
-#include "../core/arithmetic/SumByColumnTV.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for SumByColumnTV Function */
-extern "C"
-bool TestSumByColumnTV();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SUMBYCOLUMNTV_H__

source/test/TSumByColumnVT.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#include "TSumByColumnVT.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: test SumByColumnVT function
-sum of a vector (column vector) and a tensor in a column by column manner
-*/
-bool TestSumByColumnVT1()
-{
-    /* a tensor of size (2, 1) */
-    int aOrder = 2;
-    int * aDimSize = new int[aOrder];
-    aDimSize[0] = 2;
-    aDimSize[1] = 1;
-
-    int aUnitNum = 1;
-    for (int i = 0; i < aOrder; i++)
-        aUnitNum *= aDimSize[i];
-
-    /* a tensor of size (2, 4) */
-    int bOrder = 2;
-    int * bDimSize = new int[bOrder];
-    bDimSize[0] = 2;
-    bDimSize[1] = 4;
-
-    int bUnitNum = 1;
-    for (int i = 0; i < bOrder; i++)
-        bUnitNum *= bDimSize[i];
-
-    /* a tensor of size (2, 1) */
-    int cOrder = 2;
-    int * cDimSize = new int[cOrder];
-    cDimSize[0] = 2;
-    cDimSize[1] = 1;
-
-    int cUnitNum = 1;
-    for (int i = 0; i < cOrder; i++)
-        cUnitNum *= cDimSize[i];
-
-    DTYPE aData[2][1] = { {1.0F},
-                          {0.0F} };
-    DTYPE bData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE answer[2][1] = { {7.0F},
-                           {22.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a = NewTensor(aOrder, aDimSize);
-    XTensor * b = NewTensor(bOrder, bDimSize);
-    XTensor * c = NewTensor(cOrder, cDimSize);
-
-    /* initialize variables */
-    a->SetData(aData, aUnitNum);
-    b->SetData(bData, bUnitNum);
-    c->SetZeroAll();
-
-    /* call SumByColumnVT function */
-    SumByColumnVT(a, b, c);
-    
-    /* check results */
-    cpuTest = c->CheckData(answer, cUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * cGPU = NewTensor(cOrder, cDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    aGPU->SetData(aData, aUnitNum);
-    bGPU->SetData(bData, bUnitNum);
-    cGPU->SetZeroAll();
-
-    /* call SumByColumnVT function */
-    SumByColumnVT(aGPU, bGPU, cGPU);
-    
-    /* check results */
-    gpuTest = cGPU->CheckData(answer, cUnitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete c;
-    delete aGPU;
-    delete bGPU;
-    delete cGPU;
-    delete[] aDimSize;
-    delete[] bDimSize;
-    delete[] cDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete c;
-    delete[] aDimSize;
-    delete[] bDimSize;
-    delete[] cDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* 
-case 2: test SumByColumnVT function
-sum of a vector (column vector) and a tensor in a column by column manner
-*/
-bool TestSumByColumnVT2()
-{
-    /* a tensor of size (2, 1) */
-    int aOrder = 2;
-    int * aDimSize = new int[aOrder];
-    aDimSize[0] = 2;
-    aDimSize[1] = 1;
-
-    int aUnitNum = 1;
-    for (int i = 0; i < aOrder; i++)
-        aUnitNum *= aDimSize[i];
-
-    /* a tensor of size (2, 4) */
-    int bOrder = 2;
-    int * bDimSize = new int[bOrder];
-    bDimSize[0] = 2;
-    bDimSize[1] = 4;
-
-    int bUnitNum = 1;
-    for (int i = 0; i < bOrder; i++)
-        bUnitNum *= bDimSize[i];
-
-    DTYPE aData[2][1] = { {1.0F},
-                          {0.0F} };
-    DTYPE bData[2][4] = { {0.0F, 1.0F, 2.0F, 3.0F},
-                          {4.0F, 5.0F, 6.0F, 7.0F} };
-    DTYPE answer[2][1] = { {7.0F},
-                           {22.0F} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * a = NewTensor(aOrder, aDimSize);
-    XTensor * b = NewTensor(bOrder, bDimSize);
-
-    /* initialize variables */
-    a->SetData(aData, aUnitNum);
-    b->SetData(bData, bUnitNum);
-
-    /* call SumByColumnVT function */
-    SumByColumnVT(a, b);
-
-    /* check results */
-    cpuTest = a->CheckData(answer, aUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * aGPU = NewTensor(aOrder, aDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * bGPU = NewTensor(bOrder, bDimSize, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    aGPU->SetData(aData, aUnitNum);
-    bGPU->SetData(bData, bUnitNum);
-
-    /* call SumByColumnVT function */
-    SumByColumnVT(aGPU, bGPU);
-
-    /* check results */
-    gpuTest = aGPU->CheckData(answer, aUnitNum);
-
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete aGPU;
-    delete bGPU;
-    delete[] aDimSize;
-    delete[] bDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete a;
-    delete b;
-    delete[] aDimSize;
-    delete[] bDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for SumByColumnVT Function */
-bool TestSumByColumnVT() 
-{
-    XPRINT(0, stdout, "[TEST SumByColumnVT] sum of a vector (column vector) and a tensor in a column by column manner \n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestSumByColumnVT1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* case 2 test */
-    caseFlag = TestSumByColumnVT2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-        TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TSumByColumnVT.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-07-06
-*/
-
-#ifndef __TEST_SUMBYCOLUMNVT_H__
-#define __TEST_SUMBYCOLUMNVT_H__
-
-#include "../core/arithmetic/SumByColumnVT.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for SumByColumnVT Function */
-extern "C"
-bool TestSumByColumnVT();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_SUMBYCOLUMNVT_H__

source/test/TTopK.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#include "TTopK.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: get the top-k items along a given dimension.
-In this case, 
-(2, 4) -> (2, 4), dim = 0, k = 2
-(2, 4) -> (2, 4), dim = 1, k = 4
-*/
-bool TestTopK1()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a output tensor of size (2, 4) */
-    int tOrder = 2;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 4;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData[2][4] = { {5.0F, 1.0F, 2.0F, 8.0F},
-                          {4.0F, 3.0F, 7.0F, 6.0F} };
-
-    DTYPE tAnswer1[2][4] = { {5.0F, 3.0F, 7.0F, 8.0F},
-                             {4.0F, 1.0F, 2.0F, 6.0F} };
-    int indexAnswer1[2][4] = { {0, 1, 1, 0},
-                               {1, 0, 0, 1} };
-
-    DTYPE tAnswer2[2][4] = { {8.0F, 5.0F, 2.0F, 1.0F},
-                             {7.0F, 6.0F, 4.0F, 3.0F} };
-    int indexAnswer2[2][4] = { {3, 0, 2, 1},
-                               {2, 3, 0, 1} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t1 = NewTensor(tOrder, tDimSize);
-    XTensor * t2 = NewTensor(tOrder, tDimSize);
-    XTensor * index1 = NewTensor(tOrder, tDimSize, X_INT);
-    XTensor * index2 = NewTensor(tOrder, tDimSize, X_INT);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t1->SetZeroAll();
-    t2->SetZeroAll();
-    index1->SetZeroAll();
-    index2->SetZeroAll();
-
-    /* call TopK function */
-    int dim = 0;
-    int k = sDimSize[dim];
-    TopK(s, t1, index1, dim, k);
-
-    dim = 1;
-    k = sDimSize[dim];
-    TopK(s, t2, index2, dim, k);
-
-    /* check results */
-    cpuTest = t1->CheckData(tAnswer1, tUnitNum) && 
-              t2->CheckData(tAnswer2, tUnitNum) &&
-              index1->CheckData(indexAnswer1, tUnitNum) &&
-              index2->CheckData(indexAnswer2, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU1 = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU2 = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * indexGPU1 = NewTensor(tOrder, tDimSize, X_INT, 1.0F, 0);
-    XTensor * indexGPU2 = NewTensor(tOrder, tDimSize, X_INT, 1.0F, 0);
-    
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU1->SetZeroAll();
-    tGPU2->SetZeroAll();
-    indexGPU1->SetZeroAll();
-    indexGPU2->SetZeroAll();
-
-    /* call TopK function */
-    dim = 0;
-    k = sDimSize[dim];
-    TopK(sGPU, tGPU1, indexGPU1, dim, k);
-    
-    dim = 1;
-    k = sDimSize[dim];
-    TopK(sGPU, tGPU2, indexGPU2, dim, k);
-    
-    /* check results */
-    gpuTest = tGPU1->CheckData(tAnswer1, tUnitNum) && 
-              tGPU2->CheckData(tAnswer2, tUnitNum) &&
-              indexGPU1->CheckData(indexAnswer1, tUnitNum) &&
-              indexGPU2->CheckData(indexAnswer2, tUnitNum);
-
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete index1;
-    delete index2;
-    delete sGPU;
-    delete tGPU1;
-    delete tGPU2;
-    delete indexGPU1;
-    delete indexGPU2;
-    delete[] sDimSize;
-    delete[] tDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete index1;
-    delete index2;
-    delete[] sDimSize;
-    delete[] tDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/*
-case 2: get the top-k items along a given dimension.
-In this case, (2, 4) -> (2, 2), dim = 1, k = 2.
-*/
-bool TestTopK2()
-{
-    /* a input tensor of size (2, 4) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 4;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a output tensor of size (2, 2) */
-    int tOrder = 2;
-    int * tDimSize = new int[tOrder];
-    tDimSize[0] = 2;
-    tDimSize[1] = 2;
-
-    int tUnitNum = 1;
-    for (int i = 0; i < tOrder; i++)
-        tUnitNum *= tDimSize[i];
-
-    DTYPE sData[2][4] = { {5.0F, 1.0F, 2.0F, 8.0F},
-                          {4.0F, 3.0F, 7.0F, 6.0F} };
-    DTYPE tAnswer[2][2] = { {8.0F, 5.0F},
-                            {7.0F, 6.0F} };
-    int indexAnswer[2][2] = { {3, 0},
-                              {2, 3} };
-
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t = NewTensor(tOrder, tDimSize);
-    XTensor * index = NewTensor(tOrder, tDimSize, X_INT);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t->SetZeroAll();
-    index->SetZeroAll();
-
-    /* call TopK function */
-    int dim = 1;
-    int k = tDimSize[dim];
-    TopK(s, t, index, dim, k);
-
-    /* check results */
-    cpuTest = t->CheckData(tAnswer, tUnitNum) && index->CheckData(indexAnswer, tUnitNum);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensors */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU = NewTensor(tOrder, tDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * indexGPU = NewTensor(tOrder, tDimSize, X_INT, 1.0F, 0);
-    
-    /* initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU->SetZeroAll();
-    indexGPU->SetZeroAll();
-
-    /* call TopK function */
-    dim = 1;
-    k = tDimSize[dim];
-    TopK(sGPU, tGPU, indexGPU, dim, k);
-
-    /* check results */
-    gpuTest = tGPU->CheckData(tAnswer, tUnitNum) && indexGPU->CheckData(indexAnswer, tUnitNum);
-    
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete index;
-    delete sGPU;
-    delete tGPU;
-    delete indexGPU;
-    delete[] sDimSize;
-    delete[] tDimSize;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t;
-    delete index;
-    delete[] sDimSize;
-    delete[] tDimSize;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-TODO!!
-*/
-
-/* test for TopK Function */
-bool TestTopK()
-{
-    XPRINT(0, stdout, "[TEST TopK] get the top-k items along a given dimension\n");
-    bool returnFlag = true, caseFlag = true;
-    
-    /* case 1 test */
-    caseFlag = TestTopK1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-    
-    /* case 2 test */
-    caseFlag = TestTopK2();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 2 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 2 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-    }
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TTopK.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#ifndef __TEST_TOPK_H__
-#define __TEST_TOPK_H__
-
-#include "../core/sort/TopK.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for TopK Function */
-extern "C"
-bool TestTopK();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_TOPK_H__

source/test/TUnsqueeze.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-13
-*/
-
-#include "../XList.h"
-#include "TUnsqueeze.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* 
-case 1: insert a dimension by copying the blocks for x times (where x is the size of the inerted dimension)
-In this case, 
-(2, 3) -> (2, 2, 3), dim=1, dSize=2
-(2, 3) -> (2, 3, 2), dim=2, dSize=2
-*/
-bool TestUnsqueeze1()
-{
-    /* a source tensor of size (2, 3) */
-    int sOrder = 2;
-    int * sDimSize = new int[sOrder];
-    sDimSize[0] = 2;
-    sDimSize[1] = 3;
-
-    int sUnitNum = 1;
-    for (int i = 0; i < sOrder; i++)
-        sUnitNum *= sDimSize[i];
-
-    /* a target tensor of size (2, 2, 3) */
-    int tOrder1 = 3;
-    int * tDimSize1 = new int[tOrder1];
-    tDimSize1[0] = 2;
-    tDimSize1[1] = 2;
-    tDimSize1[2] = 3;
-
-    int tUnitNum1 = 1;
-    for (int i = 0; i < tOrder1; i++)
-        tUnitNum1 *= tDimSize1[i];
-
-    /* a target tensor of size (2, 3, 2) */
-    int tOrder2 = 3;
-    int * tDimSize2 = new int[tOrder2];
-    tDimSize2[0] = 2;
-    tDimSize2[1] = 3;
-    tDimSize2[2] = 2;
-
-    int tUnitNum2 = 1;
-    for (int i = 0; i < tOrder2; i++)
-        tUnitNum2 *= tDimSize2[i];
-
-    DTYPE sData[2][3] = { {0.0F, 1.0F, 2.0F},
-                          {3.0F, 4.0F, 5.0F} };
-    DTYPE answer1[2][2][3] = { { {0.0F, 1.0F, 2.0F},
-                                 {0.0F, 1.0F, 2.0F} },
-                               { {3.0F, 4.0F, 5.0F},
-                                 {3.0F, 4.0F, 5.0F} } };
-    DTYPE answer2[2][3][2] = { { {0.0F, 0.0F}, 
-                                 {1.0F, 1.0F}, 
-                                 {2.0F, 2.0F} },
-                               { {3.0F, 3.0F}, 
-                                 {4.0F, 4.0F}, 
-                                 {5.0F, 5.0F} } };
-    
-    /* CPU test */
-    bool cpuTest = true;
-
-    /* create tensors */
-    XTensor * s = NewTensor(sOrder, sDimSize);
-    XTensor * t1 = NewTensor(tOrder1, tDimSize1);
-    XTensor * t2 = NewTensor(tOrder2, tDimSize2);
-
-    /* initialize variables */
-    s->SetData(sData, sUnitNum);
-    t1->SetZeroAll();
-    t2->SetZeroAll();
-
-    /* call Unsqueeze function */
-    Unsqueeze(s, t1, 1, 2);
-    Unsqueeze(s, t2, 2, 2);
-
-    /* check results */
-    cpuTest = t1->CheckData(answer1, tUnitNum1) && t2->CheckData(answer2, tUnitNum2);
-
-#ifdef USE_CUDA
-    /* GPU test */
-    bool gpuTest = true;
-
-    /* create tensor */
-    XTensor * sGPU = NewTensor(sOrder, sDimSize, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU1 = NewTensor(tOrder1, tDimSize1, X_FLOAT, 1.0F, 0);
-    XTensor * tGPU2 = NewTensor(tOrder2, tDimSize2, X_FLOAT, 1.0F, 0);
-
-    /* Initialize variables */
-    sGPU->SetData(sData, sUnitNum);
-    tGPU1->SetZeroAll();
-    tGPU2->SetZeroAll();
-
-    /* call Unsqueeze function */
-    Unsqueeze(sGPU, tGPU1, 1, 2);
-    Unsqueeze(sGPU, tGPU2, 2, 2);
-
-    /* check results */
-    gpuTest = tGPU1->CheckData(answer1, tUnitNum1) && tGPU2->CheckData(answer2, tUnitNum2);
-
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete sGPU;
-    delete tGPU1;
-    delete tGPU2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-
-    return cpuTest && gpuTest;
-#else
-    /* destroy variables */
-    delete s;
-    delete t1;
-    delete t2;
-    delete[] sDimSize;
-    delete[] tDimSize1;
-    delete[] tDimSize2;
-
-    return cpuTest;
-#endif // USE_CUDA
-}
-
-/* other cases */
-/*
-    TODO!!
-*/
-
-/* test for Unsqueeze Function */
-bool TestUnsqueeze()
-{
-    XPRINT(0, stdout, "[TEST Unsqueeze] insert a dimension by copying the blocks for x times\n");
-    bool returnFlag = true, caseFlag = true;
-
-    /* case 1 test */
-    caseFlag = TestUnsqueeze1();
-
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    /* other cases test */
-    /*
-    TODO!!
-    */
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    XPRINT(0, stdout, "\n");
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)

source/test/TUnsqueeze.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: Xu Chen (email: hello_master1954@163.com) 2018-06-27
-*/
-
-#ifndef __TEST_UNSQUEEZE_H__
-#define __TEST_UNSQUEEZE_H__
-
-#include "../core/shape/Unsqueeze.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for Unsqueeze Function */
-extern "C"
-bool TestUnsqueeze();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_UNSQUEEZE_H__

source/test/TXMem.cpp

-/* NiuTrans.Tensor - an open-source tensor library
- * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
- * All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-6-24
- */
-
-#include "../XGlobal.h"
-#include "../XUtility.h"
-#include "TXMem.h"
-
-namespace nts{ // namespace nts(NiuTrans.Tensor)
-
-/* case 1: test memory pool class */
-bool TestXMemCase1()
-{
-    bool ok = true;
-    int caseNum = 1000;
-    int blcokSize = 16;
-    int testNum = caseNum * 10;
-
-    for(int i = 0, scalar = 1; i < 3; i++){
-        XMem mem;
-        mem.Initialize(-1, FREE_ON_THE_FLY, blcokSize * sizeof(int) * scalar * scalar, 1000, 0);
-        mem.SetIndex(10000, blcokSize * sizeof(int) / 2);
-
-        srand(907);
-
-        int ** p = new int*[caseNum];
-        int * size = new int[caseNum];
-
-        for(int i = 0; i < caseNum; i++){
-            p[i] = NULL;
-            size[i] = rand() % (2*blcokSize);
-        }
-
-        for(int i = 0; i < testNum * scalar; i++){
-            int j = rand() % caseNum;
-
-            if(p[j] == NULL){
-                p[j] = (int*)mem.AllocStandard(mem.devID, size[j] * sizeof(int));
-                for(int k = 0; k < size[j]; k++)
-                    p[j][k] = j;
-            }
-            else{
-                mem.ReleaseStandard(mem.devID, p[j]);
-                for(int k = 0; k < size[j]; k++)
-                    p[j][k] = -1;
-                p[j] = NULL;
-            }
-
-            for(int k = 0; k < caseNum; k++){
-                if(p[k] != NULL){
-                    for(int o = 0; o < size[k]; o++){
-                        if(p[k][o] != k){
-                            ok = false;
-                        }
-                    }
-                }
-            }
-        }
-
-        delete[] p;
-        delete[] size;
-        scalar *= 2;
-    }
-
-    return ok;
-}
-
-/* test for memory pool class */
-bool TestXMem()
-{
-    XPRINT(0, stdout, "[Test] Memory pool ... Began\n");
-    bool returnFlag = true;
-    bool caseFlag = true;
-
-    double startT = GetClock();
-
-    /* case 1 test */
-    caseFlag = TestXMemCase1();
-    if (!caseFlag) {
-        returnFlag = false;
-        XPRINT(0, stdout, ">> case 1 failed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> case 1 passed!\n");
-
-    if (returnFlag) {
-        XPRINT(0, stdout, ">> All Passed!\n");
-    }
-    else
-        XPRINT(0, stdout, ">> Failed!\n");
-
-    double endT = GetClock();
-
-    XPRINT1(0, stdout, "[Test] Finished (took %.3lfms)\n\n", endT - startT);
-
-    return returnFlag;
-}
-
-} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/test/TXMem.h

-/* NiuTrans.Tensor - an open-source tensor library
- * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
- * All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-6-24
- */
-
-#ifndef __TXMEM_H__
-#define __TXMEM_H__
-
-#include "../XMem.h"
-
-namespace nts{ // namespace nts(NiuTrans.Tensor)
-
-/* test for memory pool class */
-extern "C"
-bool TestXMem();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TXMEM_H__

source/test/Test.cpp

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-05-01
-*/
-
-#include "Test.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for all Function */
-bool Test()
-{
-    bool wrong = false;
-    XPRINT(0, stdout, "Testing the XTensor utilites ... \n\n");
-
-    wrong = !TestConcatenate() || wrong;
-    wrong = !TestConcatenateSolely() || wrong;
-    wrong = !TestCopyIndexed() || wrong;
-    wrong = !TestCopyValues() || wrong;
-    wrong = !TestMatrixMul() || wrong;
-    wrong = !TestMatrixMul2D() || wrong;
-    wrong = !TestMatrixMul2DParallel() || wrong;
-    //wrong = !TestMatrixMulBatched() || wrong;
-    wrong = !TestMatrixMulBatchedCPU() || wrong;
-    wrong = !TestMerge() || wrong;
-    wrong = !TestMultiply() || wrong;
-    wrong = !TestNegate() || wrong;
-    wrong = !TestNormalize() || wrong;
-    wrong = !TestPower() || wrong;
-    wrong = !TestReduceMax() || wrong;
-    wrong = !TestReduceMean() || wrong;
-    wrong = !TestReduceSum() || wrong;
-    wrong = !TestReduceSumSquared() || wrong;
-    wrong = !TestReduceVariance() || wrong;
-    wrong = !TestScaleAndShift() || wrong;
-    wrong = !TestSelect() || wrong;
-    wrong = !TestSetAscendingOrder() || wrong;
-    wrong = !TestSetData() || wrong;
-    wrong = !TestSort() || wrong;
-    wrong = !TestSplit() || wrong;
-    wrong = !TestSum() || wrong;
-    wrong = !TestSumByColumnTV() || wrong;
-    //wrong = !TestSumByColumnVT() || wrong;
-    wrong = !TestTopK() || wrong;
-    wrong = !TestUnsqueeze() || wrong;
-    wrong = !TestXMem() || wrong;
-
-    wrong = !TestHardTanH() || wrong;
-    wrong = !TestIdentity() || wrong;
-    wrong = !TestLogSoftmax() || wrong;
-    wrong = !TestLoss() || wrong;
-    wrong = !TestRectify() || wrong;
-    wrong = !TestSigmoid() || wrong;
-    wrong = !TestSoftmax() || wrong;
-
-    /* other test */
-    /*
-    TODO!!
-    */
-
-    if (wrong) {
-        XPRINT(0, stdout, "Something goes wrong! Please check the code!\n");
-        return false;
-    }
-    else {
-        XPRINT(0, stdout, "OK! Everything is good!\n");
-        return true;
-    }
-}
-
-} // namespace nts(NiuTrans.Tensor)
\ No newline at end of file

source/test/Test.h

-/* NiuTrans.Tensor - an open-source tensor library
-* Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
-* All rights reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*
-* $Created by: LI Yinqiao (li.yin.qiao.2012@hotmail.com) 2018-05-01
-*/
-
-#ifndef __TEST_H__
-#define __TEST_H__
-
-#include "TConcatenate.h"
-#include "TConcatenateSolely.h"
-#include "TCopyIndexed.h"
-#include "TCopyValues.h"
-#include "TMatrixMul.h"
-#include "TMatrixMul2D.h"
-#include "TMatrixMul2DParallel.h"
-#include "TMatrixMulBatched.h"
-#include "TMatrixMULBatchedCPU.h"
-#include "TMerge.h"
-#include "TMultiply.h"
-#include "TNegate.h"
-#include "TNormalize.h"
-#include "TPower.h"
-#include "TReduceMax.h"
-#include "TReduceMean.h"
-#include "TReduceSum.h"
-#include "TReduceSumSquared.h"
-#include "TReduceVariance.h"
-#include "TScaleAndShift.h"
-#include "TSelect.h"
-#include "TSetAscendingOrder.h"
-#include "TSetData.h"
-#include "TSort.h"
-#include "TSplit.h"
-#include "TSum.h"
-#include "TSumByColumnTV.h"
-#include "TSumByColumnVT.h"
-#include "TTopK.h"
-#include "TUnsqueeze.h"
-#include "TXMem.h"
-
-#include "THardTanH.h"
-#include "TIdentity.h"
-#include "TLogSoftmax.h"
-#include "TLoss.h"
-#include "TRectify.h"
-#include "TSigmoid.h"
-#include "TSoftmax.h"
-
-namespace nts { // namespace nts(NiuTrans.Tensor)
-
-/* test for all Function */
-extern "C"
-bool Test();
-
-} // namespace nts(NiuTrans.Tensor)
-#endif // __TEST_H__