work on the triaining processing on t2t lm

f21e1b48 · xiaotong · 5cd1be65 · f21e1b48 · f21e1b48 · f21e1b48
Commit f21e1b48 authored Aug 03, 2018 by xiaotong
--- a/source/network/Main.cpp
+++ b/source/network/Main.cpp
@@ -25,6 +25,7 @@
 #include "../tensor/function/FHeader.h"
 #include "../tensor/core/CHeader.h"
 #include "../sample/fnnlm/FNNLM.h"
+#include "../sample/transformer/Transformer.h"
 //#define CRTDBG_MAP_ALLOC
 //#include <stdlib.h>
@@ -35,6 +36,7 @@ void SumDimTest();
 using namespace nts;
 using namespace fnnlm;
+using namespace transformer;
 int main( int argc, const char ** argv )
 {
@@ -44,10 +46,11 @@ int main( int argc, const char ** argv )
    //SumDimTest();
    //return 0;
-    if(argc > 1 && !strcmp(argv[1], "-test"))
-        1;//Test();
+    if(argc > 1 && !strcmp(argv[1], "-fnnlm"))
-    else if(argc > 1 && !strcmp(argv[1], "-fnnlm"))
        FNNLMMain(argc - 1, argv + 1);
+    else if(argc > 1 && !strcmp(argv[1], "-t2t"))
+        TransformerMain(argc - 1, argv + 1); 
    else{
        fprintf(stderr, "Thanks for using NiuTrans.Network! This is a library for building\n");
        fprintf(stderr, "neural networks in an easy way. \n\n");

--- a/source/network/XBackwardMath.cpp
+++ b/source/network/XBackwardMath.cpp
@@ -614,7 +614,7 @@ void XMathGrad::GradNormalize(XTensor * node)
    XTensor * p = NewTensor(a);
    XTensor * q = NewTensor(a);
    XTensor * r = NewTensor(a);
-    DTYPE epsilon = income.GetParamInt(0);
+    DTYPE epsilon = income.GetParam(0);
    int dim = income.GetParamInt(0);
    int n = a->GetDim(dim);
@@ -742,7 +742,7 @@ void XMathGrad::GradReduceMean(XTensor * node)
    XNoder::MakeGrad(a);
    _Unsqueeze(node->grad, b, dim, n);
-    _ScaleAndShift(b, c, 1 / n);
+    _ScaleAndShift(b, c, 1.0F/n);
    _Sum(a->grad, c, a->grad);
    node->visitMark = NODE_FINISHED;

--- a/source/network/XNet.cpp
+++ b/source/network/XNet.cpp
@@ -258,10 +258,11 @@ void XNet::TarjanVisit(XTensor * node, XList &orders, const unsigned int code)
    if(node == NULL)
        return;
+    //fprintf(stderr, "%d\n", node->id);
    if(node->visitMark == code + 1){
        ShowNTErrors("There is a circle in the network\n");
    }
-    else if(node->visitMark <= code || node->visitMark >= code + 2){
+    else if(node->visitMark <= code){
        node->visitMark = code + 1;
        XLink &income = node->income;
        for(int i = 0; i < income.tailNum; i++){

--- a/source/sample/transformer/T2TAttention.cpp
+++ b/source/sample/transformer/T2TAttention.cpp
@@ -22,6 +22,7 @@
 #include <math.h>
 #include "T2TAttention.h"
 #include "T2TUtility.h"
+#include "T2TEmbedding.h"
 #include "../../tensor/core/CHeader.h"
 namespace transformer
@@ -56,9 +57,9 @@ void T2TAttention::InitModel(int argc, const char ** argv, int myDevID, XMem * m
    float minmax = 0;
    LoadParamInt(argc, argv, "nhead", &nhead, 8);
-    LoadParamInt(argc, argv, "dk", &dk, 512);
+    LoadParamInt(argc, argv, "d", &dk, DEFAULT_BEDDING_SIZE);
-    LoadParamInt(argc, argv, "dv", &dv, 512);
+    LoadParamInt(argc, argv, "d", &dv, DEFAULT_BEDDING_SIZE);
-    LoadParamInt(argc, argv, "d", &d, 512);
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_BEDDING_SIZE);
    LoadParamFloat(argc, argv, "attminmax", &minmax, 0.08F);
    InitTensor2D(&wk, d, dk, X_FLOAT, devID, mem);
@@ -104,12 +105,12 @@ XTensor * T2TAttention::Make(XTensor * k, XTensor * q, XTensor * v)
    /* scalar = softmax(Q * K^T / sqrt(dk)) * V */
    scalar = Softmax(Linear(BMMul(qheads, X_NOTRANS, kheads, X_TRANS), 1/sqrt((float)dk)), -1);
-    att = MMul(scalar, vheads);
+    att = BMMul(scalar, vheads);
    XTensor * result = new XTensor();
    /* concatenate the heads */
-    *result = Merge(att, -1);
+    *result = Merge(att, att.order - 1);
    return result;
 }

--- a/source/sample/transformer/T2TEmbedding.cpp
+++ b/source/sample/transformer/T2TEmbedding.cpp
@@ -57,7 +57,8 @@ void T2TEmbedder::InitModel(int argc, const char ** argv, int myDevID, XMem * my
    LoadParamInt(argc, argv, "vsize", &vSize, -1);
    LoadParamInt(argc, argv, "maxlen", &maxLength, 256);
-    LoadParamInt(argc, argv, "d", &d, 256);
+    LoadParamInt(argc, argv, "d", &eSize, DEFAULT_BEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_BEDDING_SIZE);
    InitTensor2D(&w, vSize, eSize, X_FLOAT, devID, mem);
@@ -74,9 +75,9 @@ length - length of the sequenc
 */
 void T2TEmbedder::MakePosEmbedding(int eSize, int d, int length)
 {
-    InitTensor2D(&posEmbedding, length, eSize, X_FLOAT, devID, mem);
+    InitTensor2D(&posEmbeddingBase, length, eSize, X_FLOAT, devID, mem);
-    float * data = new float[posEmbedding.unitNum];
+    float * data = new float[posEmbeddingBase.unitNum];
    for(int pos = 0; pos < length; pos++){
        float * dp = data + pos * eSize;
@@ -92,7 +93,7 @@ void T2TEmbedder::MakePosEmbedding(int eSize, int d, int length)
        }
    }
-    posEmbedding.SetData(data, posEmbedding.unitNum);
+    posEmbeddingBase.SetData(data, posEmbeddingBase.unitNum);
    delete[] data;
 }
@@ -105,11 +106,12 @@ XTensor * T2TEmbedder::Make(XTensor * input)
    CheckNTErrors(input->GetDim(-1) == vSize, "Wrong vocabulary size!");
    CheckNTErrors(input->order > 1, "Wrong input tensor size!");
    CheckNTErrors(input->dimSize[input->order - 2] < maxLength, "The sequence is too long!");
+    CheckNTErrors(vSize > 0, "set vocabulary size by \"-vsize\"");
+    CheckNTErrors(eSize > 0, "set embedding size by \"-esize\"");
    int dims[MAX_TENSOR_DIM_NUM];
-    memcpy(dims, input->dimSize, input->order);
+    memcpy(dims, input->dimSize, input->order * sizeof(int));
-    dims[0] = eSize;
+    dims[input->order - 1] = eSize;
    bool match = (posEmbedding.order == input->order);
    if(match){
@@ -122,17 +124,10 @@ XTensor * T2TEmbedder::Make(XTensor * input)
    /* we make positional embeddings first */
    if(!match){
        InitTensor(&posEmbedding, input->order, dims, X_FLOAT, 1.0F, devID, mem);
-        XTensor * posTMP = NewTensorBuf(2, dims, X_FLOAT, 1.0F, devID, mem);
+        XTensor * posTMP = NewTensorBuf(2, dims + 1, X_FLOAT, 1.0F, devID, mem);
-        _CopyValues(&posEmbeddingBase, 0, posTMP->unitNum, posTMP, 0);
-        int dims2[MAX_TENSOR_DIM_NUM];
+        _CopyValues(&posEmbeddingBase, 0, posTMP->unitNum, posTMP, 0);
-        dims2[0] = dims[0];
+        _Unsqueeze(posTMP, &posEmbedding, 0, dims[0]);
-        dims2[1] = dims[1];
-        dims2[2] = posEmbedding.unitNum / (dims[0] * dims[1]);
-        posEmbedding.Reshape(3, dims2);
-        _Unsqueeze(posTMP, &posEmbedding, 0, dims2[2]);
-        posEmbedding.Reshape(input->order, dims);
        DelTensorBuf(posTMP);
    }

--- a/source/sample/transformer/T2TEmbedding.h
+++ b/source/sample/transformer/T2TEmbedding.h
@@ -29,6 +29,8 @@ using namespace nts;
 namespace transformer
 {
+#define DEFAULT_BEDDING_SIZE 512
 /* 
 embedding (of word at position i):
 word embedding + positional embedding

--- a/source/sample/transformer/T2TFNN.cpp
+++ b/source/sample/transformer/T2TFNN.cpp
@@ -21,6 +21,7 @@
 #include "T2TFNN.h"
 #include "T2TUtility.h"
+#include "T2TEmbedding.h"
 #include "../../tensor/core/CHeader.h"
 #include "../../tensor/function/FHeader.h"
@@ -54,9 +55,9 @@ void T2TFNN::InitModel(int argc, const char ** argv, int myDevID, XMem * myMem)
    float minmax = 0;
-    LoadParamInt(argc, argv, "d", &inSize, 512);
+    LoadParamInt(argc, argv, "d", &inSize, DEFAULT_BEDDING_SIZE);
-    LoadParamInt(argc, argv, "d", &outSize, 512);
+    LoadParamInt(argc, argv, "d", &outSize, DEFAULT_BEDDING_SIZE);
-    LoadParamInt(argc, argv, "fnnh", &hSize, 512);
+    LoadParamInt(argc, argv, "fnnh", &hSize, DEFAULT_BEDDING_SIZE);
    LoadParamFloat(argc, argv, "fnnminmax", &minmax, 0.08F);
    InitTensor2D(&w1, inSize, hSize, X_FLOAT, devID, mem);

--- a/source/sample/transformer/T2TLayerNormal.cpp
+++ b/source/sample/transformer/T2TLayerNormal.cpp
@@ -20,6 +20,7 @@
 */
 #include "T2TLayerNormal.h"
+#include "../../tensor/core/CHeader.h"
 namespace transformer
 {
@@ -58,7 +59,32 @@ y =
 */
 XTensor * T2TLN::Make(XTensor * input)
 {
-    return NULL;
+    XTensor &x = *input;
+    XTensor mean;
+    XTensor variance;
+    XTensor standard;
+    XTensor meanFilled;
+    XTensor standardFilled;
+    XTensor * result = new XTensor();
+    /* \mu = (sum_i x_i)/m */
+    mean = ReduceSum(x, x.order - 1);
+    /* \sigma = (sum_i (x_i - \mu)^2)/m */
+    variance = ReduceVariance(x, x.order - 1, mean);
+    /* standard = sqrt(variance) */
+    standard = Power(variance, 0.5F);
+    /* unsqueeze mean and standard deviation to fit them into 
+       the same size of x */
+    meanFilled = Unsqueeze(mean, x.order - 1, x.GetDim(-1));
+    standardFilled = Unsqueeze(standard, x.order - 1, x.GetDim(-1));
+    /* x' = (x - \mu)/standard */
+    *result = (x - meanFilled)/standardFilled;
+    return result;
 }
 }
--- a/source/sample/transformer/T2TOutput.cpp
+++ b/source/sample/transformer/T2TOutput.cpp
@@ -21,6 +21,7 @@
 #include "T2TOutput.h"
 #include "T2TUtility.h"
+#include "T2TEmbedding.h"
 #include "../../tensor/core/CHeader.h"
 namespace transformer
@@ -53,10 +54,11 @@ void T2TOutput::InitModel(int argc, const char ** argv, int myDevID, XMem * myMe
    mem = myMem;
    LoadParamInt(argc, argv, "vsize", &vSize, -1);
-    LoadParamInt(argc, argv, "hsize", &inSize, 512);
+    LoadParamInt(argc, argv, "d", &inSize, DEFAULT_BEDDING_SIZE);
-    LoadParamInt(argc, argv, "hsize", &hSize, 512);
+    LoadParamInt(argc, argv, "d", &hSize, DEFAULT_BEDDING_SIZE);
-}
+    InitTensor2D(&w, hSize, vSize, X_FLOAT, devID, mem);
+}
 /* 
 make the network 

--- a/source/sample/transformer/T2TTrainer.cpp
+++ b/source/sample/transformer/T2TTrainer.cpp
@@ -31,6 +31,8 @@ namespace transformer
 /* constructor */
 T2TTrainer::T2TTrainer()
 {
+    devID = -1;
+    mem = NULL;
    seqLen = NULL;
    nseqBuf = 0;
    nextSeq = -1;
@@ -50,18 +52,19 @@ initialization
 */
 void T2TTrainer::Init(int argc, const char ** argv)
 {
+    LoadParamInt(argc, argv, "dev", &devID, -1);
    LoadParamFloat(argc, argv, "lrate", &lrate, 0.001F);
    LoadParamInt(argc, argv, "sbatch", &sBatchSize, 1);
    LoadParamInt(argc, argv, "wbatch", &wBatchSize, 1);
    LoadParamInt(argc, argv, "nepoch", &nepoch, 1);
    LoadParamInt(argc, argv, "nstep", &nstep, 1);
+    LoadParamInt(argc, argv, "vsize", &vSize, 1);
    LoadParamBool(argc, argv, "sorted", &isLenSorted, false);
+    LoadParamInt(argc, argv, "bufsize", &bufSize, 50000);
-    int maxUnitInBuf;
+    buf = new int[bufSize];
-    LoadParamInt(argc, argv, "bufsize", &maxUnitInBuf, 20000);
+    seqLen = new int[bufSize];
-    buf = new int[maxUnitInBuf];
+    seqOffset = new int[bufSize];
-    seqLen = new int[maxUnitInBuf];
-    seqOffset = new int[maxUnitInBuf];
 }
 /* 
@@ -79,6 +82,8 @@ void T2TTrainer::Train(const char * fn, T2TModel * model)
    bool isEnd = false;
    float loss = 0;
+    XNet net;
    double startT = GetClockSec();
    for(epoch = 0; epoch < nepoch; epoch++){
@@ -99,6 +104,9 @@ void T2TTrainer::Train(const char * fn, T2TModel * model)
            /* make the network */
            model->Make(&batch, &output);
+            /* back-propagation for obtaining gradients */
+            net.Backward(output, batch, CROSSENTROPY);
            /* TODO: update the model!!!! */
            /* get probabilities */
@@ -188,7 +196,7 @@ int T2TTrainer::LoadBuf(FILE * file)
        wordCount += wNum;
        lineCount++;
-        if(wordCount >= wBatchSize || lineCount >= sBatchSize)
+        if(wordCount >= bufSize - MAX_SEQUENCE_LENGTH)
            break;
    }
@@ -211,24 +219,28 @@ load a batch of sequences
 */
 int T2TTrainer::LoadBatch(FILE * file, XTensor * batch, int step, int vs, int sBatch, int wBatch, bool isSorted, int &wCount)
 {
-    if(nextSeq >= nseqBuf)
+    if(nextSeq < 0 || nextSeq >= nseqBuf)
        LoadBuf(file);
-    int seq = nextSeq;
+    int seq = MAX(nextSeq, 0);
    int wc = 0;
+    int wn = 0;
    int sc = 0;
    int max = 0;
-    while(seq < nseqBuf){
+    while(seq + sc < nseqBuf){
-        wc += seqLen[seq];
+        wn = seqLen[seq + sc];
+        wc += wn;
        sc += 1;
-        if(max < wc)
+        if(max < wn)
-            max = wc;
+            max = wn;
        if(sc >= sBatch && wc >= wBatch)
            break;
    }
+    nextSeq = seq + sc;
    if(sc > 0){
        int dims[MAX_TENSOR_DIM_NUM];
        dims[0] = sc;

--- a/source/sample/transformer/T2TTrainer.h
+++ b/source/sample/transformer/T2TTrainer.h
@@ -26,7 +26,7 @@
 #include "../../tensor/function/FHeader.h"
-#define MAX_SEQUENCE_LENGTH 1024 * 64
+#define MAX_SEQUENCE_LENGTH 1024 * 4
 using namespace nts;
@@ -46,6 +46,9 @@ public:
    /* buffer for loading words */
    int * buf;
+    /* buffer size */
+    int bufSize;
    /* length of each sequence */
    int * seqLen;

--- a/source/sample/transformer/T2TUtility.cpp
+++ b/source/sample/transformer/T2TUtility.cpp
@@ -34,8 +34,8 @@ void LoadParamString(int argc, const char ** argv, const char * name, char * p, 
    bool hit = false;
    for(int i = 0; i < argc; i++){
        if(!strcmp(argv[i], vname) && i + 1 < argc){
-            *(int*)p = atoi(argv[i + 1]);
+            strcpy(p, argv[i + 1]);
-            fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
            hit = true;
        }
    }
@@ -52,7 +52,7 @@ void LoadParamInt(int argc, const char ** argv, const char * name, int * p, int 
    for(int i = 0; i < argc; i++){
        if(!strcmp(argv[i], vname) && i + 1 < argc){
            *(int*)p = atoi(argv[i + 1]);
-            fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
            hit = true;
        }
    }
@@ -69,7 +69,8 @@ void LoadParamBool(int argc, const char ** argv, const char * name, bool * p, bo
    for(int i = 0; i < argc; i++){
        if(!strcmp(argv[i], vname)){
            *(bool*)p = true;
-            fprintf(stderr, " %s=%s\n", name, "true");
+            //fprintf(stderr, " %s=%s\n", name, "true");
+            hit = true;
        }
    }
    if(!hit)
@@ -84,12 +85,27 @@ void LoadParamFloat(int argc, const char ** argv, const char * name, float * p, 
    bool hit = false;
    for(int i = 0; i < argc; i++){
        if(!strcmp(argv[i], vname) && i + 1 < argc){
-            strcpy((char*)p, argv[i + 1]);
+            *p = (float)atof(argv[i + 1]);
-            fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            hit = true;
        }
    }
    if(!hit)
        *p = defaultP;
 }
+void ShowParams(int argc, const char ** argv)
+{
+    fprintf(stderr, "args:\n");
+    for(int i = 0; i < argc; i++){
+        if(argv[i][0] == '-'){
+            if(i + 1 < argc && argv[i + 1][0] != '-')
+                fprintf(stderr, " %s=%s\n", argv[i], argv[i + 1]);
+            else
+                fprintf(stderr, " %s=yes\n", argv[i]);
+        }
+    }
+    fprintf(stderr, "\n");
+}
 }
\ No newline at end of file
--- a/source/sample/transformer/T2TUtility.h
+++ b/source/sample/transformer/T2TUtility.h
@@ -27,12 +27,15 @@
 namespace transformer
 {
-/* load model parameters */
+/* load arguments */
 void LoadParamString(int argc, const char ** argv, const char * name, char * p, char * defaultP);
 void LoadParamInt(int argc, const char ** argv, const char * name, int * p, int defaultP);
 void LoadParamBool(int argc, const char ** argv, const char * name, bool * p, bool defaultP);
 void LoadParamFloat(int argc, const char ** argv, const char * name, float * p, float defaultP);
+/* show arguments */
+void ShowParams(int argc, const char ** argv);
 }
 #endif
\ No newline at end of file
--- a/source/sample/transformer/Transformer.cpp
+++ b/source/sample/transformer/Transformer.cpp
@@ -20,12 +20,36 @@
 */
 #include "Transformer.h"
+#include "T2TModel.h"
+#include "T2TUtility.h"
+#include "T2TTrainer.h"
 namespace transformer
 {
 int TransformerMain(int argc, const char ** argv)
 {
+    if(argc == 0)
+        return 1;
+    ShowParams(argc, argv);
+    char * trainFN = new char[MAX_LINE_LENGTH];
+    LoadParamString(argc, argv, "train", trainFN, "");
+    T2TModel model;
+    model.InitModel(argc, argv);
+    if(strcmp(trainFN, "")){
+        T2TTrainer trainer;
+        trainer.Init(argc, argv);
+        trainer.Train(trainFN, &model);
+    }
+    delete[] trainFN;
    return 0;
 }

--- a/source/sample/transformer/Transformer.h
+++ b/source/sample/transformer/Transformer.h
@@ -38,7 +38,7 @@ namespace transformer
 {
 /* entrance of the program */
-int TransformerMMain(int argc, const char ** argv);
+int TransformerMain(int argc, const char ** argv);
 }

--- a/source/tensor/XTensor.cpp
+++ b/source/tensor/XTensor.cpp
@@ -42,6 +42,8 @@
 #include "core/movement/CopyValues.h"
 #include "core/arithmetic/Sum.h"
 #include "core/arithmetic/Multiply.h"
+#include "core/arithmetic/Sub.h"
+#include "core/arithmetic/Div.h"
 #include "core/math/ScaleAndShift.h"
 #ifdef USE_CUDA
@@ -354,6 +356,18 @@ XTensor XTensor::operator* (const XTensor& tensor)
    return Multiply(*this, tensor);
 }
+/* overloading of the minus-sign */
+XTensor XTensor::operator- (const XTensor& tensor)
+{
+    return Sub(*this, tensor);
+}
+/* overloading of the division-sign */
+XTensor XTensor::operator/ (const XTensor& tensor)
+{
+    return Div(*this, tensor);
+}
 /* 
 linear transformation b = a * \scale + \shift
 >> scale - the slope
@@ -610,8 +624,8 @@ double GaussRand()
    double pi = 3.141592654;
    if (phase == 0){
-        u = rand() / (RAND_MAX + 1.0);
+        u = (rand() + 1) / (RAND_MAX + 1.0);
-        v = rand() / (RAND_MAX + 1.0);
+        v = (rand() + 1) / (RAND_MAX + 1.0);
        z = sqrt(-2.0 * log(u))* sin(2.0 * pi * v);
    }
    else{
@@ -1008,8 +1022,8 @@ set the value of a cell in a 3d tensor in default type
 bool XTensor::Set3D(DTYPE value, int d0, int d1, int d2)
 {
    CheckNTErrors((order == 3), "Cannot get a 2d cell for a tensor whose order is not 2!");
-    CheckNTErrors((d0 >= 0 && d1 < dimSize[0]), "dimension 0 is out of range!");
+    CheckNTErrors((d0 >= 0 && d0 < dimSize[0]), "dimension 0 is out of range!");
-    CheckNTErrors((d2 >= 0 && d2 < dimSize[1]), "dimension 1 is out of range!");
+    CheckNTErrors((d2 >= 0 && d1 < dimSize[1]), "dimension 1 is out of range!");
    CheckNTErrors((d2 >= 0 && d2 < dimSize[2]), "dimension 1 is out of range!");
    CheckNTErrors((dataType == DEFAULT_DTYPE), "The tensor is not in default type.");

--- a/source/tensor/XTensor.h
+++ b/source/tensor/XTensor.h
@@ -203,6 +203,12 @@ public:
    /* overloading of the multiply-sign */
    XTensor  operator* (const XTensor &tensor);
+    /* overloading of the minus-sign */
+    XTensor  operator- (const XTensor &tensor);
+    /* overloading of the division-sign */
+    XTensor  operator/ (const XTensor &tensor);
    /* linear transformation */
    XTensor Lin(DTYPE scale, DTYPE shift = 0);

--- a/source/tensor/core/arithmetic/MatrixMul.cpp
+++ b/source/tensor/core/arithmetic/MatrixMul.cpp
@@ -251,9 +251,7 @@ XTensor MatrixMul(const XTensor &a, MATRIX_TRANS_TYPE transposedA,
 /* 
 matrix multiplication with no transposition c = a * b * alpha
 >> a - tensor a
->> transposedA - indicates whether the matrices in a are transposed
 >> b - tensor b
->> transposedB - indicates whether teh matrices in b are transposed
 >> alpha - a coefficient
 >> parallelRunner - parallel processing module
 << return - the result of matrix multiplication

--- a/source/tensor/core/arithmetic/MatrixMulBatched.cpp
+++ b/source/tensor/core/arithmetic/MatrixMulBatched.cpp
@@ -326,4 +326,60 @@ XTensor MatrixMulBatched(const XTensor &a, MATRIX_TRANS_TYPE transposedA, const 
    return c;
 }
+/*
+matrix multiplication of the two tensors (do it on site)
+c = a * b * alpha
+make a new tensor to keep the result and return it
+for each 2-dimensional data array in a (denoted as ai) and
+each 2-dimensional data array in b (denoted as bi), we have
+ci = ai * bi * alpha + cm * beta
+>> a - tensor a
+>> b - tensor b
+>> alpha - a coefficient
+>> parallelRunner - parallel processing module
+<< return - the result of matrix multiplication of the two tensors
+*/
+XTensor MatrixMulBatched(const XTensor &a, const XTensor &b,
+                         DTYPE alpha, XPRunner * parallelRunner)
+{
+    CheckNTErrors(a.dataType == b.dataType, "Input tensors should have the same data type!");
+    CheckNTErrors(a.order >= 2 && b.order >= 2, "Input tensors must have a order >= 2!");
+    CheckNTErrors(a.order == b.order, "Input tensor and output tensor must have same order!");
+    int an = a.dimSizeRDI[1];
+    int am = a.dimSizeRDI[0];
+    int bn = b.dimSizeRDI[1];
+    int bm = b.dimSizeRDI[0];
+    CheckNTErrors(am == bn, "Unmatched tensors in multiplication!");
+    int order = a.order;
+    int sub = 0;
+    int * dimSize = new int[order];
+    for (int i = 0; i < a.order - 2; i++)
+        dimSize[sub++] = a.dimSize[i];
+    dimSize[sub++] = an;
+    dimSize[sub++] = bm;
+    float dr = (!a.isSparse || !b.isSparse) ? 1.0F : MAX(a.denseRatio, b.denseRatio);
+    XTensor c(order, dimSize, a.dataType, dr, a.devID, a.mem);
+    c.SetTMP();
+    /*call _MatrixMulBatched function */
+    _MatrixMulBatched(&a, X_NOTRANS, &b, X_NOTRANS, &c, alpha, 0, parallelRunner);
+    /* tensor connections */
+    XLink::MakeLink(&a, &b, &c, MATH_MATRIXMULBATCHED);
+    XLink::AddParamToHeadTrans(&c, X_NOTRANS);
+    XLink::AddParamToHeadTrans(&c, X_NOTRANS);
+    XLink::AddParamToHead(&c, alpha);
+    /* destroy variables */
+    delete[] dimSize;
+    return c;
+}
 } // namespace nts(NiuTrans.Tensor)
--- a/source/tensor/core/arithmetic/MatrixMulBatched.h
+++ b/source/tensor/core/arithmetic/MatrixMulBatched.h
@@ -73,6 +73,17 @@ where trans() returns the transposed matrix if the flag is fired
 XTensor MatrixMulBatched(const XTensor &a, MATRIX_TRANS_TYPE transposedA, const XTensor &b, MATRIX_TRANS_TYPE transposedB,
                         DTYPE alpha = (DTYPE)1.0, XPRunner * parallelRunner = NULL);
+/*
+matrix multiplication of the two tensors (return a XTensor structure) c = a * b * alpha
+make a new tensor to keep the result and return it
+for each 2-dimensional data array in a (denoted as ai) and
+each 2-dimensional data array in b (denoted as bi), we have
+ci = ai * bi * alpha + cm * beta
+*/
+XTensor MatrixMulBatched(const XTensor &a, const XTensor &b, 
+                         DTYPE alpha = (DTYPE)1.0, XPRunner * parallelRunner = NULL);
 } // namespace nts(NiuTrans.Tensor)
 #endif // __MATRIXMULBATCHED_H__
\ No newline at end of file
--- a/source/tensor/core/movement/CopyValues.cpp
+++ b/source/tensor/core/movement/CopyValues.cpp
@@ -36,7 +36,7 @@ copy s to t
 void _CopyValues(const XTensor * s, XTensor * t, XStream * stream)
 {
    CheckNTErrors((s != NULL && t != NULL), "The input tensor and output tensor must be nonempty!");
-    CheckNTErrors((s->data != NULL), "Cannot copy from an empty data array!");
+    CheckNTErrors((s->data != NULL), "Cannot copy an empty data array!");
    CheckNTErrors((t->data != NULL), "Cannot copy to an empty data array!");
    CheckNTErrors((s->unitNum == t->unitNum), "Unmatched data item number!");
@@ -82,7 +82,7 @@ copy s to t
 void _CopyValues(const XTensor * s, const int sBeg, const int sLen, XTensor * t, const int tBeg, XStream * stream)
 {
    CheckNTErrors(s != NULL && t != NULL, "The input tensor and output tensor must be nonempty!");
-    CheckNTErrors(s->data != NULL && t->data != NULL, "Cannot copy from an empty data array!");
+    CheckNTErrors(s->data != NULL && t->data != NULL, "Cannot copy an empty data array!");
    CheckNTErrors(s->unitSize == t->unitSize, "The input tensors must be of the same unit size!");
    CheckNTErrors(s->order > sBeg && sBeg >= 0 && sLen <= s->unitNum, "Wrong segment on the source side");
    CheckNTErrors(t->order > tBeg && tBeg >= 0, "Wrong segment on the target side");

--- a/source/tensor/core/shape/Split.cpp
+++ b/source/tensor/core/shape/Split.cpp
@@ -168,6 +168,8 @@ make a new tensor to keep the result and return it
 XTensor Split(const XTensor &s, int whereToSplit, int splitNum)
 {
    CheckNTErrors(&s, "Invalid tensors!");
+    CheckNTErrors(s.dimSize[whereToSplit] % splitNum == 0, 
+                  "The dimension cannot be splitted due to the inproper split number");
    int order = s.order + 1;
    int * dimSize = new int[order];

--- a/source/tensor/function/LogSoftmax.cpp
+++ b/source/tensor/function/LogSoftmax.cpp
@@ -282,6 +282,9 @@ void _LogSoftmaxBackward(XTensor * gold, XTensor * y, XTensor * x,
    CheckNTErrors((!dedx->isSparse), "The gradient matrix must be dense!");
    CheckNTErrors((gold != NULL), "The gold standard cannot be empty!");
+    if(leadDim < 0)
+        leadDim = y->order - 1;
    int leadDimRDI = y->order - leadDim - 1;
 #ifdef USE_CUDA
    if (gold->devID >= 0) {

--- a/source/tensor/function/Softmax.cpp
+++ b/source/tensor/function/Softmax.cpp
@@ -188,7 +188,11 @@ void _SoftmaxBackward(XTensor * gold, XTensor * y, XTensor * x,
    CheckNTErrors((dedx->isSparse == false), "The gradient tensor must be dense!");
    CheckNTErrors((gold != NULL), "Incorrect x gold standard tensor!");
+    if(leadDim < 0)
+        leadDim = y->order - 1;
    int leadDimRDI = y->order - leadDim - 1;
 #ifdef USE_CUDA
    if(y->devID >= 0){
        _CudaSoftmaxBackward(gold, y, x, dedy, dedx, leadDim, lossName);