init

source/network/XBackwardData.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for data operation
+ * $Created by: Xu Chen (email: hello_master1954@163.com) 2018-12-26
+ */
+
+#include "XNoder.h"
+#include "XBackwardData.h"
+#include "../tensor/XName.h"
+#include "../tensor/XUtility.h"
+#include "../tensor/core/CHeader.h"
+#include "../tensor/core/getandset/SetData.h"
+
+namespace nts{
+
+/* compute dE/dx of a node */
+void XDataGrad::MakeGrad(XTensor * node, bool isEfficent)
+{
+    CheckNTErrors(node->grad != NULL, "No gradient found!");
+
+    XLink &income = node->income;
+    int operID = income.typeID;
+
+    if(operID == GETANDSET_CONVERTDATATYPE)
+        GradConvertDataType(node, isEfficent);
+    else if(operID == GETANDSET_INDEXTOONEHOT)
+        GradIndexToOnehot(node, isEfficent);
+    else if(operID == GETANDSET_ONEHOTTOINDEX)
+        GradOnehotToIndex(node, isEfficent);
+    else{
+        ShowNTErrors("TODO!");
+    }
+}
+
+/* indicates whether the node is for a data operation */
+bool XDataGrad::IsDataOP(XTensor * node)
+{
+    XLink &income = node->income;
+    return (income.typeID & DATA_BASE) != 0;
+}
+
+/* 
+gradient computation for convert datatype
+for
+b = converdatatype(a) 
+we have
+dE/da = convertdatatype(b)
+>> node - the node (c) for backward computation
+>> isEfficient - indicates whether the computation is in
+                 an efficient manner
+*/
+void XDataGrad::GradConvertDataType(XTensor * node, bool isEfficent)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum > 0, "Wrong input tensor number for ConvertDataType!");
+
+    XTensor * input = income.tails[0];
+
+    XNoder::MakeGrad(input);
+
+	XTensor * tmp = NewTensorBuf(input->grad, input->devID, input->mem);
+    _ConvertDataType(node->grad, tmp);
+	_SumMe(input->grad, tmp);
+	DelTensorBuf(tmp);
+
+	node->visitMark = NODE_FINISHED;
+}
+
+/* 
+gradient computation for OnehotToIndex
+for
+b = OnehotToIndex(a) 
+we have
+dE/da = IndexToOnehot(b)
+>> node - the node (c) for backward computation
+>> isEfficient - indicates whether the computation is in
+                 an efficient manner
+*/
+void XDataGrad::GradOnehotToIndex(XTensor * node, bool isEfficent)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum > 0, "Wrong input tensor number for IndexToOnehot!");
+
+    XTensor * input = income.tails[0];
+
+    XNoder::MakeGrad(input);
+
+	node->visitMark = NODE_FINISHED;
+}
+
+/* 
+gradient computation for IndexToOnehot
+for
+b = IndexToOnehot(a) 
+we have
+dE/da = IndexToOnehot(b)
+>> node - the node (c) for backward computation
+>> isEfficient - indicates whether the computation is in
+                 an efficient manner
+*/
+void XDataGrad::GradIndexToOnehot(XTensor * node, bool isEfficent)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum > 0, "Wrong input tensor number for IndexToOnehot!");
+
+    XTensor * input = income.tails[0];
+
+    XNoder::MakeGrad(input);
+
+	node->visitMark = NODE_FINISHED;
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/network/XBackwardData.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for data operation
+ * $Created by: Xu Chen (email: hello_master1954@163.com) 2018-12-26
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDDATA_H__
+#define __XBACKWARDDATA_H__
+
+namespace nts{
+
+/* this class computes the gradient for tensor data operation given a node */
+class XDataGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficent);
+
+    /* indicates whether the node is for a shaping operation */
+    static
+    bool IsDataOP(XTensor * node);
+
+private:
+    
+    /* gradient computation for ConverDataType: b = converdatatype(a, datatype) */
+    static
+    void GradConvertDataType(XTensor * node, bool isEfficent);
+
+    /* gradient computation for IndexToOnehot: b = indextoonehot(a, num) */
+    static
+    void GradIndexToOnehot(XTensor * node, bool isEfficent);
+
+    /* gradient computation for OnehotToIndex: b = onehottoindex(a, num) */
+    static
+    void GradOnehotToIndex(XTensor * node, bool isEfficent);
+};
+
+} // namespace nts(NiuTrans.Tensor)
+
+#endif
\ No newline at end of file

source/network/XBackwardFunc.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for activation function
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ * Dingdang won 5 games in the GO training yesterday, hahaha ...
+ */
+
+#include "XNoder.h"
+#include "XBackwardFunc.h"
+#include "../tensor/XName.h"
+#include "../tensor/function/FHeader.h"
+
+namespace nts{
+
+/* compute dE/dx of a node */
+void XFuncGrad::MakeGrad(XTensor * node, bool isEfficient)
+{
+    XLink &income = node->income;
+    int operID = income.typeID;
+
+    CheckNTErrors(node->grad != NULL, "No gradient found!");
+    CheckNTErrors(income.tailNum == 1, "Too many input tensors for the function!");
+
+    XTensor * input = income.tails[0];
+    XTensor * output = node;
+
+    XNoder::MakeGrad(input);
+
+    if(operID == FUNC_HARDTANH)
+        _HardTanHBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_IDENTITY)
+        _IdentityBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_LOGSOFTMAX){
+        int leadDim = income.GetParamInt(0);
+        CheckNTErrors(leadDim >= 0 && leadDim < input->order, "wrong leading dimension in logsoftmax!");
+        _LogSoftmaxBackward(NULL, output, input, output->grad, input->grad, NULL, leadDim, NOLOSS);
+    }
+    else if(operID == FUNC_RECTIFY)
+        _RectifyBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_SIGMOID)
+        _SigmoidBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_SOFTMAX){
+        int leadDim = income.GetParamInt(0);
+        CheckNTErrors(leadDim >= 0 && leadDim < input->order, "wrong leading dimension in softmax!");
+        _SoftmaxBackward(NULL, output, input, output->grad, input->grad, NULL, leadDim, NOLOSS);
+    }
+    else{
+        ShowNTErrors("Wrong activation function type!");
+    }
+
+    node->visitMark = NODE_FINISHED;
+}
+
+/* indicates whether the node is for an activation function */
+bool XFuncGrad::IsFunc(XTensor * node)
+{
+    XLink &income = node->income;
+    return (income.typeID & FUNCTION_BASE) != 0;
+}
+
+}

source/network/XBackwardFunc.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for activation function
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ * Dingdang won 5 games in the GO training yesterday, hahaha ...
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDFUNC_H__
+#define __XBACKWARDFUNC_H__
+
+namespace nts{
+
+/* this class computes the gradient for activation functions given a node */
+class XFuncGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficient);
+
+    /* indicates whether the node is for an activation function */
+    static
+    bool IsFunc(XTensor * node);
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardLoss.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-17
+ */
+
+#include "XBackwardLoss.h"
+#include "XNoder.h"
+#include "../tensor/XName.h"
+#include "../tensor/function/FHeader.h"
+#include "../tensor/core/getandset/SetData.h"
+#include "../tensor/function/HardTanH.h"
+#include "../tensor/function/Identity.h"
+#include "../tensor/function/LogSoftmax.h"
+#include "../tensor/function/Rectify.h"
+#include "../tensor/function/Sigmoid.h"
+#include "../tensor/function/Softmax.h"
+
+namespace nts{
+
+
+/* compute dE/dx of a node */
+void XLossGrad::MakeGrad(XTensor * node, bool isEfficient)
+{
+    XLink &income = node->income;
+    int operID = income.typeID;
+
+    CheckNTErrors(income.tailNum >= 1, "Wrong number of tensors for loss computation!");
+
+    XTensor * output = income.tails[0];
+    XTensor * gold = NULL;
+    XTensor * weight = NULL;
+    XTensor * padding = NULL;
+    int leadingDim;
+
+    XNoder::MakeGrad(output);
+    XTensor * dedy = output->grad;
+
+    if (income.tailNum == 1) {
+        if(dedy->dataType == X_FLOAT)
+            _SetDataFixedFloat(dedy, 1.0F);
+        else if(dedy->dataType == X_DOUBLE)
+            _SetDataFixedDouble(dedy, 1.0);
+        else if(dedy->dataType == X_INT)
+            _SetDataFixedInt(dedy, 1);
+        else
+            ShowNTErrors("TODO");
+
+        return;
+    }
+
+    gold = income.tails[1];
+
+    if(operID == LOSS_CROSSENTROPY) {
+        if (income.tailNum == 3) 
+            padding = income.tails[2];
+         leadingDim = income.GetParamInt(0);
+        CheckNTErrors(leadingDim >= 0 && leadingDim < output->order, "wrong leading dimension in logsoftmax!");
+        _CrossEntropyBackward(dedy, output, gold, weight, padding, leadingDim);
+    }
+    else{
+        ShowNTErrors("Wrong activation function type!");
+    }
+
+    node->visitMark = NODE_FINISHED;
+}
+
+/* indicates whether the node is for a loss computation */
+bool XLossGrad::IsLossOP(XTensor * node)
+{
+    XLink &income = node->income;
+    return (income.typeID & LOSS_BASE) != 0;
+}
+
+/* 
+compute dE/dx for a given function y = f(x) 
+>> gold - gold standard to measure error (or loss)
+>> y - output of the function
+>> x - input of the function
+>> dedy - dE/dy
+>> dedx - dE/dx
+>> funcID - id of the function f
+>> params - parameters of the function
+>> lossName - name of the loss, e.g., cross entropy
+*/
+void XLossGrad::Compute(XTensor * gold, XTensor * y, XTensor * x, 
+                        XTensor * dedy, XTensor * dedx, XTensor * padding,
+                        int funcID, void * params,
+                        LOSS_FUNCTION_NAME lossName)
+{
+    CheckNTErrors(gold && y && x, "Empty input tensors!");
+    CheckNTErrors(dedx, "Empty gradient tensors!");
+    CheckNTErrors((funcID & FUNCTION_BASE) != 0, "Illegal function id");
+
+    if(funcID == FUNC_HARDTANH){
+        _HardTanHBackward(gold, y, x, dedy, dedx, lossName);
+    }
+    else if(funcID == FUNC_IDENTITY){
+        _IdentityBackward(gold, y, x, dedy, dedx, lossName);
+    }
+    else if(funcID == FUNC_LOGSOFTMAX){
+        int leadDim = *(int*)params;
+        _LogSoftmaxBackward(gold, y, x, dedy, dedx, padding, leadDim, lossName);
+    }
+    else if(funcID == FUNC_RECTIFY){
+        _RectifyBackward(gold, y, x, dedy, dedx, lossName);
+    }
+    else if(funcID == FUNC_SIGMOID){
+        _SigmoidBackward(gold, y, x, dedy, dedx, lossName);
+    }else if(funcID == FUNC_SOFTMAX){
+        int leadDim = *(int*)params;
+        _SoftmaxBackward(gold, y, x, dedy, dedx, padding, leadDim, lossName);
+    }
+    else{
+        ShowNTErrors("wrong function found when call the backward process!");
+    }
+
+}
+
+/* 
+compute dE/dy for variable y and error(loss) function E
+>> gold - gold standard to measure error (or loss)
+>> y - output of the function
+>> dedy - dE/dy
+>> lossName - name of the loss, e.g., cross entropy
+*/
+void XLossGrad::Compute(XTensor * gold, XTensor * y, 
+                        XTensor * dedy, XTensor * padding,
+                        LOSS_FUNCTION_NAME lossName)
+{
+    if(gold == NULL){
+        if(dedy->dataType == X_FLOAT)
+            _SetDataFixedFloat(dedy, 1.0F);
+        else if(dedy->dataType == X_DOUBLE)
+            _SetDataFixedDouble(dedy, 1.0);
+        else if(dedy->dataType == X_INT)
+            _SetDataFixedInt(dedy, 1);
+        else{
+            ShowNTErrors("TODO");
+        }
+        return;
+    }
+
+    //_LossBackward(dedy, gold, y, lossName);
+    if(lossName == CROSSENTROPY)
+        _CrossEntropyBackward(dedy, y, gold, NULL, padding);
+
+}
+
+}
\ No newline at end of file

source/network/XBackwardLoss.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-17
+ * My students worked all night to prepare a submission to CWMT. Good luck
+ * to them!
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+#include "../tensor/loss/LHeader.h"
+
+#ifndef __XBACKWARDLOSS_H__
+#define __XBACKWARDLOSS_H__
+
+namespace nts{
+
+/* this class computes the gradient (of a output node) 
+   with respect to the loss */
+class XLossGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficient);
+
+    /* indicates whether the node is for a Loss computation */
+    static
+    bool IsLossOP(XTensor * node);
+
+    /* compute dE/dx for a given function y = f(x) */
+    void Compute(XTensor * gold, XTensor * y, XTensor * x, 
+                 XTensor * dedy, XTensor * dedx, XTensor * padding,
+                 int funcID, void * params,
+                 LOSS_FUNCTION_NAME lossName);
+
+    /* compute dE/dy for variable y and error(loss) function E */
+    void Compute(XTensor * gold, XTensor * y, 
+                 XTensor * dedy, XTensor * padding,
+                 LOSS_FUNCTION_NAME lossName);
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardMath.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for math operations
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+
+#ifndef __XBACKWARDMATH_H__
+#define __XBACKWARDMATH_H__
+
+namespace nts{
+
+/* this class computes the gradient for math operations given a node */
+class XMathGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficient);
+
+    /* indicates whether the node is for a math operation */
+    static
+    bool IsMathOP(XTensor * node);
+
+private:
+    
+    /* gradient for absolute */
+    static
+    void GradAbsolute(XTensor * node, bool isEfficient);
+    
+    /* gradient for cos */
+    static
+    void GradCos(XTensor * node, bool isEfficient);
+    
+    /* gradient for exp */
+    static
+    void GradExp(XTensor * node, bool isEfficient);
+
+    /* gradient for log: c =  log(a) */
+    static
+    void GradLog(XTensor * node, bool isEfficient);
+    
+    /* gradient for round */
+    static
+    void GradRound(XTensor * node, bool isEfficient);
+    
+    /* gradient for sign */
+    static
+    void GradSign(XTensor * node, bool isEfficient);
+
+    /* gradient for sin */
+    static
+    void GradSin(XTensor * node, bool isEfficient);
+
+    /* gradient for tan */
+    static
+    void GradTan(XTensor * node, bool isEfficient);
+
+    /* gradient for clip */
+    static
+    void GradClip(XTensor * node, bool isEfficient);
+
+    /* gradient for Divide */
+    static
+    void GradDiv(XTensor * node, bool isEfficient);
+
+    /* gradient for DivideDim */
+    static
+    void GradDivDim(XTensor * node, bool isEfficient);
+
+    /* gradient for matrix multiply: c = matmul(a, b) * \alpha */
+    static
+    void GradMatrixMul(XTensor * node, bool isEfficient);
+    
+    /* gradient for matrix multiply: c = matmul(a, b) * \alpha */
+    static
+    void GradMatrixMul(XTensor * a, XTensor * deda, MATRIX_TRANS_TYPE transA,
+                       XTensor * b, XTensor * dedb, MATRIX_TRANS_TYPE transB,
+                       XTensor * dedc, DTYPE alpha, bool isEfficient);
+
+    /* gradient for matrix multiply in batch mode.
+       for each batch: c_i = matmul(a_i, b_i) * \alpha */
+    static
+    void GradMatrixMulBatched(XTensor * node, bool isEfficient);
+
+    /* gradient for multiply (dot production): c =  a * b * \alpha */
+    static
+    void GradMultiply(XTensor * node, bool isEfficient);
+
+    /* gradient for multiply one dimension: c =  a * b * \alpha 
+       where the size of b is equal to that of one dimension of a */
+    static
+    void GradMultiplyDim(XTensor * node, bool isEfficient);
+
+    /* gradient for multiply one dimension: c =  a * b
+       where some dimensions of b are of size 1 */
+    static
+    void GradMultiplyBroadcast(XTensor * node, bool isEfficient);
+
+    /* gradient for negate */
+    static
+    void GradNegate(XTensor * node, bool isEfficient);
+    
+    /* gradient for normalize */
+    static
+    void GradNormalize(XTensor * node, bool isEfficient);
+
+    /* gradient for power */
+    static
+    void GradPower(XTensor * node, bool isEfficient);
+
+    /* gradient for ScaleAndShift */
+    static
+    void GradScaleAndShift(XTensor * node, bool isEfficient);
+
+    /* gradient for Scale */
+    static
+    void GradScale(XTensor * node, bool isEfficient);
+
+    /* gradient for Shift */
+    static
+    void GradShift(XTensor * node, bool isEfficient);
+
+    /* gradient for Descale */
+    static
+    void GradDescale(XTensor * node, bool isEfficient);
+
+    /* gradient for Minus */
+    static
+    void GradSub(XTensor * node, bool isEfficient);
+    
+	/* gradient for sub with one dimension: c = a - b * \beta
+	where the size of b is equal to that of one dimension of a */
+	static
+	void GradSubDim(XTensor * node, bool isEfficient);
+
+    /* gradient for sum: c =  a + b * \beta */
+    static
+    void GradSum(XTensor * node, bool isEfficient);
+
+    /* gradient for sum with one dimension: c = a + b * \beta
+       where the size of b is equal to that of one dimension of a */
+    static
+    void GradSumDim(XTensor * node, bool isEfficient);
+
+    /* gradient for sum by broadcasting: c = a + b * \beta
+       where some dimensions of b are of size 1 */
+    static
+    void GradSumBroadcast(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceMean */
+    static
+    void GradReduceMean(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceSum */
+    static
+    void GradReduceSum(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceSumSquared */
+    static
+    void GradReduceSumSquared(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceVariance */
+    static
+    void GradReduceVariance(XTensor * node, bool isEfficient);
+
+    /* gradient for operation */
+    static
+    void GradMulAndShift(XTensor * node, bool isEfficient);
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardShape.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for shaping and data movement
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDSHAPE_H__
+#define __XBACKWARDSHAPE_H__
+
+namespace nts{
+
+/* this class computes the gradient for tensor shaping and movement given a node */
+class XShapeGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficent);
+
+    /* indicates whether the node is for a shaping operation */
+    static
+    bool IsShapeOP(XTensor * node);
+
+    /* post processing of a node */
+    static
+    void PostProcessing(XTensor * node, int typeId, bool isEfficent);
+
+private:
+    
+    /* gradient computation for copying indexed sub-tensors: b = copyindexed(a, srcIndex, indexSize, tgtIndex, copyNum) */
+    static
+    void GradCopyIndexed(XTensor * node, bool isEfficent);
+        
+    /* gradient computation for copying indexed sub-tensors: b = gather(a, index) */
+    static
+    void GradGather(XTensor * node, bool isEfficent);
+
+    /* gradient computation for dropout with indexs */
+    static
+    void GradDropoutWithIndex(XTensor * node, bool isEfficent);
+
+    /* gradient computation for merge: c = merge(a, b, ...) */
+    static
+    void GradMerge(XTensor * node, bool isEfficent);
+
+    /* gradient computation for merging a list of tensors : c = merge(list(a, b, ...)) */
+    static
+    void GradMergeList(XTensor * node, bool isEfficent);
+    
+    /* gradient computation for transposing a tensor : b = transpose(a) */
+    static
+    void GradTranspose(XTensor * node, bool isEfficent);
+
+    /* gradient computation for reshaping a tensor: c = reshape(a) */
+    static
+    void GradReshape(XTensor * node, bool isEfficent);
+
+    /* gradient computation for split: c = split(a) */
+    static
+    void GradSplit(XTensor * node, bool isEfficent);
+
+    /* gradient computation for spliting. we return the list of the splits : list(c_1, ...) = split(a) */
+    static
+    void GradSplitList(XTensor * node, bool isEfficent);
+
+    /* gradient computation for spliting. we return the list of the splits : list(c_1, ...) = split(a).
+       this method is called only when all nodes of spliting have been processed. We do this in a post-processing
+       manner because we can fuze multiple memory copy jobs one time. This is good for system speed up. */
+    static
+    void GradSplitListPost(XTensor * node, bool isEfficent);
+
+    /* gradient computation for unsqueezing a tensor : c = unsqueeze(a) */
+    static
+    void GradUnsqueeze(XTensor * node, bool isEfficent);
+
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XNet.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-12
+ * We expected a heavy rain today but a drizzle came down. Should I
+ * take a big umbrella?
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+#include "../tensor/loss/LHeader.h"
+
+#ifndef __XNET_H__
+#define __XNET_H__
+
+namespace nts{
+
+/* management of tensor net (or graph) */
+struct XNet
+{
+    /* id of the network */
+    unsigned int id;
+
+    /* tensor nodes of the network (in order) */
+    XList nodes;
+
+    /* tensor nodes to keep gradient for output (e.g., SGD)*/
+    XList gradNodes;
+
+    /* output nodes of the network */
+    XList outputs;
+
+    /* input nodes of the network */
+    XList inputs;
+
+    /* indicates whether the network just keeps the gradient for parameter tensors */
+    bool isGradEfficient;
+
+    /* constructor */
+    XNet();
+
+    /* de-constructor */
+    ~XNet();
+
+    /* clear the network */
+    void Clear();
+
+    /* backward propagation to obtain gradient */
+    void Backward(XTensor &root, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient wrt. the loss/error function */
+    void Backward(XTensor &root, XTensor &gold, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient wrt. the loss/error function */
+    void Backward(XTensor &root, XTensor &gold, XTensor &padding, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient
+       with a number of root nodes */
+    void Backward(XList &roots, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient
+       with a number of root nodes */
+    void Backward(XList &roots, XList &golds, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient wrt. the loss/error function
+       with a number of root nodes */
+    void Backward(XList &roots, XList &golds, XList &paddings, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward computation for a given node */
+    void BackwardNode(XTensor * node, bool isEfficent = false);
+
+    /* backward computation (in post processing) for a given node */
+    void BackwardNodePost(XTensor * node, bool isEfficent = false);
+
+    /* traverse the net and find the topological order by 
+       depth-first search (Tarjan's algorithm) */
+    void Traverse(XTensor &root);
+
+    /* traverse the net and find the topological order by 
+       depth-first search (Tarjan's algorithm) */
+    void Traverse(XList &roots);
+
+    /* depth-first search given a node (Tarjan's algorithm for topological ordering) */
+    void TarjanVisit(XTensor * node, XList &orders, const unsigned int code);
+
+    /* dump network information */
+    void Dump(FILE * file);
+
+    /* set the flag of gradient-efficient */
+    void SetGradEfficientFlag(bool flag = true);
+
+    /* generate the gradient-efficient flag for every node */
+    void MakeEfficientNet();
+
+    /* clear the graident information if the node is no use */
+    void ClearGrad(XTensor * node);
+
+    /* show network topology */
+    void ShowNetwork(FILE * file, XTensor * node);
+
+    /* search a node in a top-down manner by its name */
+    //static
+    //XTensor * SearchNode(XTensor * top, const char * name);
+};
+
+/* we make a unique id for every tensor */
+extern unsigned int netIDGlobal;
+extern MUTEX_HANDLE netMutex;
+extern unsigned int MakeNetID();
+extern void XNetClearAll();
+
+}
+
+#endif
\ No newline at end of file

source/network/XNoder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "XNoder.h"
+
+namespace nts{
+
+/* make gradient tensor for a node */
+void XNoder::MakeGrad(XTensor * node)
+{
+    if(node == NULL)
+        return;
+
+    if(!XTensor::IsSameShaped(node, node->grad)){
+        delete node->grad;
+        node->grad = NewTensor(node);
+        node->grad->SetZeroAll();
+    }
+}
+
+/* the node is a leaf node (intput) or not */
+bool XNoder::IsLeaf(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+
+    if(node->income.tailNum == 0)
+        return true;
+    else
+        return false;
+}
+
+/* the node is a root node (output) or not */
+bool XNoder::IsRoot(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+
+    if(node->outgo.tailNum == 0)
+        return true;
+    else
+        return false;
+}
+
+/* the node keeps the gradinent or not */
+bool XNoder::IsGrad(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+    
+    if(node->isGrad)
+        return true;
+    else
+        return false;
+}
+
+}
\ No newline at end of file

source/network/XNoder.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * low-level utilities
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+
+#ifndef __XNODER_H__
+#define __XNODER_H__
+
+namespace nts{
+
+#define NODE_UNFINISHED 0
+#define NODE_DOING      1
+#define NODE_FINISHED   2
+
+/* node management */
+class XNoder
+{
+public:
+    /* make gradient tensor for a node */
+    static
+    void MakeGrad(XTensor * node);
+
+    /* the node is a leaf node (intput) or not */
+    static
+    bool IsLeaf(XTensor * node);
+
+    /* the node is a root node (output) or not */
+    static
+    bool IsRoot(XTensor * node);
+
+    /* the node keeps the gradinent or not */
+    static
+    bool IsGrad(XTensor * node);
+};
+
+}
+
+#endif
\ No newline at end of file

source/sample/fnnlm/FNNLM.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * This is a simple impelementation of the feed-forward network-baesd language
+ * model (FNNLM). See more details about FNNLM in
+ * "A Neural Probabilistic Language Model" by Bengio et al.
+ * Journal of Machine Learning Research 3 (2003) 1137¨C1155
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-06-22
+ * Today I was awarded as the most popular teacher in our college. 
+ * It was the great honour for me!!!    
+ */
+
+#ifndef __FNNLM_H__
+#define __FNNLM_H__
+
+#include "../../tensor/XGlobal.h"
+#include "../../tensor/XTensor.h"
+#include "../../tensor/core/CHeader.h"
+
+using namespace nts;
+
+namespace fnnlm
+{
+
+#define _EXIT_(x)// exit(x)
+#define CheckErrors(x, msg) { if(!(x)) { fprintf(stderr, "Error! calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__, msg);  _EXIT_(1); } }
+#define ShowErrors(msg) { { fprintf(stderr, "Error! (%s line %d): %s\n", __FILENAME__, __LINE__, msg); _EXIT_(1); } } 
+
+#define MAX_N_GRAM 8
+#define MAX_HIDDEN_NUM 8
+
+/* an n-gram = a sequence of n words 
+   words[0..n-2] is the history, and
+   words[n-1] is the word for prediction. */
+struct NGram
+{
+    int words[MAX_N_GRAM];
+};
+
+/* fnn model */
+struct FNNModel
+{
+    /* word embedding */
+    XTensor embeddingW;
+
+    /* parameter matrix of each hidden layer 
+       hidden layer: y = f(x * w + b)
+       where x is the input, y is the output, w is
+       the tranformation (parameter) matrix, b is
+       the bias and f() is the activation function. */
+    XTensor hiddenW[MAX_HIDDEN_NUM];
+
+    /* bias of each hidden layer */
+    XTensor hiddenB[MAX_HIDDEN_NUM];
+
+    /* parameter matrix of the output layer */
+    XTensor outputW;
+
+    /* bias of the output layer */
+    XTensor outputB;
+
+    /* order of the language model */
+    int n;
+
+    /* embedding size */
+    int eSize;
+    
+    /* number of hidden layers */
+    int hDepth;
+
+    /* hidden layer size */
+    int hSize;
+
+    /* vocabulary size */
+    int vSize;
+
+    /* id of the device for running the model */
+    int devID;
+
+    /* indicates whether we use memory pool */
+    bool useMemPool;
+
+    /* memory pool */
+    XMem * mem;
+
+    FNNModel(){ n = -1; vSize = -1;hDepth = 0;devID = -1;mem = NULL;};
+    ~FNNModel(){delete mem;};
+};
+
+/* the network built on the fly */
+struct FNNNet
+{
+    /* embedding result of the previous n - 1 words */
+    XTensor embeddings[MAX_N_GRAM];
+
+    /* concatenation of embeddings */
+    XTensor embeddingCat;
+
+    /* output of the hidden layers */
+    XTensor hiddens[MAX_HIDDEN_NUM];
+
+    /* state of the hidden layers (before activation function) */
+    XTensor hiddenStates[MAX_HIDDEN_NUM];
+
+    /* state before softmax */
+    XTensor stateLast;
+
+    /* output of the net */
+    XTensor output;
+};
+
+/* entrance of the program */
+int FNNLMMain(int argc, const char ** argv);
+
+};
+
+#endif

source/sample/transformer/T2TAttention.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TAttention.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TAttention::T2TAttention()
+{
+    nhead = -1;
+    dk = -1;
+    dv = -1;
+    d  = -1;
+    isMasked = false;
+    ignored = 0;
+}
+
+/* deconstructor */
+T2TAttention::~T2TAttention()
+{
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myIgnored - number of position ignored in attention (from the begining)
+>> myIsMasked - indicates whether the attention is with a mask
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TAttention::InitModel(int argc, char ** argv, 
+                             bool myIsMasked, int myIgnored, 
+                             int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+    isMasked = myIsMasked;
+    ignored = myIgnored;
+    
+    float minmax = 0;
+
+    LoadParamInt(argc, argv, "nhead", &nhead, 8);
+    LoadParamInt(argc, argv, "d", &dk, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &dv, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_EMBEDDING_SIZE);
+    LoadParamFloat(argc, argv, "attminmax", &minmax, 0.1F);
+    LoadParamFloat(argc, argv, "dropoutatt", &dropoutP, 0);
+
+    InitTensor2D(&wk, d, dk, X_FLOAT, devID, mem);
+    InitTensor2D(&wq, d, dk, X_FLOAT, devID, mem);
+    InitTensor2D(&wv, d, dv, X_FLOAT, devID, mem);
+    InitTensor2D(&wa, d, d, X_FLOAT, devID, mem);
+    InitTensor2D(&wbig, d, 3 * d, X_FLOAT, devID, mem);
+
+    float scale = 1.0F;
+    float finfoutk = (float)sqrt(6.0F * scale/(d + dk));
+    float finfoutv = (float)sqrt(6.0F * scale/(d + dv));
+    float finfouta = (float)sqrt(6.0F * scale / (d + d));
+    float finfoutbig = (float)sqrt(6.0F * scale / (d + 3*d));
+
+    wk.SetDataRand(-finfoutk, finfoutk);
+    wq.SetDataRand(-finfoutk, finfoutk);
+    wv.SetDataRand(-finfoutv, finfoutv);
+    wa.SetDataRand(-finfouta, finfouta);
+    wbig.SetDataRand(-finfoutbig, finfoutbig);
+}
+
+/* 
+make the network 
+>> k - keys. It might be of size B * L * H
+       where B = batch size, L = sequence length, 
+       and H = vector size of each position
+>> q - queries
+>> v - values
+>> mask - as it is
+>> isTraining - indicates whether the model is used for training
+<< return - multi-attention result
+*/
+XTensor T2TAttention::Make(XTensor &k, XTensor &q, XTensor &v, XTensor &mask, bool isTraining)
+{
+    XTensor k2;
+    XTensor q2;
+    XTensor v2;
+    
+    /* linear transformation before self-attention */
+    k2 = MMul(k, wk);
+    q2 = MMul(q, wq);
+    v2 = MMul(v, wv);
+    
+    return MakeAttention(k2, q2, v2, mask, isTraining);
+}
+    
+/*
+make the network given a big tensor that keeps keys, queries and values
+>> kqv - the big tensor
+>> mask - as it is
+>> isTraining - indicates whether the model is used for training
+*/
+XTensor T2TAttention::MakeBig(XTensor &kqv, XTensor &mask, bool isTraining)
+{
+    XTensor k2;
+    XTensor q2;
+    XTensor v2;
+    XTensor kqv2;
+    XList split;
+    
+    kqv2 = MMul(kqv, wbig);
+    
+    int d1 = kqv2.GetDim(0);
+    int d2 = kqv2.GetDim(1);
+    int d3 = kqv2.GetDim(2) / 3;
+    
+    InitTensor3D(&k2, d1, d2, d3, X_FLOAT, devID, mem);
+    InitTensor3D(&q2, d1, d2, d3, X_FLOAT, devID, mem);
+    InitTensor3D(&v2, d1, d2, d3, X_FLOAT, devID, mem);
+    
+    split.Add(&q2);
+    split.Add(&k2);
+    split.Add(&v2);
+    
+    Split(kqv2, split, 2, 3);
+    
+    return MakeAttention(k2, q2, v2, mask, isTraining);
+}
+    
+/*
+make the attention network given keys, queries and values (after linear transformation)
+>> k - keys. It might be of size B * L * H
+       where B = batch size, L = sequence length,
+       and H = vector size of each position
+>> q - queries
+>> v - values
+>> mask - as it is
+>> isTraining - indicates whether the model is used for training
+*/
+XTensor T2TAttention::MakeAttention(XTensor &k, XTensor &q, XTensor &v, XTensor &mask, bool isTraining)
+{
+    XTensor kheads;
+    XTensor qheads;
+    XTensor vheads;
+    
+    /* multi head */
+    kheads = Split(k, k.order - 1, nhead);
+    qheads = Split(q, q.order - 1, nhead);
+    vheads = Split(v, v.order - 1, nhead);
+    
+    XTensor att;
+    XTensor dot;
+    XTensor scalar;
+    
+    /* scalar = softmax(Q * K^T / sqrt(dk)) * V */
+    dot = BMMul(qheads, X_NOTRANS, kheads, X_TRANS);
+    
+    if(isMasked)
+        dot = dot + mask;
+    
+    dot = Linear(dot, 1.0F/(float)sqrt((float)dk/nhead));
+    
+    scalar = Softmax(dot, -1);
+
+    if(isTraining && dropoutP > 0)
+        scalar = Dropout(scalar, dropoutP);
+    
+    att = BMMul(scalar, vheads);
+    
+    /* concatenate the heads */
+    return MMul(Merge(att, att.order - 1), wa);
+}
+
+}

source/sample/transformer/T2TAttention.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TATTENTION_H__
+#define __T2TATTENTION_H__
+
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* 
+multi-head attention 
+y(Q, K, V) = cat(head_1, head_2, ..., head_n)
+where head_i = Attention(Q * w_i^Q, K * w_i^K, V * w_i^V)
+      attention(Q, K, V) = softmax(Q * K^T/d_k^0.5) V
+      d_k = dimension size of K
+*/
+class T2TAttention
+{
+public:
+    /* device id */
+    int devID;
+    
+    /* memory pool */
+    XMem * mem;
+    
+    /* head number */
+    int nhead;
+
+    /* transformation matrix for K */
+    XTensor wk;
+
+    /* transformation matrix for Q */
+    XTensor wq;
+
+    /* transformation matrix for V */
+    XTensor wv;
+
+    /* transformation after dot-product attention */
+    XTensor wa;
+    
+    XTensor wbig;
+	
+    /* size of transformed Q and K */
+    int dk;
+
+    /* size of transformed V */
+    int dv;
+
+    /* size of input Q, K and V */
+    int d;
+
+    /* indicates whether the attention is masked */
+    bool isMasked;
+
+    /* some positions can be ignored in attention. this is useful in lm where the first position needs
+       special design for the attention model. */
+    int ignored;
+
+    /* indicates whether the model is used for training */
+    bool isTraining;
+    
+    /* dropout probability */
+    DTYPE dropoutP;
+
+public:
+    /* constructor */
+    T2TAttention();
+
+    /* de-constructor */
+    ~T2TAttention();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, 
+                   bool myIsMasked, int myIgnored, 
+                   int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the network */
+    XTensor Make(XTensor &k, XTensor &q, XTensor &v, XTensor &mask, bool isTraining);
+    
+    /* make the network given a big tensor that keeps keys, queries and values */
+    XTensor MakeBig(XTensor &kqv, XTensor &mask, bool isTraining);
+    
+    /* make the attention network given keys, queries and values (after linear transformation) */
+    XTensor MakeAttention(XTensor &k, XTensor &q, XTensor &v, XTensor &mask, bool isTraining);
+};
+
+}
+
+#endif

source/sample/transformer/T2TDecoder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-10-09
+ */
+
+#include <math.h>
+#include "T2TDecoder.h"
+#include "T2TUtility.h"
+#include "T2TLayerNormal.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+AttDecoder::AttDecoder()
+{
+    attentionsEnde = NULL;
+    attEndeLayerNorms = NULL;
+}
+
+/* de-constructor */
+AttDecoder::~AttDecoder()
+{
+    delete[] attentionsEnde;
+    delete[] attEndeLayerNorms;
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myIsMasked - indicates whether the masked attention is employed
+>> myIgnored - number of positions ignored in attention (from the start)
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void AttDecoder::InitModel(int argc, char ** argv, 
+                           bool myIsMasked, int myIgnored, 
+                           int myDevID, XMem * myMem)
+{
+    //AttEncoder::InitModel(argc, argv, myIsMasked, myIgnored, myDevID, myMem);
+
+    devID = myDevID;
+    mem = myMem;
+    ignored = myIgnored;
+
+    LoadParamInt(argc, argv, "nlayer", &nlayer, 6);
+    LoadParamInt(argc, argv, "hsize", &hSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "esize", &eSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "vsizetgt", &vSize, -1);
+    LoadParamFloat(argc, argv, "dropout", &dropoutP, 0);
+
+    CheckNTErrors(nlayer >= 1, "We have one encoding layer at least!");
+    CheckNTErrors(vSize > 1, "set vocabulary size by \"-vsize\"");
+
+    /* embedding model */
+    embedder.InitModel(argc, argv, devID, mem, false);
+
+    attentions = new T2TAttention[nlayer];
+    fnns = new T2TFNN[nlayer];
+    attLayerNorms = new T2TLN[nlayer];
+    fnnLayerNorms = new T2TLN[nlayer];
+    attentionsEnde = new T2TAttention[nlayer];
+    attEndeLayerNorms = new T2TLN[nlayer];
+
+    /* initialize the stacked layers */
+    for (int i = 0; i < nlayer; i++) {
+        attentions[i].InitModel(argc, argv, myIsMasked, myIgnored, myDevID, myMem);
+        fnns[i].InitModel(argc, argv, myDevID, myMem);
+        attLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+        fnnLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+        attentionsEnde[i].InitModel(argc, argv, true, myIgnored, myDevID, myMem);
+        attEndeLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+    }
+}
+
+/* 
+make the decoding network
+>> inputDec - the input tensor of the decoder
+>> outputEnc - the output tensor of the encoder
+>> mask - mask that indicates which position is valid
+>> maskEncDec - mask for the encoder-decoder attention
+>> isTraining - indicates whether the model is used for training
+<< return - the output tensor of the encoder
+*/
+XTensor AttDecoder::Make(XTensor &inputDec, XTensor &outputEnc, XTensor &mask, XTensor &maskEncDec, bool isTraining)
+{
+    XTensor x;
+
+    x = embedder.Make(inputDec);
+
+    /* dropout */
+    if(isTraining && dropoutP > 0)
+        x = Dropout(x, dropoutP);
+
+    for(int i = 0; i < nlayer; i++){
+        XTensor att;
+        XTensor ende;
+        XTensor ln;
+        XTensor fnn;
+        XTensor res;
+
+        /******************/
+        /* self attention */
+        att = attentions[i].MakeBig(x, mask, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            att = Dropout(att, dropoutP);
+
+        /* residual connection */
+        res = Sum(att, x);
+
+        /* layer normalization */
+        x = attLayerNorms[i].Make(res);
+
+        /*****************************/
+        /* encoder-decoder attention */
+        ende = attentionsEnde[i].Make(outputEnc, x, outputEnc, maskEncDec, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            ende = Dropout(ende, dropoutP);
+
+        /* residual connection */
+        res = Sum(ende, x);
+
+        /* layer normalization */
+        x = attEndeLayerNorms[i].Make(res);
+
+        /*******/
+        /* fnn */
+        fnn = fnns[i].Make(x, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            fnn = Dropout(fnn, dropoutP);
+
+        /* residual connection */
+        res = Sum(fnn, x);
+
+        /* layer normalization */
+        x = fnnLayerNorms[i].Make(res);
+    }
+
+    return x;
+}
+
+
+}

source/sample/transformer/T2TDecoder.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TDECODER_H__
+#define __T2TDECODER_H__
+
+#include "T2TEncoder.h"
+
+namespace transformer
+{
+
+class AttDecoder
+{
+public:
+
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* layer number */
+    int nlayer;
+
+    /* hidden layer size of the FNN layer */
+    int hSize;
+
+    /* embedding size */
+    int eSize;
+
+    /* vocabulary size */
+    int vSize;
+
+    /* dropout probability */
+    DTYPE dropoutP;
+
+    /* some positions can be ignored in attention. this is useful in lm where the first position needs
+ *     special design for the attention model. */
+    int ignored;
+
+    /* embedding of word at each position */
+    T2TEmbedder embedder;
+
+    /* FNN model of each layer */
+    T2TFNN * fnns;
+
+    /* attention model of each layer */
+    T2TAttention * attentions;
+
+    /* layer normalization for fnn */
+    T2TLN * fnnLayerNorms;
+
+    /* layer normalization for attention */
+    T2TLN * attLayerNorms;
+
+    /* input tensor of the encoder */
+    XTensor * input;
+
+    /* output tensor of the encoder */
+    XTensor * output;
+
+    /* encoder-decoder attention model of each layer */
+    T2TAttention * attentionsEnde;
+
+    /* layer normalization for encoder-decoder attention */
+    T2TLN * attEndeLayerNorms;
+public:
+    /* constructor */
+    AttDecoder();
+
+    /* deconstructor */
+    ~AttDecoder();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, 
+                   bool myIsMasked, int myIgnored, 
+                   int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the decoding network */
+    XTensor Make(XTensor &inputDec, XTensor &outputEnc, XTensor &mask, XTensor &maskEncDec, bool isTraining);
+};
+
+}
+
+#endif
\ No newline at end of file

source/sample/transformer/T2TEmbedding.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-08-01
+ */
+
+#include <math.h>
+#include "T2TEmbedding.h"
+#include "T2TUtility.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TEmbedder::T2TEmbedder()
+{
+    devID = -1;
+    mem = NULL;
+    vSize = -1;
+    maxLength = -1;
+}
+
+/* deconstructor */
+T2TEmbedder::~T2TEmbedder()
+{
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TEmbedder::InitModel(int argc, char ** argv, int myDevID, XMem * myMem, bool isEnc)
+{
+    devID = myDevID;
+    mem = myMem;
+    
+    if(isEnc){
+        LoadParamInt(argc, argv, "vsize", &vSize, -1);
+    }
+    else{
+        LoadParamInt(argc, argv, "vsizetgt", &vSize, -1);
+    }
+    //LoadParamInt(argc, argv, "vsize", &vSize, -1);
+    LoadParamInt(argc, argv, "maxlen", &maxLength, 512);
+    LoadParamInt(argc, argv, "d", &eSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_EMBEDDING_SIZE);
+
+    InitTensor2D(&w, vSize, eSize, X_FLOAT, devID, mem);
+
+    DTYPE v = 1.0F/(float)sqrt((float)eSize);
+    w.SetDataRandn(0, v);
+
+    /* create the positional embedding matrix */
+    MakePosEmbedding(eSize, d, maxLength);
+}
+
+/* 
+make positional embeddings (of size eSize * length)
+>> eSize - embedding size
+>> d - dimension size of the hidden layers
+>> length - length of the sequence
+*/
+void T2TEmbedder::MakePosEmbedding(int eSize, int d, int length)
+{
+    InitTensor2D(&posEmbeddingBase, length, eSize, X_FLOAT, devID, mem);
+
+    float * data = new float[posEmbeddingBase.unitNum];
+
+    for(int pos = 0; pos < length; pos++){
+        float * dp = data + pos * eSize;
+        
+        int channelSize = eSize / 2;
+        int offset = 0;
+        for(int i = 0; i < channelSize; i++){
+            dp[offset++] = (float)sin(pos/pow(10000.0F, 2.0F*i/(d - 2)));
+        }
+        for(int i = 0; i < channelSize; i++){
+            dp[offset++] = (float)cos(pos/pow(10000.0F, 2.0F*i/(d - 2)));
+        }
+
+        /*
+        for(int k = 0; k < eSize; k++){
+            if(k % 2 == 0){
+                int i = k/2;
+                dp[k] = (float)sin(pos/pow(10000.0F, 2.0F*i/d));
+            }
+            else{
+                int i = (k - 1)/2;
+                dp[k] = (float)cos(pos/pow(10000.0F, 2.0F*i/d));
+            }
+        }
+        */
+    }
+
+    posEmbeddingBase.SetData(data, posEmbeddingBase.unitNum);
+
+    delete[] data;
+}
+
+/* 
+make the network 
+*/
+XTensor T2TEmbedder::Make(XTensor &input)
+{
+    //CheckNTErrors(input.GetDim(-1) == vSize, "Wrong vocabulary size!");
+    CheckNTErrors(input.order > 1, "Wrong input tensor size!");
+    CheckNTErrors(input.dimSize[input.order - 1] < maxLength, "The sequence is too long!");
+    CheckNTErrors(vSize > 0, "set vocabulary size by \"-vsize\"");
+    CheckNTErrors(eSize > 0, "set embedding size by \"-esize\"");
+
+    int dims[MAX_TENSOR_DIM_NUM];
+    memcpy(dims, input.dimSize, input.order * sizeof(int));
+    dims[input.order] = eSize;
+
+    XTensor wordEmbedding;
+    XTensor posEmbedding;
+
+    bool match = (posEmbedding.order == input.order);
+    if(match){
+        for(int i = 0; i < input.order; i++){
+            if(dims[i] != posEmbedding.GetDim(i))
+                match = false;
+        }
+    }
+
+    /* we make positional embeddings first */
+    //if(!match){
+    if(true){
+        InitTensor(&posEmbedding, input.order + 1, dims, X_FLOAT, 1.0F, devID, mem);
+
+        XTensor * posTMP = NewTensorBuf(2, dims + 1, X_FLOAT, 1.0F, devID, mem);
+
+        _CopyValues(&posEmbeddingBase, 0, posTMP->unitNum, posTMP, 0);
+        _Unsqueeze(posTMP, &posEmbedding, 0, dims[0]);
+
+        DelTensorBuf(posTMP);
+    }
+
+    /* then we make word embeddings */
+    wordEmbedding = Gather(w, input);
+    wordEmbedding = Linear(wordEmbedding, (float)sqrt((float)eSize));
+
+    /* we sum over the two embeddings */
+    return wordEmbedding + posEmbedding;
+}
+
+}

source/sample/transformer/T2TEmbedding.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-08-01
+ */
+
+#ifndef __T2TEMBEDDING_H__
+#define __T2TEMBEDDING_H__
+
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+#define DEFAULT_EMBEDDING_SIZE 512
+
+/* 
+embedding (of word at position i):
+word embedding + positional embedding
+*/
+class T2TEmbedder
+{
+public:
+    /* device id */
+    int devID;
+    
+    /* memory pool */
+    XMem * mem;
+    
+    /* vocabulary size */
+    int vSize;
+
+    /* embedding size */
+    int eSize;
+
+    /* maximum length of the sequence */
+    int maxLength;
+
+    /* dimension size of the hidden layers in the t2t model */
+    int d;
+
+    /* word embedding matrix */
+    XTensor w;
+
+    /* predefined positional embeddings. It can speeds up 
+       the embedding processing by re-loading. */
+    XTensor posEmbeddingBase;
+
+public:
+    /* constructor */
+    T2TEmbedder();
+
+    /* de-constructor */
+    ~T2TEmbedder();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL, bool isEnc = true);
+
+    /* make positional embeddings */
+    void MakePosEmbedding(int eSize, int d, int length);
+
+    /* make the network */
+    XTensor Make(XTensor &input);
+};
+
+}
+
+#endif

source/sample/transformer/T2TEncoder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TEncoder.h"
+#include "T2TLayerNormal.h"
+#include "T2TUtility.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+AttEncoder::AttEncoder()
+{
+    attentions = NULL;
+    fnns = NULL;
+    attLayerNorms = NULL;
+    fnnLayerNorms = NULL;
+}
+
+/* de-constructor */
+AttEncoder::~AttEncoder()
+{
+    delete[] attentions;
+    delete[] fnns;
+    delete[] attLayerNorms;
+    delete[] fnnLayerNorms;
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myIsMasked - indicates whether the masked attention is employed
+>> myIgnored - number of positions ignored in attention (from the start)
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void AttEncoder::InitModel(int argc, char ** argv, 
+                           bool myIsMasked, int myIgnored, 
+                           int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+    ignored = myIgnored;
+    
+    LoadParamInt(argc, argv, "nlayer", &nlayer, 6);
+    LoadParamInt(argc, argv, "hsize", &hSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "esize", &eSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "vsize", &vSize, -1);
+    LoadParamFloat(argc, argv, "dropout", &dropoutP, 0);
+
+    CheckNTErrors(nlayer >= 1, "We have one encoding layer at least!");
+    CheckNTErrors(vSize > 1, "set vocabulary size by \"-vsize\"");
+
+    /* embedding model */
+    embedder.InitModel(argc, argv, devID, mem);
+
+    attentions = new T2TAttention[nlayer];
+    fnns = new T2TFNN[nlayer];
+    attLayerNorms = new T2TLN[nlayer];
+    fnnLayerNorms = new T2TLN[nlayer];
+
+    /* initialize the stacked layers */
+    for(int i = 0; i < nlayer; i++){
+        attentions[i].InitModel(argc, argv, myIsMasked, myIgnored, myDevID, myMem);
+        fnns[i].InitModel(argc, argv, myDevID, myMem);
+        attLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+        fnnLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+    }
+}
+
+/* 
+make the encoding network
+>> input - the input tensor of the encoder
+>> mask - the mask that indicate each position is valid
+>> maskEncDec - no use
+>> isTraining - indicates whether the model is used for training
+<< return - the output tensor of the encoder
+*/
+XTensor AttEncoder::Make(XTensor &input, XTensor &mask, XTensor &maskEncDec, bool isTraining)
+{
+    XTensor x;
+
+    x = embedder.Make(input);
+
+    /* dropout */
+    if(isTraining && dropoutP > 0)
+        x = Dropout(x, dropoutP);
+
+    for(int i = 0; i < nlayer; i++){
+        XTensor att;
+        XTensor ln;
+        XTensor fnn;
+        XTensor res;
+
+        /* self attention */
+        att = attentions[i].MakeBig(x, mask, isTraining);
+        
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            att = Dropout(att, dropoutP);
+
+        /* residual connection */
+        res = Sum(att, x);
+
+        /* layer normalization */
+        x = attLayerNorms[i].Make(res);
+
+        /* fnn */
+        fnn = fnns[i].Make(x, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            fnn = Dropout(fnn, dropoutP);
+
+        /* residual connection */
+        res = Sum(fnn, x);
+
+        /* layer normalization */
+        x = fnnLayerNorms[i].Make(res);
+    }
+
+    return x;
+}
+
+/*
+make the encoding network (wrapper) 
+>> input - the input tensor of the encoder
+>> mask - the mask that indicate each position is valid
+>> isTraining - indicates whether the model is used for training
+<< return - the output tensor of the encoder
+*/
+XTensor AttEncoder::Make(XTensor &input, XTensor &mask, bool isTraining)
+{
+    XTensor nothing;
+
+    return Make(input, mask, nothing, isTraining);
+}
+
+}

source/sample/transformer/T2TEncoder.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TENCODER_H__
+#define __T2TENCODER_H__
+
+#include "T2TFNN.h"
+#include "T2TAttention.h"
+#include "T2TEmbedding.h"
+#include "T2TLayerNormal.h"
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* 
+base class of the encoder 
+*/
+class T2TEncoder
+{
+public:
+    virtual
+    XTensor Make(XTensor &input, XTensor &mask, XTensor &mask2, bool isTraining) = 0;
+};
+
+/* 
+the encoder based on RNN 
+*/
+class RNNEncoder : T2TEncoder
+{
+public:
+    XTensor Make(XTensor &input, XTensor &mask, XTensor &mask2, bool isTraining);
+};
+
+
+/* 
+the encoder based on self-attention 
+*/
+class AttEncoder : T2TEncoder
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* layer number */
+    int nlayer;
+
+    /* hidden layer size of the FNN layer */
+    int hSize;
+
+    /* embedding size */
+    int eSize;
+
+    /* vocabulary size */
+    int vSize;
+
+    /* dropout probability */
+    DTYPE dropoutP;
+
+    /* some positions can be ignored in attention. this is useful in lm where the first position needs
+       special design for the attention model. */
+    int ignored;
+
+    /* embedding of word at each position */
+    T2TEmbedder embedder;
+
+    /* FNN model of each layer */
+    T2TFNN * fnns;
+
+    /* attention model of each layer */
+    T2TAttention * attentions;
+
+    /* layer normalization for fnn */
+    T2TLN * fnnLayerNorms;
+
+    /* layer normalization for attention */
+    T2TLN * attLayerNorms;
+
+    /* input tensor of the encoder */
+    XTensor * input;
+
+    /* output tensor of the encoder */
+    XTensor * output;
+    
+public:
+    /* constructor */
+    AttEncoder();
+
+    /* de-constructor */
+    ~AttEncoder();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, 
+                   bool myIsMasked, int myIgnored, 
+                   int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the encoding network */
+    XTensor Make(XTensor &input, XTensor &mask, XTensor &maskEncDec, bool isTraining);
+
+    /* make the encoding network (wrapper) */
+    XTensor Make(XTensor &input, XTensor &mask, bool isTraining);
+};
+
+
+}
+
+#endif

source/sample/transformer/T2TFNN.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TFNN.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+#include "../../tensor/function/FHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TFNN::T2TFNN()
+{
+    inSize  = -1;
+    outSize = -1;
+    hSize   = -1;
+}
+
+/* deconstructor */
+T2TFNN::~T2TFNN()
+{
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TFNN::InitModel(int argc, char ** argv, int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+    
+    float minmax = 0;
+
+    LoadParamInt(argc, argv, "d", &inSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &outSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "fnnh", &hSize, outSize * 4);
+    LoadParamFloat(argc, argv, "fnnminmax", &minmax, 0.1F);
+    LoadParamFloat(argc, argv, "dropoutfnn", &dropoutP, 0);
+
+    InitTensor2D(&w1, inSize, hSize, X_FLOAT, devID, mem);
+    InitTensor1D(&b1, hSize, X_FLOAT, devID, mem);
+
+    InitTensor2D(&w2, hSize, outSize, X_FLOAT, devID, mem);
+    InitTensor1D(&b2, outSize, X_FLOAT, devID, mem);
+
+    float scale = 1.0F;
+    float finfout1 = (float)sqrt(6.0F * scale/(inSize + hSize));
+    float finfout2 = (float)sqrt(6.0F * scale/(hSize + outSize));
+    
+    w1.SetDataRand(-finfout1, finfout1);
+    b1.SetZeroAll();
+    w2.SetDataRand(-finfout2, finfout2);
+    b2.SetZeroAll();
+}
+
+/* 
+make the network 
+y = max(0, x * w1 + b1) * w2 + b2
+>> input - the input tensor
+>> return - the output tensor 
+*/
+XTensor T2TFNN::Make(XTensor &input, bool isTraining)
+{
+    XTensor t1;
+
+    /* t1 = max(0, x * w1 + b1) */
+    //t1 = Rectify(MMul(input, w1) + b1);
+    t1 = Rectify(MulAndShift(input, w1, b1));
+    
+    if(isTraining && dropoutP > 0)
+        t1 = Dropout(t1, dropoutP);
+
+    /* result = t1 * w2 + b2 */
+    //return MMul(t1, w2) + b2;
+    return MulAndShift(t1, w2, b2);
+}
+
+
+}

source/sample/transformer/T2TFNN.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TFNN_H__
+#define __T2TFNN_H__
+
+#include "../../tensor/XTensor.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* a fnn: y = max(0, x * w1 + b1) * w2 + b2 */
+class T2TFNN
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* size of input vector */
+    int inSize;
+
+    /* size of output vector */
+    int outSize;
+
+    /* size of hidden layers */
+    int hSize;
+
+    /* matrix of transformation 1 */
+    XTensor w1;
+
+    /* bias of transformation 1 */
+    XTensor b1;
+
+    /* matrix of transformation 2 */
+    XTensor w2;
+
+    /* bias of transformation 2 */
+    XTensor b2;
+    
+    /* dropout probability */
+    DTYPE dropoutP;
+
+public:
+
+    /* constructor */
+    T2TFNN();
+
+    /* deconstructor */
+    ~T2TFNN();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the network */
+    XTensor Make(XTensor &input, bool isTraining);
+
+};
+
+}
+
+#endif

source/sample/transformer/T2TLayerNormal.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TLayerNormal.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TLN::T2TLN()
+{
+    devID = -1;
+    mem = NULL;
+    d = 0;
+}
+
+/* de-constructor */
+T2TLN::~T2TLN()
+{
+}
+
+/*
+initialize the model
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TLN::InitModel(int argc, char ** argv, int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+
+    d = 0;
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_EMBEDDING_SIZE);
+
+    InitTensor1D(&w, d, X_FLOAT, devID, mem);
+    InitTensor1D(&b, d, X_FLOAT, devID, mem);
+
+    w.SetDataRand(1.0F, 1.0F);
+    b.SetZeroAll();
+}
+
+/*
+make the network
+for each layer representation x, we have
+y =
+>> input - the input tensor
+>> return - layer normalization output
+*/
+XTensor T2TLN::Make(XTensor &input)
+{
+    XTensor &x = input;
+    XTensor xn;
+    XTensor mean;
+    XTensor variance;
+    XTensor standard;
+    XTensor meanFilled;
+    XTensor standardFilled;
+
+    /* \mu = (sum_i x_i)/m */
+    mean = ReduceMean(x, x.order - 1);
+
+    /* \sigma = (sum_i (x_i - \mu)^2)/m */
+    variance = ReduceVariance(x, x.order - 1, mean);
+
+    /* standard = sqrt(variance) */
+    standard = Power(variance, 0.5F);
+
+    /* unsqueeze mean and standard deviation to fit them into
+       the same shape of x */
+    meanFilled = Unsqueeze(mean, x.order - 1, x.GetDim(-1));
+    standardFilled = Unsqueeze(standard, x.order - 1, x.GetDim(-1));
+
+    /* x' = (x - \mu)/standard */
+    xn = (x - meanFilled) / standardFilled;
+
+    /* result = x' * w + b   */
+    return xn * w + b;
+}
+
+}

source/sample/transformer/T2TLayerNormal.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TLAYERNORMAL_H__
+#define __T2TLAYERNORMAL_H__
+
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* layer normalization: y = norm(x) * w + b 
+   where norm(x) = (x - mean)/standardDeviation */
+class T2TLN
+{
+public:
+    /* device id */
+    int devID;
+    
+    /* memory pool */
+    XMem * mem;
+
+    /* the transformation matrix w */
+    XTensor w;
+
+    /* the bias term b */
+    XTensor b;
+
+    /* dimension size of the model */
+    int d;
+    
+public:
+    /* constructor */
+    T2TLN();
+    
+    /* de-constructor */
+    ~T2TLN();
+    
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL);
+    
+    /* make the network */
+    XTensor Make(XTensor &input);
+};
+
+}
+
+#endif

source/sample/transformer/T2TModel.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TMODEL_H__
+#define __T2TMODEL_H__
+
+#include "T2TFNN.h"
+#include "T2TAttention.h"
+#include "T2TEncoder.h"
+#include "T2TDecoder.h"
+#include "T2TOutput.h"
+
+namespace transformer
+{
+
+/* a transformer model that keeps parameters of the encoder,
+   the decoder and the output layer (softmax). Also, it creates
+   the network used in transformer. */
+class T2TModel
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* the encoder */
+    AttEncoder * encoder;
+
+    /* the decoder */
+    AttDecoder * decoder;
+
+    /* output layer */
+    T2TOutput * outputLayer;
+
+    /* indicates whether the model is running for language modeling */
+    bool isLM;
+
+    /* indicates whether the model is running for machine translation */
+    bool isMT;
+
+    /* number of heads in the attention model */
+    int nhead;
+
+public:
+    /* constructor */
+    T2TModel();
+
+    /* de-constructor */
+    ~T2TModel();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv);
+
+    /* make the encoding network */
+    XTensor MakeEncoder(XTensor &input, XTensor &mask, bool isTraining);
+
+    /* make the encoding network */
+    XTensor MakeDecoder(XTensor &inputEnc, XTensor &inputDec, XTensor &mask, XTensor &MaskEncDec, bool isTraining);
+
+    /* make the network for langauge modeling (with the output softmax layer) */
+    void MakeLM(XTensor &input, XTensor &output, XTensor &padding, bool isTraining);
+
+    /* make the network for machine translation (with the output softmax layer) */
+    void MakeMT(XTensor &inputEnc, XTensor &inputDec, XTensor &output, 
+                XTensor &paddingEnc, XTensor &paddingDec, bool isTraining);
+
+    /* make the mask for training MT models */
+    void MakeMTMask(XTensor &inputEnc, XTensor &inputDec, 
+                    XTensor &paddingEnc, XTensor &paddingDec, 
+                    XTensor &maskEnc, XTensor &maskDec, XTensor &maskEncDec);
+    
+    /* make the mask of the encoder */
+    void MakeMTMaskEnc(XTensor &inputEnc, XTensor &paddingEnc, XTensor &maskEnc);
+    
+    /* make the mask of the decoder */
+    void MakeMTMaskDec(XTensor &inputEnc, XTensor &inputDec,
+                       XTensor &paddingEnc, XTensor &paddingDec,
+                       XTensor &maskDec, XTensor &maskEncDec);
+
+    /* get parameter matrics */
+    void GetParams(XList &list);
+
+    /* dump the parameters */
+    void Dump(const char * fn);
+
+    /* read the parameters */
+    void Read(const char * fn);
+};
+
+}
+
+#endif

source/sample/transformer/T2TOutput.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TOutput.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+/* constructor */
+T2TOutput::T2TOutput()
+{
+    devID = -1;
+    mem = NULL;
+    vSize = -1;
+    inSize = -1;
+    hSize = -1;
+}
+
+/* de-constructor */
+T2TOutput::~T2TOutput()
+{
+}
+
+/*
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TOutput::InitModel(int argc, char ** argv, int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+
+    float minmax = 0;
+
+    LoadParamInt(argc, argv, "vsizetgt", &vSize, -1);
+    LoadParamInt(argc, argv, "d", &inSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &hSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamFloat(argc, argv, "outputminmax", &minmax, 0.08F);
+
+    InitTensor2D(&w, hSize, vSize, X_FLOAT, devID, mem);
+    
+    float scale = 1.0F;
+    float finfout = (float)sqrt(6.0F * scale/(hSize + vSize));
+    w.SetDataRand(-finfout, finfout);
+
+    DTYPE v = 1.0F/(float)sqrt((float)hSize);
+    w.SetDataRandn(0, v);
+}
+
+/* 
+make the network 
+y = softmax(x * w)
+>> input - input tensor
+<< return - output tensor 
+*/
+XTensor T2TOutput::Make(XTensor &input)
+{
+    XTensor &x = input;
+
+    return LogSoftmax(MMul(x, w), -1);
+}
+
+/* 
+make the network (redefined output tensor) 
+>> input - input tensor
+>> output - output tensor 
+*/
+void T2TOutput::Make(XTensor &input, XTensor &output)
+{
+    XTensor &x = input;
+
+    //output = LogSoftmax(MMul(x, w), -1);
+    output = Softmax(MMul(x, w), -1);
+}
+
+}

source/sample/transformer/T2TOutput.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TOUTPUT_H__
+#define __T2TOUTPUT_H__
+
+#include "../../tensor/function/FHeader.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* output layer */
+class T2TOutput
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* vocabulary size */
+    int vSize;
+
+    /* input vector size */
+    int inSize;
+
+    /* vector size of the linear transformation */
+    int hSize;
+
+    /* transformation matrix */
+    XTensor w;
+
+public:
+    /* constructor */
+    T2TOutput();
+
+    /* de-constructor */
+    ~T2TOutput();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the network */
+    XTensor Make(XTensor &input);
+
+    /* make the network (redefined output tensor) */
+    void Make(XTensor &input, XTensor &output);
+};
+
+
+}
+
+#endif
\ No newline at end of file

source/sample/transformer/T2TTrainer.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-08-02
+ */
+
+#ifndef __T2TTRAINER_H__
+#define __T2TTRAINER_H__
+
+#include "T2TModel.h"
+
+#include "../../tensor/function/FHeader.h"
+
+#define MAX_SEQUENCE_LENGTH 1024 * 4
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* node to keep batch information */
+struct BatchNode
+{
+    /* begining position */
+    int beg;
+
+    /* end position */
+    int end;
+
+    /* maximum word number on the encoder side */
+    int maxEnc;
+
+    /* maximum word number on the decoder side */
+    int maxDec;
+
+    /* a key for sorting */
+    int key;
+};
+
+/* trainer of the T2T model */
+class T2TTrainer
+{
+public:
+    /* paramter number */
+    int argNum;
+
+    /* parameter array */
+    char ** argArray;
+
+    /* buffer for loading words */
+    int * buf;
+
+    /* another buffer */
+    int * buf2;
+
+    /* batch buf */
+    BatchNode * bufBatch;
+
+    /* buffer size */
+    int bufSize;
+
+    /* size of batch buffer */
+    int bufBatchSize;
+
+    /* length of each sequence */
+    int * seqLen;
+
+    /* another array */
+    int * seqLen2;
+
+    /* offset of the first word for each sequence */
+    int * seqOffset;
+
+    /* number of sequences in the buffer */
+    int nseqBuf;
+
+    /* offset for next sequence in the buffer */
+    int nextSeq;
+
+    /* offset for next batch */
+    int nextBatch;
+    
+    /* indicates whether the sequence is sorted by length */
+    bool isLenSorted;
+    
+    /* dimension size of each inner layer */
+    int d;
+    
+    /* step number of warm-up for training */
+    int nwarmup;
+
+    /* vocabulary size of the source side */
+    int vSize;
+
+    /* vocabulary size of the target side */
+    int vSizeTgt;
+
+    /* learning rate */
+    float lrate;
+    
+    /* the parameter that controls the maximum learning rate in training */
+    float lrbias;
+
+    /* sentence batch size */
+    int sBatchSize;
+
+    /* word batch size */
+    int wBatchSize;
+
+    /* training epoch number */
+    int nepoch;
+
+    /* traing step number */
+    int nstep;
+
+    /* indicates whether we use adam */
+    bool useAdam;
+
+    /* hyper parameters of adam*/
+    float adamBeta1;
+    float adamBeta2;
+    float adamDelta;
+    float adamBeta1T;
+    float adamBeta2T;
+
+    /* list of the moment of the parameter matrics */
+    XList moments;
+
+    /* list of the 2nd order moment of the parameter matrics */
+    XList moments2nd;
+
+    /* indicates whether the data file is shuffled for training */
+    bool isShuffled;
+    
+    /* the factor of label smoothing */
+    DTYPE labelSmoothingP;
+
+    /* number of steps after which we make a checkpoint */
+    int nStepCheckpoint;
+
+    /* indicates whether we make a checkpoint after each traing epoch */
+    bool useEpochCheckpoint;
+    
+    /* number of batches on which we do model update */
+    int updateStep;
+    
+    /* indicates whether we double the </s> symbol for the output of lms */
+    bool isDoubledEnd;
+    
+    /* indicates whether we use batchsize = max * sc
+       rather rather than batchsize = word-number, where max is the maximum
+       length and sc is the sentence number */
+    bool isSmallBatch;
+
+    /* counterpart of "isSmallBatch" */
+    bool isBigBatch;
+
+    /* randomize batches */
+    bool isRandomBatch;
+
+    /* indicates whether we intend to debug the net */
+    bool isDebugged;
+
+    /* bucket size */
+    int bucketSize;
+
+public:
+    /* constructor */
+    T2TTrainer();
+
+    /* de-constructor */
+    ~T2TTrainer();
+
+    /* initialize the trainer */
+    void Init(int argc, char ** argv);
+
+    /* train the model */
+    void Train(const char * fn, const char * validFN, const char * modelFN, T2TModel * model);
+
+    /* test the model */
+    void Test(const char * fn, const char * ofn, T2TModel * model);
+
+    /* make a checkpoint */
+    void MakeCheckpoint(T2TModel * model, const char * validFN, const char * modelFN, const char * label, int id);
+
+    /* load data to buffer */
+    int LoadBuf(FILE * file, bool isSorted, int step);
+
+    /* clear data buffer */
+    void ClearBuf();
+
+    /* load a batch of sequences */
+    int LoadBatch(FILE * file, bool isLM,
+                  XTensor * batchEnc, XTensor * paddingEnc, 
+                  XTensor * batchDec, XTensor * paddingDec,
+                  XTensor * gold, XTensor * label,
+                  int * seqs,
+                  int vsEnc, int vsDec, int sBatch, int wBatch, 
+                  bool isSorted, int &ws, int &wCount,
+                  int devID, XMem * mem, 
+				  bool isTraining);
+
+    /* load a batch of sequences (for language modeling) */
+    int LoadBatchLM(FILE * file, 
+                    XTensor * batchEnc, XTensor * paddingEnc,
+                    XTensor * batchDec, XTensor * paddingDec,
+                    XTensor * gold, XTensor * label,
+                    int * seqs, int vs, int sBatch, int wBatch, 
+                    bool isSorted, int &wCount,
+                    int devID, XMem * mem, 
+					bool isTraining);
+
+    /* load a batch of sequences (for machine translation) */
+    int LoadBatchMT(FILE * file, 
+                    XTensor * batchEnc, XTensor * paddingEnc, 
+                    XTensor * batchDec, XTensor * paddingDec,
+                    XTensor * gold, XTensor * label,
+                    int * seqs, int vsEnc, int vsDec, int sBatch, int wBatch, 
+                    bool isSorted, int &ws, int &wCount,
+                    int devID, XMem * mem, 
+					bool isTraining);
+
+    /* shuffle the data file */
+    void Shuffle(const char * srcFile, const char * tgtFile);
+    
+    /* get word probabilities for a batch of sequences */
+    float GetProb(XTensor * output, XTensor * gold, XTensor * wordProbs);
+
+    /* update the model by delta rule */
+    void Update(T2TModel * model, const float lr);
+
+    /* prepare model for training */
+    void PrepareModel(T2TModel * model);
+
+    /* do padding on the output */
+    void PadOutput(XTensor * output, XTensor * gold, XTensor * padding);
+    
+    /* recale the output and gold tensors for normalized loss */
+    void RescaleOutput(XTensor * output, XTensor * gold, XTensor * padding);
+    
+    /* perform label smoothing */
+    void LabelSmooth(XTensor * gold, XTensor * smoothed, DTYPE p);
+};
+
+
+}
+
+#endif

source/sample/transformer/T2TUtility.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+namespace transformer
+{
+
+FILE * tmpFILE;
+int llnum = 0;
+FILE * tf = NULL;
+
+void LoadParamString(int argc, char ** argv, const char * name, char * p, const char * defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname) && i + 1 < argc){
+            strcpy(p, argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            hit = true;
+        }
+    }
+    if(!hit)
+        strcpy(p, defaultP);
+}
+
+void LoadParamInt(int argc, char ** argv, const char * name, int * p, int defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname) && i + 1 < argc){
+            *(int*)p = atoi(argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            hit = true;
+        }
+    }
+    if(!hit)
+        *p = defaultP;
+}
+
+void LoadParamBool(int argc, char ** argv, const char * name, bool * p, bool defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname)){
+            *(bool*)p = true;
+            //fprintf(stderr, " %s=%s\n", name, "true");
+            hit = true;
+        }
+    }
+    if(!hit)
+        *p = defaultP;
+}
+
+void LoadParamFloat(int argc, char ** argv, const char * name, float * p, float defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname) && i + 1 < argc){
+            *p = (float)atof(argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            hit = true;
+        }
+    }
+    if(!hit)
+        *p = defaultP;
+}
+
+void ShowParams(int argc, char ** argv)
+{
+    fprintf(stderr, "args:\n");
+    for(int i = 0; i < argc; i++){
+        if(argv[i][1] == 0)
+            continue;
+        if(argv[i][0] == '-' && (argv[i][1] < '1' || argv[i][1] > '9')){
+            if(i + 1 < argc && argv[i + 1][0] != '-')
+                fprintf(stderr, " %s=%s\n", argv[i], argv[i + 1]);
+            else
+                fprintf(stderr, " %s=yes\n", argv[i]);
+        }
+    }
+    fprintf(stderr, "\n");
+}
+
+}

source/sample/transformer/T2TUtility.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TUTILITY_H__
+#define __T2TUTILITY_H__
+
+#include <stdio.h>
+
+namespace transformer
+{
+
+extern FILE * tmpFILE;
+
+/* load arguments */
+void LoadParamString(int argc, char ** argv, const char * name, char * p, const char * defaultP);
+void LoadParamInt(int argc, char ** argv, const char * name, int * p, int defaultP);
+void LoadParamBool(int argc, char ** argv, const char * name, bool * p, bool defaultP);
+void LoadParamFloat(int argc, char ** argv, const char * name, float * p, float defaultP);
+
+/* show arguments */
+void ShowParams(int argc, char ** argv);
+
+extern int llnum;
+extern FILE * tf;
+
+}
+
+#endif

source/sample/transformer/Transformer.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include <time.h>
+#include "Transformer.h"
+#include "T2TModel.h"
+#include "T2TUtility.h"
+#include "T2TTrainer.h"
+#include "../../tensor/XDevice.h"
+#include "../../tensor/XUtility.h"
+#include "../../tensor/XGlobal.h"
+
+namespace transformer
+{
+
+int TransformerMain(int argc, const char ** argv)
+{
+    if(argc == 0)
+        return 1;
+
+    char ** args = new char*[argc];
+    for(int i = 0; i < argc; i++){
+        args[i] = new char[strlen(argv[i]) + 1];
+        strcpy(args[i], argv[i]);
+    }
+
+    tmpFILE = fopen("tmp.txt", "wb");
+
+    ShowParams(argc, args);
+
+    char * trainFN = new char[MAX_LINE_LENGTH];
+    char * modelFN = new char[MAX_LINE_LENGTH];
+    char * testFN = new char[MAX_LINE_LENGTH];
+    char * outputFN = new char[MAX_LINE_LENGTH];
+
+    LoadParamString(argc, args, "train", trainFN, "");
+    LoadParamString(argc, args, "model", modelFN, "");
+    LoadParamString(argc, args, "test", testFN, "");
+    LoadParamString(argc, args, "output", outputFN, "");
+
+    srand((unsigned int)time(NULL));
+
+    T2TTrainer trainer;
+    trainer.Init(argc, args);
+
+    T2TModel model;
+    model.InitModel(argc, args);
+
+    //if(strcmp(modelFN, ""))
+    //    model.Read(modelFN);
+    
+    /* learn model parameters */
+    if(strcmp(trainFN, ""))
+        trainer.Train(trainFN, testFN, strcmp(modelFN, "") ? modelFN : "checkpoint.model", &model);
+    
+    /* save the final model */
+    //if(strcmp(modelFN, "") && strcmp(trainFN, ""))
+        //model.Dump(modelFN);
+    
+    /* load the model if neccessary */
+    //if(strcmp(modelFN, ""))
+        //model.Read(modelFN);
+
+    T2TTrainer tester;
+    tester.Init(argc, args);
+
+    /* test the model on the new data */
+    if(strcmp(testFN, "") && strcmp(outputFN, ""))
+        tester.Test(testFN, outputFN, &model);
+
+    delete[] trainFN;
+    delete[] modelFN;
+    delete[] testFN;
+    delete[] outputFN;
+
+    for(int i = 0; i < argc; i++)
+        delete[] args[i];
+    delete[] args;
+
+    fclose(tmpFILE);
+
+    return 0;
+}
+
+}

source/sample/transformer/Transformer.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * An impelementation of the transformer system. See more details 
+ * about FNNLM in 
+ * "Attention Is All You Need" by Vaswani et al.
+ * https://arxiv.org/pdf/1706.03762.pdf
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ * I start writing the code related to NMT - a long time since my last coding 
+ * work on MT
+ */
+
+#ifndef __TRANSFORMER_H__
+#define __TRANSFORMER_H__
+
+#include "../../tensor/XGlobal.h"
+#include "../../tensor/XTensor.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* entrance of the program */
+int TransformerMain(int argc, const char ** argv);
+
+}
+
+#endif
\ No newline at end of file

source/tensor/Main.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+/*
+ *
+ * This is the entrance of the low-level tensor library : NiuTrans.Tensor
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2015-12-14
+ *
+ */
+
+#include <stdio.h>
+#include <math.h>
+#include <time.h>
+#include "XTensor.h"
+#include "XDevice.h"
+#include "./test/Test.h"
+#include "./core/CHeader.h"
+#include "./loss/CrossEntropy.h"
+
+//#define CRTDBG_MAP_ALLOC
+//#include <stdlib.h>  
+//#include <crtdbg.h> 
+
+using namespace nts;
+
+void SmallTest();
+void TransposeTest();
+void LittleTest();
+void T2TTest();
+void T2TTest2();
+void PowerTest();
+
+int main( int argc, const char ** argv )
+{
+    //PowerTest();
+    //LittleTest();
+
+    //T2TTest();
+    //T2TTest2();
+
+    //return 0;
+    //_CrtSetBreakAlloc(123);
+
+    /* a tiny test */
+    //SmallTest();
+
+    //_CrtDumpMemoryLeaks();
+    //return 0;
+
+    if(argc > 1 && !strcmp(argv[1], "-test"))
+        Test();
+    else{
+        fprintf(stderr, "Thanks for using NiuTrans.Tensor! This is a library that eases the\n");
+        fprintf(stderr, "use of tensors. All you need is to ... \n\n");
+        fprintf(stderr, "Run this program with \"-test\" for unit test!\n");
+    }
+
+    //_CrtDumpMemoryLeaks();
+
+    return 0;
+}
+
+void myRead(XTensor * tensor, const char * filename, const char * label)
+{
+    FILE * file = fopen(filename, "rb");
+    if(file == NULL)
+        printf("%s\n", filename);
+    tensor->Read(file, label);
+}
+
+void myDump(XTensor * tensor, const char * filename, const char * label)
+{
+    FILE * file = fopen(filename, "wb");
+    if(file == NULL)
+        printf("%s\n", filename);
+    tensor->Dump(file, label);
+}
+
+void PowerTest()
+{
+    XTensor input;
+    XTensor output;
+    InitTensor2D(&input, 256, 10000, X_FLOAT, 0);
+    InitTensor2D(&output, 256, 10000, X_FLOAT, 0);
+    myRead(&input, "1.txt", "");
+
+    _Power(&input, &output, 2);
+    output.Dump(stderr, "", 200);
+}
+
+void SmallTest()
+{
+    XTensor a;
+    XTensor b;
+    XTensor c;
+    XTensor d;
+
+    InitTensor2D(&a, 2, 2);
+    InitTensor2D(&b, 2, 2);
+    a.SetZeroAll();
+    b.SetZeroAll();
+    a.Set2D(1.0F, 0, 0);
+    a.Set2D(2.0F, 1, 1);
+
+    b = Sum(a, Multiply(a, a));
+
+    /* this is prohibited !!!!!!!!!!!!! */
+    //XTensor c = a * b + a;
+    //XTensor d = a + b + c.Lin(0.5F);
+    
+    c = a * b + a;
+    d = a + b + c.Lin(0.5F);
+
+    XLink::CheckNetwork(&d);
+    //XLink::ShowNetwork(stderr, &d);
+        
+    a.Dump(stderr, "a:");
+    b.Dump(stderr, "b:");
+    c.Dump(stderr, "c:");
+    d.Dump(stderr, "d:");
+}
+
+void TransposeTest()
+{
+    XTensor a;
+    XTensor b;
+
+    int I = 2;
+    int J = 3;
+
+    InitTensor4D(&a, 2, 3, 4, 5);
+
+    int * dims = new int[a.order];
+    memcpy(dims, a.dimSize, sizeof(int) * a.order);
+    dims[I] = a.dimSize[J];
+    dims[J] = a.dimSize[I];
+
+    InitTensor(&b, 4, dims);
+
+    a.SetZeroAll();
+    b.SetZeroAll();
+
+    float * data = new float[a.unitNum];
+    for(int i = 0; i < a.unitNum; i++)
+        data[i] = (float)i;
+
+    a.SetData(data, a.unitNum, 0);
+
+    _Transpose(&a, &b, I, J);
+    b.Dump(stderr, "b:");
+
+    delete[] data;
+}
+
+void LittleTest()
+{
+    int a = 5000;
+    int b = 100000;
+    int c = a*b;
+    printf("%d\n", c);
+
+    exit(1);
+}
+
+void T2TTest()
+{
+    XTensor * input;
+    XTensor * weight;
+    XTensor * output;
+    XTensor * gold;
+    XTensor * dedy;
+    XTensor * dedx;
+    XTensor * dedxTmp;
+    XTensor * dedw;
+    XTensor * padding;
+
+    DTYPE loss;
+
+    int * dimSize = new int[2];
+    dimSize[0] = 256;
+    dimSize[1] = 10001;
+
+    int * dimSize2 = new int[3];
+    dimSize2[0] = 2;
+    dimSize2[1] = 31;
+    dimSize2[2] = 256;
+   
+    int * dimSize3 = new int[3];
+    dimSize3[0] = 2;
+    dimSize3[1] = 31;
+    dimSize3[2] = 10001;
+
+    int * dimSize4 = new int[2];
+    dimSize4[0] = 2;
+    dimSize4[1] = 31;
+
+    input = NewTensor(3, dimSize2, X_FLOAT, 1.0F, 0);
+    weight = NewTensor(2, dimSize, X_FLOAT, 1.0F, 0);
+    dedw = NewTensor(2, dimSize, X_FLOAT, 1.0F, 0);
+    gold = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    output = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    dedy = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    dedx = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    dedxTmp = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    padding = NewTensor(2, dimSize4, X_FLOAT, 1.0F, 0);
+
+    //weight = NewTensor(2, dimSize);
+    //dedw = NewTensor(2, dimSize);
+    //input = NewTensor(3, dimSize2);
+    //gold = NewTensor(3, dimSize3);
+    //output = NewTensor(3, dimSize3);
+    //dedy = NewTensor(3, dimSize3);
+    //dedx = NewTensor(3, dimSize3);
+    //dedxTmp = NewTensor(3, dimSize3);
+    //padding = NewTensor(2, dimSize4);
+
+    myRead(input, "x.txt", "x");
+    myRead(weight, "w.txt", "w");
+    myRead(gold, "gold.txt", "gold");
+    myRead(padding, "padding.txt", "padding");
+
+    XTensor inter;
+    inter = MMul(*input, *weight);
+
+    _Softmax(&inter, output, 2);
+
+    //_LogMe(output);
+    loss = _CrossEntropyFast(output, gold, REDUCE_MEAN, NULL, padding);
+
+    printf("loss: %f\n", loss);
+
+    _CrossEntropyBackward(dedy, output, gold, NULL);
+    //_CrossEntropyBackward(dedy, output, gold, NULL, padding);
+
+    myDump(dedy, "dedy.txt", "dedy");
+
+    _SoftmaxBackward(NULL, output, input, dedy, dedx, NULL, -1, NOLOSS);
+    _Sub(output, gold, dedxTmp);
+
+    myDump(dedx, "dedx.txt", "dedx");
+    dedx->Dump(stderr, "dedx", 200);
+    dedxTmp->Dump(stderr, "dedxTmp", 200);
+
+    input->Reshape(input->unitNum/input->GetDim(-1), input->GetDim(-1));
+    dedx->Reshape(dedx->unitNum/dedx->GetDim(-1), dedx->GetDim(-1));
+
+    _MatrixMulBatched(input, X_TRANS, dedx, X_NOTRANS, dedw);
+
+    myDump(dedw, "dedw.txt", "dedw");
+}
+
+void T2TTest2()
+{
+    int dimSize[3];
+    dimSize[0] = 161;
+    dimSize[1] = 47;
+    dimSize[2] = 10001;
+    XTensor * probs = NewTensor(3, dimSize, X_FLOAT, 1.0F, 0);
+    //XTensor * probs = NewTensor(3, dimSize, X_FLOAT, 1.0F, -1);
+
+    //myRead(probs, "probs.txt", " ");
+    _SetDataFixedFloat(probs, 1.0F);
+
+    probs->Reshape(1, probs->unitNum);
+
+    DTYPE sum = _ReduceSumAll(probs);
+    printf("%e\n", sum);
+
+    //XTensor tmp;
+    //tmp = IsNonZero(*probs);
+    //DTYPE nonZeroNum = ReduceSumAll(tmp);
+    //printf("%f\n", nonZeroNum);
+    //
+    //DTYPE gpu = ReduceSum(*probs, 1).Get2D(0, 0);
+
+    //printf("%e\n", gpu);
+}
+

source/tensor/XBLAS.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * This is a wrapper of the BLAS (Basic Linear Algebra Subprograms http://www.netlib.org/blas/) 
+ * libraries. By using BLAS, we can access very fast matrix operations although they
+ * are also implemented in NiuTrans in a native manner. To use BLAS, 
+ * set USE_BLAS. 
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-04-08
+ *
+ */
+
+#ifdef WIN32
+#include <wtypes.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "XBLAS.h"
+#include "XGlobal.h"
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+#ifdef WIN32
+HINSTANCE hBLASDll;
+#endif
+
+
+/* single-precision floating matrix-matrix multiplication */
+void (*XBLAS_SGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                    OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float,  
+                    OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT,
+                    OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float, 
+                    float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating matrix-matrix multiplication */
+void (*XBLAS_DGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                    OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double,  
+                    OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT,
+                    OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double, 
+                    double *, OPENBLAS_CONST BLASINT);
+
+/* single-precision floating vector-vector multiplication (rank-1) */
+void (*XBLAS_SGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST float  alpha, 
+                   OPENBLAS_CONST float *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, 
+                   float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating vector-vector multiplication (rank-1) */
+void (*XBLAS_DGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST double  alpha, 
+                   OPENBLAS_CONST double *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, 
+                   double *, OPENBLAS_CONST BLASINT);
+
+/* set the number of threads */
+void (*XBLAS_SET_THREAD_NUM)(int);
+
+/* get the number of threads */
+//int (*XBLAS_GET_THREAD_NUM)();
+
+
+/* get the number of physical processors (cores).*/
+int (*XBLAS_GET_CORE_NUM)();
+
+
+/* get the CPU corename */
+//char * (*XBLAS_GET_CORE_NAME)();
+
+/* get the parallelization type used by OpenBLAS */
+//int (*XBLAS_GET_PARALLEL_TYPE)(void);
+
+
+#if defined(USE_BLAS)
+
+/* load some stuff for BLAS */
+void LoadBLAS(const char * dllFileName)
+{
+#ifndef CUDA_BLAS
+#ifdef _WIN32
+
+#if defined(OPENBLAS)
+    /* non-ascii characters are not supported yet */
+    wchar_t * fn = new wchar_t[strlen(dllFileName) + 1];
+    memset(fn, 0, sizeof(wchar_t) * (strlen(dllFileName) + 1));
+    for(int i = 0; i < strlen(dllFileName); i++)
+        fn[i] = dllFileName[i];
+
+    hBLASDll = LoadLibrary((LPCWSTR)fn);
+
+    if(!hBLASDll){
+        XPRINT1(0, stderr, "[LoadBLAS] Error! Cannot load dll %s!\n", dllFileName);
+        exit(1);
+    }
+
+    /* matrix-matrix multiplicatoin */
+    (FARPROC&)XBLAS_SGEMM = GetProcAddress(hBLASDll, "cblas_sgemm");
+    (FARPROC&)XBLAS_DGEMM = GetProcAddress(hBLASDll, "cblas_dgemm");
+
+    /* vector-vector multiplication */
+    (FARPROC&)XBLAS_SGER = GetProcAddress(hBLASDll, "cblas_sger");
+    (FARPROC&)XBLAS_DGER = GetProcAddress(hBLASDll, "cblas_dger");
+
+    /* multi-threading */
+    (FARPROC&)XBLAS_SET_THREAD_NUM = GetProcAddress(hBLASDll, "openblas_set_num_threads");
+    //(FARPROC&)XBLAS_SET_THREAD_NUM = GetProcAddress(hBLASDll, "goto_set_num_threads");
+    //(FARPROC&)XBLAS_GET_THREAD_NUM = GetProcAddress(hBLASDll, "openblas_get_num_threads");
+    (FARPROC&)XBLAS_GET_CORE_NUM = GetProcAddress(hBLASDll, "openblas_get_num_procs");
+    //(FARPROC&)XBLAS_GET_CORE_NAME = GetProcAddress(hBLASDll, "openblas_get_corename");
+    //(FARPROC&)XBLAS_GET_PARALLEL_TYPE = GetProcAddress(hBLASDll, "openblas_get_parallel");
+
+    delete[] fn;
+#endif // defined(OPENBLAS)
+
+#if defined(MKL)
+    /* non-ascii characters are not supported yet */
+    wchar_t * fn = new wchar_t[strlen(dllFileName) + 1];
+    memset(fn, 0, sizeof(wchar_t) * (strlen(dllFileName) + 1));
+    for(int i = 0; i < strlen(dllFileName); i++)
+        fn[i] = dllFileName[i];
+
+    hBLASDll = LoadLibrary((LPCWSTR)fn);
+
+    if(!hBLASDll){
+        XPRINT1(0, stderr, "[LoadBLAS] Error! Cannot load dll %s!\n", dllFileName);
+        exit(1);
+    }
+
+    /* matrix-matrix multiplicatoin */
+    (FARPROC&)XBLAS_SGEMM = GetProcAddress(hBLASDll, "cblas_sgemm");
+    (FARPROC&)XBLAS_DGEMM = GetProcAddress(hBLASDll, "cblas_dgemm");
+
+    /* vector-vector multiplication */
+    (FARPROC&)XBLAS_SGER = GetProcAddress(hBLASDll, "cblas_sger");
+    (FARPROC&)XBLAS_DGER = GetProcAddress(hBLASDll, "cblas_dger");
+
+    /* multi-threading */
+    (FARPROC&)XBLAS_SET_THREAD_NUM = GetProcAddress(hBLASDll, "MKL_Set_Num_Threads");
+    (FARPROC&)XBLAS_GET_CORE_NUM   = GetProcAddress(hBLASDll, "MKL_Get_Max_Threads");
+#endif // defined(MKL)
+
+#else // _WIN32
+
+    XBLAS_SGEMM = &cblas_sgemm;
+    XBLAS_DGEMM = &cblas_dgemm;
+    XBLAS_SGER  = &cblas_sger;
+    XBLAS_DGER  = &cblas_dger;
+#if defined(OPENBLAS)
+    XBLAS_SET_THREAD_NUM    = &openblas_set_num_threads;
+    XBLAS_GET_CORE_NUM      = &openblas_get_num_procs;
+#endif // defined(OPENBLAS)
+#if defined(MKL)
+    XBLAS_SET_THREAD_NUM    = &mkl_set_num_threads;
+    XBLAS_GET_CORE_NUM      = &mkl_get_max_num_threads;
+#endif // defined(MKL)
+
+#endif // _WIN32
+
+    XBLAS_SET_THREAD_NUM(1);
+#endif // ndef(CUDA_BLAS)
+}
+
+/* unload the libs */
+void UnloadBLAS()
+{
+#ifdef _WIN32
+
+    if(!FreeLibrary(hBLASDll)){
+        XPRINT(0, stderr, "[UnloadBLAS] Error! Cannot free the BLAS dll!\n");
+        exit(1);
+    }
+
+#else
+
+#endif
+}
+
+#else  // undefined(USE_BLAS) || undefined(OPENBLAS)
+
+void LoadBLAS(const char * dllFileName)
+{
+    XPRINT(0, stderr, "[LoadBLAS] Error! No Blas lib is available. Please use OPENBLAS or MKL!\n");
+    exit(1);
+}
+
+void UnloadBLAS()
+{
+    XPRINT(0, stderr, "[UnloadBLAS] Error! No Blas lib is available. Please use OPENBLAS or MKL!\n");
+    exit(1);
+}
+
+#endif // defined(USE_BLAS) && defined(OPENBLAS)
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
\ No newline at end of file

source/tensor/XBLAS.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.b
+ */
+
+/*
+ *
+ * This is a wrapper of the BLAS (Basic Linear Algebra Subprograms http://www.netlib.org/blas/) 
+ * libraries. By using BLAS, we can access very fast matrix operations although they
+ * are also implemented in NiuTrans in a native manner. To use BLAS, 
+ * specify USE_BLAS when compiling the code. 
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-04-08
+ *
+ */
+
+#ifndef __XBLAS_H__
+#define __XBLAS_H__
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+/* some of the code below is from OpenBLAS (https://github.com/xianyi/OpenBLAS) */
+
+//#define OPENBLAS
+
+#define OPENBLAS_CONST const
+typedef int BLASINT;
+typedef enum CBLAS_ORDER     {CblasRowMajor=101, CblasColMajor=102} CBLAS_ORDER;
+typedef enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113, CblasConjNoTrans=114} CBLAS_TRANSPOSE;
+typedef enum CBLAS_UPLO      {CblasUpper=121, CblasLower=122} CBLAS_UPLO;
+typedef enum CBLAS_DIAG      {CblasNonUnit=131, CblasUnit=132} CBLAS_DIAG;
+typedef enum CBLAS_SIDE      {CblasLeft=141, CblasRight=142} CBLAS_SIDE;
+
+
+#if defined(USE_BLAS)
+
+/* 
+single/double-precision floating matrix-matrix multiplication (rank-3)
+- SGEMM (ORDER, TRANSA, TRANSB, M, N, K, ALPHA, A, LDA, B, LDB, BETA, C, LDC)
+It implements C = \alpha * op(A)*op(B) + \beta * C
+where A, B and C are matrices,
+      \alpha and \beta are coefficients,
+      TRANSA specifies we need a transposed matrix (op(A)=A**T); otherwise op(A) = A,
+      M specifies the row number of op(A),
+      N specifies the column number of op(B),
+      K specifies the column number of op(A),
+      LDA(=K) specifies the size of the first(or leading) dimension of A as declared in the calling (sub) program,
+              E.g., if we are using CblasRowMajor, the leading dimension is the number of columns.
+      LDB(=N) specifies the size of the first dimension of B as declared in the calling (sub) program,
+      and LDC(=N) specifies the size of the first dimension of C as declared in the calling (sub) program.
+*/
+extern "C" void (*XBLAS_SGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                               OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float,  
+                               OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT,
+                               OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float, 
+                               float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating matrix-matrix multiplication */
+extern "C" void (*XBLAS_DGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                               OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double,  
+                               OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT,
+                               OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double, 
+                               double *, OPENBLAS_CONST BLASINT);
+
+/* 
+single/double-precision floating vector-vector multiplication (rank-2)
+- SGER (ORDER,M, N, ALPHA, X, INCX, Y, INCY, A, LDA)
+It implements A = \alpha * X * (Y^T) + A
+where X and Y are vectors with m and n elements respectively,
+      A is an m by n matrix,
+      \alpha is the scalar,
+      INCX specifies the increment for the elements of X,
+      INCY specifies the increment for the elements of Y,
+      LDA specifies the size of the first(or leading) dimension of A as declared in the calling (sub) program,
+          E.g., if we are using CblasRowMajor, the leading dimension is the number of columns of A.
+
+*/
+extern "C" void (*XBLAS_SGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST float  alpha, 
+                              OPENBLAS_CONST float *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, 
+                              float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating vector-vector multiplication (rank-1) */
+extern "C" void (*XBLAS_DGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST double  alpha, 
+                              OPENBLAS_CONST double *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, 
+                              double *, OPENBLAS_CONST BLASINT);
+
+/* set the number of threads */
+extern "C" void (*XBLAS_SET_THREAD_NUM)(int);
+
+/* get the number of threads */
+//extern "C" int (*XBLAS_GET_THREAD_NUM)();
+
+
+/* get the number of physical processors (cores).*/
+extern "C" int (*XBLAS_GET_CORE_NUM)();
+
+/* get the CPU corename */
+//extern "C" char * (*XBLAS_GET_CORE_NAME)();
+
+/* get the parallelization type used by OpenBLAS */
+//extern "C" int (*XBLAS_GET_PARALLEL_TYPE)(void);
+
+/* linux systems */
+#ifndef _WIN32
+
+/* cblas functions that are imported from the lib. See cblas.h in OpenBlas for more information */
+extern "C" void cblas_sgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, 
+                        OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST BLASINT K, OPENBLAS_CONST float alpha, 
+                        OPENBLAS_CONST float *A, OPENBLAS_CONST BLASINT lda, 
+                        OPENBLAS_CONST float *B, OPENBLAS_CONST BLASINT ldb, 
+                        OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST BLASINT ldc);
+extern "C" void cblas_dgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, 
+                        OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST BLASINT K, OPENBLAS_CONST double alpha, 
+                        OPENBLAS_CONST double *A, OPENBLAS_CONST BLASINT lda, 
+                        OPENBLAS_CONST double *B, OPENBLAS_CONST BLASINT ldb, 
+                        OPENBLAS_CONST double beta, double *C, OPENBLAS_CONST BLASINT ldc);
+extern "C" void cblas_sger (OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST float  alpha, 
+                        OPENBLAS_CONST float  *X, OPENBLAS_CONST BLASINT incX, OPENBLAS_CONST float  *Y, OPENBLAS_CONST BLASINT incY, 
+                        float  *A, OPENBLAS_CONST BLASINT lda);
+extern "C" void cblas_dger (OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST double alpha, 
+                        OPENBLAS_CONST double *X, OPENBLAS_CONST BLASINT incX, OPENBLAS_CONST double *Y, OPENBLAS_CONST BLASINT incY, 
+                        double *A, OPENBLAS_CONST BLASINT lda);
+
+#if defined(OPENBLAS)
+/* better control of multi-threading */
+extern "C" void  openblas_set_num_threads(int num_threads);
+extern "C" void  goto_set_num_threads(int num_threads);
+//extern "C" int   openblas_get_num_threads(void);
+extern "C" int   openblas_get_num_procs(void);
+//extern "C" char* openblas_get_config(void);
+//extern "C" char* openblas_get_corename(void);
+//extern "C" int   openblas_get_parallel(void);
+#endif
+
+#endif
+
+#if defined(MKL)
+
+
+/* better control of multi-threading */
+//_Mkl_Api(void,MKL_Set_Num_Threads,(int nth))
+//_Mkl_Api(int,MKL_Get_Max_Threads,(void))
+extern "C" void  MKL_Set_Num_Threads(int num_threads);
+extern "C" int  MKL_Get_Max_Threads();
+
+
+#define mkl_set_num_threads MKL_Set_Num_Threads
+#define mkl_get_max_num_threads MKL_Get_Max_Threads
+
+//extern "C" void  mkl_set_num_threads(int num_threads);
+//extern "C" void  omp_set_num_threads(int num_threads);
+//extern "C" int  mkl_get_max_num_threads();
+
+#endif
+
+#if defined(CUDA_BLAS)
+
+// Utilities and system includes
+#include <assert.h>
+#include <helper_string.h>  // helper for shared functions common to CUDA Samples
+
+// CUDA runtime
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+
+// CUDA and CUBLAS functions
+#include <helper_functions.h>
+#include <helper_cuda.h>
+
+/* Matrix multiplication */
+extern void BLASMatrixMULS(int deviceID, float * a, float * b, float * c, int na, int ma, int nb, int mb, int nc, int mc, float alpha = 1.0F, float beta = 0);
+extern void BLASMatrixMULD(int deviceID, double * a, double * b, double * c, int na, int ma, int nb, int mb, int nc, int mc, double alpha = 1.0F, double beta = 0);
+
+#endif
+
+#endif
+
+#ifdef _WIN32
+
+#include "windows.h"
+
+extern HINSTANCE hBLASDll;
+
+#else
+
+#endif
+
+/* load some stuff for BLAS */
+extern void LoadBLAS(const char * dllFileName);
+
+/* unload the libs */
+extern void UnloadBLAS();
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif

source/tensor/XDataType.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-05-30
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "XDataType.h"
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+const char * GetDataTypeName(TENSOR_DATA_TYPE type)
+{
+    if (type == X_INT)
+        return "X_INT";
+    else if (type == X_INT8)
+        return "X_INT8";
+    else if (type == X_FLOAT)
+        return "X_FLOAT";
+    else if (type == X_FLOAT16)
+        return "X_FLOAT16";
+    else if (type == X_DOUBLE)
+        return "X_DOUBLE";
+    return "NULL";
+}
+
+TENSOR_DATA_TYPE GetDataType(const char * typeName)
+{
+    if (!strcmp(typeName, "X_INT"))
+        return X_INT;
+    else if (!strcmp(typeName, "X_INT8"))
+        return X_INT8;
+    else if (!strcmp(typeName, "X_FLOAT"))
+        return X_FLOAT;
+    else if (!strcmp(typeName, "X_FLOAT16"))
+        return X_FLOAT16;
+    else if (!strcmp(typeName, "X_DOUBLE"))
+        return X_DOUBLE;
+    else {
+        ShowNTErrors("Unknown data type!");
+    }
+}
+
+/****************************************************
+Below is for calling CPU BLAS for fast matrix operations
+I'm not sure how fast it is. But it seems that other
+guys are crazy about this. So I decided to have a try.
+*/
+
+/* float -> float16 */
+_XINLINE_ unsigned short FloatToFloat16(float f)
+{
+    unsigned int x = *((unsigned int*)&f);
+    unsigned short h = ((x>>16)&0x8000)|((((x&0x7f800000)-0x38000000)>>13)&0x7c00)|((x>>13)&0x03ff);
+    return h;
+}
+
+/* float16 -> float */
+_XINLINE_ float Float16ToFloat(unsigned short h)
+{
+    float f = float(((h&0x8000)<<16) | (((h&0x7c00)+0x1C000)<<13) | ((h&0x03FF)<<13));
+    return f;
+}
+
+/* 
+data type conversion
+>> devID - device id
+>> s - source data array
+>> typeS - source data type
+>> t - target data array
+>> typeT - target data type
+>> size - number of the items in s (and t)
+*/
+void ConvertDataType(int devID, void * s, TENSOR_DATA_TYPE typeS, void * t, TENSOR_DATA_TYPE typeT, int size)
+{
+    CheckNTErrors((devID < 0), "This code must be run on CPUs!");
+
+    if(typeS == typeT)
+        return;
+
+    if(typeS == X_FLOAT && typeT == X_FLOAT16){
+        for(int i = 0; i < size; i++){
+            ((unsigned short*)t)[i] = FloatToFloat16(((float*)s)[i]);
+        }
+    }
+    else if(typeS == X_FLOAT16 && typeT == X_FLOAT){
+        for(int i = 0; i < size; i++){
+            ((float*)t)[i] = Float16ToFloat(((unsigned short*)s)[i]);
+        }
+    }
+    else{
+        ShowNTErrors("Unsupported data types for conversion!");
+    }
+}
+
+} /* end of the nts (NiuTrans.Tensor) namespace */

source/tensor/XDataType.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-05-30
+ *
+ */
+
+#ifndef __XDATATYPE_H__
+#define __XDATATYPE_H__
+
+#include "XGlobal.h"
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+/* data type of the tensor, e.g., int, float, and double. */
+enum TENSOR_DATA_TYPE {X_INT, X_INT8, X_FLOAT, X_FLOAT16, X_DOUBLE};
+
+/* transposed matrix type */
+enum MATRIX_TRANS_TYPE{X_TRANS, X_NOTRANS};
+
+/* default data type */
+#ifdef DOUBELPRICSION
+#define DEFAULT_DTYPE X_DOUBLE
+#else
+#define DEFAULT_DTYPE X_FLOAT
+#endif
+
+/* get data type name */
+extern const char * GetDataTypeName(TENSOR_DATA_TYPE type);
+extern TENSOR_DATA_TYPE GetDataType(const char * typeName);
+
+/* data conversion (for lower precision computation) */
+unsigned short FloatToFloat16(float f);
+float Float16ToFloat(unsigned short h);
+void ConvertDataType(int devID, 
+                     void * s, TENSOR_DATA_TYPE typeS, 
+                     void * t, TENSOR_DATA_TYPE typeT, int size);
+
+#ifdef USE_CUDA
+void CudaConvertDataType(int devID, 
+                         void * s, TENSOR_DATA_TYPE typeS, 
+                         void * t, TENSOR_DATA_TYPE typeT, int size);
+#endif
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif
\ No newline at end of file

source/tensor/XDevice.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-06-23
+ *
+ */
+
+#ifndef __XDEVICE_H__
+#define __XDEVICE_H__
+
+#include "XThread.h"
+#include "XStream.h"
+
+#ifdef USE_CUDA
+
+/* the CUDA stuff */
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cuda.h>
+
+#endif
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+#define MAX_LENGTH_OF_DEVICE_NAME 64
+#define MAX_CPU_NUM 16
+#define MAX_GPU_NUM 16
+#define MAX_DEVICE_NUM MAX_CPU_NUM+MAX_GPU_NUM
+#define INVALID_DEVICE_ID -1000
+#define CURRENT_GPU 1000
+//#define CUDA_UVA 1 // Unified Virtual Address Space of Cuda
+
+/*
+a class that records the basic information for each GPU/CPU device
+e.g., the memory limit, warp size of a GPU and etc.
+*/
+class XDevice
+{
+public:
+    /* 
+    device id 
+    <0:  CPU memory
+    >=0: GPU device ID
+    */
+    int devID;
+
+    /* size of the memory */
+    int memSize;
+
+    /* warp size of an (Navida) GPU */
+    int GPUWarpSize;
+
+    /* indicates whether the device class has been initialized */
+    bool isInitialized;
+
+    /* 
+    max grid size (or number of blocks) of an (Navida) GPU 
+    NOTE: the grid size is alone with three dimensions (x, y, z)
+    */
+    int GPUMaxGridSize[3];
+
+    /*
+    max block size (or number of threads per block) of an (Navida) GPU 
+    NOTE: the block size is alone with three dimensions (x, y, z)
+    */
+    int GPUMaxBlockSize[3];
+
+    /* max thread number that is supported */
+    int GPUMaxThreadNum;
+
+    /* max (and optimal) thread number for a block */
+    int GPUMaxThreadNumPerBlock;
+
+    /* name of the device */
+    char name[MAX_LENGTH_OF_DEVICE_NAME];
+
+    /* name of the device */
+    char name2[MAX_LENGTH_OF_DEVICE_NAME];
+
+    /* specify whether Unified Virtual Address Space (UVA) is supported */
+    bool isUVASupported;
+
+    /* default stream for the device */
+    XStream * stream;
+
+    /* seed for random number generation */
+    int seed;
+    
+#ifdef USE_CUDA
+    /* mutex for handle (GPU cublas) */
+    MUTEX_HANDLE cublasMutex;
+
+    /* handle used for cublas */
+    cublasHandle_t cublasHandle;
+
+    /* specify if the handle is initialized */
+    bool isHandleReady;
+
+    /* generater of random numbers */
+    curandGenerator_t gen;
+#endif
+
+
+public:
+    /* constructor */
+    XDevice();
+
+    /* de-constructor */
+    ~XDevice();
+
+    /* initialize it and get the device information */
+    void Init(int myDevID);
+
+    /* clear it */
+    void Clear();
+
+#ifdef USE_CUDA
+    /* get cublas handle */
+    cublasHandle_t * GetCublasHandle();
+
+    /* get the stream of cuda */
+    cudaStream_t * GetCudaStream();
+#endif
+
+    /* switch to a device */
+    static
+    void SetGPUDevice(int devID);
+
+    /* switch to a device (with fast GPU execution mode) */
+    static
+    void SetGPUDeviceFast(int devID);
+
+    /* switch to a get current dev */
+    static
+    int GetGPUDevice();
+
+    /* reset cuda flag for more efficient cuda execution */
+    static
+    void SetFastFlags();
+
+    /* reset cuda flag for more efficient cuda execution (all devices) */
+    static
+    void SetFastFlagsAllDevices();
+};
+
+/*
+a class for the management of devices
+*/
+class XDevManager
+{
+public:
+    /* CPU device information */
+    XDevice CPUs[MAX_CPU_NUM];
+
+    /* number of CPUs */
+    int nCPU;
+
+    /* GPU device information */
+    XDevice GPUs[MAX_GPU_NUM];
+
+    /* number of GPUs */
+    int nGPU;
+
+public:
+    /* constructor */
+    XDevManager();
+
+    /* de-constructor */
+    ~XDevManager();
+
+    /* initialize it and get the CPU and GPU information */
+    void Init();
+
+    /* clear it */
+    void Clear();
+
+#ifdef USE_CUDA
+    /* get the handle of GPU */
+    cublasHandle_t * GetCudaHandle(const int devID);
+
+    /* get the stream of cuda */
+    cudaStream_t * GetCudaStream(const int devID);
+#endif
+
+    /* get grid and block sizes that max potential */
+    int GetCudaThread(const int devID, const int n, int * gridSize, int * blockSize);
+
+    /* get grid and block sizes that max potential (2-dimension assignment) */
+    int GetCudaThread2D(const int devID, const int n, const int m, int nLimit, int * gridSize, int * blockSize);
+
+    /* get device ids for the given device information */
+    int GetDeviceIDs(char * devInfo, int * devIDs);
+
+    /* show id sequence */
+    void ShowDeviceIDs(char * devInfo, char * msg);
+
+    /* show device information */
+    void ShowDevInfo();
+
+    /* get the device information in string */
+    char * GetDevString(int devID);
+};
+
+/* managing the devices */
+extern XDevManager GDevs;
+
+/* keep the device config */
+
+#define ProtectCudaDev(devID, devIDBackup) \
+{ \
+    cudaGetDevice(&devIDBackup); \
+    if(devIDBackup != devID) \
+        cudaSetDevice(devID); \
+} \
+
+#define BacktoCudaDev(devID, devIDBackup) \
+{ \
+    if(devIDBackup != devID) \
+        cudaSetDevice(devIDBackup); \
+} \
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif

source/tensor/XGlobal.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-01-20
+ *
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "XGlobal.h"
+
+#if !defined( WIN32 ) && !defined( _WIN32 )
+    #include "sys/time.h"
+    #include "time.h"
+    #include "iconv.h"
+#else
+    #include "time.h"
+    #include "windows.h"
+    #include "process.h"
+#endif
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+/* memory pool setting */
+int MAX_MEM_BLOCK_NUM = 1024;
+int MAX_MEM_BLOCK_SIZE = 1024 * 1024 * 256;
+int MIN_MEM_BLOCK_SIZE = 1024 * 1024 * 64;
+int MINOR_MEM_BLOCK_SIZE = 1024 * 1024 * 256;
+int MAX_MEM_BUF_SIZE = 1024 * 1024 * 256;
+int MIN_MEM_BUF_SIZE = 1024 * 1024 * 32;
+int TRAINING_SAMPLE_BUF_SIZE = 1024 * 1024 * 16;
+int CONST_MINUSONE = -1;
+bool CONST_TRUE = true;
+
+int verboseLevel = 0;
+bool useBLAS = false;
+bool useCUDA = false;
+
+FILE * tmpLog = NULL;
+double myTime = 0;
+double myTime2 = 0;
+double myTime3 = 0;
+double myTime4 = 0;
+double myTime5 = 0;
+double myTime6 = 0;
+double myTime7 = 0;
+double myTime8 = 0;
+double myTime9 = 0;
+double myTimeForward1 = 0;
+double myTimeForward2 = 0;
+double myTimeForward3 = 0;
+double myTimeBackward1 = 0;
+double myTimeBackward2 = 0;
+double myTimeBackward3 = 0;
+double myTimeBackward4 = 0;
+
+int dEdWCount = 0;
+FILE * tF;
+
+/* initialization of the global stuff */
+void InitGlobalAll()
+{
+    srand((unsigned int)time(NULL));
+}
+
+} /* end of the nts (NiuTrans.Tensor) namespace */

source/tensor/XGlobal.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-01-20
+ *
+ */
+
+#ifndef __XGLOBAL_H__
+#define __XGLOBAL_H__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <float.h>
+
+#ifndef WIN32
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+// the CUDA stuff
+#ifdef USE_CUDA
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#endif
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts {
+
+#if (__cplusplus >= 201103L || _MSC_VER >= 1700)
+#define USE_CPP11
+#endif
+
+#define _XINLINE_  
+
+//#define DOUBELPRICSION
+
+#ifdef DOUBELPRICSION
+#define DTYPE double
+#define DTYPE_MIN (DTYPE)-1.79E+308
+#else
+#define DTYPE float
+#define DTYPE_MIN (DTYPE)-3.40E+38
+#endif
+
+#define LOGPROB_MIN (DTYPE)-2E+1
+#define GRAD_MAX (DTYPE)1E+5
+
+#if WIN32
+#define DELIMITER '\\'
+#else
+#define DELIMITER '/'
+#endif
+
+#ifndef MIN
+#define MIN(a,b) ((a) < (b) ? a : b)
+#endif
+#ifndef MAX
+#define MAX(a,b) ((a) > (b) ? a : b)
+#endif
+
+#define __FILENAME__ ( strrchr(__FILE__, DELIMITER) != NULL ? strrchr(__FILE__, DELIMITER)+1 : __FILE__ )
+
+#define CheckNTErrors(x, msg) \
+{ \
+    if(!(x)) \
+    { \
+        fprintf(stderr, "[ERROR] calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__, msg); \
+        exit(1); \
+    } \
+} \
+
+#define CheckNTErrorsV0(x) \
+{ \
+    if(!(x)) \
+    { \
+        fprintf(stderr, "[ERROR] calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__); \
+        exit(1); \
+    } \
+} \
+
+#define ShowNTErrors(msg) \
+{ \
+    { \
+        fprintf(stderr, "[ERROR] (%s line %d): %s\n", __FILENAME__, __LINE__, msg); \
+        exit(1); \
+    } \
+} \
+
+#define MAX_FILE_NAME_LENGTH 1024 * 2
+#define MAX_LINE_LENGTH 1024*1024
+#define MAX_SENTENCE_LEN 512
+#define X_MILLION 1000000
+#define MAX_INT 2147483647
+#define MAX_FLOAT FLT_MAX
+#define FIELD_SEP " ||| "
+#define FLOAT_MIN float(-1.0E38)
+#define FLOAT16_MIN float(-65504)
+#define MILLION 1000000
+#define LOG_E_10 2.302585
+#define LEADING_DIM 1
+
+/* cuda setting */
+#define MAX_CUDA_THREAD_NUM_PER_BLOCK 512
+#define MIN_CUDA_SHARED_MEM_COL_SIZE 8
+#define MAX_MODEL_NUM 512
+#define SHARED_MEMORY_SIZE (48 << 10)
+
+/* memory pool setting */
+extern int MAX_MEM_BLOCK_NUM;
+extern int MAX_MEM_BLOCK_SIZE;
+extern int MIN_MEM_BLOCK_SIZE;
+extern int MINOR_MEM_BLOCK_SIZE;
+extern int MAX_MEM_BUF_SIZE;
+extern int MIN_MEM_BUF_SIZE;
+extern int TRAINING_SAMPLE_BUF_SIZE;
+
+extern int CONST_MINUSONE;
+extern bool CONST_TRUE;
+
+//#define USE_CUDA_RESURSION 1
+
+#define NIUTRANSNNDEBUG
+
+extern int verboseLevel;
+extern bool useBLAS;
+extern bool useCUDA;
+
+#define FFLUSH(FILEH) \
+{ \
+    fflush(FILEH); \
+} \
+
+#define XPRINT(VERBOSE,FILEH,STR) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR);FFLUSH(FILEH);}}
+#define XPRINT1(VERBOSE,FILEH,STR,ARG) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG);FFLUSH(FILEH);}}
+#define XPRINT2(VERBOSE,FILEH,STR,ARG,ARG2) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2);FFLUSH(FILEH);}}
+#define XPRINT3(VERBOSE,FILEH,STR,ARG,ARG2,ARG3) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3);FFLUSH(FILEH);}}
+#define XPRINT4(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4);FFLUSH(FILEH);}}
+#define XPRINT5(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5);FFLUSH(FILEH);}}
+#define XPRINT6(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6);FFLUSH(FILEH);}}
+#define XPRINT7(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7);FFLUSH(FILEH);}}
+#define XPRINT8(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7,ARG8) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7,ARG8);FFLUSH(FILEH);}}
+
+#define B2I(V) V==0?false:true
+
+/* BLAS interfaces */
+#ifdef DOUBELPRICSION
+#define GEMM XBLAS_DGEMM
+#else
+#define GEMM XBLAS_SGEMM
+#endif
+
+extern void InitGlobalAll();
+
+extern FILE * tmpLog;
+
+extern int dEdWCount;
+extern FILE * tF;
+extern int tmpCountV2;
+extern int nnnTotal;
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif