更新部分功能的float16

source/network/XBackwardData.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for data operation
+ * $Created by: Xu Chen (email: hello_master1954@163.com) 2018-12-26
+ */
+
+#include "XNoder.h"
+#include "XBackwardData.h"
+#include "../tensor/XName.h"
+#include "../tensor/XUtility.h"
+#include "../tensor/core/CHeader.h"
+#include "../tensor/core/getandset/SetData.h"
+
+namespace nts{
+
+/* compute dE/dx of a node */
+void XDataGrad::MakeGrad(XTensor * node, bool isEfficent)
+{
+    CheckNTErrors(node->grad != NULL, "No gradient found!");
+
+    XLink &income = node->income;
+    int operID = income.typeID;
+
+    if(operID == GETANDSET_CONVERTDATATYPE)
+        GradConvertDataType(node, isEfficent);
+    else if(operID == GETANDSET_INDEXTOONEHOT)
+        GradIndexToOnehot(node, isEfficent);
+    else if(operID == GETANDSET_ONEHOTTOINDEX)
+        GradOnehotToIndex(node, isEfficent);
+    else{
+        ShowNTErrors("TODO!");
+    }
+}
+
+/* indicates whether the node is for a data operation */
+bool XDataGrad::IsDataOP(XTensor * node)
+{
+    XLink &income = node->income;
+    return (income.typeID & DATA_BASE) != 0;
+}
+
+/* 
+gradient computation for convert datatype
+for
+b = converdatatype(a) 
+we have
+dE/da = convertdatatype(b)
+>> node - the node (c) for backward computation
+>> isEfficient - indicates whether the computation is in
+                 an efficient manner
+*/
+void XDataGrad::GradConvertDataType(XTensor * node, bool isEfficent)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum > 0, "Wrong input tensor number for ConvertDataType!");
+
+    XTensor * input = income.tails[0];
+
+    XNoder::MakeGrad(input);
+
+    XTensor * tmp = NewTensorBuf(input->grad, input->devID, input->mem);
+    _ConvertDataType(node->grad, tmp);
+    _SumMe(input->grad, tmp);
+    DelTensorBuf(tmp);
+
+    node->visitMark = NODE_FINISHED;
+}
+
+/* 
+gradient computation for OnehotToIndex
+for
+b = OnehotToIndex(a) 
+we have
+dE/da = IndexToOnehot(b)
+>> node - the node (c) for backward computation
+>> isEfficient - indicates whether the computation is in
+                 an efficient manner
+*/
+void XDataGrad::GradIndexToOnehot(XTensor * node, bool isEfficent)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum > 0, "Wrong input tensor number for IndexToOnehot!");
+
+    XTensor * input = income.tails[0];
+
+    XNoder::MakeGrad(input);
+
+    node->visitMark = NODE_FINISHED;
+
+}
+
+void XDataGrad::GradOnehotToIndex(XTensor * node, bool isEfficent)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum > 0, "Wrong input tensor number for IndexToOnehot!");
+
+    XTensor * input = income.tails[0];
+
+    XNoder::MakeGrad(input);
+
+    node->visitMark = NODE_FINISHED;
+
+}
+
+} // namespace nts(NiuTrans.Tensor)

source/network/XBackwardData.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for data operation
+ * $Created by: Xu Chen (email: hello_master1954@163.com) 2018-12-26
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDDATA_H__
+#define __XBACKWARDDATA_H__
+
+namespace nts{
+
+/* this class computes the gradient for tensor data operation given a node */
+class XDataGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficent);
+
+    /* indicates whether the node is for a shaping operation */
+    static
+    bool IsDataOP(XTensor * node);
+
+private:
+    
+    /* gradient computation for ConverDataType: b = converdatatype(a, datatype) */
+    static
+    void GradConvertDataType(XTensor * node, bool isEfficent);
+
+    /* gradient computation for IndexToOnehot: b = indextoonehot(a, num) */
+    static
+    void GradIndexToOnehot(XTensor * node, bool isEfficent);
+
+    /* gradient computation for OnehotToIndex: b = onehottoindex(a, num) */
+    static
+    void GradOnehotToIndex(XTensor * node, bool isEfficent);
+};
+
+} // namespace nts(NiuTrans.Tensor)
+
+#endif
\ No newline at end of file

source/network/XBackwardFunc.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for activation function
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ * Dingdang won 5 games in the GO training yesterday, hahaha ...
+ */
+
+#include "XNoder.h"
+#include "XBackwardFunc.h"
+#include "../tensor/XName.h"
+#include "../tensor/function/FHeader.h"
+
+namespace nts{
+
+/* compute dE/dx of a node */
+void XFuncGrad::MakeGrad(XTensor * node, bool isEfficient)
+{
+    XLink &income = node->income;
+    int operID = income.typeID;
+
+    CheckNTErrors(node->grad != NULL, "No gradient found!");
+    CheckNTErrors(income.tailNum == 1, "Too many input tensors for the function!");
+
+    XTensor * input = income.tails[0];
+    XTensor * output = node;
+
+    XNoder::MakeGrad(input);
+
+    if(operID == FUNC_HARDTANH)
+        _HardTanHBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_IDENTITY)
+        _IdentityBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_LOGSOFTMAX){
+        int leadDim = income.GetParamInt(0);
+        CheckNTErrors(leadDim >= 0 && leadDim < input->order, "wrong leading dimension in logsoftmax!");
+        _LogSoftmaxBackward(NULL, output, input, output->grad, input->grad, NULL, leadDim, NOLOSS);
+    }
+    else if(operID == FUNC_RECTIFY)
+        _RectifyBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_SIGMOID)
+        _SigmoidBackward(NULL, output, input, output->grad, input->grad, NOLOSS);
+    else if(operID == FUNC_SOFTMAX){
+        int leadDim = income.GetParamInt(0);
+        CheckNTErrors(leadDim >= 0 && leadDim < input->order, "wrong leading dimension in softmax!");
+        _SoftmaxBackward(NULL, output, input, output->grad, input->grad, NULL, leadDim, NOLOSS);
+    }
+    else{
+        ShowNTErrors("Wrong activation function type!");
+    }
+
+    node->visitMark = NODE_FINISHED;
+}
+
+/* indicates whether the node is for an activation function */
+bool XFuncGrad::IsFunc(XTensor * node)
+{
+    XLink &income = node->income;
+    return (income.typeID & FUNCTION_BASE) != 0;
+}
+
+}

source/network/XBackwardFunc.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for activation function
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ * Dingdang won 5 games in the GO training yesterday, hahaha ...
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDFUNC_H__
+#define __XBACKWARDFUNC_H__
+
+namespace nts{
+
+/* this class computes the gradient for activation functions given a node */
+class XFuncGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficient);
+
+    /* indicates whether the node is for an activation function */
+    static
+    bool IsFunc(XTensor * node);
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardLoss.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-17
+ */
+
+#include "XBackwardLoss.h"
+#include "../tensor/XName.h"
+#include "../tensor/core/getandset/SetData.h"
+#include "../tensor/function/HardTanH.h"
+#include "../tensor/function/Identity.h"
+#include "../tensor/function/LogSoftmax.h"
+#include "../tensor/function/Rectify.h"
+#include "../tensor/function/Sigmoid.h"
+#include "../tensor/function/Softmax.h"
+
+namespace nts{
+
+/* 
+compute dE/dx for a given function y = f(x) 
+>> gold - gold standard to measure error (or loss)
+>> y - output of the function
+>> x - input of the function
+>> dedy - dE/dy
+>> dedx - dE/dx
+>> funcID - id of the function f
+>> params - parameters of the function
+>> lossName - name of the loss, e.g., cross entropy
+*/
+void XLossGrad::Compute(XTensor * gold, XTensor * y, XTensor * x, 
+                        XTensor * dedy, XTensor * dedx, XTensor * padding,
+                        int funcID, void * params,
+                        LOSS_FUNCTION_NAME lossName)
+{
+    CheckNTErrors(gold && y && x, "Empty input tensors!");
+    CheckNTErrors(dedx, "Empty gradient tensors!");
+    CheckNTErrors((funcID & FUNCTION_BASE) != 0, "Illegal function id");
+
+    if(funcID == FUNC_HARDTANH){
+        _HardTanHBackward(gold, y, x, dedy, dedx, lossName);
+    }
+    else if(funcID == FUNC_IDENTITY){
+        _IdentityBackward(gold, y, x, dedy, dedx, lossName);
+    }
+    else if(funcID == FUNC_LOGSOFTMAX){
+        int leadDim = *(int*)params;
+        _LogSoftmaxBackward(gold, y, x, dedy, dedx, padding, leadDim, lossName);
+    }
+    else if(funcID == FUNC_RECTIFY){
+        _RectifyBackward(gold, y, x, dedy, dedx, lossName);
+    }
+    else if(funcID == FUNC_SIGMOID){
+        _SigmoidBackward(gold, y, x, dedy, dedx, lossName);
+    }else if(funcID == FUNC_SOFTMAX){
+        int leadDim = *(int*)params;
+        _SoftmaxBackward(gold, y, x, dedy, dedx, padding, leadDim, lossName);
+    }
+    else{
+        ShowNTErrors("wrong function found when call the backward process!");
+    }
+
+}
+
+/* 
+compute dE/dy for variable y and error(loss) function E
+>> gold - gold standard to measure error (or loss)
+>> y - output of the function
+>> dedy - dE/dy
+>> lossName - name of the loss, e.g., cross entropy
+*/
+void XLossGrad::Compute(XTensor * gold, XTensor * y, 
+                        XTensor * dedy, XTensor * padding,
+                        LOSS_FUNCTION_NAME lossName)
+{
+    //return;
+
+    if(gold == NULL){
+        if(dedy->dataType == X_FLOAT)
+            _SetDataFixedFloat(dedy, 1.0F);
+        else if(dedy->dataType == X_DOUBLE)
+            _SetDataFixedDouble(dedy, 1.0);
+        else if(dedy->dataType == X_INT)
+            _SetDataFixedInt(dedy, 1);
+        else{
+            //ShowNTErrors("TODO");
+        }
+        return;
+    }
+
+    //_LossBackward(dedy, gold, y, lossName);
+    if(lossName == CROSSENTROPY)
+        _CrossEntropyBackward(dedy, y, gold, NULL, padding);
+
+}
+
+}
\ No newline at end of file

source/network/XBackwardLoss.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-17
+ * My students worked all night to prepare a submission to CWMT. Good luck
+ * to them!
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDLOSS_H__
+#define __XBACKWARDLOSS_H__
+
+namespace nts{
+
+/* this class computes the gradient (of a output node) 
+   with respect to the loss */
+class XLossGrad
+{
+public:
+    /* compute dE/dx for a given function y = f(x) */
+    void Compute(XTensor * gold, XTensor * y, XTensor * x, 
+                 XTensor * dedy, XTensor * dedx, XTensor * padding,
+                 int funcID, void * params,
+                 LOSS_FUNCTION_NAME lossName);
+
+    /* compute dE/dy for variable y and error(loss) function E */
+    void Compute(XTensor * gold, XTensor * y, 
+                 XTensor * dedy, XTensor * padding,
+                 LOSS_FUNCTION_NAME lossName);
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardMath.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for math operations
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+
+#ifndef __XBACKWARDMATH_H__
+#define __XBACKWARDMATH_H__
+
+namespace nts{
+
+/* this class computes the gradient for math operations given a node */
+class XMathGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficient);
+
+    /* indicates whether the node is for a math operation */
+    static
+    bool IsMathOP(XTensor * node);
+
+private:
+    
+    /* gradient for absolute */
+    static
+    void GradAbsolute(XTensor * node, bool isEfficient);
+    
+    /* gradient for cos */
+    static
+    void GradCos(XTensor * node, bool isEfficient);
+    
+    /* gradient for exp */
+    static
+    void GradExp(XTensor * node, bool isEfficient);
+
+    /* gradient for log: c =  log(a) */
+    static
+    void GradLog(XTensor * node, bool isEfficient);
+    
+    /* gradient for round */
+    static
+    void GradRound(XTensor * node, bool isEfficient);
+    
+    /* gradient for sign */
+    static
+    void GradSign(XTensor * node, bool isEfficient);
+
+    /* gradient for sin */
+    static
+    void GradSin(XTensor * node, bool isEfficient);
+
+    /* gradient for tan */
+    static
+    void GradTan(XTensor * node, bool isEfficient);
+
+    /* gradient for clip */
+    static
+    void GradClip(XTensor * node, bool isEfficient);
+
+    /* gradient for Divide */
+    static
+    void GradDiv(XTensor * node, bool isEfficient);
+
+    /* gradient for DivideDim */
+    static
+    void GradDivDim(XTensor * node, bool isEfficient);
+
+    /* gradient for matrix multiply: c = matmul(a, b) * \alpha */
+    static
+    void GradMatrixMul(XTensor * node, bool isEfficient);
+    
+    /* gradient for matrix multiply: c = matmul(a, b) * \alpha */
+    static
+    void GradMatrixMul(XTensor * a, XTensor * deda, MATRIX_TRANS_TYPE transA,
+                       XTensor * b, XTensor * dedb, MATRIX_TRANS_TYPE transB,
+                       XTensor * dedc, DTYPE alpha, bool isEfficient);
+
+    /* gradient for matrix multiply in batch mode.
+       for each batch: c_i = matmul(a_i, b_i) * \alpha */
+    static
+    void GradMatrixMulBatched(XTensor * node, bool isEfficient);
+
+    /* gradient for multiply (dot production): c =  a * b * \alpha */
+    static
+    void GradMultiply(XTensor * node, bool isEfficient);
+
+    /* gradient for multiply one dimension: c =  a * b * \alpha 
+       where the size of b is equal to that of one dimension of a */
+    static
+    void GradMultiplyDim(XTensor * node, bool isEfficient);
+
+    /* gradient for negate */
+    static
+    void GradNegate(XTensor * node, bool isEfficient);
+    
+    /* gradient for normalize */
+    static
+    void GradNormalize(XTensor * node, bool isEfficient);
+
+    /* gradient for power */
+    static
+    void GradPower(XTensor * node, bool isEfficient);
+
+    /* gradient for ScaleAndShift */
+    static
+    void GradScaleAndShift(XTensor * node, bool isEfficient);
+
+    /* gradient for Minus */
+    static
+    void GradSub(XTensor * node, bool isEfficient);
+    
+	/* gradient for sub with one dimension: c = a - b * \beta
+	where the size of b is equal to that of one dimension of a */
+	static
+	void GradSubDim(XTensor * node, bool isEfficient);
+
+    /* gradient for sum: c =  a + b * \beta */
+    static
+    void GradSum(XTensor * node, bool isEfficient);
+
+    /* gradient for sum with one dimension: c = a + b * \beta
+       where the size of b is equal to that of one dimension of a */
+    static
+    void GradSumDim(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceMean */
+    static
+    void GradReduceMean(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceSum */
+    static
+    void GradReduceSum(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceSumSquared */
+    static
+    void GradReduceSumSquared(XTensor * node, bool isEfficient);
+
+    /* gradient for reduceVariance */
+    static
+    void GradReduceVariance(XTensor * node, bool isEfficient);
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardShape.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for shaping and data movement
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDSHAPE_H__
+#define __XBACKWARDSHAPE_H__
+
+namespace nts{
+
+/* this class computes the gradient for tensor shaping and movement given a node */
+class XShapeGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node, bool isEfficent);
+
+    /* indicates whether the node is for a shaping operation */
+    static
+    bool IsShapeOP(XTensor * node);
+
+    /* post processing of a node */
+    static
+    void PostProcessing(XTensor * node, int typeId, bool isEfficent);
+
+private:
+    
+    /* gradient computation for copying indexed sub-tensors: b = copyindexed(a, srcIndex, indexSize, tgtIndex, copyNum) */
+    static
+    void GradCopyIndexed(XTensor * node, bool isEfficent);
+        
+    /* gradient computation for copying indexed sub-tensors: b = gather(a, index) */
+    static
+    void GradGather(XTensor * node, bool isEfficent);
+
+    /* gradient computation for merge: c = merge(a, b, ...) */
+    static
+    void GradMerge(XTensor * node, bool isEfficent);
+
+    /* gradient computation for merging a list of tensors : c = merge(list(a, b, ...)) */
+    static
+    void GradMergeList(XTensor * node, bool isEfficent);
+    
+    /* gradient computation for transposing a tensor : b = transpose(a) */
+    static
+    void GradTranspose(XTensor * node, bool isEfficent);
+
+    /* gradient computation for reshaping a tensor: c = reshape(a) */
+    static
+    void GradReshape(XTensor * node, bool isEfficent);
+
+    /* gradient computation for split: c = split(a) */
+    static
+    void GradSplit(XTensor * node, bool isEfficent);
+
+    /* gradient computation for spliting. we return the list of the splits : list(c_1, ...) = split(a) */
+    static
+    void GradSplitList(XTensor * node, bool isEfficent);
+
+    /* gradient computation for spliting. we return the list of the splits : list(c_1, ...) = split(a).
+       this method is called only when all nodes of spliting have been processed. We do this in a post-processing
+       manner because we can fuze multiple memory copy jobs one time. This is good for system speed up. */
+    static
+    void GradSplitListPost(XTensor * node, bool isEfficent);
+
+    /* gradient computation for unsqueezing a tensor : c = unsqueeze(a) */
+    static
+    void GradUnsqueeze(XTensor * node, bool isEfficent);
+
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XNet.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-12
+ * We expected a heavy rain today but a drizzle came down. Should I
+ * take a big umbrella?
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XNET_H__
+#define __XNET_H__
+
+namespace nts{
+
+/* management of tensor net (or graph) */
+struct XNet
+{
+    /* id of the network */
+    unsigned int id;
+
+    /* tensor nodes of the network (in order) */
+    XList nodes;
+
+    /* tensor nodes to keep gradient for output (e.g., SGD)*/
+    XList gradNodes;
+
+    /* output nodes of the network */
+    XList outputs;
+
+    /* input nodes of the network */
+    XList inputs;
+
+    /* indicates whether the network just keeps the gradient for parameter tensors */
+    bool isGradEfficient;
+
+    /* constructor */
+    XNet();
+
+    /* de-constructor */
+    ~XNet();
+
+    /* clear the network */
+    void Clear();
+
+    /* backward propagation to obtain gradient */
+    void Backward(XTensor &root, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient wrt. the loss/error function */
+    void Backward(XTensor &root, XTensor &gold, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient wrt. the loss/error function */
+    void Backward(XTensor &root, XTensor &gold, XTensor &padding, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient
+       with a number of root nodes */
+    void Backward(XList &roots, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient
+       with a number of root nodes */
+    void Backward(XList &roots, XList &golds, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward propagation to obtain gradient wrt. the loss/error function
+       with a number of root nodes */
+    void Backward(XList &roots, XList &golds, XList &paddings, LOSS_FUNCTION_NAME loss = NOLOSS);
+
+    /* backward computation for a given node */
+    void BackwardNode(XTensor * node, bool isEfficent = false);
+
+    /* backward computation (in post processing) for a given node */
+    void BackwardNodePost(XTensor * node, bool isEfficent = false);
+
+    /* traverse the net and find the topological order by 
+       depth-first search (Tarjan's algorithm) */
+    void Traverse(XTensor &root);
+
+    /* traverse the net and find the topological order by 
+       depth-first search (Tarjan's algorithm) */
+    void Traverse(XList &roots);
+
+    /* depth-first search given a node (Tarjan's algorithm for topological ordering) */
+    void TarjanVisit(XTensor * node, XList &orders, const unsigned int code);
+
+    /* dump network information */
+    void Dump(FILE * file);
+
+    /* set the flag of gradient-efficient */
+    void SetGradEfficientFlag(bool flag = true);
+
+    /* generate the gradient-efficient flag for every node */
+    void MakeEfficientNet();
+
+    /* clear the graident information if the node is no use */
+    void ClearGrad(XTensor * node);
+
+    /* show network topology */
+    void ShowNetwork(FILE * file, XTensor * node);
+};
+
+/* we make a unique id for every tensor */
+extern unsigned int netIDGlobal;
+extern MUTEX_HANDLE netMutex;
+extern unsigned int MakeNetID();
+extern void XNetClearAll();
+
+}
+
+#endif
\ No newline at end of file

source/network/XNoder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "XNoder.h"
+
+namespace nts{
+
+/* make gradient tensor for a node */
+void XNoder::MakeGrad(XTensor * node)
+{
+    if(node == NULL)
+        return;
+
+    if(!XTensor::IsSameShaped(node, node->grad)){
+        delete node->grad;
+        node->grad = NewTensor(node);
+        node->grad->SetZeroAll();
+    }
+}
+
+/* the node is a leaf node (intput) or not */
+bool XNoder::IsLeaf(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+
+    if(node->income.tailNum == 0)
+        return true;
+    else
+        return false;
+}
+
+/* the node is a root node (output) or not */
+bool XNoder::IsRoot(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+
+    if(node->outgo.tailNum == 0)
+        return true;
+    else
+        return false;
+}
+
+/* the node keeps the gradinent or not */
+bool XNoder::IsGrad(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+    
+    if(node->isGrad)
+        return true;
+    else
+        return false;
+}
+
+}
\ No newline at end of file

source/network/XNoder.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * low-level utilities
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+
+#ifndef __XNODER_H__
+#define __XNODER_H__
+
+namespace nts{
+
+#define NODE_UNFINISHED 0
+#define NODE_DOING      1
+#define NODE_FINISHED   2
+
+/* node management */
+class XNoder
+{
+public:
+    /* make gradient tensor for a node */
+    static
+    void MakeGrad(XTensor * node);
+
+    /* the node is a leaf node (intput) or not */
+    static
+    bool IsLeaf(XTensor * node);
+
+    /* the node is a root node (output) or not */
+    static
+    bool IsRoot(XTensor * node);
+
+    /* the node keeps the gradinent or not */
+    static
+    bool IsGrad(XTensor * node);
+};
+
+}
+
+#endif
\ No newline at end of file

source/sample/fnnlm/FNNLM.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * This is a simple impelementation of the feed-forward network-baesd language
+ * model (FNNLM). See more details about FNNLM in
+ * "A Neural Probabilistic Language Model" by Bengio et al.
+ * Journal of Machine Learning Research 3 (2003) 1137¨C1155
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-06-22
+ * Today I was awarded as the most popular teacher in our college. 
+ * It was the great honour for me!!!    
+ */
+
+#ifndef __FNNLM_H__
+#define __FNNLM_H__
+
+#include "../../tensor/XGlobal.h"
+#include "../../tensor/XTensor.h"
+#include "../../tensor/core/CHeader.h"
+
+using namespace nts;
+
+namespace fnnlm
+{
+
+#define _EXIT_(x)// exit(x)
+#define CheckErrors(x, msg) { if(!(x)) { fprintf(stderr, "Error! calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__, msg);  _EXIT_(1); } }
+#define ShowErrors(msg) { { fprintf(stderr, "Error! (%s line %d): %s\n", __FILENAME__, __LINE__, msg); _EXIT_(1); } } 
+
+#define MAX_N_GRAM 8
+#define MAX_HIDDEN_NUM 8
+
+/* an n-gram = a sequence of n words 
+   words[0..n-2] is the history, and
+   words[n-1] is the word for prediction. */
+struct NGram
+{
+    int words[MAX_N_GRAM];
+};
+
+/* fnn model */
+struct FNNModel
+{
+    /* word embedding */
+    XTensor embeddingW;
+
+    /* parameter matrix of each hidden layer 
+       hidden layer: y = f(x * w + b)
+       where x is the input, y is the output, w is
+       the tranformation (parameter) matrix, b is
+       the bias and f() is the activation function. */
+    XTensor hiddenW[MAX_HIDDEN_NUM];
+
+    /* bias of each hidden layer */
+    XTensor hiddenB[MAX_HIDDEN_NUM];
+
+    /* parameter matrix of the output layer */
+    XTensor outputW;
+
+    /* bias of the output layer */
+    XTensor outputB;
+
+    /* order of the language model */
+    int n;
+
+    /* embedding size */
+    int eSize;
+    
+    /* number of hidden layers */
+    int hDepth;
+
+    /* hidden layer size */
+    int hSize;
+
+    /* vocabulary size */
+    int vSize;
+
+    /* id of the device for running the model */
+    int devID;
+
+    /* indicates whether we use memory pool */
+    bool useMemPool;
+
+    /* memory pool */
+    XMem * mem;
+
+    FNNModel(){ n = -1; vSize = -1;hDepth = 0;devID = -1;mem = NULL;};
+    ~FNNModel(){delete mem;};
+};
+
+/* the network built on the fly */
+struct FNNNet
+{
+    /* embedding result of the previous n - 1 words */
+    XTensor embeddings[MAX_N_GRAM];
+
+    /* concatenation of embeddings */
+    XTensor embeddingCat;
+
+    /* output of the hidden layers */
+    XTensor hiddens[MAX_HIDDEN_NUM];
+
+    /* state of the hidden layers (before activation function) */
+    XTensor hiddenStates[MAX_HIDDEN_NUM];
+
+    /* state before softmax */
+    XTensor stateLast;
+
+    /* output of the net */
+    XTensor output;
+};
+
+/* entrance of the program */
+int FNNLMMain(int argc, const char ** argv);
+
+};
+
+#endif

source/sample/transformer/T2TAttention.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TAttention.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TAttention::T2TAttention()
+{
+    nhead = -1;
+    dk = -1;
+    dv = -1;
+    d  = -1;
+    isMasked = false;
+    ignored = 0;
+}
+
+/* deconstructor */
+T2TAttention::~T2TAttention()
+{
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myIgnored - number of position ignored in attention (from the begining)
+>> myIsMasked - indicates whether the attention is with a mask
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+
+void T2TAttention::InitModel(int argc, char ** argv,
+    bool myIsMasked, int myIgnored,
+    int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+    isMasked = myIsMasked;
+    ignored = myIgnored;
+
+    float minmax = 0;
+
+    LoadParamInt(argc, argv, "nhead", &nhead, 8);
+    LoadParamInt(argc, argv, "d", &dk, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &dv, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_EMBEDDING_SIZE);
+    LoadParamFloat(argc, argv, "attminmax", &minmax, 0.1F);
+    LoadParamFloat(argc, argv, "dropoutatt", &dropoutP, 0);
+
+    InitTensor2D(&wk, d, dk, X_FLOAT16, devID, mem);
+    InitTensor2D(&wq, d, dk, X_FLOAT16, devID, mem);
+    InitTensor2D(&wv, d, dv, X_FLOAT16, devID, mem);
+    InitTensor2D(&wa, d, d, X_FLOAT16, devID, mem);
+
+    float scale = 1.0F;
+    float finfoutk = (float)sqrt(6.0F * scale / (d + dk));
+    float finfoutv = (float)sqrt(6.0F * scale / (d + dv));
+    float finfouta = (float)sqrt(6.0F * scale / (d + d));
+
+    wk.SetDataRand(-finfoutk, finfoutk);
+    wq.SetDataRand(-finfoutk, finfoutk);
+    wv.SetDataRand(-finfoutv, finfoutv);
+    wa.SetDataRand(-finfouta, finfouta);
+    //_SetDataRand(&wk, -finfoutk, finfoutk);
+    //_SetDataRand(&wq, -finfoutk, finfoutk);
+    //_SetDataRand(&wv, -finfoutv, finfoutv);
+    //_SetDataRand(&wa, -finfouta, finfouta);
+}
+
+/* 
+make the network 
+>> k - keys. It might be of size B * L * H
+       where B = batch size, L = sequence length, 
+       and H = vector size of each position
+>> q - queries
+>> v - values
+>> mask - as it is
+>> isTraining - indicates whether the model is used for training
+<< return - multi-attention result
+*/
+
+XTensor T2TAttention::Make(XTensor &k, XTensor &q, XTensor &v, XTensor &mask, bool isTraining)
+{
+    XTensor halfK;
+
+    XTensor halfK2;
+    XTensor halfQ2;
+    XTensor halfV2;
+
+    halfK = ConvertDataType(k, X_FLOAT16);
+
+    halfK2 = MMul(halfK, wk);
+    halfQ2 = MMul(halfK, wq);
+    halfV2 = MMul(halfK, wv);
+
+    XTensor halfKheads;
+    XTensor halfQheads;
+    XTensor halfVheads;
+
+    halfKheads = Split(halfK2, halfK2.order - 1, nhead);
+    halfQheads = Split(halfQ2, halfQ2.order - 1, nhead);
+    halfVheads = Split(halfV2, halfV2.order - 1, nhead);
+
+    XTensor halfMask;
+    XTensor halfDot;
+    XTensor halfScalar;
+    XTensor halfAtt;
+
+    halfMask = ConvertDataType(mask, X_FLOAT16);
+
+    /* scalar = softmax(Q * K^T / sqrt(dk)) * V */
+    halfDot = BMMul(halfQheads, X_NOTRANS, halfKheads, X_TRANS);
+
+    if (isMasked) {
+        halfDot = Sum(halfDot, halfMask);
+    }
+
+    halfDot = Linear(halfDot, 1.0F / (float)sqrt((float)dk / nhead));
+
+    halfScalar = Softmax(halfDot, -1);
+
+    if (isTraining && dropoutP > 0)
+        halfScalar = Dropout(halfScalar, dropoutP);
+
+    /*att = BMMul(scalar, vheads);*/
+    halfAtt = BMMul(halfScalar, halfVheads);
+
+    /* concatenate the heads */
+    return ConvertDataType(MMul(Merge(halfAtt, halfAtt.order - 1), wa), X_FLOAT);
+
+}
+
+
+}

source/sample/transformer/T2TAttention.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TATTENTION_H__
+#define __T2TATTENTION_H__
+
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* 
+multi-head attention 
+y(Q, K, V) = cat(head_1, head_2, ..., head_n)
+where head_i = Attention(Q * w_i^Q, K * w_i^K, V * w_i^V)
+      attention(Q, K, V) = softmax(Q * K^T/d_k^0.5) V
+      d_k = dimension size of K
+*/
+class T2TAttention
+{
+public:
+    /* device id */
+    int devID;
+    
+    /* memory pool */
+    XMem * mem;
+    
+    /* head number */
+    int nhead;
+
+    /* transformation matrix for K */
+    XTensor wk;
+
+    /* transformation matrix for Q */
+    XTensor wq;
+
+    /* transformation matrix for V */
+    XTensor wv;
+
+    /* transformation after dot-product attention */
+    XTensor wa;
+
+    /* size of transformed Q and K */
+    int dk;
+
+    /* size of transformed V */
+    int dv;
+
+    /* size of input Q, K and V */
+    int d;
+
+    /* indicates whether the attention is masked */
+    bool isMasked;
+
+    /* some positions can be ignored in attention. this is useful in lm where the first position needs
+       special design for the attention model. */
+    int ignored;
+
+    /* indicates whether the model is used for training */
+    bool isTraining;
+    
+    /* dropout probability */
+    DTYPE dropoutP;
+
+public:
+    /* constructor */
+    T2TAttention();
+
+    /* de-constructor */
+    ~T2TAttention();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, 
+                   bool myIsMasked, int myIgnored, 
+                   int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the network */
+    XTensor Make(XTensor &k, XTensor &q, XTensor &v, XTensor &mask, bool isTraining);
+};
+
+}
+
+#endif

source/sample/transformer/T2TDecoder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-10-09
+ */
+
+#include <math.h>
+#include "T2TDecoder.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+AttDecoder::AttDecoder()
+{
+    attentionsEnde = NULL;
+    attEndeLayerNorms = NULL;
+}
+
+/* de-constructor */
+AttDecoder::~AttDecoder()
+{
+    delete[] attentionsEnde;
+    delete[] attEndeLayerNorms;
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myIsMasked - indicates whether the masked attention is employed
+>> myIgnored - number of positions ignored in attention (from the start)
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void AttDecoder::InitModel(int argc, char ** argv, 
+                           bool myIsMasked, int myIgnored, 
+                           int myDevID, XMem * myMem)
+{
+    AttEncoder::InitModel(argc, argv, myIsMasked, myIgnored, myDevID, myMem);
+
+    attentionsEnde = new T2TAttention[nlayer];
+    attEndeLayerNorms = new T2TLN[nlayer];
+
+    /* initialize the stacked layers */
+    for(int i = 0; i < nlayer; i++){
+        attentionsEnde[i].InitModel(argc, argv, myIsMasked, myIgnored, myDevID, myMem);
+        attEndeLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+    }
+}
+
+/* 
+make the decoding network
+>> inputDec - the input tensor of the decoder
+>> outputEnc - the output tensor of the encoder
+>> mask - mask that indicates which position is valid
+>> maskEncDec - mask for the encoder-decoder attention
+>> isTraining - indicates whether the model is used for training
+<< return - the output tensor of the encoder
+*/
+XTensor AttDecoder::Make(XTensor &inputDec, XTensor &outputEnc, XTensor &mask, XTensor &maskEncDec, bool isTraining)
+{
+    XTensor x;
+
+    x = embedder.Make(inputDec);
+
+    /* dropout */
+    if(isTraining && dropoutP > 0)
+        x = Dropout(x, dropoutP);
+
+    for(int i = 0; i < nlayer; i++){
+        XTensor att;
+        XTensor ende;
+        XTensor ln;
+        XTensor fnn;
+        XTensor res;
+
+        /******************/
+        /* self attention */
+        att = attentions[i].Make(x, x, x, mask, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            att = Dropout(att, dropoutP);
+
+        /* residual connection */
+        res = Sum(att, x);
+
+        /* layer normalization */
+        x = attLayerNorms[i].Make(res);
+
+        /*****************************/
+        /* encoder-decoder attention */
+        ende = attentionsEnde[i].Make(outputEnc, x, outputEnc, maskEncDec, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            ende = Dropout(ende, dropoutP);
+
+        /* residual connection */
+        res = Sum(ende, x);
+
+        /* layer normalization */
+        x = attEndeLayerNorms[i].Make(res);
+
+        /*******/
+        /* fnn */
+        fnn = fnns[i].Make(x, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            fnn = Dropout(fnn, dropoutP);
+
+        /* residual connection */
+        res = Sum(fnn, x);
+
+        /* layer normalization */
+        x = fnnLayerNorms[i].Make(res);
+    }
+
+    return x;
+}
+
+
+}

source/sample/transformer/T2TDecoder.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TDECODER_H__
+#define __T2TDECODER_H__
+
+#include "T2TEncoder.h"
+
+namespace transformer
+{
+
+class AttDecoder : public AttEncoder
+{
+public:
+    /* encoder-decoder attention model of each layer */
+    T2TAttention * attentionsEnde;
+
+    /* layer normalization for encoder-decoder attention */
+    T2TLN * attEndeLayerNorms;
+public:
+    /* constructor */
+    AttDecoder();
+
+    /* deconstructor */
+    ~AttDecoder();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, 
+                   bool myIsMasked, int myIgnored, 
+                   int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the decoding network */
+    XTensor Make(XTensor &inputDec, XTensor &outputEnc, XTensor &mask, XTensor &maskEncDec, bool isTraining);
+};
+
+}
+
+#endif
\ No newline at end of file

source/sample/transformer/T2TEmbedding.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-08-01
+ */
+
+#include <math.h>
+#include "T2TEmbedding.h"
+#include "T2TUtility.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TEmbedder::T2TEmbedder()
+{
+    devID = -1;
+    mem = NULL;
+    vSize = -1;
+    maxLength = -1;
+}
+
+/* deconstructor */
+T2TEmbedder::~T2TEmbedder()
+{
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TEmbedder::InitModel(int argc, char ** argv, int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+    
+    LoadParamInt(argc, argv, "vsize", &vSize, -1);
+    LoadParamInt(argc, argv, "maxlen", &maxLength, 512);
+    LoadParamInt(argc, argv, "d", &eSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_EMBEDDING_SIZE);
+
+    InitTensor2D(&w, vSize, eSize, X_FLOAT, devID, mem);
+
+    DTYPE v = 1.0F/(float)sqrt((float)eSize);
+    w.SetDataRandn(0, v);
+
+    /* create the positional embedding matrix */
+    MakePosEmbedding(eSize, d, maxLength);
+}
+
+/* 
+make positional embeddings (of size eSize * length)
+>> eSize - embedding size
+>> d - dimension size of the hidden layers
+>> length - length of the sequence
+*/
+void T2TEmbedder::MakePosEmbedding(int eSize, int d, int length)
+{
+    InitTensor2D(&posEmbeddingBase, length, eSize, X_FLOAT, devID, mem);
+
+    float * data = new float[posEmbeddingBase.unitNum];
+
+    for(int pos = 0; pos < length; pos++){
+        float * dp = data + pos * eSize;
+        
+        int channelSize = eSize / 2;
+        int offset = 0;
+        for(int i = 0; i < channelSize; i++){
+            dp[offset++] = (float)sin(pos/pow(10000.0F, 2.0F*i/(d - 2)));
+        }
+        for(int i = 0; i < channelSize; i++){
+            dp[offset++] = (float)cos(pos/pow(10000.0F, 2.0F*i/(d - 2)));
+        }
+
+        /*
+        for(int k = 0; k < eSize; k++){
+            if(k % 2 == 0){
+                int i = k/2;
+                dp[k] = (float)sin(pos/pow(10000.0F, 2.0F*i/d));
+            }
+            else{
+                int i = (k - 1)/2;
+                dp[k] = (float)cos(pos/pow(10000.0F, 2.0F*i/d));
+            }
+        }
+        */
+    }
+
+    posEmbeddingBase.SetData(data, posEmbeddingBase.unitNum);
+
+    delete[] data;
+}
+
+
+/* 
+make the network 
+*/
+XTensor T2TEmbedder::Make(XTensor &input)
+{
+    //CheckNTErrors(input.GetDim(-1) == vSize, "Wrong vocabulary size!");
+    CheckNTErrors(input.order > 1, "Wrong input tensor size!");
+    CheckNTErrors(input.dimSize[input.order - 1] < maxLength, "The sequence is too long!");
+    CheckNTErrors(vSize > 0, "set vocabulary size by \"-vsize\"");
+    CheckNTErrors(eSize > 0, "set embedding size by \"-esize\"");
+
+    int dims[MAX_TENSOR_DIM_NUM];
+    memcpy(dims, input.dimSize, input.order * sizeof(int));
+    dims[input.order] = eSize;
+
+    XTensor wordEmbedding;
+    XTensor posEmbedding;
+
+    bool match = (posEmbedding.order == input.order);
+    if(match){
+        for(int i = 0; i < input.order; i++){
+            if(dims[i] != posEmbedding.GetDim(i))
+                match = false;
+        }
+    }
+
+    /* we make positional embeddings first */
+    //if(!match){
+    if(true){
+        InitTensor(&posEmbedding, input.order + 1, dims, X_FLOAT, 1.0F, devID, mem);
+
+        XTensor * posTMP = NewTensorBuf(2, dims + 1, X_FLOAT, 1.0F, devID, mem);
+
+        _CopyValues(&posEmbeddingBase, 0, posTMP->unitNum, posTMP, 0);
+        _Unsqueeze(posTMP, &posEmbedding, 0, dims[0]);
+
+        DelTensorBuf(posTMP);
+    }
+
+    /* then we make word embeddings */
+    wordEmbedding = Gather(w, input);
+    wordEmbedding = Linear(wordEmbedding, (float)sqrt((float)eSize));
+
+    /* we sum over the two embeddings */
+    return wordEmbedding + posEmbedding;
+}
+
+}

source/sample/transformer/T2TEmbedding.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-08-01
+ */
+
+#ifndef __T2TEMBEDDING_H__
+#define __T2TEMBEDDING_H__
+
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+#define DEFAULT_EMBEDDING_SIZE 512
+
+/* 
+embedding (of word at position i):
+word embedding + positional embedding
+*/
+class T2TEmbedder
+{
+public:
+    /* device id */
+    int devID;
+    
+    /* memory pool */
+    XMem * mem;
+    
+    /* vocabulary size */
+    int vSize;
+
+    /* embedding size */
+    int eSize;
+
+    /* maximum length of the sequence */
+    int maxLength;
+
+    /* dimension size of the hidden layers in the t2t model */
+    int d;
+
+    /* word embedding matrix */
+    XTensor w;
+
+    /* predefined positional embeddings. It can speeds up 
+       the embedding processing by re-loading. */
+    XTensor posEmbeddingBase;
+
+public:
+    /* constructor */
+    T2TEmbedder();
+
+    /* de-constructor */
+    ~T2TEmbedder();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL);
+
+    /* make positional embeddings */
+    void MakePosEmbedding(int eSize, int d, int length);
+
+    /* make the network */
+    XTensor Make(XTensor &input);
+};
+
+}
+
+#endif

source/sample/transformer/T2TEncoder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TEncoder.h"
+#include "T2TLayerNormal.h"
+#include "T2TUtility.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+AttEncoder::AttEncoder()
+{
+    attentions = NULL;
+    fnns = NULL;
+    attLayerNorms = NULL;
+    fnnLayerNorms = NULL;
+}
+
+/* de-constructor */
+AttEncoder::~AttEncoder()
+{
+    delete[] attentions;
+    delete[] fnns;
+    delete[] attLayerNorms;
+    delete[] fnnLayerNorms;
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myIsMasked - indicates whether the masked attention is employed
+>> myIgnored - number of positions ignored in attention (from the start)
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void AttEncoder::InitModel(int argc, char ** argv, 
+                           bool myIsMasked, int myIgnored, 
+                           int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+    ignored = myIgnored;
+    
+    LoadParamInt(argc, argv, "nlayer", &nlayer, 6);
+    LoadParamInt(argc, argv, "hsize", &hSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "esize", &eSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "vsize", &vSize, -1);
+    LoadParamFloat(argc, argv, "dropout", &dropoutP, 0);
+
+    CheckNTErrors(nlayer >= 1, "We have one encoding layer at least!");
+    CheckNTErrors(vSize > 1, "set vocabulary size by \"-vsize\"");
+
+    /* embedding model */
+    embedder.InitModel(argc, argv, devID, mem);
+
+    attentions = new T2TAttention[nlayer];
+    fnns = new T2TFNN[nlayer];
+    attLayerNorms = new T2TLN[nlayer];
+    fnnLayerNorms = new T2TLN[nlayer];
+
+    /* initialize the stacked layers */
+    for(int i = 0; i < nlayer; i++){
+        attentions[i].InitModel(argc, argv, myIsMasked, myIgnored, myDevID, myMem);
+        fnns[i].InitModel(argc, argv, myDevID, myMem);
+        attLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+        fnnLayerNorms[i].InitModel(argc, argv, myDevID, myMem);
+    }
+}
+
+/* 
+make the encoding network
+>> input - the input tensor of the encoder
+>> mask - the mask that indicate each position is valid
+>> maskEncDec - no use
+>> isTraining - indicates whether the model is used for training
+<< return - the output tensor of the encoder
+*/
+XTensor AttEncoder::Make(XTensor &input, XTensor &mask, XTensor &maskEncDec, bool isTraining)
+{
+    XTensor x;
+
+    x = embedder.Make(input);
+
+    //x.Dump(tmpFILE, "embedding: ");
+
+    /* dropout */
+    if(isTraining && dropoutP > 0)
+        x = Dropout(x, dropoutP);
+
+    for(int i = 0; i < nlayer; i++){
+        XTensor att;
+        XTensor ln;
+        XTensor fnn;
+        XTensor res;
+
+        /* self attention */
+        att = attentions[i].Make(x, x, x, mask, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            att = Dropout(att, dropoutP);
+
+        /* residual connection */
+        res = Sum(att, x);
+
+        /* layer normalization */
+        x = attLayerNorms[i].Make(res);
+
+        /* fnn */
+        fnn = fnns[i].Make(x, isTraining);
+
+        /* dropout */
+        if(isTraining && dropoutP > 0)
+            fnn = Dropout(fnn, dropoutP);
+
+        /* residual connection */
+        res = Sum(fnn, x);
+
+        /* layer normalization */
+        x = fnnLayerNorms[i].Make(res);
+    }
+
+    return x;
+}
+
+/*
+make the encoding network (wrapper) 
+>> input - the input tensor of the encoder
+>> mask - the mask that indicate each position is valid
+>> isTraining - indicates whether the model is used for training
+<< return - the output tensor of the encoder
+*/
+XTensor AttEncoder::Make(XTensor &input, XTensor &mask, bool isTraining)
+{
+    XTensor nothing;
+
+    return Make(input, mask, nothing, isTraining);
+}
+
+}

source/sample/transformer/T2TEncoder.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TENCODER_H__
+#define __T2TENCODER_H__
+
+#include "T2TFNN.h"
+#include "T2TAttention.h"
+#include "T2TEmbedding.h"
+#include "T2TLayerNormal.h"
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* 
+base class of the encoder 
+*/
+class T2TEncoder
+{
+public:
+    virtual
+    XTensor Make(XTensor &input, XTensor &mask, XTensor &mask2, bool isTraining) = 0;
+};
+
+/* 
+the encoder based on RNN 
+*/
+class RNNEncoder : T2TEncoder
+{
+public:
+    XTensor Make(XTensor &input, XTensor &mask, XTensor &mask2, bool isTraining);
+};
+
+
+/* 
+the encoder based on self-attention 
+*/
+class AttEncoder : T2TEncoder
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* layer number */
+    int nlayer;
+
+    /* hidden layer size of the FNN layer */
+    int hSize;
+
+    /* embedding size */
+    int eSize;
+
+    /* vocabulary size */
+    int vSize;
+
+    /* dropout probability */
+    DTYPE dropoutP;
+
+    /* some positions can be ignored in attention. this is useful in lm where the first position needs
+       special design for the attention model. */
+    int ignored;
+
+    /* embedding of word at each position */
+    T2TEmbedder embedder;
+
+    /* FNN model of each layer */
+    T2TFNN * fnns;
+
+    /* attention model of each layer */
+    T2TAttention * attentions;
+
+    /* layer normalization for fnn */
+    T2TLN * fnnLayerNorms;
+
+    /* layer normalization for attention */
+    T2TLN * attLayerNorms;
+
+    /* input tensor of the encoder */
+    XTensor * input;
+
+    /* output tensor of the encoder */
+    XTensor * output;
+    
+public:
+    /* constructor */
+    AttEncoder();
+
+    /* de-constructor */
+    ~AttEncoder();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, 
+                   bool myIsMasked, int myIgnored, 
+                   int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the encoding network */
+    XTensor Make(XTensor &input, XTensor &mask, XTensor &maskEncDec, bool isTraining);
+
+    /* make the encoding network (wrapper) */
+    XTensor Make(XTensor &input, XTensor &mask, bool isTraining);
+};
+
+
+}
+
+#endif

source/sample/transformer/T2TFNN.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TFNN.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+#include "../../tensor/function/FHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TFNN::T2TFNN()
+{
+    inSize  = -1;
+    outSize = -1;
+    hSize   = -1;
+}
+
+/* deconstructor */
+T2TFNN::~T2TFNN()
+{
+}
+
+/* 
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TFNN::InitModel(int argc, char ** argv, int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+    
+    float minmax = 0;
+
+    LoadParamInt(argc, argv, "d", &inSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &outSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "fnnh", &hSize, outSize * 4);
+    LoadParamFloat(argc, argv, "fnnminmax", &minmax, 0.1F);
+    LoadParamFloat(argc, argv, "dropoutfnn", &dropoutP, 0);
+
+    InitTensor2D(&w1, inSize, hSize, X_FLOAT, devID, mem);
+    InitTensor1D(&b1, hSize, X_FLOAT, devID, mem);
+
+    InitTensor2D(&w2, hSize, outSize, X_FLOAT, devID, mem);
+    InitTensor1D(&b2, outSize, X_FLOAT, devID, mem);
+
+    float scale = 1.0F;
+    float finfout1 = (float)sqrt(6.0F * scale/(inSize + hSize));
+    float finfout2 = (float)sqrt(6.0F * scale/(hSize + outSize));
+    
+    w1.SetDataRand(-finfout1, finfout1);
+    b1.SetZeroAll();
+    w2.SetDataRand(-finfout2, finfout2);
+    b2.SetZeroAll();
+}
+
+/* 
+make the network 
+y = max(0, x * w1 + b1) * w2 + b2
+>> input - the input tensor
+>> return - the output tensor 
+*/
+XTensor T2TFNN::Make(XTensor &input, bool isTraining)
+{
+    XTensor t1;
+
+    /* t1 = max(0, x * w1 + b1) */
+    t1 = Rectify(MMul(input, w1) + b1);
+    
+    if(isTraining && dropoutP > 0)
+        t1 = Dropout(t1, dropoutP);
+
+    /* result = t1 * w2 + b2 */
+    return MMul(t1, w2) + b2;
+}
+
+
+}

source/sample/transformer/T2TFNN.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TFNN_H__
+#define __T2TFNN_H__
+
+#include "../../tensor/XTensor.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* a fnn: y = max(0, x * w1 + b1) * w2 + b2 */
+class T2TFNN
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* size of input vector */
+    int inSize;
+
+    /* size of output vector */
+    int outSize;
+
+    /* size of hidden layers */
+    int hSize;
+
+    /* matrix of transformation 1 */
+    XTensor w1;
+
+    /* bias of transformation 1 */
+    XTensor b1;
+
+    /* matrix of transformation 2 */
+    XTensor w2;
+
+    /* bias of transformation 2 */
+    XTensor b2;
+    
+    /* dropout probability */
+    DTYPE dropoutP;
+
+public:
+
+    /* constructor */
+    T2TFNN();
+
+    /* deconstructor */
+    ~T2TFNN();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the network */
+    XTensor Make(XTensor &input, bool isTraining);
+
+};
+
+}
+
+#endif

source/sample/transformer/T2TLayerNormal.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TLayerNormal.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* constructor */
+T2TLN::T2TLN()
+{
+    devID = -1;
+    mem = NULL;
+    d = 0;
+}
+
+/* de-constructor */
+T2TLN::~T2TLN()
+{
+}
+
+/*
+initialize the model
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TLN::InitModel(int argc, char ** argv, int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+
+    d = 0;
+    LoadParamInt(argc, argv, "d", &d, DEFAULT_EMBEDDING_SIZE);
+
+    InitTensor1D(&w, d, X_FLOAT, devID, mem);
+    InitTensor1D(&b, d, X_FLOAT, devID, mem);
+
+    w.SetDataRand(1.0F, 1.0F);
+    b.SetZeroAll();
+}
+
+/*
+make the network
+for each layer representation x, we have
+y =
+>> input - the input tensor
+>> return - layer normalization output
+*/
+XTensor T2TLN::Make(XTensor &input)
+{
+    XTensor &x = input;
+    XTensor xn;
+    XTensor mean;
+    XTensor variance;
+    XTensor standard;
+    XTensor meanFilled;
+    XTensor standardFilled;
+
+    /* \mu = (sum_i x_i)/m */
+    mean = ReduceMean(x, x.order - 1);
+
+    /* \sigma = (sum_i (x_i - \mu)^2)/m */
+    variance = ReduceVariance(x, x.order - 1, mean);
+
+    /* standard = sqrt(variance) */
+    standard = Power(variance, 0.5F);
+
+    /* unsqueeze mean and standard deviation to fit them into
+       the same shape of x */
+    meanFilled = Unsqueeze(mean, x.order - 1, x.GetDim(-1));
+    standardFilled = Unsqueeze(standard, x.order - 1, x.GetDim(-1));
+
+    /* x' = (x - \mu)/standard */
+    xn = (x - meanFilled) / standardFilled;
+
+    /* result = x' * w + b   */
+    return xn * w + b;
+}
+
+}

source/sample/transformer/T2TLayerNormal.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TLAYERNORMAL_H__
+#define __T2TLAYERNORMAL_H__
+
+#include "../../network/XNet.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* layer normalization: y = norm(x) * w + b 
+   where norm(x) = (x - mean)/standardDeviation */
+class T2TLN
+{
+public:
+    /* device id */
+    int devID;
+    
+    /* memory pool */
+    XMem * mem;
+
+    /* the transformation matrix w */
+    XTensor w;
+
+    /* the bias term b */
+    XTensor b;
+
+    /* dimension size of the model */
+    int d;
+    
+public:
+    /* constructor */
+    T2TLN();
+    
+    /* de-constructor */
+    ~T2TLN();
+    
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL);
+    
+    /* make the network */
+    XTensor Make(XTensor &input);
+};
+
+}
+
+#endif

source/sample/transformer/T2TModel.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TMODEL_H__
+#define __T2TMODEL_H__
+
+#include "T2TFNN.h"
+#include "T2TAttention.h"
+#include "T2TEncoder.h"
+#include "T2TDecoder.h"
+#include "T2TOutput.h"
+
+namespace transformer
+{
+
+class T2TModel
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* the encoder */
+    AttEncoder * encoder;
+
+    /* the decoder */
+    AttDecoder * decoder;
+
+    /* output layer */
+    T2TOutput * outputLayer;
+
+    /* indicates whether the model is running for language modeling */
+    bool isLM;
+
+    /* indicates whether the model is running for machine translation */
+    bool isMT;
+
+    /* number of heads in the attention model */
+    int nhead;
+
+public:
+    /* constructor */
+    T2TModel();
+
+    /* de-constructor */
+    ~T2TModel();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv);
+
+    /* make the encoding network */
+    XTensor MakeEncoder(XTensor &input, XTensor &mask, bool isTraining);
+
+    /* make the encoding network */
+    XTensor MakeDecoder(XTensor &inputEnc, XTensor &inputDec, XTensor &mask, XTensor &MaskEncDec, bool isTraining);
+
+    /* make the network for langauge modeling (with the output softmax layer) */
+    void MakeLM(XTensor &input, XTensor &output, XTensor &padding, bool isTraining);
+
+    /* make the network for machine translation (with the output softmax layer) */
+    void MakeMT(XTensor &inputEnc, XTensor &inputDec, XTensor &output, XTensor &paddingEnc, XTensor &paddingDec, bool isTraining);
+
+    /* get parameter matrics */
+    void GetParams(XList &list);
+
+    /* dump the parameters */
+    void Dump(const char * fn);
+
+    /* read the parameters */
+    void Read(const char * fn);
+};
+
+}
+
+#endif

source/sample/transformer/T2TOutput.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include "T2TOutput.h"
+#include "T2TUtility.h"
+#include "T2TEmbedding.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+/* constructor */
+T2TOutput::T2TOutput()
+{
+    devID = -1;
+    mem = NULL;
+    vSize = -1;
+    inSize = -1;
+    hSize = -1;
+}
+
+/* de-constructor */
+T2TOutput::~T2TOutput()
+{
+}
+
+/*
+initialize the model 
+>> argc - number of arguments
+>> argv - list of pointers to the arguments
+>> myDevID - device id
+>> myMem - the memory pool
+*/
+void T2TOutput::InitModel(int argc, char ** argv, int myDevID, XMem * myMem)
+{
+    devID = myDevID;
+    mem = myMem;
+
+    float minmax = 0;
+
+    LoadParamInt(argc, argv, "vsize", &vSize, -1);
+    LoadParamInt(argc, argv, "d", &inSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamInt(argc, argv, "d", &hSize, DEFAULT_EMBEDDING_SIZE);
+    LoadParamFloat(argc, argv, "outputminmax", &minmax, 0.08F);
+
+    InitTensor2D(&w, hSize, vSize, X_FLOAT, devID, mem);
+    
+    float scale = 1.0F;
+    float finfout = (float)sqrt(6.0F * scale/(hSize + vSize));
+    w.SetDataRand(-finfout, finfout);
+
+    DTYPE v = 1.0F/(float)sqrt((float)hSize);
+    w.SetDataRandn(0, v);
+}
+
+/* 
+make the network 
+y = softmax(x * w)
+>> input - input tensor
+<< return - output tensor 
+*/
+XTensor T2TOutput::Make(XTensor &input)
+{
+    XTensor &x = input;
+
+    return LogSoftmax(MMul(x, w), -1);
+}
+
+/* 
+make the network (redefined output tensor) 
+>> input - input tensor
+>> output - output tensor 
+*/
+void T2TOutput::Make(XTensor &input, XTensor &output)
+{
+    XTensor &x = input;
+
+    output = LogSoftmax(MMul(x, w), -1);
+    //output = Softmax(MMul(x, w), -1);
+}
+
+}

source/sample/transformer/T2TOutput.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TOUTPUT_H__
+#define __T2TOUTPUT_H__
+
+#include "../../tensor/function/FHeader.h"
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* output layer */
+class T2TOutput
+{
+public:
+    /* device id */
+    int devID;
+
+    /* memory pool */
+    XMem * mem;
+
+    /* vocabulary size */
+    int vSize;
+
+    /* input vector size */
+    int inSize;
+
+    /* vector size of the linear transformation */
+    int hSize;
+
+    /* transformation matrix */
+    XTensor w;
+
+public:
+    /* constructor */
+    T2TOutput();
+
+    /* de-constructor */
+    ~T2TOutput();
+
+    /* initialize the model */
+    void InitModel(int argc, char ** argv, int myDevID = -1, XMem * myMem = NULL);
+
+    /* make the network */
+    XTensor Make(XTensor &input);
+
+    /* make the network (redefined output tensor) */
+    void Make(XTensor &input, XTensor &output);
+};
+
+
+}
+
+#endif
\ No newline at end of file

source/sample/transformer/T2TTrainer.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-08-02
+ */
+
+#ifndef __T2TTRAINER_H__
+#define __T2TTRAINER_H__
+
+#include "T2TModel.h"
+
+#include "../../tensor/function/FHeader.h"
+
+#define MAX_SEQUENCE_LENGTH 1024 * 4
+
+using namespace nts;
+
+namespace transformer
+{
+
+/* trainer of the T2T model */
+class T2TTrainer
+{
+public:
+    /* paramter number */
+    int argNum;
+
+    /* parameter array */
+    char ** argArray;
+
+    /* buffer for loading words */
+    int * buf;
+
+    /* another buffer */
+    int * buf2;
+
+    /* buffer size */
+    int bufSize;
+
+    /* length of each sequence */
+    int * seqLen;
+
+    /* another array */
+    int * seqLen2;
+
+    /* offset of the first word for each sequence */
+    int * seqOffset;
+
+    /* number of sequences in the buffer */
+    int nseqBuf;
+
+    /* offset for next sequence in the buffer */
+    int nextSeq;
+    
+    /* indicates whether the sequence is sorted by length */
+    bool isLenSorted;
+    
+    /* dimension size of each inner layer */
+    int d;
+    
+    /* step number of warm-up for training */
+    int nwarmup;
+
+    /* vocabulary size of the source side */
+    int vSize;
+
+    /* vocabulary size of the target side */
+    int vSizeTgt;
+
+    /* learning rate */
+    float lrate;
+    
+    /* the parameter that controls the maximum learning rate in training */
+    float lrbias;
+
+    /* sentence batch size */
+    int sBatchSize;
+
+    /* word batch size */
+    int wBatchSize;
+
+    /* training epoch number */
+    int nepoch;
+
+    /* traing step number */
+    int nstep;
+
+    /* indicates whether we use adam */
+    bool useAdam;
+
+    /* hyper parameters of adam*/
+    float adamBeta1;
+    float adamBeta2;
+    float adamDelta;
+    float adamBeta1T;
+    float adamBeta2T;
+
+    /* list of the moment of the parameter matrics */
+    XList moments;
+
+    /* list of the 2nd order moment of the parameter matrics */
+    XList moments2nd;
+
+    /* indicates whether the data file is shuffled for training */
+    bool isShuffled;
+    
+    /* the factor of label smoothing */
+    DTYPE labelSmoothingP;
+
+    /* number of steps after which we make a checkpoint */
+    int nStepCheckpoint;
+
+    /* indicates whether we make a checkpoint after each traing epoch */
+    bool useEpochCheckpoint;
+    
+    /* number of batches on which we do model update */
+    int updateStep;
+    
+    /* indicates whether we double the </s> symbol for the output of lms */
+    bool isDoubledEnd;
+    
+    /* indicates whether we use batchsize = max * sc
+       rather rather than batchsize = word-number, where max is the maximum
+       length and sc is the sentence number */
+    bool isSmallBatch;
+
+    /* counterpart of "isSmallBatch" */
+    bool isBigBatch;
+
+    /* indicates whether we use small memory footprint for backward process */
+    bool isSmallFootprint;
+
+public:
+    /* constructor */
+    T2TTrainer();
+
+    /* de-constructor */
+    ~T2TTrainer();
+
+    /* initialize the trainer */
+    void Init(int argc, char ** argv);
+
+    /* train the model */
+    void Train(const char * fn, const char * validFN, const char * modelFN, T2TModel * model);
+
+    /* test the model */
+    void Test(const char * fn, const char * ofn, T2TModel * model);
+
+    /* make a checkpoint */
+    void MakeCheckpoint(T2TModel * model, const char * validFN, const char * modelFN, const char * label, int id);
+
+    /* load data to buffer */
+    int LoadBuf(FILE * file, bool isSorted, int step);
+
+    /* clear data buffer */
+    void ClearBuf();
+
+    /* load a batch of sequences */
+    int LoadBatch(FILE * file, bool isLM,
+                  XTensor * batchEnc, XTensor * paddingEnc, 
+                  XTensor * batchDec, XTensor * paddingDec,
+                  XTensor * gold,
+                  int * seqs,
+                  int vsEnc, int vsDec, int sBatch, int wBatch, 
+                  bool isSorted, int &wCount,
+                  int devID, XMem * mem, 
+				  bool isTraining);
+
+    /* load a batch of sequences (for language modeling) */
+    int LoadBatchLM(FILE * file, 
+                    XTensor * batchEnc, XTensor * paddingEnc,
+                    XTensor * batchDec, XTensor * paddingDec,
+                    XTensor * gold,
+                    int * seqs, int vs, int sBatch, int wBatch, 
+                    bool isSorted, int &wCount,
+                    int devID, XMem * mem, 
+					bool isTraining);
+
+    /* load a batch of sequences (for machine translation) */
+    int LoadBatchMT(FILE * file, 
+                    XTensor * batchEnc, XTensor * paddingEnc, 
+                    XTensor * batchDec, XTensor * paddingDec,
+                    XTensor * gold,
+                    int * seqs, int vsEnc, int vsDec, int sBatch, int wBatch, 
+                    bool isSorted, int &wCount,
+                    int devID, XMem * mem, 
+					bool isTraining);
+
+    /* shuffle the data file */
+    void Shuffle(const char * srcFile, const char * tgtFile);
+    
+    /* get word probabilities for a batch of sequences */
+    float GetProb(XTensor * output, XTensor * gold, XTensor * wordProbs);
+
+    /* update the model by delta rule */
+    void Update(T2TModel * model, const float lr);
+
+    /* prepare model for training */
+    void PrepareModel(T2TModel * model);
+
+    /* do padding on the output */
+    void PadOutput(XTensor * output, XTensor * gold, XTensor * padding);
+    
+    /* recale the output and gold tensors for normalized loss */
+    void RescaleOutput(XTensor * output, XTensor * gold, XTensor * padding);
+    
+    /* perform label smoothing */
+    void LabelSmooth(XTensor * gold, XTensor * smoothed, DTYPE p);
+};
+
+
+}
+
+#endif

source/sample/transformer/T2TUtility.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+namespace transformer
+{
+
+FILE * tmpFILE;
+int llnum = 0;
+FILE * tf = NULL;
+
+void LoadParamString(int argc, char ** argv, const char * name, char * p, const char * defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname) && i + 1 < argc){
+            strcpy(p, argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            hit = true;
+        }
+    }
+    if(!hit)
+        strcpy(p, defaultP);
+}
+
+void LoadParamInt(int argc, char ** argv, const char * name, int * p, int defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname) && i + 1 < argc){
+            *(int*)p = atoi(argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            hit = true;
+        }
+    }
+    if(!hit)
+        *p = defaultP;
+}
+
+void LoadParamBool(int argc, char ** argv, const char * name, bool * p, bool defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname)){
+            *(bool*)p = true;
+            //fprintf(stderr, " %s=%s\n", name, "true");
+            hit = true;
+        }
+    }
+    if(!hit)
+        *p = defaultP;
+}
+
+void LoadParamFloat(int argc, char ** argv, const char * name, float * p, float defaultP)
+{
+    char vname[128];
+    vname[0] = '-';
+    strcpy(vname + 1, name);
+    bool hit = false;
+    for(int i = 0; i < argc; i++){
+        if(!strcmp(argv[i], vname) && i + 1 < argc){
+            *p = (float)atof(argv[i + 1]);
+            //fprintf(stderr, " %s=%s\n", name, argv[i + 1]);
+            hit = true;
+        }
+    }
+    if(!hit)
+        *p = defaultP;
+}
+
+void ShowParams(int argc, char ** argv)
+{
+    fprintf(stderr, "args:\n");
+    for(int i = 0; i < argc; i++){
+        if(argv[i][1] == 0)
+            continue;
+        if(argv[i][0] == '-' && (argv[i][1] < '1' || argv[i][1] > '9')){
+            if(i + 1 < argc && argv[i + 1][0] != '-')
+                fprintf(stderr, " %s=%s\n", argv[i], argv[i + 1]);
+            else
+                fprintf(stderr, " %s=yes\n", argv[i]);
+        }
+    }
+    fprintf(stderr, "\n");
+}
+
+}

source/sample/transformer/T2TUtility.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#ifndef __T2TUTILITY_H__
+#define __T2TUTILITY_H__
+
+#include <stdio.h>
+
+namespace transformer
+{
+
+extern FILE * tmpFILE;
+
+/* load arguments */
+void LoadParamString(int argc, char ** argv, const char * name, char * p, const char * defaultP);
+void LoadParamInt(int argc, char ** argv, const char * name, int * p, int defaultP);
+void LoadParamBool(int argc, char ** argv, const char * name, bool * p, bool defaultP);
+void LoadParamFloat(int argc, char ** argv, const char * name, float * p, float defaultP);
+
+/* show arguments */
+void ShowParams(int argc, char ** argv);
+
+extern int llnum;
+extern FILE * tf;
+
+}
+
+#endif

source/sample/transformer/Transformer.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ */
+
+#include <math.h>
+#include <time.h>
+#include "Transformer.h"
+#include "T2TModel.h"
+#include "T2TUtility.h"
+#include "T2TTrainer.h"
+#include "../../tensor/XDevice.h"
+#include "../../tensor/XUtility.h"
+#include "../../tensor/XGlobal.h"
+
+namespace transformer
+{
+
+int TransformerMain(int argc, const char ** argv)
+{
+    if(argc == 0)
+        return 1;
+    
+    fprintf(stderr, "%e\n", log(1e-8F));
+
+    char ** args = new char*[argc];
+    for(int i = 0; i < argc; i++){
+        args[i] = new char[strlen(argv[i]) + 1];
+        strcpy(args[i], argv[i]);
+    }
+
+    tmpFILE = fopen("tmp.txt", "wb");
+
+    ShowParams(argc, args);
+
+    char * trainFN = new char[MAX_LINE_LENGTH];
+    char * modelFN = new char[MAX_LINE_LENGTH];
+    char * testFN = new char[MAX_LINE_LENGTH];
+    char * outputFN = new char[MAX_LINE_LENGTH];
+
+    LoadParamString(argc, args, "train", trainFN, "");
+    LoadParamString(argc, args, "model", modelFN, "");
+    LoadParamString(argc, args, "test", testFN, "");
+    LoadParamString(argc, args, "output", outputFN, "");
+
+    srand((unsigned int)time(NULL));
+    T2TTrainer trainer;
+    trainer.Init(argc, args);
+
+    T2TModel model;
+    model.InitModel(argc, args);
+    
+    /* learn model parameters */
+    if(strcmp(trainFN, ""))
+        trainer.Train(trainFN, testFN, strcmp(modelFN, "") ? modelFN : "checkpoint.model", &model);
+    
+    /* save the final model */
+    if(strcmp(modelFN, "") && strcmp(trainFN, ""))
+        model.Dump(modelFN);
+    
+    /* load the model if neccessary */
+    if(strcmp(modelFN, ""))
+        model.Read(modelFN);
+
+    T2TTrainer tester;
+    tester.Init(argc, args);
+
+    /* test the model on the new data */
+    if(strcmp(testFN, "") && strcmp(outputFN, ""))
+        tester.Test(testFN, outputFN, &model);
+
+    delete[] trainFN;
+    delete[] modelFN;
+    delete[] testFN;
+    delete[] outputFN;
+
+    for(int i = 0; i < argc; i++)
+        delete[] args[i];
+    delete[] args;
+
+    fclose(tmpFILE);
+
+    return 0;
+}
+
+}

source/sample/transformer/Transformer.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * An impelementation of the transformer system. See more details 
+ * about FNNLM in 
+ * "Attention Is All You Need" by Vaswani et al.
+ * https://arxiv.org/pdf/1706.03762.pdf
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-31
+ * I start writing the code related to NMT - a long time since my last coding 
+ * work on MT
+ */
+
+#ifndef __TRANSFORMER_H__
+#define __TRANSFORMER_H__
+
+#include "../../tensor/XGlobal.h"
+#include "../../tensor/XTensor.h"
+#include "../../tensor/core/CHeader.h"
+
+namespace transformer
+{
+
+/* entrance of the program */
+int TransformerMain(int argc, const char ** argv);
+
+}
+
+#endif
\ No newline at end of file

source/tensor/Main.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+/*
+ *
+ * This is the entrance of the low-level tensor library : NiuTrans.Tensor
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2015-12-14
+ *
+ */
+
+#include <stdio.h>
+#include <math.h>
+#include <time.h>
+#include "XTensor.h"
+#include "XDevice.h"
+#include "./test/Test.h"
+#include "./core/CHeader.h"
+
+//#define CRTDBG_MAP_ALLOC
+//#include <stdlib.h>  
+//#include <crtdbg.h> 
+
+using namespace nts;
+
+void SmallTest();
+void TransposeTest();
+void LittleTest();
+void T2TTest();
+void T2TTest2();
+void PowerTest();
+
+int main( int argc, const char ** argv )
+{
+    //PowerTest();
+    //LittleTest();
+
+    //T2TTest();
+    //T2TTest2();
+
+    //return 0;
+    //_CrtSetBreakAlloc(123);
+
+    /* a tiny test */
+    //SmallTest();
+
+    //_CrtDumpMemoryLeaks();
+    //return 0;
+
+    if(argc > 1 && !strcmp(argv[1], "-test"))
+        Test();
+    else{
+        fprintf(stderr, "Thanks for using NiuTrans.Tensor! This is a library that eases the\n");
+        fprintf(stderr, "use of tensors. All you need is to ... \n\n");
+        fprintf(stderr, "Run this program with \"-test\" for unit test!\n");
+    }
+
+    //_CrtDumpMemoryLeaks();
+
+    return 0;
+}
+
+void myRead(XTensor * tensor, const char * filename, const char * label)
+{
+    FILE * file = fopen(filename, "rb");
+    if(file == NULL)
+        printf("%s\n", filename);
+    tensor->Read(file, label);
+}
+
+void myDump(XTensor * tensor, const char * filename, const char * label)
+{
+    FILE * file = fopen(filename, "wb");
+    if(file == NULL)
+        printf("%s\n", filename);
+    tensor->Dump(file, label);
+}
+
+void PowerTest()
+{
+    XTensor input;
+    XTensor output;
+    InitTensor2D(&input, 256, 10000, X_FLOAT, 0);
+    InitTensor2D(&output, 256, 10000, X_FLOAT, 0);
+    myRead(&input, "1.txt", "");
+
+    _Power(&input, &output, 2);
+    output.Dump(stderr, "", 200);
+}
+
+void SmallTest()
+{
+    XTensor a;
+    XTensor b;
+    XTensor c;
+    XTensor d;
+
+    InitTensor2D(&a, 2, 2);
+    InitTensor2D(&b, 2, 2);
+    a.SetZeroAll();
+    b.SetZeroAll();
+    a.Set2D(1.0F, 0, 0);
+    a.Set2D(2.0F, 1, 1);
+
+    b = Sum(a, Multiply(a, a));
+
+    /* this is prohibited !!!!!!!!!!!!! */
+    //XTensor c = a * b + a;
+    //XTensor d = a + b + c.Lin(0.5F);
+    
+    c = a * b + a;
+    d = a + b + c.Lin(0.5F);
+
+    XLink::CheckNetwork(&d);
+    //XLink::ShowNetwork(stderr, &d);
+        
+    a.Dump(stderr, "a:");
+    b.Dump(stderr, "b:");
+    c.Dump(stderr, "c:");
+    d.Dump(stderr, "d:");
+}
+
+void TransposeTest()
+{
+    XTensor a;
+    XTensor b;
+
+    int I = 2;
+    int J = 3;
+
+    InitTensor4D(&a, 2, 3, 4, 5);
+
+    int * dims = new int[a.order];
+    memcpy(dims, a.dimSize, sizeof(int) * a.order);
+    dims[I] = a.dimSize[J];
+    dims[J] = a.dimSize[I];
+
+    InitTensor(&b, 4, dims);
+
+    a.SetZeroAll();
+    b.SetZeroAll();
+
+    float * data = new float[a.unitNum];
+    for(int i = 0; i < a.unitNum; i++)
+        data[i] = (float)i;
+
+    a.SetData(data, a.unitNum, 0);
+
+    _Transpose(&a, &b, I, J);
+    b.Dump(stderr, "b:");
+
+    delete[] data;
+}
+
+void LittleTest()
+{
+    int a = 5000;
+    int b = 100000;
+    int c = a*b;
+    printf("%d\n", c);
+
+    exit(1);
+}
+
+void T2TTest()
+{
+    XTensor * input;
+    XTensor * weight;
+    XTensor * output;
+    XTensor * gold;
+    XTensor * dedy;
+    XTensor * dedx;
+    XTensor * dedxTmp;
+    XTensor * dedw;
+    XTensor * padding;
+
+    DTYPE loss;
+
+    int * dimSize = new int[2];
+    dimSize[0] = 256;
+    dimSize[1] = 10001;
+
+    int * dimSize2 = new int[3];
+    dimSize2[0] = 2;
+    dimSize2[1] = 31;
+    dimSize2[2] = 256;
+   
+    int * dimSize3 = new int[3];
+    dimSize3[0] = 2;
+    dimSize3[1] = 31;
+    dimSize3[2] = 10001;
+
+    int * dimSize4 = new int[2];
+    dimSize4[0] = 2;
+    dimSize4[1] = 31;
+
+    input = NewTensor(3, dimSize2, X_FLOAT, 1.0F, 0);
+    weight = NewTensor(2, dimSize, X_FLOAT, 1.0F, 0);
+    dedw = NewTensor(2, dimSize, X_FLOAT, 1.0F, 0);
+    gold = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    output = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    dedy = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    dedx = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    dedxTmp = NewTensor(3, dimSize3, X_FLOAT, 1.0F, 0);
+    padding = NewTensor(2, dimSize4, X_FLOAT, 1.0F, 0);
+
+    //weight = NewTensor(2, dimSize);
+    //dedw = NewTensor(2, dimSize);
+    //input = NewTensor(3, dimSize2);
+    //gold = NewTensor(3, dimSize3);
+    //output = NewTensor(3, dimSize3);
+    //dedy = NewTensor(3, dimSize3);
+    //dedx = NewTensor(3, dimSize3);
+    //dedxTmp = NewTensor(3, dimSize3);
+    //padding = NewTensor(2, dimSize4);
+
+    myRead(input, "x.txt", "x");
+    myRead(weight, "w.txt", "w");
+    myRead(gold, "gold.txt", "gold");
+    myRead(padding, "padding.txt", "padding");
+
+    XTensor inter;
+    inter = MMul(*input, *weight);
+
+    _Softmax(&inter, output, 2);
+
+    //_LogMe(output);
+    loss = _CrossEntropyFast(output, gold, REDUCE_MEAN, NULL, padding);
+
+    printf("loss: %f\n", loss);
+
+    _CrossEntropyBackward(dedy, output, gold, NULL);
+    //_CrossEntropyBackward(dedy, output, gold, NULL, padding);
+
+    myDump(dedy, "dedy.txt", "dedy");
+
+    _SoftmaxBackward(NULL, output, input, dedy, dedx, NULL, -1, NOLOSS);
+    _Sub(output, gold, dedxTmp);
+
+    myDump(dedx, "dedx.txt", "dedx");
+    dedx->Dump(stderr, "dedx", 200);
+    dedxTmp->Dump(stderr, "dedxTmp", 200);
+
+    input->Reshape(input->unitNum/input->GetDim(-1), input->GetDim(-1));
+    dedx->Reshape(dedx->unitNum/dedx->GetDim(-1), dedx->GetDim(-1));
+
+    _MatrixMulBatched(input, X_TRANS, dedx, X_NOTRANS, dedw);
+
+    myDump(dedw, "dedw.txt", "dedw");
+}
+
+void T2TTest2()
+{
+    int dimSize[3];
+    dimSize[0] = 161;
+    dimSize[1] = 47;
+    dimSize[2] = 10001;
+    XTensor * probs = NewTensor(3, dimSize, X_FLOAT, 1.0F, 0);
+    //XTensor * probs = NewTensor(3, dimSize, X_FLOAT, 1.0F, -1);
+
+    //myRead(probs, "probs.txt", " ");
+    _SetDataFixedFloat(probs, 1.0F);
+
+    probs->Reshape(1, probs->unitNum);
+
+    DTYPE sum = _ReduceSumAll(probs);
+    printf("%e\n", sum);
+
+    //XTensor tmp;
+    //tmp = IsNonZero(*probs);
+    //DTYPE nonZeroNum = ReduceSumAll(tmp);
+    //printf("%f\n", nonZeroNum);
+    //
+    //DTYPE gpu = ReduceSum(*probs, 1).Get2D(0, 0);
+
+    //printf("%e\n", gpu);
+}
+

source/tensor/XBLAS.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * This is a wrapper of the BLAS (Basic Linear Algebra Subprograms http://www.netlib.org/blas/) 
+ * libraries. By using BLAS, we can access very fast matrix operations although they
+ * are also implemented in NiuTrans in a native manner. To use BLAS, 
+ * set USE_BLAS. 
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-04-08
+ *
+ */
+
+#ifdef WIN32
+#include <wtypes.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "XBLAS.h"
+#include "XGlobal.h"
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+#ifdef WIN32
+HINSTANCE hBLASDll;
+#endif
+
+
+/* single-precision floating matrix-matrix multiplication */
+void (*XBLAS_SGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                    OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float,  
+                    OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT,
+                    OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float, 
+                    float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating matrix-matrix multiplication */
+void (*XBLAS_DGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                    OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double,  
+                    OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT,
+                    OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double, 
+                    double *, OPENBLAS_CONST BLASINT);
+
+/* single-precision floating vector-vector multiplication (rank-1) */
+void (*XBLAS_SGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST float  alpha, 
+                   OPENBLAS_CONST float *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, 
+                   float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating vector-vector multiplication (rank-1) */
+void (*XBLAS_DGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST double  alpha, 
+                   OPENBLAS_CONST double *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, 
+                   double *, OPENBLAS_CONST BLASINT);
+
+/* set the number of threads */
+void (*XBLAS_SET_THREAD_NUM)(int);
+
+/* get the number of threads */
+//int (*XBLAS_GET_THREAD_NUM)();
+
+
+/* get the number of physical processors (cores).*/
+int (*XBLAS_GET_CORE_NUM)();
+
+
+/* get the CPU corename */
+//char * (*XBLAS_GET_CORE_NAME)();
+
+/* get the parallelization type used by OpenBLAS */
+//int (*XBLAS_GET_PARALLEL_TYPE)(void);
+
+
+#if defined(USE_BLAS)
+
+/* load some stuff for BLAS */
+void LoadBLAS(const char * dllFileName)
+{
+#ifndef CUDA_BLAS
+#ifdef _WIN32
+
+#if defined(OPENBLAS)
+    /* non-ascii characters are not supported yet */
+    wchar_t * fn = new wchar_t[strlen(dllFileName) + 1];
+    memset(fn, 0, sizeof(wchar_t) * (strlen(dllFileName) + 1));
+    for(int i = 0; i < strlen(dllFileName); i++)
+        fn[i] = dllFileName[i];
+
+    hBLASDll = LoadLibrary((LPCWSTR)fn);
+
+    if(!hBLASDll){
+        XPRINT1(0, stderr, "[LoadBLAS] Error! Cannot load dll %s!\n", dllFileName);
+        exit(1);
+    }
+
+    /* matrix-matrix multiplicatoin */
+    (FARPROC&)XBLAS_SGEMM = GetProcAddress(hBLASDll, "cblas_sgemm");
+    (FARPROC&)XBLAS_DGEMM = GetProcAddress(hBLASDll, "cblas_dgemm");
+
+    /* vector-vector multiplication */
+    (FARPROC&)XBLAS_SGER = GetProcAddress(hBLASDll, "cblas_sger");
+    (FARPROC&)XBLAS_DGER = GetProcAddress(hBLASDll, "cblas_dger");
+
+    /* multi-threading */
+    (FARPROC&)XBLAS_SET_THREAD_NUM = GetProcAddress(hBLASDll, "openblas_set_num_threads");
+    //(FARPROC&)XBLAS_SET_THREAD_NUM = GetProcAddress(hBLASDll, "goto_set_num_threads");
+    //(FARPROC&)XBLAS_GET_THREAD_NUM = GetProcAddress(hBLASDll, "openblas_get_num_threads");
+    (FARPROC&)XBLAS_GET_CORE_NUM = GetProcAddress(hBLASDll, "openblas_get_num_procs");
+    //(FARPROC&)XBLAS_GET_CORE_NAME = GetProcAddress(hBLASDll, "openblas_get_corename");
+    //(FARPROC&)XBLAS_GET_PARALLEL_TYPE = GetProcAddress(hBLASDll, "openblas_get_parallel");
+
+    delete[] fn;
+#endif // defined(OPENBLAS)
+
+#if defined(MKL)
+    /* non-ascii characters are not supported yet */
+    wchar_t * fn = new wchar_t[strlen(dllFileName) + 1];
+    memset(fn, 0, sizeof(wchar_t) * (strlen(dllFileName) + 1));
+    for(int i = 0; i < strlen(dllFileName); i++)
+        fn[i] = dllFileName[i];
+
+    hBLASDll = LoadLibrary((LPCWSTR)fn);
+
+    if(!hBLASDll){
+        XPRINT1(0, stderr, "[LoadBLAS] Error! Cannot load dll %s!\n", dllFileName);
+        exit(1);
+    }
+
+    /* matrix-matrix multiplicatoin */
+    (FARPROC&)XBLAS_SGEMM = GetProcAddress(hBLASDll, "cblas_sgemm");
+    (FARPROC&)XBLAS_DGEMM = GetProcAddress(hBLASDll, "cblas_dgemm");
+
+    /* vector-vector multiplication */
+    (FARPROC&)XBLAS_SGER = GetProcAddress(hBLASDll, "cblas_sger");
+    (FARPROC&)XBLAS_DGER = GetProcAddress(hBLASDll, "cblas_dger");
+
+    /* multi-threading */
+    (FARPROC&)XBLAS_SET_THREAD_NUM = GetProcAddress(hBLASDll, "MKL_Set_Num_Threads");
+    (FARPROC&)XBLAS_GET_CORE_NUM   = GetProcAddress(hBLASDll, "MKL_Get_Max_Threads");
+#endif // defined(MKL)
+
+#else // _WIN32
+
+    XBLAS_SGEMM = &cblas_sgemm;
+    XBLAS_DGEMM = &cblas_dgemm;
+    XBLAS_SGER  = &cblas_sger;
+    XBLAS_DGER  = &cblas_dger;
+#if defined(OPENBLAS)
+    XBLAS_SET_THREAD_NUM    = &openblas_set_num_threads;
+    XBLAS_GET_CORE_NUM      = &openblas_get_num_procs;
+#endif // defined(OPENBLAS)
+#if defined(MKL)
+    XBLAS_SET_THREAD_NUM    = &mkl_set_num_threads;
+    XBLAS_GET_CORE_NUM      = &mkl_get_max_num_threads;
+#endif // defined(MKL)
+
+#endif // _WIN32
+
+    XBLAS_SET_THREAD_NUM(1);
+#endif // ndef(CUDA_BLAS)
+}
+
+/* unload the libs */
+void UnloadBLAS()
+{
+#ifdef _WIN32
+
+    if(!FreeLibrary(hBLASDll)){
+        XPRINT(0, stderr, "[UnloadBLAS] Error! Cannot free the BLAS dll!\n");
+        exit(1);
+    }
+
+#else
+
+#endif
+}
+
+#else  // undefined(USE_BLAS) || undefined(OPENBLAS)
+
+void LoadBLAS(const char * dllFileName)
+{
+    XPRINT(0, stderr, "[LoadBLAS] Error! No Blas lib is available. Please use OPENBLAS or MKL!\n");
+    exit(1);
+}
+
+void UnloadBLAS()
+{
+    XPRINT(0, stderr, "[UnloadBLAS] Error! No Blas lib is available. Please use OPENBLAS or MKL!\n");
+    exit(1);
+}
+
+#endif // defined(USE_BLAS) && defined(OPENBLAS)
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
\ No newline at end of file

source/tensor/XBLAS.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.b
+ */
+
+/*
+ *
+ * This is a wrapper of the BLAS (Basic Linear Algebra Subprograms http://www.netlib.org/blas/) 
+ * libraries. By using BLAS, we can access very fast matrix operations although they
+ * are also implemented in NiuTrans in a native manner. To use BLAS, 
+ * specify USE_BLAS when compiling the code. 
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-04-08
+ *
+ */
+
+#ifndef __XBLAS_H__
+#define __XBLAS_H__
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+/* some of the code below is from OpenBLAS (https://github.com/xianyi/OpenBLAS) */
+
+//#define OPENBLAS
+
+#define OPENBLAS_CONST const
+typedef int BLASINT;
+typedef enum CBLAS_ORDER     {CblasRowMajor=101, CblasColMajor=102} CBLAS_ORDER;
+typedef enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113, CblasConjNoTrans=114} CBLAS_TRANSPOSE;
+typedef enum CBLAS_UPLO      {CblasUpper=121, CblasLower=122} CBLAS_UPLO;
+typedef enum CBLAS_DIAG      {CblasNonUnit=131, CblasUnit=132} CBLAS_DIAG;
+typedef enum CBLAS_SIDE      {CblasLeft=141, CblasRight=142} CBLAS_SIDE;
+
+
+#if defined(USE_BLAS)
+
+/* 
+single/double-precision floating matrix-matrix multiplication (rank-3)
+- SGEMM (ORDER, TRANSA, TRANSB, M, N, K, ALPHA, A, LDA, B, LDB, BETA, C, LDC)
+It implements C = \alpha * op(A)*op(B) + \beta * C
+where A, B and C are matrices,
+      \alpha and \beta are coefficients,
+      TRANSA specifies we need a transposed matrix (op(A)=A**T); otherwise op(A) = A,
+      M specifies the row number of op(A),
+      N specifies the column number of op(B),
+      K specifies the column number of op(A),
+      LDA(=K) specifies the size of the first(or leading) dimension of A as declared in the calling (sub) program,
+              E.g., if we are using CblasRowMajor, the leading dimension is the number of columns.
+      LDB(=N) specifies the size of the first dimension of B as declared in the calling (sub) program,
+      and LDC(=N) specifies the size of the first dimension of C as declared in the calling (sub) program.
+*/
+extern "C" void (*XBLAS_SGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                               OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float,  
+                               OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT,
+                               OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float, 
+                               float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating matrix-matrix multiplication */
+extern "C" void (*XBLAS_DGEMM)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST enum CBLAS_TRANSPOSE, OPENBLAS_CONST enum CBLAS_TRANSPOSE,
+                               OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double,  
+                               OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT,
+                               OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double, 
+                               double *, OPENBLAS_CONST BLASINT);
+
+/* 
+single/double-precision floating vector-vector multiplication (rank-2)
+- SGER (ORDER,M, N, ALPHA, X, INCX, Y, INCY, A, LDA)
+It implements A = \alpha * X * (Y^T) + A
+where X and Y are vectors with m and n elements respectively,
+      A is an m by n matrix,
+      \alpha is the scalar,
+      INCX specifies the increment for the elements of X,
+      INCY specifies the increment for the elements of Y,
+      LDA specifies the size of the first(or leading) dimension of A as declared in the calling (sub) program,
+          E.g., if we are using CblasRowMajor, the leading dimension is the number of columns of A.
+
+*/
+extern "C" void (*XBLAS_SGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST float  alpha, 
+                              OPENBLAS_CONST float *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST float *, OPENBLAS_CONST BLASINT, 
+                              float *, OPENBLAS_CONST BLASINT);
+
+/* double-precision floating vector-vector multiplication (rank-1) */
+extern "C" void (*XBLAS_DGER)(OPENBLAS_CONST enum CBLAS_ORDER, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST double  alpha, 
+                              OPENBLAS_CONST double *Y, OPENBLAS_CONST BLASINT, OPENBLAS_CONST double *, OPENBLAS_CONST BLASINT, 
+                              double *, OPENBLAS_CONST BLASINT);
+
+/* set the number of threads */
+extern "C" void (*XBLAS_SET_THREAD_NUM)(int);
+
+/* get the number of threads */
+//extern "C" int (*XBLAS_GET_THREAD_NUM)();
+
+
+/* get the number of physical processors (cores).*/
+extern "C" int (*XBLAS_GET_CORE_NUM)();
+
+/* get the CPU corename */
+//extern "C" char * (*XBLAS_GET_CORE_NAME)();
+
+/* get the parallelization type used by OpenBLAS */
+//extern "C" int (*XBLAS_GET_PARALLEL_TYPE)(void);
+
+/* linux systems */
+#ifndef _WIN32
+
+/* cblas functions that are imported from the lib. See cblas.h in OpenBlas for more information */
+extern "C" void cblas_sgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, 
+                        OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST BLASINT K, OPENBLAS_CONST float alpha, 
+                        OPENBLAS_CONST float *A, OPENBLAS_CONST BLASINT lda, 
+                        OPENBLAS_CONST float *B, OPENBLAS_CONST BLASINT ldb, 
+                        OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST BLASINT ldc);
+extern "C" void cblas_dgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, 
+                        OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST BLASINT K, OPENBLAS_CONST double alpha, 
+                        OPENBLAS_CONST double *A, OPENBLAS_CONST BLASINT lda, 
+                        OPENBLAS_CONST double *B, OPENBLAS_CONST BLASINT ldb, 
+                        OPENBLAS_CONST double beta, double *C, OPENBLAS_CONST BLASINT ldc);
+extern "C" void cblas_sger (OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST float  alpha, 
+                        OPENBLAS_CONST float  *X, OPENBLAS_CONST BLASINT incX, OPENBLAS_CONST float  *Y, OPENBLAS_CONST BLASINT incY, 
+                        float  *A, OPENBLAS_CONST BLASINT lda);
+extern "C" void cblas_dger (OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST BLASINT M, OPENBLAS_CONST BLASINT N, OPENBLAS_CONST double alpha, 
+                        OPENBLAS_CONST double *X, OPENBLAS_CONST BLASINT incX, OPENBLAS_CONST double *Y, OPENBLAS_CONST BLASINT incY, 
+                        double *A, OPENBLAS_CONST BLASINT lda);
+
+#if defined(OPENBLAS)
+/* better control of multi-threading */
+extern "C" void  openblas_set_num_threads(int num_threads);
+extern "C" void  goto_set_num_threads(int num_threads);
+//extern "C" int   openblas_get_num_threads(void);
+extern "C" int   openblas_get_num_procs(void);
+//extern "C" char* openblas_get_config(void);
+//extern "C" char* openblas_get_corename(void);
+//extern "C" int   openblas_get_parallel(void);
+#endif
+
+#endif
+
+#if defined(MKL)
+
+
+/* better control of multi-threading */
+//_Mkl_Api(void,MKL_Set_Num_Threads,(int nth))
+//_Mkl_Api(int,MKL_Get_Max_Threads,(void))
+extern "C" void  MKL_Set_Num_Threads(int num_threads);
+extern "C" int  MKL_Get_Max_Threads();
+
+
+#define mkl_set_num_threads MKL_Set_Num_Threads
+#define mkl_get_max_num_threads MKL_Get_Max_Threads
+
+//extern "C" void  mkl_set_num_threads(int num_threads);
+//extern "C" void  omp_set_num_threads(int num_threads);
+//extern "C" int  mkl_get_max_num_threads();
+
+#endif
+
+#if defined(CUDA_BLAS)
+
+// Utilities and system includes
+#include <assert.h>
+#include <helper_string.h>  // helper for shared functions common to CUDA Samples
+
+// CUDA runtime
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+
+// CUDA and CUBLAS functions
+#include <helper_functions.h>
+#include <helper_cuda.h>
+
+/* Matrix multiplication */
+extern void BLASMatrixMULS(int deviceID, float * a, float * b, float * c, int na, int ma, int nb, int mb, int nc, int mc, float alpha = 1.0F, float beta = 0);
+extern void BLASMatrixMULD(int deviceID, double * a, double * b, double * c, int na, int ma, int nb, int mb, int nc, int mc, double alpha = 1.0F, double beta = 0);
+
+#endif
+
+#endif
+
+#ifdef _WIN32
+
+#include "windows.h"
+
+extern HINSTANCE hBLASDll;
+
+#else
+
+#endif
+
+/* load some stuff for BLAS */
+extern void LoadBLAS(const char * dllFileName);
+
+/* unload the libs */
+extern void UnloadBLAS();
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif

source/tensor/XDataType.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-05-30
+ *
+ */
+
+#ifndef __XDATATYPE_H__
+#define __XDATATYPE_H__
+
+#include "XGlobal.h"
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+/* data type of the tensor, e.g., int, float, and double. */
+enum TENSOR_DATA_TYPE {X_INT, X_INT8, X_FLOAT, X_FLOAT16, X_DOUBLE};
+
+/* transposed matrix type */
+enum MATRIX_TRANS_TYPE{X_TRANS, X_NOTRANS};
+
+/* default data type */
+#ifdef DOUBELPRICSION
+#define DEFAULT_DTYPE X_DOUBLE
+#else
+#define DEFAULT_DTYPE X_FLOAT
+#endif
+
+/* get data type name */
+extern const char * GetDataTypeName(TENSOR_DATA_TYPE type);
+extern TENSOR_DATA_TYPE GetDataType(const char * typeName);
+
+/* data conversion (for lower precision computation) */
+inline unsigned short cal_complement(unsigned short sig, unsigned short tal);
+unsigned short Float16Add(unsigned short a, unsigned short b);
+unsigned short Float16Sub(unsigned short a, unsigned short b);
+unsigned short Float16Mul(unsigned short a, unsigned short b);
+unsigned short Float16Div(unsigned short a, unsigned short b);
+unsigned short FloatToFloat16(float f);
+float Float16ToFloat(unsigned short h);
+unsigned short FloatbitsToHalfbits(float ff);
+float HalfbitsToFloatbits(unsigned short h);
+void ConvertDataType(int devID, 
+                     void * s, TENSOR_DATA_TYPE typeS, 
+                     void * t, TENSOR_DATA_TYPE typeT, int size);
+
+#ifdef USE_CUDA
+void CudaConvertDataType(int devID, 
+                         void * s, TENSOR_DATA_TYPE typeS, 
+                         void * t, TENSOR_DATA_TYPE typeT, int size);
+#endif
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif
\ No newline at end of file

source/tensor/XDevice.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-06-23
+ *
+ */
+
+#ifndef __XDEVICE_H__
+#define __XDEVICE_H__
+
+#include "XThread.h"
+#include "XStream.h"
+
+#ifdef USE_CUDA
+
+/* the CUDA stuff */
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cuda.h>
+
+#endif
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+#define MAX_LENGTH_OF_DEVICE_NAME 64
+#define MAX_CPU_NUM 16
+#define MAX_GPU_NUM 16
+#define MAX_DEVICE_NUM MAX_CPU_NUM+MAX_GPU_NUM
+#define INVALID_DEVICE_ID -1000
+#define CURRENT_GPU 1000
+//#define CUDA_UVA 1 // Unified Virtual Address Space of Cuda
+
+/*
+a class that records the basic information for each GPU/CPU device
+e.g., the memory limit, warp size of a GPU and etc.
+*/
+class XDevice
+{
+public:
+    /* 
+    device id 
+    <0:  CPU memory
+    >=0: GPU device ID
+    */
+    int devID;
+
+    /* size of the memory */
+    int memSize;
+
+    /* warp size of an (Navida) GPU */
+    int GPUWarpSize;
+
+    /* indicates whether the device class has been initialized */
+    bool isInitialized;
+
+    /* 
+    max grid size (or number of blocks) of an (Navida) GPU 
+    NOTE: the grid size is alone with three dimensions (x, y, z)
+    */
+    int GPUMaxGridSize[3];
+
+    /*
+    max block size (or number of threads per block) of an (Navida) GPU 
+    NOTE: the block size is alone with three dimensions (x, y, z)
+    */
+    int GPUMaxBlockSize[3];
+
+    /* max thread number that is supported */
+    int GPUMaxThreadNum;
+
+    /* max (and optimal) thread number for a block */
+    int GPUMaxThreadNumPerBlock;
+
+    /* name of the device */
+    char name[MAX_LENGTH_OF_DEVICE_NAME];
+
+    /* name of the device */
+    char name2[MAX_LENGTH_OF_DEVICE_NAME];
+
+    /* specify whether Unified Virtual Address Space (UVA) is supported */
+    bool isUVASupported;
+
+    /* default stream for the device */
+    XStream * stream;
+    
+#ifdef USE_CUDA
+    /* mutex for handle (GPU cublas) */
+    MUTEX_HANDLE cublasMutex;
+
+    /* handle used for cublas */
+    cublasHandle_t cublasHandle;
+
+    /* specify if the handle is initialized */
+    bool isHandleReady;
+#endif
+
+
+public:
+    /* constructor */
+    XDevice();
+
+    /* de-constructor */
+    ~XDevice();
+
+    /* initialize it and get the device information */
+    void Init(int myDevID);
+
+    /* clear it */
+    void Clear();
+
+#ifdef USE_CUDA
+    /* get cublas handle */
+    cublasHandle_t * GetCublasHandle();
+
+    /* get the stream of cuda */
+    cudaStream_t * GetCudaStream();
+#endif
+
+    /* switch to a device */
+    static
+    void SetGPUDevice(int devID);
+
+    /* switch to a device (with fast GPU execution mode) */
+    static
+    void SetGPUDeviceFast(int devID);
+
+    /* switch to a get current dev */
+    static
+    int GetGPUDevice();
+
+    /* reset cuda flag for more efficient cuda execution */
+    static
+    void SetFastFlags();
+
+    /* reset cuda flag for more efficient cuda execution (all devices) */
+    static
+    void SetFastFlagsAllDevices();
+};
+
+/*
+a class for the management of devices
+*/
+class XDevManager
+{
+public:
+    /* CPU device information */
+    XDevice CPUs[MAX_CPU_NUM];
+
+    /* number of CPUs */
+    int nCPU;
+
+    /* GPU device information */
+    XDevice GPUs[MAX_GPU_NUM];
+
+    /* number of GPUs */
+    int nGPU;
+
+public:
+    /* constructor */
+    XDevManager();
+
+    /* de-constructor */
+    ~XDevManager();
+
+    /* initialize it and get the CPU and GPU information */
+    void Init();
+
+    /* clear it */
+    void Clear();
+
+#ifdef USE_CUDA
+    /* get the handle of GPU */
+    cublasHandle_t * GetCudaHandle(const int devID);
+
+    /* get the stream of cuda */
+    cudaStream_t * GetCudaStream(const int devID);
+#endif
+
+    /* get grid and block sizes that max potential */
+    int GetCudaThread(const int devID, const int n, int * gridSize, int * blockSize);
+
+    /* get grid and block sizes that max potential (2-dimension assignment) */
+    int GetCudaThread2D(const int devID, const int n, const int m, int nLimit, int * gridSize, int * blockSize);
+
+    /* get device ids for the given device information */
+    int GetDeviceIDs(char * devInfo, int * devIDs);
+
+    /* show id sequence */
+    void ShowDeviceIDs(char * devInfo, char * msg);
+
+    /* show device information */
+    void ShowDevInfo();
+
+    /* get the device information in string */
+    char * GetDevString(int devID);
+};
+
+/* managing the devices */
+extern XDevManager GDevs;
+
+/* keep the device config */
+
+#define ProtectCudaDev(devID, devIDBackup) \
+{ \
+    cudaGetDevice(&devIDBackup); \
+    if(devIDBackup != devID) \
+        cudaSetDevice(devID); \
+} \
+
+#define BacktoCudaDev(devID, devIDBackup) \
+{ \
+    if(devIDBackup != devID) \
+        cudaSetDevice(devIDBackup); \
+} \
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif

source/tensor/XGlobal.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-01-20
+ *
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "XGlobal.h"
+
+#if !defined( WIN32 ) && !defined( _WIN32 )
+    #include "sys/time.h"
+    #include "time.h"
+    #include "iconv.h"
+#else
+    #include "time.h"
+    #include "windows.h"
+    #include "process.h"
+#endif
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+/* memory pool setting */
+int MAX_MEM_BLOCK_NUM = 1024;
+int MAX_MEM_BLOCK_SIZE = 1024 * 1024 * 256;
+int MIN_MEM_BLOCK_SIZE = 1024 * 1024 * 64;
+int MINOR_MEM_BLOCK_SIZE = 1024 * 1024 * 256;
+int MAX_MEM_BUF_SIZE = 1024 * 1024 * 256;
+int MIN_MEM_BUF_SIZE = 1024 * 1024 * 32;
+int TRAINING_SAMPLE_BUF_SIZE = 1024 * 1024 * 16;
+int CONST_MINUSONE = -1;
+bool CONST_TRUE = true;
+
+int verboseLevel = 0;
+bool useBLAS = false;
+bool useCUDA = false;
+
+FILE * tmpLog = NULL;
+double myTime = 0;
+double myTime2 = 0;
+double myTime3 = 0;
+double myTime4 = 0;
+double myTime5 = 0;
+double myTime6 = 0;
+double myTime7 = 0;
+double myTime8 = 0;
+double myTime9 = 0;
+double myTimeForward1 = 0;
+double myTimeForward2 = 0;
+double myTimeForward3 = 0;
+double myTimeBackward1 = 0;
+double myTimeBackward2 = 0;
+double myTimeBackward3 = 0;
+double myTimeBackward4 = 0;
+
+int dEdWCount = 0;
+FILE * tF;
+
+/* initialization of the global stuff */
+void InitGlobalAll()
+{
+    srand((unsigned int)time(NULL));
+}
+
+} /* end of the nts (NiuTrans.Tensor) namespace */

source/tensor/XGlobal.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2016-01-20
+ *
+ */
+
+#ifndef __XGLOBAL_H__
+#define __XGLOBAL_H__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <float.h>
+
+#ifndef WIN32
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+// the CUDA stuff
+#ifdef USE_CUDA
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#endif
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts {
+
+#define _XINLINE_  
+
+//#define DOUBELPRICSION
+
+#ifdef DOUBELPRICSION
+#define DTYPE double
+#define DTYPE_MIN (DTYPE)1.79E+308
+#else
+#define DTYPE float
+#define DTYPE_MIN (DTYPE)-3.40E+38
+#endif
+
+#define LOGPROB_MIN (DTYPE)-2E+1
+#define GRAD_MAX (DTYPE)1E+5
+
+#if WIN32
+#define DELIMITER '\\'
+#else
+#define DELIMITER '/'
+#endif
+
+#ifndef MIN
+#define MIN(a,b) ((a) < (b) ? a : b)
+#endif
+#ifndef MAX
+#define MAX(a,b) ((a) > (b) ? a : b)
+#endif
+
+#define __FILENAME__ ( strrchr(__FILE__, DELIMITER) != NULL ? strrchr(__FILE__, DELIMITER)+1 : __FILE__ )
+
+#define CheckNTErrors(x, msg) \
+{ \
+    if(!(x)) \
+    { \
+        fprintf(stderr, "[ERROR] calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__, msg); \
+        exit(1); \
+    } \
+} \
+
+#define CheckNTErrorsV0(x) \
+{ \
+    if(!(x)) \
+    { \
+        fprintf(stderr, "[ERROR] calling '%s' (%s line %d): %s\n", #x, __FILENAME__, __LINE__); \
+        exit(1); \
+    } \
+} \
+
+#define ShowNTErrors(msg) \
+{ \
+    { \
+        fprintf(stderr, "[ERROR] (%s line %d): %s\n", __FILENAME__, __LINE__, msg); \
+        exit(1); \
+    } \
+} \
+
+#define MAX_FILE_NAME_LENGTH 1024 * 2
+#define MAX_LINE_LENGTH 1024*1024
+#define MAX_SENTENCE_LEN 512
+#define X_MILLION 1000000
+#define MAX_INT 2147483647
+#define MAX_FLOAT FLT_MAX
+#define FIELD_SEP " ||| "
+#define FLOAT_MIN float(-1.0E38)
+#define FLOAT16_MIN float(-65504)
+#define MILLION 1000000
+#define LOG_E_10 2.302585
+#define LEADING_DIM 1
+
+/* cuda setting */
+#define MAX_CUDA_THREAD_NUM_PER_BLOCK 512
+#define MIN_CUDA_SHARED_MEM_COL_SIZE 8
+#define MAX_MODEL_NUM 512
+#define SHARED_MEMORY_SIZE (48 << 10)
+
+/* memory pool setting */
+extern int MAX_MEM_BLOCK_NUM;
+extern int MAX_MEM_BLOCK_SIZE;
+extern int MIN_MEM_BLOCK_SIZE;
+extern int MINOR_MEM_BLOCK_SIZE;
+extern int MAX_MEM_BUF_SIZE;
+extern int MIN_MEM_BUF_SIZE;
+extern int TRAINING_SAMPLE_BUF_SIZE;
+
+extern int CONST_MINUSONE;
+extern bool CONST_TRUE;
+
+//#define USE_CUDA_RESURSION 1
+
+#define NIUTRANSNNDEBUG
+
+extern int verboseLevel;
+extern bool useBLAS;
+extern bool useCUDA;
+
+#define FFLUSH(FILEH) \
+{ \
+    fflush(FILEH); \
+} \
+
+#define XPRINT(VERBOSE,FILEH,STR) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR);FFLUSH(FILEH);}}
+#define XPRINT1(VERBOSE,FILEH,STR,ARG) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG);FFLUSH(FILEH);}}
+#define XPRINT2(VERBOSE,FILEH,STR,ARG,ARG2) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2);FFLUSH(FILEH);}}
+#define XPRINT3(VERBOSE,FILEH,STR,ARG,ARG2,ARG3) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3);FFLUSH(FILEH);}}
+#define XPRINT4(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4);FFLUSH(FILEH);}}
+#define XPRINT5(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5);FFLUSH(FILEH);}}
+#define XPRINT6(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6);FFLUSH(FILEH);}}
+#define XPRINT7(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7);FFLUSH(FILEH);}}
+#define XPRINT8(VERBOSE,FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7,ARG8) {if(VERBOSE<=verboseLevel) {fprintf(FILEH,STR,ARG,ARG2,ARG3,ARG4,ARG5,ARG6,ARG7,ARG8);FFLUSH(FILEH);}}
+
+#define B2I(V) V==0?false:true
+
+/* BLAS interfaces */
+#ifdef DOUBELPRICSION
+#define GEMM XBLAS_DGEMM
+#else
+#define GEMM XBLAS_SGEMM
+#endif
+
+extern void InitGlobalAll();
+
+extern FILE * tmpLog;
+
+extern int dEdWCount;
+extern FILE * tF;
+extern int tmpCountV2;
+extern int nnnTotal;
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif

source/tensor/XHeap.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * As it is, this is a heap.
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2017-12-20
+ *
+ */
+
+#include "XGlobal.h"
+#include "XHeap.h"
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+/* constructor */
+template<HeapType hType, typename T>
+XHeap<hType, T>::XHeap(int mySize, XMem * myMem)
+{
+    mem = myMem;
+    size = mySize;
+    count = 0;
+    if (mem == NULL)
+        items = new HeapNode<T>[mySize];
+    else
+        mem->Alloc(mem->devID, mySize * sizeof(T));
+}
+
+/* deconstructor */
+template<HeapType hType, typename T>
+XHeap<hType, T>::~XHeap()
+{
+    delete[] items;
+}
+
+template<HeapType hType, typename T>
+void XHeap<hType, T>::Clear(T initValue)
+{
+    count = 0;
+    for (int i = 0; i < size; i++) {
+        items[i].index = 0;
+        items[i].value = initValue;
+    }
+}
+
+/* compare node i and node j */
+template<HeapType hType, typename T>
+_XINLINE_ bool XHeap<hType, T>::Compare(int i, int j)
+{
+    if (hType == MIN_HEAP)
+        return items[i].value < items[j].value;
+    else
+        return items[j].value < items[i].value;
+}
+
+/* top most item */
+template<HeapType hType, typename T>
+_XINLINE_ HeapNode<T> XHeap<hType, T>::Top()
+{
+    HeapNode<T> node = items[0];
+    return node;
+}
+
+/* last item */
+template<HeapType hType, typename T>
+_XINLINE_ HeapNode<T> XHeap<hType, T>::End()
+{
+    HeapNode<T> node = items[count - 1];
+    return node;
+}
+
+/* push an item into the heap */
+template<HeapType hType, typename T>
+_XINLINE_ void XHeap<hType, T>::Push(HeapNode<T> node)
+{
+    //CheckNTErrors((count < size), "Heap is full!");
+    items[count] = node;
+    Up(count);
+    count++;
+}
+
+/* replace the top-most item and update the heap */
+template<HeapType hType, typename T>
+_XINLINE_ void XHeap<hType, T>::ReplaceTop(HeapNode<T> node)
+{
+    items[0] = node;
+    Down(0);
+}
+
+/* pop the top most item */
+template<HeapType hType, typename T>
+_XINLINE_ HeapNode<T> XHeap<hType, T>::Pop()
+{
+    //CheckNTErrors((size > 0), "Empty heap!");
+    HeapNode<T> node = items[0];
+    items[0] = items[count - 1];
+    count--;
+    items[count].index = 0;
+    items[count].value = 0;
+    Down(0);
+    return node;
+}
+
+/* move item k down the tree */
+template<HeapType hType, typename T>
+_XINLINE_ void XHeap<hType, T>::Down(int k)
+{
+    int i = k;
+    while (2 * i + 1 < count) {
+        int l = 2 * i + 1, r = 2 * i + 2;
+        int m = (r >= count || Compare(l, r)) ? l : r;
+        if (Compare(i, m))
+            break;
+        HeapNode<T> tmp = items[i];
+        items[i] = items[m];
+        items[m] = tmp;
+        i = m;
+    }
+}
+
+/* move item k up the tree */
+template<HeapType hType, typename T>
+_XINLINE_ void XHeap<hType, T>::Up(int k)
+{
+    int i = k;
+    int parent = (i - 1) / 2;
+    while (i > 0 && !Compare(parent, i)) {
+        HeapNode<T> tmp = items[i];
+        items[i] = items[parent];
+        items[parent] = tmp;
+        i = parent;
+        parent = (i - 1) / 2;
+    }
+}
+
+/* explicit instantiation */
+template class XHeap<MAX_HEAP, float>;
+template class XHeap<MAX_HEAP, double>;
+template class XHeap<MAX_HEAP, int>;
+template class XHeap<MIN_HEAP, float>;
+template class XHeap<MIN_HEAP, double>;
+template class XHeap<MIN_HEAP, int>;
+
+} /* end of the nts (NiuTrans.Tensor) namespace */

source/tensor/XHeap.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ *
+ * As it is, this is a heap.
+ *
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2017-12-20
+ * Wedding anniversary !!! 
+ *
+*/
+
+#ifndef __XHEAP_H__
+#define __XHEAP_H__
+
+#include "XMem.h"
+
+/* the nts (NiuTrans.Tensor) namespace */
+namespace nts{
+
+enum HeapType{MIN_HEAP, MAX_HEAP};
+
+/* an item in the heap */
+template <typename T>
+struct HeapNode
+{
+    /* node index */
+    int index;
+
+    /* value of the node */
+    T value;
+
+    HeapNode()
+    {
+        index = -1;
+        value = 0;
+    };
+
+    HeapNode(int i, T v)
+    {
+        index = i;
+        value = v;
+    };
+};
+
+/* a heap that keeps a data array of T */
+template<HeapType hType, typename T>
+class XHeap
+{
+public:
+    /* memory pool */
+    XMem * mem;
+
+    /* number of the items the heap keeps */
+    int size;
+
+    /* number of the items that are already in the heap */
+    int count;
+
+    /* items */
+    HeapNode<T> * items;
+
+public:
+    /* constructor */
+    XHeap(int mySize, XMem * myMem = NULL);
+
+    /* deconstructor */
+    ~XHeap();
+
+    /* clear the data */
+    void Clear(T initValue);
+
+    /* compare node i and node j */
+    bool Compare(int i, int j);
+
+    /* top most item */
+    HeapNode<T> Top();
+
+    /* last item */
+    HeapNode<T> End();
+
+    /* push an item into the heap */
+    void Push(HeapNode<T> node);
+
+    /* replace the top-most item and update the heap */
+    void ReplaceTop(HeapNode<T> node);
+
+    /* pop the top most item */
+    HeapNode<T> Pop();
+
+    /* move item k down the tree */
+    void Down(int k);
+
+    /* move item k up the tree */
+    void Up(int k);
+};
+
+} /* end of the nts (NiuTrans.Tensor) namespace */
+
+#endif

source/tensor/XLink.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-04
+ */
+
+#include <stdio.h>
+#include "XGlobal.h"
+#include "XTensor.h"
+
+#ifndef __XLINK_H__
+#define __XLINK_H__
+
+#include "XGlobal.h"
+
+namespace nts{ // namespace nts(NiuTrans.Tensor)
+
+/* cross reference */
+struct XTensor;
+
+#define MAX_OP_NAME_LENGTH 16
+#define PARAM_UNTI_SIZE    64
+
+/*
+This defines the link among tensors in networks. XLink can be
+cast as a hyperedge in a graph. when we compute on tensors, we actually create a
+network where nodes are tensors and edges the connections among them. Each connection is
+a hyperedge whose head is the output tensor and tails are input tensors. E.g,
+c = a + b
+represents a network with three nodes (a, b and c) and a hyperedge that links a and b (tails) to c (head).
+ 
+   + (=c)
+  / \
+ a   b
+ 
+for c, we have a incoming edge (a, b) -> c
+for a, we also have a edge c -> a in the reverse order (in a view of acyclic directed graphs)
+*/
+struct XLink
+{
+    /* head of the hyperedge */
+    XTensor *  head;
+
+    /* tails of the hyperedge */
+    XTensor ** tails;
+
+    /* number of tails */
+    int tailNum;
+
+    /* parameters used. e.g., c = a * b * \alpha 
+       scalar \alpha is the parameter */
+    void * params;
+
+    /* number of parameters */
+    int paramNum;
+
+    /* size of each parameter */
+    static int paramSize;
+
+    /* name of the hyperedge type. e.g., sum, mul ... */
+    char type[MAX_OP_NAME_LENGTH];
+    
+    /* type id */
+    int typeID;
+
+    /* caculator (pointer to the class for computation) */
+    void * caculator;
+    
+    /* constuctor */
+    XLink();
+    
+    /* deconstructor */
+    ~XLink();
+
+    /* reset it */
+    void Reset();
+
+    /* clear it */
+    void Clear();
+
+    /* clear tails */
+    void ClearTail();
+
+    /* clear the incoming node list of tensor node */
+    static
+    void ClearIncoming(XTensor * node);
+    
+    /* clear the outgoing node list of tensor node */
+    static
+    void ClearOutgoing(XTensor * node);
+
+    /* set edge type id and name */
+    void SetType(int id);
+
+    /* set head */
+    void SetHead(XTensor * h);
+
+    /* add a tail */
+    void AddTail(XTensor * t);
+
+    /* add two tails in one time */
+    void AddTwoTails(XTensor * t1, XTensor * t2);
+
+    /* add a parameter in default type */
+    void AddParam(DTYPE param);
+
+    /* add a parameter */
+    void AddParam(void * param, int size);
+
+    /* get a paramter in default type */
+    DTYPE GetParam(int i);
+
+    /* get a paramter in integer */
+    int GetParamInt(int i);
+
+    /* get a paramter in pointer */
+    void * GetParamPointer(int i);
+    
+    /* get a parameter in MATRIX_TRANS_TYPE */
+    MATRIX_TRANS_TYPE GetParamTrans(int i);
+
+    /* create a hyper edge with two input tensors and a output tensor */
+    static
+    void MakeLink(const XTensor * t1, const XTensor * t2, XTensor * h, int id);
+
+    /* create a hyper edge with a list of input tensors and a output tensor */
+    static
+    void MakeLink(const XList * list, XTensor * h, int id);
+
+    /* create a hyper edge with a input tensors and a list of output tensors */
+    static
+    void MakeLink(XTensor * h, XList * list, int id);
+
+    /* add a parameter */
+    static
+    void AddParamToHead(XTensor * h, DTYPE param);
+
+    /* add an integer parameter */
+    static
+    void AddParamToHeadInt(XTensor * h, int param);
+
+    /* add a MATRIX_TRANS_TYPE parameter */
+    static
+    void AddParamToHeadTrans(XTensor * h, MATRIX_TRANS_TYPE param);
+
+    /* add a boolean parameter */
+    static
+    void AddParamToHeadBool(XTensor * h, bool param);
+
+    /* add a pointer parameter */
+    static
+    void AddParamToHeadPointer(XTensor * h, void * param);
+
+    /* replace a node with another, i.e., we redirect the links to the new node */
+    static 
+    void Replace(const XTensor * oldOne, XTensor * newOne);
+
+    /* copy links of a given node */
+    static
+    void CopyIncoming(const XTensor * reference, XTensor * target);
+
+    /* check the correctness of the network encoded in a root node (tensor) */
+    static
+    void CheckNetwork(XTensor * root);
+
+    /* show a node */
+    static
+    void ShowNode(FILE * file, XTensor * node);
+};
+    
+} // namespace nts(NiuTrans.Tensor)
+
+#endif // __XLINK_H__