restructure the code

source/network/XBackwardFunc.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for activation function
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ * Dingdang won 5 games in the GO training yesterday, hahaha ...
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDFUNC_H__
+#define __XBACKWARDFUNC_H__
+
+namespace nts{
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardMath.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for math operations
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "XNoder.h"
+#include "XBackwardMath.h"
+#include "../tensor/XName.h"
+#include "../tensor/core/CHeader.h"
+
+namespace nts{
+
+/* compute dE/dx of a node */
+void XMathGrad::MakeGrad(XTensor * node)
+{
+    CheckNTErrors(node->grad != NULL, "No gradient found!");
+
+    XLink &income = node->income;
+    int operID = income.typeID;
+
+    if(operID == MATH_SUM)
+        GradSum(node);
+    else if(operID == MATH_MULTIPLY)
+        GradMultiply(node);
+    else{
+        ShowNTErrors("TODO!");
+    }
+}
+
+/* indicates whether the node is for a math operation */
+bool XMathGrad::IsMathOP(XTensor * node)
+{
+    XLink &income = node->income;
+    return (income.typeID & MATH_BASE) != 0;
+}
+
+/* 
+gradient for sum
+for 
+c =  a + b * \beta
+we have
+dE/da = dE/dc 
+dE/db = dE/dc * \beta
+>> node - the node (c) for backward computation
+*/
+void XMathGrad::GradSum(XTensor * node)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum == 2, "Wrong input tensor number for SUM!");
+
+    XTensor * a = income.tails[0]; 
+    XTensor * b = income.tails[1];
+    DTYPE beta = income.GetParam(0);
+    XNoder::MakeGrad(a);
+    XNoder::MakeGrad(b);
+
+    _CopyValues(node->grad, a->grad);
+    if(beta != 1.0F)
+        _ScaleAndShift(node->grad, a->grad, beta);
+    else
+        _CopyValues(node->grad, b->grad);
+}
+
+/* 
+gradient for multiply (dot production)
+for
+c =  a * b 
+we have
+dE/da = dE/dc * b
+dE/db = dE/dc * a 
+>> node - the node (c) for backward computation
+*/
+void XMathGrad::GradMultiply(XTensor * node)
+{
+    XLink &income = node->income;
+    CheckNTErrors(income.tailNum == 2, "Wrong input tensor number for MULTIPLY!");
+
+    XTensor * a = income.tails[0]; 
+    XTensor * b = income.tails[1];
+    XNoder::MakeGrad(a);
+    XNoder::MakeGrad(b);
+
+    CheckNTErrors(XTensor::IsIdentical(a, b), "Wrong sized input tensors!");
+    _Multiply(node->grad, b, a->grad);
+    _Multiply(node->grad, a, b->grad);
+}
+
+}
\ No newline at end of file

source/network/XBackwardMath.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for math operations
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+
+#ifndef __XBACKWARDMATH_H__
+#define __XBACKWARDMATH_H__
+
+namespace nts{
+
+/* this class computes the gradient for math operations given a node */
+class XMathGrad
+{
+public:
+    /* compute dE/dx of a node */
+    static
+    void MakeGrad(XTensor * node);
+
+    /* indicates whether the node is for a math operation */
+    static
+    bool IsMathOP(XTensor * node);
+
+private:
+    /* gradient for sum: c =  a + b * \beta */
+    static
+    void GradSum(XTensor * node);
+
+    /* gradient for multiply (dot production): c =  a * b */
+    static
+    void GradMultiply(XTensor * node);
+};
+
+}
+
+#endif
\ No newline at end of file

source/network/XBackwardShape.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * backward computation for shaping and data movement
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+#include "../tensor/function/FHeader.h"
+
+#ifndef __XBACKWARDSHAPE_H__
+#define __XBACKWARDSHAPE_H__
+
+namespace nts{
+
+}
+
+#endif
\ No newline at end of file

source/network/XNet.cpp

@@ -20,8 +20,9 @@
  */
 
 #include "XNet.h"
+#include "XNoder.h"
 #include "XBackwardLoss.h"
-#include "XBackwardNode.h"
+#include "XBackwardMath.h"
 #include "../tensor/XName.h"
 
 namespace nts{
@@ -131,6 +132,23 @@ void XNet::Backward(XList &roots, XList &golds, LOSS_FUNCTION_NAME loss)
 }
 
 /* 
+backward computation for a given node 
+>> node - the node keeps the result of an operation (e.g., activation function)
+*/
+void XNet::BackwardNode(XTensor * node)
+{
+    if(node == NULL || node->visitMark == NODE_FINISHED)
+        return;
+
+    if(!XNoder::IsLeaf(node)){
+        if(XMathGrad::IsMathOP(node))
+            XMathGrad::MakeGrad(node);
+    }
+
+    node->visitMark = NODE_FINISHED;
+}
+
+/* 
 traverse the net and find the topological order by 
 depth-first search (Tarjan's algorithm) 
 >> root - root node (or output of the net)

source/network/XNet.h

@@ -63,6 +63,9 @@ struct XNet
        with a number of root nodes */
     void Backward(XList &roots, XList &golds = NULLList, LOSS_FUNCTION_NAME loss = NOLOSS);
 
+    /* backward computation for a given node */
+    void BackwardNode(XTensor * node);
+
     /* traverse the net and find the topological order by 
        depth-first search (Tarjan's algorithm) */
     void Traverse(XTensor &root);

source/network/XNoder.cpp

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "XNoder.h"
+
+namespace nts{
+
+/* make gradient tensor for a node */
+void XNoder::MakeGrad(XTensor * node)
+{
+    if(node == NULL)
+        return;
+
+    if(!XTensor::IsIdentical(node, node->grad)){
+        delete node->grad;
+        node->grad = NewTensor(node);
+    }
+}
+
+/* the node is a leaf node (intput) or not */
+bool XNoder::IsLeaf(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+
+    if(node->income.tailNum == 0)
+        return true;
+    else
+        return false;
+}
+
+/* the node is a root node (output) or not */
+bool XNoder::IsRoot(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+
+    if(node->outgo.tailNum == 0)
+        return true;
+    else
+        return false;
+}
+
+/* the node keeps the gradinent or not */
+bool XNoder::IsGrad(XTensor * node)
+{
+    if(node == NULL)
+        return false;
+    
+    if(node->isGrad)
+        return true;
+    else
+        return false;
+}
+
+}
\ No newline at end of file

source/network/XNoder.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * low-level utilities
+ * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-18
+ */
+
+#include "../tensor/XTensor.h"
+
+#ifndef __XNODER_H__
+#define __XNODER_H__
+
+namespace nts{
+
+#define NODE_UNFINISHED 0
+#define NODE_DOING      1
+#define NODE_FINISHED   2
+
+/* node management */
+class XNoder
+{
+public:
+    /* make gradient tensor for a node */
+    static
+    void MakeGrad(XTensor * node);
+
+    /* the node is a leaf node (intput) or not */
+    static
+    bool IsLeaf(XTensor * node);
+
+    /* the node is a root node (output) or not */
+    static
+    bool IsRoot(XTensor * node);
+
+    /* the node keeps the gradinent or not */
+    static
+    bool IsGrad(XTensor * node);
+};
+
+}
+
+#endif
\ No newline at end of file