update hardtanh

source/sample/fnnlm/FNNLM.cpp

@@ -783,7 +783,7 @@ void Forward(XTensor inputs[], XTensor &output, FNNModel &model, FNNNet &net)
         InitModelTensor2D(embedding, batchSize, model.eSize, model);
 
         /* generate word embedding of position i:
-           embedding = input * w   */
+           embedding = input * w */
         _MatrixMul(&input, X_NOTRANS, &w, X_NOTRANS, &embedding);
 
         eList.Add(&net.embeddings[i]);

source/tensor/test/THardTanH.cpp

@@ -17,7 +17,7 @@
 
 /*
 * $Created by: Lin Ye (email: linye2015@outlook.com) 2018-06-20
-* $Update by: Lin Ye (email: linye2015@outlook.com) 2019-07-06 float16 added
+* $Update by: Lin Ye (email: linye2015@outlook.com) 2019-07-15 float16 added
 */
 
 #include "../XTensor.h"
@@ -297,6 +297,108 @@ bool TestHardTanH3()
 #endif // USE_CUDA
 }
 
+/*
+case 4: float16 test backward computation of HardTanH function.
+dE/dx = dE/dy * dy/dx
+hard tanh: y =  1    if x > 1
+                x    if -1 <= x <= 1
+               -1    if x< -1
+
+   and dy/dx =  1    if -1 <= x <= 1
+                0    otherwise
+In this case, lossName=SQUAREDERROR.
+*/
+bool TestHardTanH4()
+{
+    /* a tensor of size (2, 3) */
+    int order = 2;
+    int * dimSize = new int[order];
+    dimSize[0] = 2;
+    dimSize[1] = 3;
+
+    int unitNum = 1;
+    for (int i = 0; i < order; i++)
+        unitNum *= dimSize[i];
+
+    DTYPE xData[2][3] = { {0.5F, -1.0F, 2.0F},
+                          {3.5F, -4.5F, 1.0F} };
+    DTYPE goldData[2][3] = { {1.0F, 1.0F, 1.0F},
+                             {1.0F, 1.0F, 1.0F} };
+    DTYPE yAnswer[2][3] = { {0.5F, -1.0F, 1.0F},
+                            {1.0F, -1.0F, 1.0F} };
+    DTYPE dedyAnswer[2][3] = { {-0.5F, -2.0F, 0.0F},
+                               {0.0F, -2.0F, 0.0F} };
+    DTYPE dedxAnswer[2][3] = { {-0.5F, -2.0F, 0.0F},
+                               {0.0F, 0.0F, -0.0F} };
+
+    /* CPU test */
+    bool cpuTest = true;
+
+#ifdef USE_CUDA
+    /* GPU test */
+    bool gpuTest = true;
+
+    /* create tensors */
+    XTensor * xGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * yGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * goldGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedyGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+    XTensor * dedxGPU = NewTensor(order, dimSize, X_FLOAT, 1.0F, 0);
+
+    /* create float16 tensors */
+    XTensor xHalfGPU;
+    XTensor yHalfGPU;
+    XTensor goldHalfGPU;
+    XTensor dedyHalfGPU;
+    XTensor dedxHalfGPU;
+
+    /* initialize variables */
+    xGPU->SetData(xData, unitNum);
+    goldGPU->SetData(goldData, unitNum);
+    yGPU->SetZeroAll();
+    dedyGPU->SetZeroAll();
+    dedxGPU->SetZeroAll();
+
+    /* convert data type from float to float16 */
+    xHalfGPU = ConvertDataType(*xGPU, X_FLOAT16);
+    yHalfGPU = ConvertDataType(*yGPU, X_FLOAT16);
+    goldHalfGPU = ConvertDataType(*goldGPU, X_FLOAT16);
+    dedyHalfGPU = ConvertDataType(*dedyGPU, X_FLOAT16);
+    dedxHalfGPU = ConvertDataType(*dedxGPU, X_FLOAT16);
+
+    /* call hardtanh function */
+    _HardTanH(&xHalfGPU, &yHalfGPU);
+
+    /* call hardtanhbackward function */
+    _HardTanHBackward(&goldHalfGPU, &yHalfGPU, &xHalfGPU, &dedyHalfGPU, &dedxHalfGPU, SQUAREDERROR);
+
+    /* convert data type from float16 to float */
+    _ConvertDataType(&yHalfGPU, yGPU);
+    _ConvertDataType(&dedyHalfGPU, dedyGPU);
+    _ConvertDataType(&dedxHalfGPU, dedxGPU);
+
+    /* check results */
+    gpuTest = yGPU->CheckData(yAnswer, unitNum, 1e-4F) &&
+              dedxGPU->CheckData(dedxAnswer, unitNum, 1e-4F) &&
+              dedyGPU->CheckData(dedyAnswer, unitNum, 1e-4F);
+
+    /* destroy variables */
+    delete xGPU;
+    delete yGPU;
+    delete goldGPU;
+    delete dedxGPU;
+    delete dedyGPU;
+    delete[] dimSize;
+
+    return cpuTest && gpuTest;
+#else
+    /* destroy variables */
+    delete[] dimSize;
+
+    return cpuTest;
+#endif // USE_CUDA
+}
+
 
 /* other cases */
 /*
@@ -339,6 +441,16 @@ bool TestHardTanH()
     else
         XPRINT(0, stdout, ">> case 3 passed!\n");
 
+    /* case 4 test */
+    caseFlag = TestHardTanH4();
+
+    if (!caseFlag) {
+        returnFlag = false;
+        XPRINT(0, stdout, ">> case 4 failed!\n");
+    }
+    else
+        XPRINT(0, stdout, ">> case 4 passed!\n");
+
 	/* other cases test */
 	/*
 	TODO!!