Support fp16 data type for more operations and fix the minor errors.

source/Main.cpp

 /* NiuTrans.Tensor - an open-source tensor library
  * Copyright (C) 2018, Natural Language Processing Lab, Northeastern University.
- * All rights reserved.
+* All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

source/tensor/XGlobal.h

@@ -55,6 +55,8 @@ namespace nts {
 #define DTYPE_MIN (DTYPE)-3.40E+38
 #endif
 
+#define MY_HALF_MIN 6.10E-5
+
 #define LOGPROB_MIN (DTYPE)-2E+1
 #define GRAD_MAX (DTYPE)1E+5
 

source/tensor/XList.cpp

@@ -254,6 +254,9 @@ template <typename T>
 void TensorListBase<T>::Clear()
 {
     count = 0;
+    delete[] items;
+    maxNum = 0;
+    items = NULL;
 }
 
 /*
@@ -288,12 +291,13 @@ inline void TensorListBase<T>::Reverse()
 
 /* remove the item at position i */
 template <typename T>
-void TensorListBase<T>::Remove(int i)
+void TensorListBase<T>::Remove(int idx)
 {
-    if (i >= count || i < 0)
-        return;
+    CheckNTErrors(idx < count && idx > -1, "invalid index");
 
-    memcpy(items + i, items + i + 1, sizeof(T*) * (count - i - 1));
+    for (int i = idx; i < count - 1; i++) {
+        items[i] = items[i + 1];
+    }
 
     count--;
 }
@@ -355,6 +359,20 @@ void TensorListBase<T>::Shuffle(int nround, int beg, int len)
     }
 }
 
+/* sum a range of values */
+template<>
+int TensorListBase<int>::SumRange(int begin, int end)
+{
+    CheckNTErrors(begin < count && end <= count && end > begin, "invalid index of begin or end");
+    int value = items[begin];
+    int i = begin + 1;
+    while (i < end) {
+        value += items[i];
+        i++;
+    }
+    return value;
+}
+
 /* specializations and typedef of list */
 template struct TensorListBase<int>;
 template struct TensorListBase<char>;

source/tensor/XList.h

@@ -130,6 +130,9 @@ public:
     };
     T& Get(int i) { return GetItem(i); };
     void Set(int i, T item) { SetItem(i, item); };
+
+    /* sum a range of values */
+    T SumRange(int begin, int end);
 };
 
 struct XTensor;

source/tensor/XMem.cpp

@@ -1583,16 +1583,16 @@ MTYPE XMemManager::GetAvailableGPUMemory(int devID)
 void XMemManager::GetBufferSize(MTYPE freeMem, MTYPE * myBufSize)
 {
     *myBufSize = 0;
-    if (freeMem >= MILLION * 128){
-        *myBufSize = MILLION * 32;
-        if (freeMem >= MILLION * 256){
-            *myBufSize = MILLION * 64;
-            if (freeMem >= MILLION * 512){
-                *myBufSize = MILLION * 128;
-                if (freeMem >= MILLION * 1024) {
-                    *myBufSize = MILLION * 128;
-                    if (freeMem >= MILLION * 2048)
-                        *myBufSize = MILLION * 128;
+    if (freeMem >= MILLION * 128ULL){
+        *myBufSize = MILLION * 32ULL;
+        if (freeMem >= MILLION * 256ULL){
+            *myBufSize = MILLION * 64ULL;
+            if (freeMem >= MILLION * 512ULL){
+                *myBufSize = MILLION * 128ULL;
+                if (freeMem >= MILLION * 1024ULL) {
+                    *myBufSize = MILLION * 128ULL;
+                    if (freeMem >= MILLION * 2048ULL)
+                        *myBufSize = MILLION * 128ULL;
                 }
             }
         }

source/tensor/XTensor.cpp

@@ -50,6 +50,7 @@
 #include "core/getandset/SetData.h"
 #include "function/Identity.h"
 #include "core/CHeader.h"
+//#include "XHalf.hpp"
 
 #ifdef USE_CUDA
 
@@ -179,12 +180,14 @@ XTensor::XTensor(const XTensor& reference)
         _CopyValues(&reference, this);
     }
 
-    if(reference.isTmp)
+    if (reference.enableGrad) {
+        if (reference.isTmp)
             XLink::Replace(&reference, this);
-    else{
+        else {
             CheckNTErrors(outgo.tailNum == 0, "The node has outgoing edge to other nodes!");
             XLink::CopyIncoming(&reference, this);
         }
+    }
 
     isInit = true;
     isTmp = reference.isTmp;
@@ -212,7 +215,9 @@ XTensor::XTensor(const XTensor&& reference)
        This is VERY tricky and there might be better solutions :) */
     *reference.dataP = NULL;
 
+    if (reference.enableGrad) {
         XLink::Replace(&reference, this);
+    }
 
     isInit = true;
     isTmp = reference.isTmp;
@@ -235,12 +240,14 @@ XTensor::~XTensor()
         newTensor->data = data;
         data = NULL;
 
+        if (enableGrad)
             XLink::Replace(this, newTensor);
     }
 
+    if (enableGrad) {
         XLink::ClearOutgoing(this);
         XLink::ClearIncoming(this);
-    
+    }
     DestroyData();
 
     if(grad != NULL)
@@ -341,9 +348,11 @@ XTensor& XTensor::operator= (const XTensor& tensor)
         newTensor->dataHost = dataHost;
         newTensor->signature = tensor.signature;
         
+        if (enableGrad) {
             XLink::Replace(this, newTensor);
             XLink::ClearOutgoing(this);
             XLink::ClearIncoming(this);
+        }
         newTensor->ShallowCopy(this);
 
         data = NULL;
@@ -354,16 +363,20 @@ XTensor& XTensor::operator= (const XTensor& tensor)
         /* NOTE: this might lead to additional data copy by Mac LLVM compilers */
         /* we make an identity transformation here */
         
-        if(outgo.tailNum > 0)
+        if (enableGrad) {
+            if (outgo.tailNum > 0)
                 XLink::ClearOutgoing(this);
             XLink::ClearIncoming(this);
+        }
         
         if(!_IsSameShaped(this, &tensor))
             Resize(tensor.order, tensor.dimSize, tensor.dataType, tensor.denseRatio);
         
         _Identity(&tensor, this);
+        if (enableGrad) {
             XLink::MakeLink(&tensor, NULL, this, FUNC_IDENTITY);
         }
+    }
     else{
         /* hard copy of the data array */
         int size = unitNum * unitSize;
@@ -396,8 +409,10 @@ XTensor& XTensor::operator= (const XTensor& tensor)
         CheckNTErrors(outgo.tailNum == 0, "The node has outgoing edge to other nodes!");
 
         /* create tensor links for the new tensor */
+        if (enableGrad) {
             XLink::Copy(&tensor, this);
         }
+    }
 
     return *this;
 }
@@ -418,9 +433,11 @@ XTensor& XTensor::operator= (const XTensor&& tensor)
         newTensor->dataHost = dataHost;
         newTensor->signature = tensor.signature;
         
+        if (enableGrad) {
             XLink::Replace(this, newTensor);
             XLink::ClearOutgoing(this);
             XLink::ClearIncoming(this);
+        }
         newTensor->ShallowCopy(this);
 
         data = NULL;
@@ -444,7 +461,9 @@ XTensor& XTensor::operator= (const XTensor&& tensor)
        This is VERY tricky and there might be better solutions :) */
     *tensor.dataP = NULL;
 
+    if (enableGrad) {
         XLink::Copy(&tensor, this);
+    }
 
     return *this;
 }
@@ -1697,6 +1716,17 @@ void XTensor::Dump(FILE* file, const char* label, const int n, const int beg, co
                     fprintf(file, " %d", f);
             }
         }
+        /*else if (dataType == X_FLOAT16) {
+            int end = MIN(n > 0 ? beg + n : beg + unitNum, unitNum);
+            for(int i = beg; i < end; i++){
+                DTYPE f = ((half_float::half*)d)[i];
+                if(i == beg)
+                    fprintf(file, "%e", f);
+                else
+                    fprintf(file, " %e", f);
+
+            }
+        }*/
         else
             ShowNTErrors("TODO!");
     }
@@ -1751,7 +1781,12 @@ void XTensor::BinaryDump(FILE* file)
     switch (dataType) {
         case X_INT: {
             fwrite(tmp.data, sizeof(int), unitNum, file);
+            break;
         }
+        /*case X_FLOAT16: {
+            fwrite(tmp.data, sizeof(half_float::half), unitNum, file);
+            break;
+        }*/
         default: {
             fwrite(tmp.data, sizeof(float), unitNum, file);
         }
@@ -1877,14 +1912,21 @@ read data from a binary file
 */
 void XTensor::BinaryRead(FILE* file, size_t offset)
 {
-    fseek(file, offset, 0);
     switch (dataType) {
         case X_INT: {
             int* d = new int[unitNum];
             fread(d, sizeof(int), unitNum, file);
             SetData(d, unitNum);
             delete[] d;
+            break;
         }
+        /*case X_FLOAT16: {
+            half_float::half* d = new half_float::half[unitNum];
+            fread(d, sizeof(half_float::half), unitNum, file);
+            SetData(d, unitNum);
+            delete[] d;
+            break;
+        }*/
         default: {
             float* d = new float[unitNum];
             fread(d, sizeof(float), unitNum, file);

source/tensor/XTensor.h

@@ -175,7 +175,7 @@ public:
     XLink outgo;
 
     /********************
-     XTensor untilities
+     XTensor utilities
     ********************/
     
     /* constructor */
@@ -440,7 +440,7 @@ public:
     void Read(FILE * file, const char * label = NULL);
 
     /* read data from a binary file */
-    void BinaryRead(FILE * file, size_t offset);
+    void BinaryRead(FILE * file, size_t offset = 0);
 
     /* flush the data to the target device */
     void FlushToMem(XMem * targetMem);