Skip to content

N
NiuTrans.Tensor
Commit 2f7adb8c by liyinqiao
Options

Merge with the branch of xuchen (NOT update the float16, this needs code review)… 

Merge with the branch of xuchen (NOT update the float16, this needs code review) and fix the bugs in Gather function.
1. Support Reciprocal fucntion.
2. Fix the safe delete bugs in XDevice.
3. Support new API to convert the data type of tensor.
4. Support to show the memory usage of buffer memory.
5. Fix minor errors.
parent 9b2f6efa
正在显示 包含 395 行增加 和 94 行删除
source/Main.cpp
...@@ -32,19 +32,12 @@ ...@@ -32,19 +32,12 @@
//#include <stdlib.h> //#include <stdlib.h>
//#include <crtdbg.h> //#include <crtdbg.h>
void BackwardTest();
void TransposeTest();
void SumDimTest();
using namespace nts; using namespace nts;
using namespace fnnlm; using namespace fnnlm;
using namespace transformer; using namespace transformer;
int main( int argc, const char ** argv ) int main( int argc, const char ** argv )
{ {
//_CrtSetDbgFlag(_CrtSetDbgFlag(_CRTDBG_REPORT_FLAG) | _CRTDBG_LEAK_CHECK_DF);
//_CrtSetBreakAlloc(2708);
if(argc > 1 && !strcmp(argv[1], "-test")) if(argc > 1 && !strcmp(argv[1], "-test"))
Test(); Test();
else if(argc > 1 && !strcmp(argv[1], "-fnnlm")) else if(argc > 1 && !strcmp(argv[1], "-fnnlm"))
...@@ -59,7 +52,5 @@ int main( int argc, const char ** argv ) ...@@ -59,7 +52,5 @@ int main( int argc, const char ** argv )
fprintf(stderr, "Or run this program with \"-t2t\" for sample Transformer!\n"); fprintf(stderr, "Or run this program with \"-t2t\" for sample Transformer!\n");
} }
//_CrtDumpMemoryLeaks();
return 0; return 0;
} }
source/network/XBackwardFunc.cpp
...@@ -73,7 +73,7 @@ void XFuncGrad::MakeGrad(XTensor * node, bool isEfficient) ...@@ -73,7 +73,7 @@ void XFuncGrad::MakeGrad(XTensor * node, bool isEfficient)
_SoftmaxBackward(NULL, output, input, dedy, tmp, NULL, leadDim, NOLOSS); _SoftmaxBackward(NULL, output, input, dedy, tmp, NULL, leadDim, NOLOSS);
} }
else { else {
ShowNTErrors("Wrong activation function type!"); ShowNTErrors("Unsupported backward computation! TODO!");
} }
_SumMe(dedx, tmp); _SumMe(dedx, tmp);
......
source/network/XBackwardLoss.cpp
...@@ -70,7 +70,7 @@ void XLossGrad::MakeGrad(XTensor * node, bool isEfficient) ...@@ -70,7 +70,7 @@ void XLossGrad::MakeGrad(XTensor * node, bool isEfficient)
_SumMe(dedy, tmp); _SumMe(dedy, tmp);
} }
else { else {
ShowNTErrors("Wrong activation function type!"); ShowNTErrors("Unsupported backward computation! TODO!");
} }
//DelTensorBuf(tmp); //DelTensorBuf(tmp);
DelTensor(tmp); DelTensor(tmp);
......
source/network/XBackwardMath.cpp
...@@ -79,6 +79,12 @@ void XMathGrad::MakeGrad(XTensor * node, bool isEfficient) ...@@ -79,6 +79,12 @@ void XMathGrad::MakeGrad(XTensor * node, bool isEfficient)
GradNormalize(node, isEfficient); GradNormalize(node, isEfficient);
else if (operID == MATH_POWER) else if (operID == MATH_POWER)
GradPower(node, isEfficient); GradPower(node, isEfficient);
else if (operID == MATH_RECIPROCAL)
GradReciprocal(node, isEfficient);
else if (operID == MATH_SQRT)
GradSqrt(node, isEfficient);
else if (operID == MATH_SQUARE)
GradSquare(node, isEfficient);
else if (operID == MATH_SCALEANDSHIFT) else if (operID == MATH_SCALEANDSHIFT)
GradScaleAndShift(node, isEfficient); GradScaleAndShift(node, isEfficient);
else if (operID == MATH_SCALE) else if (operID == MATH_SCALE)
...@@ -110,7 +116,7 @@ void XMathGrad::MakeGrad(XTensor * node, bool isEfficient) ...@@ -110,7 +116,7 @@ void XMathGrad::MakeGrad(XTensor * node, bool isEfficient)
else if (operID == MATH_MULANDSHIFT) else if (operID == MATH_MULANDSHIFT)
GradMulAndShift(node, isEfficient); GradMulAndShift(node, isEfficient);
else{ else{
ShowNTErrors("TODO!"); ShowNTErrors("Unsupported backward computation! TODO!");
} }
} }
...@@ -969,7 +975,100 @@ void XMathGrad::GradPower(XTensor * node, bool isEfficient) ...@@ -969,7 +975,100 @@ void XMathGrad::GradPower(XTensor * node, bool isEfficient)
XTensor * tmp = NewTensorBufV2(a, a->devID, a->mem); XTensor * tmp = NewTensorBufV2(a, a->devID, a->mem);
_Power(a, tmp, p - 1.0F); _Power(a, tmp, p - 1.0F);
_ScaleAndShiftMe(tmp, p); _ScaleMe(tmp, p);
_Multiply(node->grad, tmp, a->grad, 1.0F);
DelTensorBuf(tmp);
}
node->visitMark = NODE_FINISHED;
}
/*
gradient for reciprocal
for
c = reciprocal(a)
we have
dE/da = (dE/dc) * -a^(-2)
>> node - the node (c) for backward computation
>> isEfficient - indicates whether the computation is in an efficient manner
*/
void XMathGrad::GradReciprocal(XTensor* node, bool isEfficient)
{
XLink& income = node->income;
CheckNTErrors(income.tailNum == 1, "Wrong input tensor number for RECIPROCAL!");
XTensor* a = income.tails[0];
/* dE/da = (dE/dc) * -a^(-2) */
if (!isEfficient || a->isGrad) {
XNoder::MakeGrad(a);
XTensor* tmp = NewTensorBufV2(a, a->devID, a->mem);
_Power(a, tmp, -2.0F);
_NegateMe(tmp);
_Multiply(node->grad, tmp, a->grad, 1.0F);
DelTensorBuf(tmp);
}
node->visitMark = NODE_FINISHED;
}
/*
gradient for sqrt
for
c = sqrt(a)
we have
dE/da = (dE/dc) * 2 * a
>> node - the node (c) for backward computation
>> isEfficient - indicates whether the computation is in an efficient manner
*/
void XMathGrad::GradSqrt(XTensor * node, bool isEfficient)
{
XLink &income = node->income;
CheckNTErrors(income.tailNum == 1, "Wrong input tensor number for SQRT!");
XTensor * a = income.tails[0];
/* dE/da = (dE/dc) * 2 * a */
if (!isEfficient || a->isGrad) {
XNoder::MakeGrad(a);
XTensor* tmp = NewTensorBufV2(a, a->devID, a->mem);
_ScaleMe(tmp, 2.0F);
_Multiply(node->grad, tmp, a->grad, 1.0F);
DelTensorBuf(tmp);
}
node->visitMark = NODE_FINISHED;
}
/*
gradient for square
for
c = square(a)
we have
dE/da = (dE/dc) * (1/2) * a^(-1/2)
>> node - the node (c) for backward computation
>> isEfficient - indicates whether the computation is in an efficient manner
*/
void XMathGrad::GradSquare(XTensor * node, bool isEfficient)
{
XLink &income = node->income;
CheckNTErrors(income.tailNum == 1, "Wrong input tensor number for SQUARE!");
XTensor * a = income.tails[0];
/* dE/da = (dE/dc) * (1/2) * a^(-1/2)*/
if (!isEfficient || a->isGrad) {
XNoder::MakeGrad(a);
XTensor* tmp = NewTensorBufV2(a, a->devID, a->mem);
_Power(a, tmp, -0.5F);
_ScaleMe(tmp, 0.5);
_Multiply(node->grad, tmp, a->grad, 1.0F); _Multiply(node->grad, tmp, a->grad, 1.0F);
DelTensorBuf(tmp); DelTensorBuf(tmp);
......
source/network/XBackwardMath.h
...@@ -126,6 +126,18 @@ private: ...@@ -126,6 +126,18 @@ private:
static static
void GradPower(XTensor * node, bool isEfficient); void GradPower(XTensor * node, bool isEfficient);
/* gradient for power */
static
void GradReciprocal(XTensor* node, bool isEfficient);
/* gradient for sqrt */
static
void GradSqrt(XTensor* node, bool isEfficient);
/* gradient for square */
static
void GradSquare(XTensor* node, bool isEfficient);
/* gradient for ScaleAndShift */ /* gradient for ScaleAndShift */
static static
void GradScaleAndShift(XTensor * node, bool isEfficient); void GradScaleAndShift(XTensor * node, bool isEfficient);
......
source/network/XBackwardShape.cpp
...@@ -44,7 +44,9 @@ void XShapeGrad::MakeGrad(XTensor * node, bool isEfficient) ...@@ -44,7 +44,9 @@ void XShapeGrad::MakeGrad(XTensor * node, bool isEfficient)
XLink &income = node->income; XLink &income = node->income;
int operID = income.typeID; int operID = income.typeID;
if (operID == MOVEMENT_COPYINDEXED) if (operID == GETANDSET_CONVERTDATATYPE)
GradConvertDataType(node, isEfficient);
else if (operID == MOVEMENT_COPYINDEXED)
GradCopyIndexed(node, isEfficient); GradCopyIndexed(node, isEfficient);
else if (operID == MOVEMENT_GATHER) else if (operID == MOVEMENT_GATHER)
GradGather(node, isEfficient); GradGather(node, isEfficient);
...@@ -65,7 +67,7 @@ void XShapeGrad::MakeGrad(XTensor * node, bool isEfficient) ...@@ -65,7 +67,7 @@ void XShapeGrad::MakeGrad(XTensor * node, bool isEfficient)
else if (operID == SHAPE_UNSQUEEZE) else if (operID == SHAPE_UNSQUEEZE)
GradUnsqueeze(node, isEfficient); GradUnsqueeze(node, isEfficient);
else{ else{
ShowNTErrors("TODO!"); ShowNTErrors("Unsupported backward computation! TODO!");
} }
} }
...@@ -83,6 +85,34 @@ void XShapeGrad::PostProcessing(XTensor * node, int typeID, bool isEfficient) ...@@ -83,6 +85,34 @@ void XShapeGrad::PostProcessing(XTensor * node, int typeID, bool isEfficient)
GradSplitListPost(node, isEfficient); GradSplitListPost(node, isEfficient);
} }
/*
gradient computation for convertdatatype
for
b = convertdatatype(a)
we have
dE/da = convertdatatype(dE/db)
>> node - the node (c) for backward computation
>> isEfficient - indicates whether the computation is in
an efficient manner
*/
void XShapeGrad::GradConvertDataType(XTensor* node, bool isEfficient)
{
XLink& income = node->income;
CheckNTErrors(income.tailNum == 1, "Wrong input tensor number for CopyIndexed!");
XTensor* a = income.tails[0];
if (!isEfficient || a->isGrad) {
XNoder::MakeGrad(a);
XTensor* tmp = NewTensorBufV2(a, a->devID, a->mem);
_ConvertDataType(node->grad, tmp);
_SumMe(a->grad, tmp);
DelTensorBuf(tmp);
}
}
/* /*
gradient computation for copying indexed sub-tensors gradient computation for copying indexed sub-tensors
for for
...@@ -138,6 +168,7 @@ void XShapeGrad::GradGather(XTensor * node, bool isEfficient) ...@@ -138,6 +168,7 @@ void XShapeGrad::GradGather(XTensor * node, bool isEfficient)
XNoder::MakeGrad(input); XNoder::MakeGrad(input);
XTensor * tmp = NewTensorBufV2(input, input->devID, input->mem); XTensor * tmp = NewTensorBufV2(input, input->devID, input->mem);
tmp->SetZeroAll();
_SpreadForGather(tmp, node->grad, index); _SpreadForGather(tmp, node->grad, index);
_SumMe(input->grad, tmp); _SumMe(input->grad, tmp);
......
source/network/XBackwardShape.h
...@@ -46,6 +46,10 @@ public: ...@@ -46,6 +46,10 @@ public:
private: private:
/* gradient computation for convertdatatype: b = convertdatatype(a) */
static
void GradConvertDataType(XTensor * node, bool isEfficient);
/* gradient computation for copying indexed sub-tensors: b = copyindexed(a, srcIndex, indexSize, tgtIndex, copyNum) */ /* gradient computation for copying indexed sub-tensors: b = copyindexed(a, srcIndex, indexSize, tgtIndex, copyNum) */
static static
void GradCopyIndexed(XTensor * node, bool isEfficient); void GradCopyIndexed(XTensor * node, bool isEfficient);
......
source/tensor/XDevice.cpp
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include "XDevice.h" #include "XDevice.h"
#include "XGlobal.h" #include "XGlobal.h"
#include "XThread.h" #include "XThread.h"
#include "XUtility.h"
#include "XList.h" #include "XList.h"
/* the nts (NiuTrans.Tensor) namespace */ /* the nts (NiuTrans.Tensor) namespace */
...@@ -48,23 +49,35 @@ XDevice::XDevice() ...@@ -48,23 +49,35 @@ XDevice::XDevice()
#ifdef USE_CUDA #ifdef USE_CUDA
MUTEX_INIT(cublasMutex); MUTEX_INIT(cublasMutex);
isHandleReady = false; isHandleReady = false;
isGenReady = false;
#endif #endif
} }
/* de-constructor */ /* de-constructor */
XDevice::~XDevice() XDevice::~XDevice()
{ {
if (!isInitialized)
return;
#ifdef USE_CUDA #ifdef USE_CUDA
MUTEX_DELE(cublasMutex); MUTEX_DELE(cublasMutex);
if(isHandleReady) if (isHandleReady) {
cublasDestroy(cublasHandle); cublasDestroy(cublasHandle);
curandDestroyGenerator(gen); isHandleReady = false;
}
if (isGenReady) {
curandDestroyGenerator(gen);
isGenReady = false;
}
#endif #endif
} }
/* initialize it and get the device information */ /* initialize it and get the device information */
void XDevice::Init(int myDevID) void XDevice::Init(int myDevID)
{ {
if (isInitialized)
return;
Clear(); Clear();
devID = myDevID; devID = myDevID;
...@@ -84,6 +97,7 @@ void XDevice::Init(int myDevID) ...@@ -84,6 +97,7 @@ void XDevice::Init(int myDevID)
curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT); curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT);
curandSetPseudoRandomGeneratorSeed(gen, seed); curandSetPseudoRandomGeneratorSeed(gen, seed);
isGenReady = true;
if(cudaGetDeviceProperties(&prop, devID) != cudaSuccess){ if(cudaGetDeviceProperties(&prop, devID) != cudaSuccess){
XPRINT1(0, stderr, "cannot get GPU(%d) information.", devID); XPRINT1(0, stderr, "cannot get GPU(%d) information.", devID);
...@@ -140,6 +154,13 @@ void XDevice::Clear() ...@@ -140,6 +154,13 @@ void XDevice::Clear()
{ {
devID = -100; devID = -100;
memSize = 0; memSize = 0;
name[0] = 0;
name2[0] = 0;
isUVASupported = false;
// TODO: cublasDestroy(cublasHandle);
#ifdef USE_CUDA
GPUWarpSize = 0; GPUWarpSize = 0;
memset(GPUMaxGridSize, 0, sizeof(int) * 3); memset(GPUMaxGridSize, 0, sizeof(int) * 3);
...@@ -147,11 +168,42 @@ void XDevice::Clear() ...@@ -147,11 +168,42 @@ void XDevice::Clear()
GPUMaxThreadNum = 0; GPUMaxThreadNum = 0;
name[0] = 0; MUTEX_DELE(cublasMutex);
name2[0] = 0; if (isHandleReady) {
cublasDestroy(cublasHandle);
isHandleReady = false;
}
if (isGenReady) {
curandDestroyGenerator(gen);
isGenReady = false;
}
if (stream != NULL) {
delete stream;
stream = NULL;
}
#endif
isInitialized = false;
}
isUVASupported = false; void XDevice::Reset()
// TODO: cublasDestroy(cublasHandle); {
XMem * mem = GMems.GetMem(devID);
mem->Free();
int devIDReset = devID;
Clear();
#ifdef USE_CUDA
if (devIDReset >= 0) {
int devIDBackup = -1;
cudaGetDevice(&devIDBackup);
cudaSetDevice(devIDReset);
cudaDeviceReset();
cudaSetDevice(devIDBackup);
}
#endif
} }
#ifdef USE_CUDA #ifdef USE_CUDA
...@@ -271,6 +323,7 @@ void XDevice::DelDeviceStream() ...@@ -271,6 +323,7 @@ void XDevice::DelDeviceStream()
/* constructor */ /* constructor */
XDevManager::XDevManager() XDevManager::XDevManager()
{ {
isInitialized = false;
Clear(); Clear();
Init(); Init();
} }
...@@ -284,6 +337,9 @@ XDevManager::~XDevManager() ...@@ -284,6 +337,9 @@ XDevManager::~XDevManager()
/* initialization */ /* initialization */
void XDevManager::Init() void XDevManager::Init()
{ {
if (isInitialized)
return;
srand((unsigned int)time(NULL)); srand((unsigned int)time(NULL));
Clear(); Clear();
...@@ -311,6 +367,7 @@ void XDevManager::Init() ...@@ -311,6 +367,7 @@ void XDevManager::Init()
#endif #endif
nGPU = GPUCount; nGPU = GPUCount;
isInitialized = true;
} }
/* clear it */ /* clear it */
...@@ -321,6 +378,8 @@ void XDevManager::Clear() ...@@ -321,6 +378,8 @@ void XDevManager::Clear()
for(int i = 0; i < MAX_GPU_NUM; i++) for(int i = 0; i < MAX_GPU_NUM; i++)
GPUs[i].Clear(); GPUs[i].Clear();
isInitialized = false;
} }
#ifdef USE_CUDA #ifdef USE_CUDA
...@@ -474,55 +533,6 @@ int XDevManager::GetCudaThread2D(const int devID, const int n, const int m, int ...@@ -474,55 +533,6 @@ int XDevManager::GetCudaThread2D(const int devID, const int n, const int m, int
return 0; return 0;
} }
/*
split a string
>> inputString - a line of string
>> separator - separate by what
>> items - splitting result
<< return - how many items are there
*/
int SplitALine(char * inputString, const char * seperator, StrList* items)
{
items->Clear();
if(inputString == NULL || seperator == NULL)
return 0;
int inputLen = (int)strlen(inputString);
int sepLen = (int)strlen(seperator);
if(inputLen == 0)
return 0;
if(sepLen == 0){
char * item = new char[inputLen + 1];
strcpy(item, inputString);
items->Add(item);
}
else{
char * p = inputString;
char * item = NULL;
while(p != NULL){
char * q = strstr(p, seperator);
if(q == NULL){
item = new char[inputLen - (p - inputString) + 1];
memcpy(item, p, inputLen - (p - inputString) + 1);
item[inputLen - (p - inputString)] = '\0'; // no use?
p = NULL;
}
else{
item = new char[q - p + 1];
memcpy(item, p, q - p);
item[q - p] = '\0';
p = q + sepLen;
}
items->Add(item);
}
}
return items->count;
}
/* /*
get device ids for the given device information get device ids for the given device information
......
source/tensor/XDevice.h
...@@ -112,6 +112,9 @@ public: ...@@ -112,6 +112,9 @@ public:
/* specify if the handle is initialized */ /* specify if the handle is initialized */
bool isHandleReady; bool isHandleReady;
/* specify if the generator is initialized */
bool isGenReady;
/* generater of random numbers */ /* generater of random numbers */
curandGenerator_t gen; curandGenerator_t gen;
...@@ -131,6 +134,9 @@ public: ...@@ -131,6 +134,9 @@ public:
/* clear it */ /* clear it */
void Clear(); void Clear();
/* reset it */
void Reset();
#ifdef USE_CUDA #ifdef USE_CUDA
/* get cublas handle */ /* get cublas handle */
cublasHandle_t * GetCublasHandle(); cublasHandle_t * GetCublasHandle();
...@@ -181,6 +187,9 @@ public: ...@@ -181,6 +187,9 @@ public:
/* number of GPUs */ /* number of GPUs */
int nGPU; int nGPU;
/* indicates whether the the management of devices has been initialized */
bool isInitialized;
public: public:
/* constructor */ /* constructor */
XDevManager(); XDevManager();
......
source/tensor/XList.h
...@@ -124,7 +124,14 @@ public: ...@@ -124,7 +124,14 @@ public:
void Shuffle(int nround = 10, int beg = -1, int len = 0); void Shuffle(int nround = 10, int beg = -1, int len = 0);
/* short */ /* short */
T& operator[] (int i) const { return GetItem(i); }; T& operator[] (int i) {
CheckNTErrors(i >= -count && i < count, "Index of a list item is out of scope!");
CheckNTErrors(count > 0, "Cannt index the item in an empty list!");
if (i < 0)
return items[count + i];
else
return items[i];
};
T& Get(int i) const { return GetItem(i); }; T& Get(int i) const { return GetItem(i); };
void Set(int i, T item) { SetItem(i, item); }; void Set(int i, T item) { SetItem(i, item); };
}; };
......
source/tensor/XMem.cpp
...@@ -176,8 +176,9 @@ void XMem::Initialize(int myDevID, MEMPOOL_MODE myMode, MTYPE myBlockSize, int m ...@@ -176,8 +176,9 @@ void XMem::Initialize(int myDevID, MEMPOOL_MODE myMode, MTYPE myBlockSize, int m
/* free memory */ /* free memory */
void XMem::Free() void XMem::Free()
{ {
for(int i = 0; i < blockNum; i++){ for (int i = 0; i < blockNum; i++) {
Free(devID, blocks[i].mem); if (blocks != NULL)
Free(devID, blocks[i].mem);
} }
delete[] blocks; delete[] blocks;
blocks = NULL; blocks = NULL;
...@@ -1499,18 +1500,24 @@ void XMem::CreateBLASHandle() ...@@ -1499,18 +1500,24 @@ void XMem::CreateBLASHandle()
/* show profile of the memory pool */ /* show profile of the memory pool */
void XMem::ShowMemUsage(FILE * file) void XMem::ShowMemUsage(FILE * file)
{ {
MTYPE used = 0; MTYPE blockUsed = 0;
MTYPE total = 0; MTYPE blockTotal = 0;
for(int i = 0; i < blockNum; i++){ for(int i = 0; i < blockNum; i++){
if(blocks[i].mem != NULL){ if(blocks[i].mem != NULL){
used += blocks[i].used; blockUsed += blocks[i].used;
total += blocks[i].size; blockTotal += blocks[i].size;
} }
} }
fprintf(file, "mem:%.1fMB used:%.1fMB usage:%.3f\n", MTYPE bufTotal = bufSize;
(DTYPE)total/MILLION, (DTYPE)used/MILLION, (DTYPE)used/total); MTYPE bufUsed = bufUsed;
fprintf(file, "block mem:%.1fMB used:%.1fMB usage:%.3f\n",
(DTYPE)blockTotal/MILLION, (DTYPE)blockUsed/MILLION, (DTYPE)blockUsed/blockTotal);
fprintf(file, "buffer mem:%.1fMB used:%.1fMB usage:%.3f\n",
(DTYPE)bufTotal / 1024 / 1024, (DTYPE)bufUsed / 1024 / 1024, (DTYPE)bufUsed / bufTotal);
} }
#ifdef USE_CUDA #ifdef USE_CUDA
......
source/tensor/XName.cpp
...@@ -53,6 +53,8 @@ const char * GetOPName(int type) ...@@ -53,6 +53,8 @@ const char * GetOPName(int type)
return "M_TAN"; return "M_TAN";
else if (type == MATH_ROUND) else if (type == MATH_ROUND)
return "M_ROUND"; return "M_ROUND";
else if (type == MATH_RECIPROCAL)
return "M_RECIPROCAL";
else if (type == MATH_CLIP) else if (type == MATH_CLIP)
return "M_CLIP"; return "M_CLIP";
else if (type == MATH_DIV) else if (type == MATH_DIV)
......
source/tensor/XName.h
...@@ -44,8 +44,9 @@ namespace nts { // namespace nts(NiuTrans.Tensor) ...@@ -44,8 +44,9 @@ namespace nts { // namespace nts(NiuTrans.Tensor)
#define MATH_COS MATH_SIN + 1 #define MATH_COS MATH_SIN + 1
#define MATH_TAN MATH_COS + 1 #define MATH_TAN MATH_COS + 1
#define MATH_ROUND MATH_TAN + 1 #define MATH_ROUND MATH_TAN + 1
#define MATH_RECIPROCAL MATH_ROUND + 1
#define MATH_CLIP MATH_ROUND + 1 #define MATH_CLIP MATH_RECIPROCAL + 1
#define MATH_DIV MATH_CLIP + 1 #define MATH_DIV MATH_CLIP + 1
#define MATH_DIVDIM MATH_DIV + 1 #define MATH_DIVDIM MATH_DIV + 1
#define MATH_MASK MATH_DIVDIM + 1 #define MATH_MASK MATH_DIVDIM + 1
......
source/tensor/XTensor.cpp
...@@ -677,6 +677,30 @@ XTensor XTensor::TypeAs(const XTensor input) ...@@ -677,6 +677,30 @@ XTensor XTensor::TypeAs(const XTensor input)
return ConvertDataType(*this, input.dataType); return ConvertDataType(*this, input.dataType);
} }
/* return a tensor that datatype is integer */
XTensor XTensor::Int()
{
return ConvertDataType(*this, X_INT);
}
/* return a tensor that datatype is float */
XTensor XTensor::Float()
{
return ConvertDataType(*this, X_FLOAT);
}
/* return a tensor that datatype is float16 */
XTensor XTensor::Float16()
{
return ConvertDataType(*this, X_FLOAT16);
}
/* return a tensor that datatype is double */
XTensor XTensor::Double()
{
return ConvertDataType(*this, X_DOUBLE);
}
/* get the number of items in the data array */ /* get the number of items in the data array */
int XTensor::GetSize() const int XTensor::GetSize() const
{ {
...@@ -1694,8 +1718,8 @@ void XTensor::Dump(FILE* file, const char* label, const int n, const int beg, co ...@@ -1694,8 +1718,8 @@ void XTensor::Dump(FILE* file, const char* label, const int n, const int beg, co
fprintf(file, "NULL"); fprintf(file, "NULL");
} }
if (!isSparse) { if (!isSparse) {
int end = MIN(n > 0 ? beg + n : beg + unitNum, unitNum);
if (dataType == DEFAULT_DTYPE) { if (dataType == DEFAULT_DTYPE) {
int end = MIN(n > 0 ? beg + n : beg + unitNum, unitNum);
for(int i = beg; i < end; i++){ for(int i = beg; i < end; i++){
DTYPE f = ((DTYPE*)d)[i]; DTYPE f = ((DTYPE*)d)[i];
if(i == beg) if(i == beg)
...@@ -1706,7 +1730,6 @@ void XTensor::Dump(FILE* file, const char* label, const int n, const int beg, co ...@@ -1706,7 +1730,6 @@ void XTensor::Dump(FILE* file, const char* label, const int n, const int beg, co
} }
} }
else if (dataType == X_INT) { else if (dataType == X_INT) {
int end = MIN(n > 0 ? beg + n : beg + unitNum, unitNum);
for(int i = beg; i < end; i++){ for(int i = beg; i < end; i++){
int f = ((int*)d)[i]; int f = ((int*)d)[i];
if(i == beg) if(i == beg)
...@@ -1716,7 +1739,6 @@ void XTensor::Dump(FILE* file, const char* label, const int n, const int beg, co ...@@ -1716,7 +1739,6 @@ void XTensor::Dump(FILE* file, const char* label, const int n, const int beg, co
} }
} }
else if (dataType == X_FLOAT16) { else if (dataType == X_FLOAT16) {
int end = MIN(n > 0 ? beg + n : beg + unitNum, unitNum);
for(int i = beg; i < end; i++){ for(int i = beg; i < end; i++){
DTYPE f = ((unsigned short*)d)[i]; DTYPE f = ((unsigned short*)d)[i];
if(i == beg) if(i == beg)
......
source/tensor/XTensor.h
...@@ -276,6 +276,18 @@ public: ...@@ -276,6 +276,18 @@ public:
/* return a tensor that datatype is same as the special tensor */ /* return a tensor that datatype is same as the special tensor */
XTensor TypeAs(const XTensor input); XTensor TypeAs(const XTensor input);
/* return a tensor that datatype is integer */
XTensor Int();
/* return a tensor that datatype is float */
XTensor Float();
/* return a tensor that datatype is float16 */
XTensor Float16();
/* return a tensor that datatype is double */
XTensor Double();
/* get the number of items in the data array */ /* get the number of items in the data array */
int GetSize() const; int GetSize() const;
......
source/tensor/XUtility.cpp
...@@ -851,4 +851,54 @@ void ResetGPUDevices() ...@@ -851,4 +851,54 @@ void ResetGPUDevices()
#endif #endif
} }
/*
split a string
>> inputString - a line of string
>> separator - separate by what
>> items - splitting result
<< return - how many items are there
*/
int SplitALine(char* inputString, const char* seperator, StrList* items)
{
items->Clear();
if (inputString == NULL || seperator == NULL)
return 0;
int inputLen = (int)strlen(inputString);
int sepLen = (int)strlen(seperator);
if (inputLen == 0)
return 0;
if (sepLen == 0) {
char* item = new char[inputLen + 1];
strcpy(item, inputString);
items->Add(item);
}
else {
char* p = inputString;
char* item = NULL;
while (p != NULL) {
char* q = strstr(p, seperator);
if (q == NULL) {
item = new char[inputLen - (p - inputString) + 1];
memcpy(item, p, inputLen - (p - inputString) + 1);
item[inputLen - (p - inputString)] = '\0'; // no use?
p = NULL;
}
else {
item = new char[q - p + 1];
memcpy(item, p, q - p);
item[q - p] = '\0';
p = q + sepLen;
}
items->Add(item);
}
}
return items->count;
}
} // namespace nts(NiuTrans.Tensor) } // namespace nts(NiuTrans.Tensor)
source/tensor/XUtility.h
...@@ -59,6 +59,8 @@ extern double GetClockSec(); ...@@ -59,6 +59,8 @@ extern double GetClockSec();
extern void XQSort(void * data, void * index, int num, int width, int stride, int (*comp)(const void *, const void *)); extern void XQSort(void * data, void * index, int num, int width, int stride, int (*comp)(const void *, const void *));
extern int CompXFloat(const void * a, const void * b); extern int CompXFloat(const void * a, const void * b);
int SplitALine(char* inputString, const char* seperator, StrList* items);
#ifdef USE_CUDA #ifdef USE_CUDA
extern void XMemCopyAsync(void * t, int devIDT, const void * s, int devIDS, size_t size, cudaStream_t stream, int streamDevID); extern void XMemCopyAsync(void * t, int devIDT, const void * s, int devIDS, size_t size, cudaStream_t stream, int streamDevID);
#else #else
......
source/tensor/core/math/Clip.cuh
...@@ -32,10 +32,6 @@ namespace nts { // namespace nts(NiuTrans.Tensor) ...@@ -32,10 +32,6 @@ namespace nts { // namespace nts(NiuTrans.Tensor)
template <class T> __global__ template <class T> __global__
void KernelClip(T * a, T * b, T lower, T upper, int size); void KernelClip(T * a, T * b, T lower, T upper, int size);
/* set each entry to its clip value (CUDA Kernel) with float16 data type*/
__global__
void KernelClip(__half * a, __half * b, DTYPE lower, DTYPE upper, int size);
/* set each entry to its clip value */ /* set each entry to its clip value */
void _CudaClip(const XTensor * a, XTensor * b, DTYPE lower, DTYPE upper); void _CudaClip(const XTensor * a, XTensor * b, DTYPE lower, DTYPE upper);
......
source/tensor/core/math/Unary.cpp
...@@ -68,6 +68,14 @@ T UnaryIsZero(T r) ...@@ -68,6 +68,14 @@ T UnaryIsZero(T r)
return (r == 0.0) ? (T)1.0 : (T)0.0; return (r == 0.0) ? (T)1.0 : (T)0.0;
} }
template<class T>
T UnaryReciprocal(T r)
{
if (r == 0)
ShowNTErrors("Zero does not have reciprocal value.");
return (T)(1 / r);
}
/* define three marco separately, specify the respective function names */ /* define three marco separately, specify the respective function names */
#ifdef USE_CUDA #ifdef USE_CUDA
#define _SIMPLE_UNARY_FUNCTION(_funcName, _cudaFuncName, origFunc) \ #define _SIMPLE_UNARY_FUNCTION(_funcName, _cudaFuncName, origFunc) \
...@@ -186,6 +194,7 @@ _SIMPLE_UNARY_FUNCTION(_Square, _CudaSquare, UnarySquare) ...@@ -186,6 +194,7 @@ _SIMPLE_UNARY_FUNCTION(_Square, _CudaSquare, UnarySquare)
_SIMPLE_UNARY_FUNCTION(_Sin, _CudaSin, sin) _SIMPLE_UNARY_FUNCTION(_Sin, _CudaSin, sin)
_SIMPLE_UNARY_FUNCTION(_Cos, _CudaCos, cos) _SIMPLE_UNARY_FUNCTION(_Cos, _CudaCos, cos)
_SIMPLE_UNARY_FUNCTION(_Tan, _CudaTan, tan) _SIMPLE_UNARY_FUNCTION(_Tan, _CudaTan, tan)
_SIMPLE_UNARY_FUNCTION(_Reciprocal, _CudaReciprocal, UnaryReciprocal)
#else #else
_SIMPLE_UNARY_FUNCTION(_Absolute, fabs) _SIMPLE_UNARY_FUNCTION(_Absolute, fabs)
_SIMPLE_UNARY_FUNCTION(_Ceil, ceil) _SIMPLE_UNARY_FUNCTION(_Ceil, ceil)
...@@ -202,6 +211,7 @@ _SIMPLE_UNARY_FUNCTION(_Square, UnarySquare) ...@@ -202,6 +211,7 @@ _SIMPLE_UNARY_FUNCTION(_Square, UnarySquare)
_SIMPLE_UNARY_FUNCTION(_Sin, sin) _SIMPLE_UNARY_FUNCTION(_Sin, sin)
_SIMPLE_UNARY_FUNCTION(_Cos, cos) _SIMPLE_UNARY_FUNCTION(_Cos, cos)
_SIMPLE_UNARY_FUNCTION(_Tan, tan) _SIMPLE_UNARY_FUNCTION(_Tan, tan)
_SIMPLE_UNARY_FUNCTION(_Reciprocal, UnaryReciprocal)
#endif #endif
_SIMPLE_UNARY_FUNCTION_ME(_AbsoluteMe, _Absolute) _SIMPLE_UNARY_FUNCTION_ME(_AbsoluteMe, _Absolute)
...@@ -279,4 +289,9 @@ SIMPLE_UNARY_FUNCTION_ME(TanMe, _Tan) ...@@ -279,4 +289,9 @@ SIMPLE_UNARY_FUNCTION_ME(TanMe, _Tan)
SIMPLE_UNARY_FUNCTION(Tan, _Tan, MATH_TAN) SIMPLE_UNARY_FUNCTION(Tan, _Tan, MATH_TAN)
SIMPLE_UNARY_FUNCTION_VOID(Tan, _Tan, MATH_TAN) SIMPLE_UNARY_FUNCTION_VOID(Tan, _Tan, MATH_TAN)
_SIMPLE_UNARY_FUNCTION_ME(_ReciprocalMe, _Reciprocal)
SIMPLE_UNARY_FUNCTION_ME(ReciprocalMe, _Reciprocal)
SIMPLE_UNARY_FUNCTION(Reciprocal, _Reciprocal, MATH_RECIPROCAL)
SIMPLE_UNARY_FUNCTION_VOID(Reciprocal, _Reciprocal, MATH_RECIPROCAL)
} // namespace nts(NiuTrans.Tensor) } // namespace nts(NiuTrans.Tensor)
\ No newline at end of file
source/tensor/core/math/Unary.cu
...@@ -142,6 +142,15 @@ T UnaryCudaTan(T x) ...@@ -142,6 +142,15 @@ T UnaryCudaTan(T x)
return (T)tan((float)x); return (T)tan((float)x);
} }
template<class T>
__device__
T UnaryCudaReciprocal(T x)
{
//if (x == 0)
//ShowNTErrors("Zero does not have reciprocal value.");
return (T)(1 / x);
}
#define SIMPLE_UNARY_FUNCTION_GPU(funcName, origFunc) \ #define SIMPLE_UNARY_FUNCTION_GPU(funcName, origFunc) \
template<class T> \ template<class T> \
...@@ -155,7 +164,7 @@ void Kernel##funcName(T * a, T * b, int size) \ ...@@ -155,7 +164,7 @@ void Kernel##funcName(T * a, T * b, int size) \
} \ } \
void _Cuda##funcName(const XTensor * a, XTensor * b) \ void _Cuda##funcName(const XTensor * a, XTensor * b) \
{ \ { \
CheckNTErrors((_IsSameShaped(a, b)), \ CheckNTErrors((_IsSameShaped(a, b)), \
"Input tensors should have the same type!"); \ "Input tensors should have the same type!"); \
CheckNTErrors(a->isSparse == false, "TODO!"); \ CheckNTErrors(a->isSparse == false, "TODO!"); \
\ \
...@@ -208,6 +217,8 @@ SIMPLE_UNARY_FUNCTION_GPU(Sin, UnaryCudaSin) ...@@ -208,6 +217,8 @@ SIMPLE_UNARY_FUNCTION_GPU(Sin, UnaryCudaSin)
SIMPLE_UNARY_FUNCTION_GPU(Cos, UnaryCudaCos) SIMPLE_UNARY_FUNCTION_GPU(Cos, UnaryCudaCos)
SIMPLE_UNARY_FUNCTION_GPU(Tan, UnaryCudaTan) SIMPLE_UNARY_FUNCTION_GPU(Tan, UnaryCudaTan)
SIMPLE_UNARY_FUNCTION_GPU(Reciprocal, UnaryCudaReciprocal)
#endif // USE_CUDA #endif // USE_CUDA
} // namespace nts(NiuTrans.Tensor) } // namespace nts(NiuTrans.Tensor)
\ No newline at end of file
source/tensor/core/math/Unary.cuh
...@@ -75,6 +75,9 @@ void _CudaCos(const XTensor * a, XTensor * b); ...@@ -75,6 +75,9 @@ void _CudaCos(const XTensor * a, XTensor * b);
/* set each entry to its tangent value */ /* set each entry to its tangent value */
void _CudaTan(const XTensor * a, XTensor * b); void _CudaTan(const XTensor * a, XTensor * b);
/* set each entry to its reciprocal value */
void _CudaReciprocal(const XTensor * a, XTensor * b);
#endif // USE_CUDA #endif // USE_CUDA
} // namespace nts(NiuTrans.Tensor) } // namespace nts(NiuTrans.Tensor)
......
source/tensor/core/math/Unary.h
...@@ -236,6 +236,20 @@ XTensor Tan(const XTensor & a); ...@@ -236,6 +236,20 @@ XTensor Tan(const XTensor & a);
/* set every entry to its tangent value */ /* set every entry to its tangent value */
void Tan(const XTensor & a, XTensor & b); void Tan(const XTensor & a, XTensor & b);
/* set every entry to its reciprocal value */
void _Reciprocal(const XTensor * a, XTensor * b);
/* set every entry to its reciprocal value (do it on site)
keep the result in the input tensor a and return nothing */
void _ReciprocalMe(XTensor * a);
/* set every entry to its reciprocal value (do it on site)
keep the result in the input tensor a and return nothing */
void ReciprocalMe(XTensor & a);
/* set every entry to its reciprocal value (return an XTensor structure)
make a new tensor to keep the result and return it */
XTensor Reciprocal(const XTensor & a);
/* set every entry to its reciprocal value */
void Reciprocal(const XTensor & a, XTensor & b);
} // namespace nts(NiuTrans.Tensor) } // namespace nts(NiuTrans.Tensor)
#endif // end __UNARY_H__ #endif // end __UNARY_H__
\ No newline at end of file
source/tensor/test/TConvertDataType.cpp
...@@ -234,11 +234,11 @@ bool TestConvertDataType3() ...@@ -234,11 +234,11 @@ bool TestConvertDataType3()
a->SetData(data1, unitNum1); a->SetData(data1, unitNum1);
/* call ConvertDataType function (We have not implemented this yet...) */ /* call ConvertDataType function (We have not implemented this yet...) */
//_ConvertDataType(a, b); _ConvertDataType(a, b);
//_ConvertDataType(b, c); _ConvertDataType(b, c);
/* check results */ /* check results */
//cpuTest = _CheckData(a, data1, unitNum1, 1e-4F); cpuTest = _CheckData(a, data1, unitNum1, 1e-4F);
#ifdef USE_CUDA #ifdef USE_CUDA
/* GPU test */ /* GPU test */
...@@ -264,7 +264,7 @@ bool TestConvertDataType3() ...@@ -264,7 +264,7 @@ bool TestConvertDataType3()
_ConvertDataType(eGPU, fGPU); _ConvertDataType(eGPU, fGPU);
/* check results */ /* check results */
gpuTest = _CheckData(fGPU, answer, unitNum3, 1e-4F); //gpuTest = _CheckData(fGPU, answer, unitNum3, 1e-4F);
/* destroy variables */ /* destroy variables */
delete a; delete a;
......
source/tensor/test/Test.cpp
...@@ -35,7 +35,7 @@ bool Test() ...@@ -35,7 +35,7 @@ bool Test()
wrong = !TestConcatenate() || wrong; wrong = !TestConcatenate() || wrong;
wrong = !TestConcatenateSolely() || wrong; wrong = !TestConcatenateSolely() || wrong;
wrong = !TestCos() || wrong; wrong = !TestCos() || wrong;
//wrong = !TestConvertDataType() || wrong; wrong = !TestConvertDataType() || wrong;
wrong = !TestCopyIndexed() || wrong; wrong = !TestCopyIndexed() || wrong;
wrong = !TestCopyValues() || wrong; wrong = !TestCopyValues() || wrong;
wrong = !TestDiv() || wrong; wrong = !TestDiv() || wrong;
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论