Skip to content

N
NiuTrans.Tensor
Commit 4c7d8dde by xiaotong
Options

updates

parent ecc8c6ed
正在显示 包含 225 行增加 和 114 行删除
source/train/XLeader.cpp
...@@ -393,6 +393,8 @@ int XLeader::CountModels() ...@@ -393,6 +393,8 @@ int XLeader::CountModels()
ShowNTErrors("TODO: support a new XWorker type!"); ShowNTErrors("TODO: support a new XWorker type!");
} }
} }
CheckNTErrors(modelCount == jworkers.count, "We assume that a worker just has one model!");
return modelCount; return modelCount;
} }
......
source/train/XLeaderAllReduce.cpp
...@@ -43,22 +43,44 @@ XLeaderAllReduce::XLeaderAllReduce() ...@@ -43,22 +43,44 @@ XLeaderAllReduce::XLeaderAllReduce()
/* deconstructor */ /* deconstructor */
XLeaderAllReduce::~XLeaderAllReduce() XLeaderAllReduce::~XLeaderAllReduce()
{ {
Clear();
}
/* clear */
void XLeaderAllReduce::Clear()
{
for(int i = 1; i < optimizers.count; i++){
XOptimizer * opt = (XOptimizer*)optimizers[i];
delete opt;
}
optimizers.Clear();
} }
/* /*
create workers and other stuff create workers and other stuff
>> config - configuration >> config - configuration
>> model - the model that we run >> model - the model that we run
>> optimizer - the optimizer
>> devIDs - device ids of the workers (the first id is for server) >> devIDs - device ids of the workers (the first id is for server)
>> jobWorkerNum - number of job workers >> jobWorkerNum - number of job workers
*/ */
void XLeaderAllReduce::MakeAll(XConfig * config, XModel * model, const int * devIDs, const int jobWorkerNum) void XLeaderAllReduce::MakeAll(XConfig * config, XModel * model, XOptimizer * optimizer,
const int * devIDs, const int jobWorkerNum)
{ {
Clear();
Init(); Init();
AddJobWorker(model, jobWorkerNum, devIDs); AddJobWorker(model, jobWorkerNum, devIDs);
AddCollectWorker(); AddCollectWorker();
AddUpdateWorker();
AddAuxiliaryWorker(CountModels()); /* Model updaters. One updater for each model. */
AddUpdateWorker(jobWorkerNum);
/* Optimizers of updating models. One optimizer for each model. */
optimizers.Add(optimizer);
for(int i = 1; i < jobWorkerNum; i++){
XOptimizer * opt = optimizer->Clone(devIDs[i]);
optimizers.Add(opt);
}
XLeader::MakeAll(config, model); XLeader::MakeAll(config, model);
} }
...@@ -193,8 +215,10 @@ update the model in a standard server-worker manner ...@@ -193,8 +215,10 @@ update the model in a standard server-worker manner
*/ */
void XLeaderAllReduce::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* active) void XLeaderAllReduce::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* active)
{ {
XWorkerCollect* collecter = (XWorkerCollect*)cworkers.GetItem(0); XWorkerCollect * collecter = (XWorkerCollect*)cworkers.GetItem(0);
XWorkerUpdate * updaterPrime = (XWorkerUpdate*)uworkers.GetItem(0);
CheckNTErrors(modelNum == jworkers.count, "We assume that a worker has one model only!");
CheckNTErrors(uworkers.count >= modelNum, "No enough updaters!"); CheckNTErrors(uworkers.count >= modelNum, "No enough updaters!");
/* parameter map */ /* parameter map */
...@@ -253,60 +277,62 @@ void XLeaderAllReduce::RunUpdate(XConfig* config, XOptimizer* optimizer, const i ...@@ -253,60 +277,62 @@ void XLeaderAllReduce::RunUpdate(XConfig* config, XOptimizer* optimizer, const i
paramServer.grad = paramServer.tensor->grad; paramServer.grad = paramServer.tensor->grad;
/* check if all the models (or part of them) are ready */ /* check if all the models (or part of them) are ready */
for (int n = 0, i = 0; n < jworkers.count; n++) { for (int i = 0; i < jworkers.count; i++) {
XWorkerJob* worker = (XWorkerJob*)jworkers[n];
for (int m = 0; m < worker->GetModelNum(); m++, i++) { /* skip the inactive model */
if (modelFlag[i] == 0)
continue;
/* skip the inactive model */ XTensorKeeper& paramWorker = paramMap[j][i];
if (modelFlag[i] == 0)
continue;
XTensorKeeper& paramWorker = paramMap[j][i]; /* isGradFinished is true only if the model finishes the computation
(in another thread) */
if (paramWorker.flag == PARAM_STATE_NOT_READY && paramWorker.tensor->isGradFinished) {
/* isGradFinished is true only if the model finishes the computation /* get the gradient */
//paramWorker.grad = paramWorker.tensor->grad;
/* data transmit */
//collecter->AddJobCollectDataP2P(NULL, paramWorker.grad, paramServer.grad);
//collecter->AddJobEnqueueFinished();
/* We keep the worker parameter in a list. It would be used when we broadcast
the updated paramter to the workers, that is, this is a list of worker
parameters. */
paramList[j].Add(&paramWorker);
/* reset the flag */
paramWorker.flag = PARAM_STATE_COLLECTED;
finished++;
finishedCount[j]++;
/* we call model update (in another thread) and then
broadcast the new parameters to member models
(in another thread) */ (in another thread) */
if (paramWorker.flag == PARAM_STATE_NOT_READY && paramWorker.tensor->isGradFinished) { if (finishedCount[j] == activeModelCount) {
paramServer.flag = PARAM_STATE_COLLECTED;
/* get the gradient */
//paramWorker.grad = paramWorker.tensor->grad; XList grads;
for(int k = 0; k < modelNum; k++)
/* data transmit */ grads.Add(&paramMap[j][i]);
//collecter->AddJobCollectDataP2P(NULL, paramWorker.grad, paramServer.grad);
//collecter->AddJobEnqueueFinished(); /* run the all-reduce procedure to collect the gradient and share
the gradient sum across models */
/* We keep the worker parameter in a list. It would be used when we broadcast collecter->AddJobCollectGradAllReduce(NULL, &grads);
the updated paramter to the workers, that is, this is a list of worker
parameters. */ /* update on every model. NOTE THAT we do not worry about the
paramList[j].Add(&paramWorker); inconsistence issue of updated parameters across models because
the all-reduce method can guarantee that all the models share
/* reset the flag */ the same copy of the gradient. */
paramWorker.flag = PARAM_STATE_COLLECTED; for (int k = 0; k < modelNum; k++) {
finished++; XWorkerUpdate * updater = (XWorkerUpdate*)uworkers[k];
finishedCount[j]++; updater->AddJobUpdate(NULL, &paramServer, optimizer);
updater->AddJobEnqueueFinished();
/* we call model update (in another thread) and then
broadcast the new parameters to member models
(in another thread) */
if (finishedCount[j] == activeModelCount) {
paramServer.flag = PARAM_STATE_COLLECTED;
/* run the all-reduce procedure to collect the gradient and share
the gradient sum across models */
/* update on every model. NOTE THAT we do not worry about the
inconsistence issue of updated parameters across models because
the all-reduce method can guarantee that all the models share
the same copy of the gradient. */
for (int k = 0; k < modelNum; k++) {
XWorkerUpdate * updater = (XWorkerUpdate*)uworkers[k];
updater->AddJobUpdate(NULL, &paramServer, optimizer);
updater->AddJobEnqueueFinished();
}
}
else if (finishedCount[j] > activeModelCount) {
ShowNTErrors("Something is wrong with finishedCount!");
} }
} }
else if (finishedCount[j] > activeModelCount) {
ShowNTErrors("Something is wrong with finishedCount!");
}
} }
} }
} }
......
source/train/XLeaderAllReduce.h
...@@ -43,15 +43,22 @@ namespace nts { // namespace nts(NiuTrans.Tensor) ...@@ -43,15 +43,22 @@ namespace nts { // namespace nts(NiuTrans.Tensor)
/* parameter server */ /* parameter server */
class XLeaderAllReduce : public XLeader class XLeaderAllReduce : public XLeader
{ {
protected:
/* optimizer for each model */
XList optimizers;
public: public:
/* constructor */ /* constructor */
XLeaderAllReduce(); XLeaderAllReduce();
/* deconstructor */ /* deconstructor */
~XLeaderAllReduce(); ~XLeaderAllReduce();
/* clear */
void Clear();
/* create workers and other stuff used in training */ /* create workers and other stuff used in training */
void MakeAll(XConfig * config, XModel * model, const int * devIDs, const int jobWorkerNum); void MakeAll(XConfig * config, XModel * model, XOptimizer * optimizer, const int * devIDs, const int jobWorkerNum);
/* wait for finished states (i.e., all workers finish their jobs) */ /* wait for finished states (i.e., all workers finish their jobs) */
void WaitForFinishing(const int * activeJobWorkers, const int isToUpdate); void WaitForFinishing(const int * activeJobWorkers, const int isToUpdate);
...@@ -68,4 +75,4 @@ public: ...@@ -68,4 +75,4 @@ public:
} }
#endif #endif
\ No newline at end of file
source/train/XLeaderPS.cpp
...@@ -223,6 +223,7 @@ void XLeaderPS::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* act ...@@ -223,6 +223,7 @@ void XLeaderPS::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* act
jobQueues.Add(worker->GetJobQueue()); jobQueues.Add(worker->GetJobQueue());
} }
CheckNTErrors(modelNum == jworkers.count, "We assume that a worker has one model only!");
CheckNTErrors(jobQueues.count == serverModel.paramNum, "Incompatiable model!"); CheckNTErrors(jobQueues.count == serverModel.paramNum, "Incompatiable model!");
/* jobs in queue 2 (say jobQueue): collect the (gradient) data. /* jobs in queue 2 (say jobQueue): collect the (gradient) data.
...@@ -253,10 +254,6 @@ void XLeaderPS::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* act ...@@ -253,10 +254,6 @@ void XLeaderPS::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* act
} }
} }
if (activeModelCount != jworkers.count) {
int nnn = 0;
}
XList* paramList = new XList[serverModel.paramNum]; XList* paramList = new XList[serverModel.paramNum];
CheckNTErrors(modelCount == modelNum, "Wrong model number!"); CheckNTErrors(modelCount == modelNum, "Wrong model number!");
...@@ -277,60 +274,57 @@ void XLeaderPS::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* act ...@@ -277,60 +274,57 @@ void XLeaderPS::RunUpdate(XConfig* config, XOptimizer* optimizer, const int* act
paramServer.grad = paramServer.tensor->grad; paramServer.grad = paramServer.tensor->grad;
/* check if all the models (or part of them) are ready */ /* check if all the models (or part of them) are ready */
for (int n = 0, i = 0; n < jworkers.count; n++) { for (int i = 0; i < jworkers.count; i++) {
XWorkerJob* worker = (XWorkerJob*)jworkers[n];
for (int m = 0; m < worker->GetModelNum(); m++, i++) { /* skip the inactive model */
if (modelFlag[i] == 0)
/* skip the inactive model */ continue;
if (modelFlag[i] == 0)
continue; XTensorKeeper& paramWorker = paramMap[j][i];
/* isGradFinished is true only if the model finishes the computation
(in another thread) */
if (paramWorker.flag == PARAM_STATE_NOT_READY && paramWorker.tensor->isGradFinished) {
/* get the gradient */
paramWorker.grad = paramWorker.tensor->grad;
/* the job queue of updating parameter j */
XQueue* jobQueue = (XQueue*)jobQueues.GetItem(j);
XTensorKeeper& paramWorker = paramMap[j][i]; /* data transmit */
collecter->AddJobCollectGradP2P(jobQueue, &paramWorker, &paramServer);
collecter->AddJobEnqueueFinished(jobQueue);
/* isGradFinished is true only if the model finishes the computation /* We keep the worker parameter in a list. It would be used when we broadcast
the updated paramter to the workers, that is, this is a list of worker
parameters. */
paramList[j].Add(&paramWorker);
/* reset the flag */
paramWorker.flag = PARAM_STATE_COLLECTED;
finished++;
finishedCount[j]++;
/* we call model update (in another thread) and then
broadcast the new parameters to member models
(in another thread) */ (in another thread) */
if (paramWorker.flag == PARAM_STATE_NOT_READY && paramWorker.tensor->isGradFinished) { if (finishedCount[j] == activeModelCount) {
paramServer.flag = PARAM_STATE_COLLECTED;
/* get the gradient */ if (updater != NULL) {
paramWorker.grad = paramWorker.tensor->grad;
/* update the parameters */
/* the job queue of updating parameter j */ updater->AddJobUpdate(jobQueue, &paramServer, optimizer);
XQueue* jobQueue = (XQueue*)jobQueues.GetItem(j); updater->AddJobEnqueueFinished(jobQueue);
/* data transmit */ /* broadcast the new parameter to other models */
collecter->AddJobCollectGradP2P(jobQueue, &paramWorker, &paramServer); broadcaster->AddJobBroadcast(jobQueue, &paramServer, &paramList[j]);
collecter->AddJobEnqueueFinished(jobQueue); broadcaster->AddJobEnqueueFinished(jobQueue);
/* We keep the worker parameter in a list. It would be used when we broadcast
the updated paramter to the workers, that is, this is a list of worker
parameters. */
paramList[j].Add(&paramWorker);
/* reset the flag */
paramWorker.flag = PARAM_STATE_COLLECTED;
finished++;
finishedCount[j]++;
/* we call model update (in another thread) and then
broadcast the new parameters to member models
(in another thread) */
if (finishedCount[j] == activeModelCount) {
paramServer.flag = PARAM_STATE_COLLECTED;
if (updater != NULL) {
/* update the parameters */
updater->AddJobUpdate(jobQueue, &paramServer, optimizer);
updater->AddJobEnqueueFinished(jobQueue);
/* broadcast the new parameter to other models */
broadcaster->AddJobBroadcast(jobQueue, &paramServer, &paramList[j]);
broadcaster->AddJobEnqueueFinished(jobQueue);
}
}
else if (finishedCount[j] > activeModelCount) {
ShowNTErrors("Something is wrong with finishedCount!");
} }
} }
else if (finishedCount[j] > activeModelCount) {
ShowNTErrors("Something is wrong with finishedCount!");
}
} }
} }
} }
......
source/train/XOptimizer.cpp
...@@ -65,6 +65,32 @@ void XOptimizer::Clear() ...@@ -65,6 +65,32 @@ void XOptimizer::Clear()
void XOptimizer::Reset() void XOptimizer::Reset()
{ {
} }
/* clone the optimizer (with the data in it) */
XOptimizer * XOptimizer::Clone(int devID)
{
XOptimizer * opt = new XOptimizer();
Copy(this, opt, devID);
return opt;
}
/*
copy data
>> source - where we copy the data from
>> target - where we copy the data to
>> devID - the device where place the new data
*/
void XOptimizer::Copy(XOptimizer * source, XOptimizer * target, int devID)
{
CheckNTErrors(source != NULL, "No input source optimizer");
CheckNTErrors(target != NULL, "No input source optimizer");
target->nstep = source->nstep;
target->nepoch = source->nepoch;
target->lrate = source->lrate;
}
void XOptimizer::ShowSettings() void XOptimizer::ShowSettings()
{ {
......
source/train/XOptimizer.h
...@@ -67,6 +67,14 @@ public: ...@@ -67,6 +67,14 @@ public:
virtual virtual
void Reset(); void Reset();
/* clone the optimizer (with the data in it) */
virtual
XOptimizer * Clone(int devID);
/* copy data */
virtual
void Copy(XOptimizer * source, XOptimizer * target, int devID);
/* show settings */ /* show settings */
virtual virtual
void ShowSettings(); void ShowSettings();
...@@ -88,4 +96,4 @@ public: ...@@ -88,4 +96,4 @@ public:
} }
#endif #endif
\ No newline at end of file
source/train/optimizer/Adam.cpp
...@@ -92,6 +92,52 @@ void Adam::Reset() ...@@ -92,6 +92,52 @@ void Adam::Reset()
adamBeta1T = 1.0F; adamBeta1T = 1.0F;
adamBeta2T = 1.0F; adamBeta2T = 1.0F;
} }
/*
clone the optimizer (with the data in it)
>> devID - device where we place the data
*/
XOptimizer * Adam::Clone(int devID)
{
Adam * opt = new Adam();
Copy(this, opt, devID);
return (XOptimizer*)opt;
}
/* copy data */
void Adam::Copy(XOptimizer * source, XOptimizer * target, int devID)
{
XOptimizer::Copy(source, target, devID);
Adam * s = (Adam*)source;
Adam * t = (Adam*)target;
t->adamBeta1 = s->adamBeta1;
t->adamBeta2 = s->adamBeta2;
t->adamDelta = s->adamDelta;
t->adamBeta1T = s->adamBeta1T;
t->adamBeta2T = s->adamBeta2T;
t->moments.Clear();
for(int i = 0; i < s->moments.count; i++){
XTensor * st = s->moments[i];
XTensor * stNew = new XTensor();
InitTensorV2(stNew, st->order, st->dimSize, st->dataType, st->denseRatio, devID);
_CopyValues(st, stNew);
t->moments.Add(stNew);
}
t->moments2nd.Clear();
for(int i = 0; i < s->moments2nd.count; i++){
XTensor * st = s->moments2nd[i];
XTensor * stNew = new XTensor();
InitTensorV2(stNew, st->order, st->dimSize, st->dataType, st->denseRatio, devID);
_CopyValues(st, stNew);
t->moments2nd.Add(stNew);
}
}
/* show settings */ /* show settings */
void Adam::ShowSettings() void Adam::ShowSettings()
...@@ -149,4 +195,4 @@ void Adam::UpdateParam(XTensor * param, XTensor * grad, int pid) ...@@ -149,4 +195,4 @@ void Adam::UpdateParam(XTensor * param, XTensor * grad, int pid)
GMems.GetMem(v->devID)->UnlockBuf(); GMems.GetMem(v->devID)->UnlockBuf();
} }
} }
\ No newline at end of file
source/train/optimizer/Adam.h
...@@ -66,6 +66,12 @@ public: ...@@ -66,6 +66,12 @@ public:
/* reset the optimizer (re-start) */ /* reset the optimizer (re-start) */
void Reset(); void Reset();
/* clone the optimizer (with the data in it) */
XOptimizer * Clone(int devID);
/* copy data */
void Copy(XOptimizer * source, XOptimizer * target, int devID);
/* show settings */ /* show settings */
void ShowSettings(); void ShowSettings();
...@@ -80,4 +86,4 @@ public: ...@@ -80,4 +86,4 @@ public:
} }
#endif #endif
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论