updates

source/Main.cpp

@@ -39,6 +39,10 @@ using namespace nmt;
 
 int main( int argc, const char ** argv )
 {
+    XConfig config;
+    config.Create(argc - 1, argv + 1);
+    verboseLevel = config.GetInt("verbose", 1);
+
     if (argc > 1 && !strcmp(argv[1], "-test"))
         Test();
     else if (argc > 1 && !strcmp(argv[1], "-testtrain"))

source/train/TTrain.cpp

@@ -21,11 +21,11 @@
 
 /*
 * We test XTrain here. It is simple, we design a simple task in that we
-* make the model to predict an integer D (0-100) from three input integers
-* A, B and C (0-100). We generate a number of samples with different values
-* of A, B and C. The gold standard is
+* make the model to predict an integer D (0-100) from four input integers
+* A, B, C and D (0-100). We generate a number of samples with different values
+* of A, B, C and D. The gold standard is
 *
-*          D = (int)(sqrt(A * B) + C)/2
+*          D = (int)(sqrt(A * B) + abs(C - D))/2
 *
 * Our model is a two-layer feed-forward neural network. It can be treated
 * as a classifier rather than a regression model.
@@ -47,7 +47,7 @@ void GeneateTTrainData(const char * fileName)
     FILE * file = fopen(fileName, "wb");
     CheckNTErrors(file, "Cannot open the file");
 
-    fprintf(stderr, "Generating data ... ");
+    XPRINT(1, stderr, "[INFO] Generating data ... ");
 
     int sampleNum = MAX_SAMPLE_NUM_IN_TTRAIN;
     int range = MAX_INT_IN_TTRAIN;
@@ -60,11 +60,12 @@ void GeneateTTrainData(const char * fileName)
         int A = (int)(((float)rand() / RAND_MAX) * range);
         int B = (int)(((float)rand() / RAND_MAX) * range);
         int C = (int)(((float)rand() / RAND_MAX) * range);
-        int D = (int)((sqrt(A * B) + C) / 2);
-        fprintf(file, "%d %d %d %d\n", A, B, C, D);
+        int D = (int)(((float)rand() / RAND_MAX) * range);
+        int E = (int)((sqrt(A * B) + abs(C - D)) / 2);
+        fprintf(file, "%d %d %d %d %d\n", A, B, C, D, E);
     }
 
-    fprintf(stderr, "%d samples in \"%s\" [done]\n", sampleNum, fileName);
+    XPRINT2(1, stderr, "%d samples in \"%s\" [DONE]\n", sampleNum, fileName);
     
     fclose(file);
 }
@@ -76,7 +77,9 @@ void TestTrain()
 
     XConfig config;
     config.Add("dev", -1);
-    config.Add("lrate", 0.1F);
+    config.Add("lrate", 0.001F);
+    config.Add("nstep", 10000);
+    config.Add("nepoch", 5);
 
     TTDataLoader loader;
     loader.SetFileName("ttrain.txt");
@@ -165,30 +168,31 @@ bool TTDataLoader::GetBatchSimple(XList * inputs, XList * golds)
     char * line = new char[MAX_SAMPLE_LINE_LENGTH];
     int * inputBatch = new int[batchSize * sampleSize];
     int * goldBatch = new int[batchSize];
-    int A, B, C, D;
+    int A, B, C, D, E;
     
     while (fgets(line, MAX_SAMPLE_LINE_LENGTH, file)) {
 
         if (count == batchSize)
             break;
 
-        if (sscanf(line, "%d %d %d %d", &A, &B, &C, &D) < 4) {
+        if (sscanf(line, "%d %d %d %d %d", &A, &B, &C, &D, &E) < sampleSize + 1) {
             ShowNTErrors("Wrong format in the training file!");
         }
 
-        inputBatch[count * 3] = A;
-        inputBatch[count * 3 + 1] = B;
-        inputBatch[count * 3 + 2] = C;
-        goldBatch[count] = D;
+        inputBatch[count * sampleSize] = A;
+        inputBatch[count * sampleSize + 1] = B;
+        inputBatch[count * sampleSize + 2] = C;
+        inputBatch[count * sampleSize + 3] = D;
+        goldBatch[count] = E;
 
         count++;
     }
 
     if (count > 0) {
-        InitTensor2D(input, count, 3, X_INT);
+        InitTensor2D(input, count, 4, X_INT);
         InitTensor2D(gold, count, 1, X_INT);
 
-        input->SetData(inputBatch, count * 3);
+        input->SetData(inputBatch, count * 4);
         gold->SetData(goldBatch, count);
     }
 
@@ -237,7 +241,7 @@ void TTModel::Init(XConfig &myConfig, int devID)
     hSize = config.GetInt("hsize", TT_HIDDEN_SIZE);
 
     InitTensor2D(&embeddingW, vSize, eSize, X_FLOAT, devID);
-    InitTensor2D(&hiddenW, 3 * eSize, hSize, X_FLOAT, devID);
+    InitTensor2D(&hiddenW, MAX_SAMPLE_SIZE * eSize, hSize, X_FLOAT, devID);
     InitTensor2D(&outputW, hSize, vSize, X_FLOAT, devID);
 
     embeddingW.SetName("embeddingw");
@@ -306,7 +310,7 @@ run the neural network
 */
 bool TTModel::RunSimple(XList * inputs, XList * outputs, XList * golds, XList* losses)
 {
-    fprintf(stderr, "run simple 0\n");
+    //fprintf(stderr, "run simple 0\n");
     CheckNTErrors(inputs != NULL && inputs->count >= 1, "Wrong arguments!");
     CheckNTErrors(outputs != NULL && outputs->count >= 1, "Wrong arguments!");
     CheckNTErrors(golds != NULL && golds->count >= 1, "Wrong arguments!");
@@ -326,7 +330,7 @@ bool TTModel::RunSimple(XList * inputs, XList * outputs, XList * golds, XList* l
     /* gold standard in ong-hot representaiton */
     goldOneHot = IndexToOnehot(*gold, vSize, 0.0F);
 
-    int* dims = new int[goldOneHot.order];
+    int * dims = new int[goldOneHot.order];
     for (int i = 0; i < goldOneHot.order - 2; i++)
         dims[i] = goldOneHot.GetDim(i);
     dims[goldOneHot.order - 2] = goldOneHot.GetDim(goldOneHot.order - 1);
@@ -340,7 +344,7 @@ bool TTModel::RunSimple(XList * inputs, XList * outputs, XList * golds, XList* l
 
     delete[] dims;
     
-    fprintf(stderr, "run simple 1\n");
+    //fprintf(stderr, "run simple 1\n");
 
     return true;
 }

source/train/TTrain.h

@@ -44,12 +44,12 @@
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
-#define MAX_SAMPLE_NUM_IN_TTRAIN 100000
+#define MAX_SAMPLE_NUM_IN_TTRAIN 200000
 #define MAX_INT_IN_TTRAIN 100
 #define MAX_SAMPLE_LINE_LENGTH 128
-#define MAX_SAMPLE_SIZE 3
+#define MAX_SAMPLE_SIZE 4
 #define TT_BATCH_SIZE 256
-#define TT_EMBEDDING_SIZE 256
+#define TT_EMBEDDING_SIZE 64
 #define TT_HIDDEN_SIZE 256
 
 extern XTensor * tmpTT;

source/train/XLeader.cpp

@@ -418,12 +418,7 @@ void XLeader::WaitForFinishing(int sleepTime)
         if (finished)
             break;
 
-#ifdef _WIN32
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
-
+        XSleep(sleepTime);
     }
 }
 

source/train/XLeader.h

@@ -48,7 +48,7 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 #define MAX_NUM_OF_WORKERS 1024
-#define SLEEP_TIME_IN_WAITING_FOR_JOBS 10
+#define SLEEP_TIME_IN_WAITING_FOR_JOBS 20
 
 /* 
 conmmunication mode of a leader. This offers a way of organizing a hierachy of the work

source/train/XOptimizer.cpp

@@ -60,6 +60,14 @@ void XOptimizer::Clear()
     lrate = 0;
 }
 
+void XOptimizer::ShowSettings()
+{
+    XPRINT(1, stderr, "[INFO] Optimizer Setup:\n");
+    XPRINT1(1, stderr, "       nstep = %d\n", nstep);
+    XPRINT1(1, stderr, "       nepoch = %d\n", nepoch);
+    XPRINT1(1, stderr, "       lrate = %.3f\n", lrate);
+}
+
 /* 
 prepare for the update 
 >> model - the model that we want to update

source/train/XOptimizer.h

@@ -63,6 +63,10 @@ public:
     virtual
     void Clear();
     
+    /* show settings */
+    virtual
+    void ShowSettings();
+
     /* prepare for the update */
     virtual
     void Prepare(XModel * model);

source/train/XTrainer.cpp

@@ -94,8 +94,6 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
     CheckNTErrors(dataDistributor != NULL, "No input data distributor!");
     CheckNTErrors(model != NULL, "No input neural network!");
 
-    int nepoch = config->GetInt("nepoch", 50);
-    int nstep = config->GetInt("nstep", 100000);
     int epoch = 0;
     int step = 0;
     int jobNum = 0;
@@ -103,6 +101,8 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
     int * ids = new int[MAX_DEVICE_NUM_TRAINING];
     GetDevIDs(config, ids, jobNum, MAX_DEVICE_NUM_TRAINING);
 
+    optimizer->ShowSettings();
+
     /* create the server and workers */
     XLeader leader;
     leader.Init();
@@ -114,8 +114,12 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
     leader.SetServerModel(config, model);
     leader.Start();
 
+    double startT = GetClockSec();
+
+    XPRINT(1, stderr, "[INFO] Initializing the model ... [DONE]\n");
+
     /* train the model */
-    for (epoch = 0; epoch < nepoch; epoch++) {
+    for (epoch = 0; epoch < optimizer->nepoch; epoch++) {
 
         bool ok = true;
         dataDistributor->Start();
@@ -127,23 +131,21 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
 
             float loss = leader.GetLoss() / leader.GetSampleNum();
 
-            if ((step + 1) % 1 == 0)
-                fprintf(stderr, "epoch:%d step:%d sample:%d loss:%f predict:%d\n",
-                        epoch + 1, step + 1, leader.GetSampleNum(), loss, leader.GetPredictNum());
+            if ((step + 1) % 100 == 0)
+                XPRINT5(1, stderr, "[INFO] elapsed=%.1fs epoch:%d step:%d sample:%d loss:%f\n",
+                        GetClockSec() - startT, epoch + 1, step + 1, leader.GetSampleNum(), loss);
 
-            if (step++ >= nstep)
+            if (step++ >= optimizer->nstep)
                 break;
         }
 
         dataDistributor->End();
 
-        if (step >= nstep)
+        if (step >= optimizer->nstep)
             break;   
     }
 
     delete[] ids;
-
-    fprintf(stderr, "epoch:%d step:%d\n", epoch, step);
 }
 
 } /* end of the nts (NiuTrans.Tensor) namespace */

source/train/XWorker.h

@@ -32,6 +32,7 @@
 #define __XWORKER_H__
 
 #include "../tensor/XQueue.h"
+#include "../tensor/XUtility.h"
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 

source/train/XWorkerBroadcast.cpp

@@ -90,11 +90,7 @@ void XWorkerBroadcast::BroadcastData(XModel * source, XList * targetList, long s
         if (finished == sp.count * targetList->count)
             break;
 
-#ifdef _WIN32
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
+        XSleep(sleepTime);
     }
 
     delete[] finishedFlag;
@@ -106,7 +102,7 @@ wrapper of BroadcastData
 */
 void XWorkerBroadcast::Broadcast(XList * args)
 {
-    fprintf(stderr, "broadcast 0\n");
+    //fprintf(stderr, "broadcast 0\n");
     XWorkerBroadcast * broadcaster = (XWorkerBroadcast*)args->GetItem(0);
     XModel * source = (XModel*)args->GetItem(1);
 
@@ -119,7 +115,7 @@ void XWorkerBroadcast::Broadcast(XList * args)
     }
 
     broadcaster->BroadcastData(source, &target, SLEEP_TIME_IN_BROADCASTING);
-    fprintf(stderr, "broadcast 1\n");
+    //fprintf(stderr, "broadcast 1\n");
 }
 
 /* 

source/train/XWorkerBroadcast.h

@@ -35,7 +35,7 @@
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
-#define SLEEP_TIME_IN_BROADCASTING 10
+#define SLEEP_TIME_IN_BROADCASTING 20
 
 /*
 data broadcasting method

source/train/XWorkerCollect.cpp

@@ -160,11 +160,7 @@ void XWorkerCollect::CollectData(XList * sourceList, XModel * target, long sleep
         if (finished == tp.count * sourceList->count)
             break;
 
-#ifdef _WIN32
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
+        XSleep(sleepTime);
     }
 
     /* reset the flags */
@@ -175,7 +171,7 @@ void XWorkerCollect::CollectData(XList * sourceList, XModel * target, long sleep
 /* wrapper of CollectData */
 void XWorkerCollect::Collect(XList * args)
 {
-    fprintf(stderr, "collect data 0\n");
+    //fprintf(stderr, "collect data 0\n");
 
     XWorkerCollect * collecter = (XWorkerCollect*)args->GetItem(0);
     int sourceNum = args->GetItemInt(1);
@@ -192,7 +188,7 @@ void XWorkerCollect::Collect(XList * args)
 
     collecter->CollectData(&source, target, SLEEP_TIME_IN_COLLECTING);
 
-    fprintf(stderr, "collect data 1\n");
+    //fprintf(stderr, "collect data 1\n");
 }
 
 /* 
@@ -298,11 +294,7 @@ void XWorkerCollect::CollectOtherData(XList* sourceList, XNNRecord* target, long
         if (finished == sourceList->count)
             break;
 
-#ifdef _WIN32
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
+        XSleep(sleepTime);
     }
 
     delete[] flags;

source/train/XWorkerCollect.h

@@ -35,8 +35,8 @@
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
-#define SLEEP_TIME_IN_COLLECTING 10
-#define SLEEP_TIME_IN_COLLECTING_OTHER 10
+#define SLEEP_TIME_IN_COLLECTING 20
+#define SLEEP_TIME_IN_COLLECTING_OTHER 40
 
 /*
 data collection method

source/train/XWorkerUpdate.cpp

@@ -86,11 +86,7 @@ void XWorkerUpdate::UpdateModel(XModel * model, XOptimizer * optimizer, long sle
         if (finished == params.count)
             break;
 
-#ifdef _WIN32
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
+        XSleep(sleepTime);
     }
 
     optimizer->Note(model);
@@ -102,7 +98,7 @@ wrapper of UpdateModel
 */
 void XWorkerUpdate::Update(XList * args)
 {
-    fprintf(stderr, "update 0\n");
+    //fprintf(stderr, "update 0\n");
 
     CheckNTErrors(args != NULL && args->count >= 3, "Illegal argument list!");
 
@@ -112,7 +108,7 @@ void XWorkerUpdate::Update(XList * args)
 
     updater->UpdateModel(model, optimizer, SLEEP_TIME_IN_MODEL_UPDATE);
 
-    fprintf(stderr, "update 1\n");
+    //fprintf(stderr, "update 1\n");
 }
 
 /* 

source/train/XWorkerUpdate.h

@@ -33,7 +33,7 @@
 
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
-#define SLEEP_TIME_IN_MODEL_UPDATE 10
+#define SLEEP_TIME_IN_MODEL_UPDATE 20
 
 /* The class defines the model-update worker */
 class XWorkerUpdate : public XWorker