updates

08bd5aec · xiaotong · 412e53a8 · 08bd5aec · 08bd5aec · 08bd5aec
Commit 08bd5aec authored Mar 10, 2021 by xiaotong
--- a/source/Main.cpp
+++ b/source/Main.cpp
@@ -39,6 +39,10 @@ using namespace nmt;
 int main( int argc, const char ** argv )
 {
+    XConfig config;
+    config.Create(argc - 1, argv + 1);
+    verboseLevel = config.GetInt("verbose", 1);
    if (argc > 1 && !strcmp(argv[1], "-test"))
        Test();
    else if (argc > 1 && !strcmp(argv[1], "-testtrain"))

--- a/source/train/TTrain.cpp
+++ b/source/train/TTrain.cpp
@@ -21,11 +21,11 @@
 /*
 * We test XTrain here. It is simple, we design a simple task in that we
-* make the model to predict an integer D (0-100) from three input integers
+* make the model to predict an integer D (0-100) from four input integers
-* A, B and C (0-100). We generate a number of samples with different values
+* A, B, C and D (0-100). We generate a number of samples with different values
-* of A, B and C. The gold standard is
+* of A, B, C and D. The gold standard is
 *
-*          D = (int)(sqrt(A * B) + C)/2
+*          D = (int)(sqrt(A * B) + abs(C - D))/2
 *
 * Our model is a two-layer feed-forward neural network. It can be treated
 * as a classifier rather than a regression model.
@@ -47,7 +47,7 @@ void GeneateTTrainData(const char * fileName)
    FILE * file = fopen(fileName, "wb");
    CheckNTErrors(file, "Cannot open the file");
-    fprintf(stderr, "Generating data ... ");
+    XPRINT(1, stderr, "[INFO] Generating data ... ");
    int sampleNum = MAX_SAMPLE_NUM_IN_TTRAIN;
    int range = MAX_INT_IN_TTRAIN;
@@ -60,11 +60,12 @@ void GeneateTTrainData(const char * fileName)
        int A = (int)(((float)rand() / RAND_MAX) * range);
        int B = (int)(((float)rand() / RAND_MAX) * range);
        int C = (int)(((float)rand() / RAND_MAX) * range);
-        int D = (int)((sqrt(A * B) + C) / 2);
+        int D = (int)(((float)rand() / RAND_MAX) * range);
-        fprintf(file, "%d %d %d %d\n", A, B, C, D);
+        int E = (int)((sqrt(A * B) + abs(C - D)) / 2);
+        fprintf(file, "%d %d %d %d %d\n", A, B, C, D, E);
    }
-    fprintf(stderr, "%d samples in \"%s\" [done]\n", sampleNum, fileName);
+    XPRINT2(1, stderr, "%d samples in \"%s\" [DONE]\n", sampleNum, fileName);
    fclose(file);
 }
@@ -76,7 +77,9 @@ void TestTrain()
    XConfig config;
    config.Add("dev", -1);
-    config.Add("lrate", 0.1F);
+    config.Add("lrate", 0.001F);
+    config.Add("nstep", 10000);
+    config.Add("nepoch", 5);
    TTDataLoader loader;
    loader.SetFileName("ttrain.txt");
@@ -165,30 +168,31 @@ bool TTDataLoader::GetBatchSimple(XList * inputs, XList * golds)
    char * line = new char[MAX_SAMPLE_LINE_LENGTH];
    int * inputBatch = new int[batchSize * sampleSize];
    int * goldBatch = new int[batchSize];
-    int A, B, C, D;
+    int A, B, C, D, E;
    while (fgets(line, MAX_SAMPLE_LINE_LENGTH, file)) {
        if (count == batchSize)
            break;
-        if (sscanf(line, "%d %d %d %d", &A, &B, &C, &D) < 4) {
+        if (sscanf(line, "%d %d %d %d %d", &A, &B, &C, &D, &E) < sampleSize + 1) {
            ShowNTErrors("Wrong format in the training file!");
        }
-        inputBatch[count * 3] = A;
+        inputBatch[count * sampleSize] = A;
-        inputBatch[count * 3 + 1] = B;
+        inputBatch[count * sampleSize + 1] = B;
-        inputBatch[count * 3 + 2] = C;
+        inputBatch[count * sampleSize + 2] = C;
-        goldBatch[count] = D;
+        inputBatch[count * sampleSize + 3] = D;
+        goldBatch[count] = E;
        count++;
    }
    if (count > 0) {
-        InitTensor2D(input, count, 3, X_INT);
+        InitTensor2D(input, count, 4, X_INT);
        InitTensor2D(gold, count, 1, X_INT);
-        input->SetData(inputBatch, count * 3);
+        input->SetData(inputBatch, count * 4);
        gold->SetData(goldBatch, count);
    }
@@ -237,7 +241,7 @@ void TTModel::Init(XConfig &myConfig, int devID)
    hSize = config.GetInt("hsize", TT_HIDDEN_SIZE);
    InitTensor2D(&embeddingW, vSize, eSize, X_FLOAT, devID);
-    InitTensor2D(&hiddenW, 3 * eSize, hSize, X_FLOAT, devID);
+    InitTensor2D(&hiddenW, MAX_SAMPLE_SIZE * eSize, hSize, X_FLOAT, devID);
    InitTensor2D(&outputW, hSize, vSize, X_FLOAT, devID);
    embeddingW.SetName("embeddingw");
@@ -306,7 +310,7 @@ run the neural network
 */
 bool TTModel::RunSimple(XList * inputs, XList * outputs, XList * golds, XList* losses)
 {
-    fprintf(stderr, "run simple 0\n");
+    //fprintf(stderr, "run simple 0\n");
    CheckNTErrors(inputs != NULL && inputs->count >= 1, "Wrong arguments!");
    CheckNTErrors(outputs != NULL && outputs->count >= 1, "Wrong arguments!");
    CheckNTErrors(golds != NULL && golds->count >= 1, "Wrong arguments!");
@@ -326,7 +330,7 @@ bool TTModel::RunSimple(XList * inputs, XList * outputs, XList * golds, XList* l
    /* gold standard in ong-hot representaiton */
    goldOneHot = IndexToOnehot(*gold, vSize, 0.0F);
-    int* dims = new int[goldOneHot.order];
+    int * dims = new int[goldOneHot.order];
    for (int i = 0; i < goldOneHot.order - 2; i++)
        dims[i] = goldOneHot.GetDim(i);
    dims[goldOneHot.order - 2] = goldOneHot.GetDim(goldOneHot.order - 1);
@@ -340,7 +344,7 @@ bool TTModel::RunSimple(XList * inputs, XList * outputs, XList * golds, XList* l
    delete[] dims;
-    fprintf(stderr, "run simple 1\n");
+    //fprintf(stderr, "run simple 1\n");
    return true;
 }

--- a/source/train/TTrain.h
+++ b/source/train/TTrain.h
@@ -44,12 +44,12 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
-#define MAX_SAMPLE_NUM_IN_TTRAIN 100000
+#define MAX_SAMPLE_NUM_IN_TTRAIN 200000
 #define MAX_INT_IN_TTRAIN 100
 #define MAX_SAMPLE_LINE_LENGTH 128
-#define MAX_SAMPLE_SIZE 3
+#define MAX_SAMPLE_SIZE 4
 #define TT_BATCH_SIZE 256
-#define TT_EMBEDDING_SIZE 256
+#define TT_EMBEDDING_SIZE 64
 #define TT_HIDDEN_SIZE 256
 extern XTensor * tmpTT;

--- a/source/train/XLeader.cpp
+++ b/source/train/XLeader.cpp
@@ -418,12 +418,7 @@ void XLeader::WaitForFinishing(int sleepTime)
        if (finished)
            break;
-#ifdef _WIN32
+        XSleep(sleepTime);
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
    }
 }

--- a/source/train/XLeader.h
+++ b/source/train/XLeader.h
@@ -48,7 +48,7 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
 #define MAX_NUM_OF_WORKERS 1024
-#define SLEEP_TIME_IN_WAITING_FOR_JOBS 10
+#define SLEEP_TIME_IN_WAITING_FOR_JOBS 20
 /* 
 conmmunication mode of a leader. This offers a way of organizing a hierachy of the work

--- a/source/train/XOptimizer.cpp
+++ b/source/train/XOptimizer.cpp
@@ -60,6 +60,14 @@ void XOptimizer::Clear()
    lrate = 0;
 }
+void XOptimizer::ShowSettings()
+{
+    XPRINT(1, stderr, "[INFO] Optimizer Setup:\n");
+    XPRINT1(1, stderr, "       nstep = %d\n", nstep);
+    XPRINT1(1, stderr, "       nepoch = %d\n", nepoch);
+    XPRINT1(1, stderr, "       lrate = %.3f\n", lrate);
+}
 /* 
 prepare for the update 
 >> model - the model that we want to update

--- a/source/train/XOptimizer.h
+++ b/source/train/XOptimizer.h
@@ -62,6 +62,10 @@ public:
    /* clear the optimizer */
    virtual
    void Clear();
+    /* show settings */
+    virtual
+    void ShowSettings();
    /* prepare for the update */
    virtual

--- a/source/train/XTrainer.cpp
+++ b/source/train/XTrainer.cpp
@@ -94,8 +94,6 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
    CheckNTErrors(dataDistributor != NULL, "No input data distributor!");
    CheckNTErrors(model != NULL, "No input neural network!");
-    int nepoch = config->GetInt("nepoch", 50);
-    int nstep = config->GetInt("nstep", 100000);
    int epoch = 0;
    int step = 0;
    int jobNum = 0;
@@ -103,6 +101,8 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
    int * ids = new int[MAX_DEVICE_NUM_TRAINING];
    GetDevIDs(config, ids, jobNum, MAX_DEVICE_NUM_TRAINING);
+    optimizer->ShowSettings();
    /* create the server and workers */
    XLeader leader;
    leader.Init();
@@ -114,8 +114,12 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
    leader.SetServerModel(config, model);
    leader.Start();
+    double startT = GetClockSec();
+    XPRINT(1, stderr, "[INFO] Initializing the model ... [DONE]\n");
    /* train the model */
-    for (epoch = 0; epoch < nepoch; epoch++) {
+    for (epoch = 0; epoch < optimizer->nepoch; epoch++) {
        bool ok = true;
        dataDistributor->Start();
@@ -127,23 +131,21 @@ void XTrainer::Run(XConfig * config, DataDistributeBase * dataDistributor,
            float loss = leader.GetLoss() / leader.GetSampleNum();
-            if ((step + 1) % 1 == 0)
+            if ((step + 1) % 100 == 0)
-                fprintf(stderr, "epoch:%d step:%d sample:%d loss:%f predict:%d\n",
+                XPRINT5(1, stderr, "[INFO] elapsed=%.1fs epoch:%d step:%d sample:%d loss:%f\n",
-                        epoch + 1, step + 1, leader.GetSampleNum(), loss, leader.GetPredictNum());
+                        GetClockSec() - startT, epoch + 1, step + 1, leader.GetSampleNum(), loss);
-            if (step++ >= nstep)
+            if (step++ >= optimizer->nstep)
                break;
        }
        dataDistributor->End();
-        if (step >= nstep)
+        if (step >= optimizer->nstep)
            break;   
    }
    delete[] ids;
-    fprintf(stderr, "epoch:%d step:%d\n", epoch, step);
 }
 } /* end of the nts (NiuTrans.Tensor) namespace */
--- a/source/train/XWorker.h
+++ b/source/train/XWorker.h
@@ -32,6 +32,7 @@
 #define __XWORKER_H__
 #include "../tensor/XQueue.h"
+#include "../tensor/XUtility.h"
 namespace nts { // namespace nts(NiuTrans.Tensor)

--- a/source/train/XWorkerBroadcast.cpp
+++ b/source/train/XWorkerBroadcast.cpp
@@ -90,11 +90,7 @@ void XWorkerBroadcast::BroadcastData(XModel * source, XList * targetList, long s
        if (finished == sp.count * targetList->count)
            break;
-#ifdef _WIN32
+        XSleep(sleepTime);
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
    }
    delete[] finishedFlag;
@@ -106,7 +102,7 @@ wrapper of BroadcastData
 */
 void XWorkerBroadcast::Broadcast(XList * args)
 {
-    fprintf(stderr, "broadcast 0\n");
+    //fprintf(stderr, "broadcast 0\n");
    XWorkerBroadcast * broadcaster = (XWorkerBroadcast*)args->GetItem(0);
    XModel * source = (XModel*)args->GetItem(1);
@@ -119,7 +115,7 @@ void XWorkerBroadcast::Broadcast(XList * args)
    }
    broadcaster->BroadcastData(source, &target, SLEEP_TIME_IN_BROADCASTING);
-    fprintf(stderr, "broadcast 1\n");
+    //fprintf(stderr, "broadcast 1\n");
 }
 /* 

--- a/source/train/XWorkerBroadcast.h
+++ b/source/train/XWorkerBroadcast.h
@@ -35,7 +35,7 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
-#define SLEEP_TIME_IN_BROADCASTING 10
+#define SLEEP_TIME_IN_BROADCASTING 20
 /*
 data broadcasting method

--- a/source/train/XWorkerCollect.cpp
+++ b/source/train/XWorkerCollect.cpp
@@ -160,11 +160,7 @@ void XWorkerCollect::CollectData(XList * sourceList, XModel * target, long sleep
        if (finished == tp.count * sourceList->count)
            break;
-#ifdef _WIN32
+        XSleep(sleepTime);
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
    }
    /* reset the flags */
@@ -175,7 +171,7 @@ void XWorkerCollect::CollectData(XList * sourceList, XModel * target, long sleep
 /* wrapper of CollectData */
 void XWorkerCollect::Collect(XList * args)
 {
-    fprintf(stderr, "collect data 0\n");
+    //fprintf(stderr, "collect data 0\n");
    XWorkerCollect * collecter = (XWorkerCollect*)args->GetItem(0);
    int sourceNum = args->GetItemInt(1);
@@ -192,7 +188,7 @@ void XWorkerCollect::Collect(XList * args)
    collecter->CollectData(&source, target, SLEEP_TIME_IN_COLLECTING);
-    fprintf(stderr, "collect data 1\n");
+    //fprintf(stderr, "collect data 1\n");
 }
 /* 
@@ -298,11 +294,7 @@ void XWorkerCollect::CollectOtherData(XList* sourceList, XNNRecord* target, long
        if (finished == sourceList->count)
            break;
-#ifdef _WIN32
+        XSleep(sleepTime);
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
    }
    delete[] flags;

--- a/source/train/XWorkerCollect.h
+++ b/source/train/XWorkerCollect.h
@@ -35,8 +35,8 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
-#define SLEEP_TIME_IN_COLLECTING 10
+#define SLEEP_TIME_IN_COLLECTING 20
-#define SLEEP_TIME_IN_COLLECTING_OTHER 10
+#define SLEEP_TIME_IN_COLLECTING_OTHER 40
 /*
 data collection method

--- a/source/train/XWorkerUpdate.cpp
+++ b/source/train/XWorkerUpdate.cpp
@@ -86,11 +86,7 @@ void XWorkerUpdate::UpdateModel(XModel * model, XOptimizer * optimizer, long sle
        if (finished == params.count)
            break;
-#ifdef _WIN32
+        XSleep(sleepTime);
-        Sleep((DWORD)sleepTime);
-#else
-        sleep((unsigned)sleepTime / 1000);
-#endif
    }
    optimizer->Note(model);
@@ -102,7 +98,7 @@ wrapper of UpdateModel
 */
 void XWorkerUpdate::Update(XList * args)
 {
-    fprintf(stderr, "update 0\n");
+    //fprintf(stderr, "update 0\n");
    CheckNTErrors(args != NULL && args->count >= 3, "Illegal argument list!");
@@ -112,7 +108,7 @@ void XWorkerUpdate::Update(XList * args)
    updater->UpdateModel(model, optimizer, SLEEP_TIME_IN_MODEL_UPDATE);
-    fprintf(stderr, "update 1\n");
+    //fprintf(stderr, "update 1\n");
 }
 /* 

--- a/source/train/XWorkerUpdate.h
+++ b/source/train/XWorkerUpdate.h
@@ -33,7 +33,7 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
-#define SLEEP_TIME_IN_MODEL_UPDATE 10
+#define SLEEP_TIME_IN_MODEL_UPDATE 20
 /* The class defines the model-update worker */
 class XWorkerUpdate : public XWorker