larger buffer of a line

source/sample/transformer/T2TBatchLoader.cpp

@@ -62,7 +62,7 @@ initialization
 */
 void T2TBatchLoader::Init(int argc, char ** argv)
 {
-    LoadParamInt(argc, argv, "bufsize", &bufSize, 50000);
+    LoadParamInt(argc, argv, "bufsize", &bufSize, 65000);
     LoadParamBool(argc, argv, "doubledend", &isDoubledEnd, false);
     LoadParamBool(argc, argv, "smallbatch", &isSmallBatch, true);
     LoadParamBool(argc, argv, "bigbatch", &isBigBatch, false);
@@ -167,7 +167,8 @@ int T2TBatchLoader::LoadBuf(FILE * file, bool isSorted, int step)
         if(wordCount >= bufSize - MAX_SEQUENCE_LENGTH)
             break;
 
-        CheckNTErrors(seqCount % step == 0, "Wrong number of sequences!");
+        CheckNTErrors(seqCount % step == 0, 
+                      "Wrong number of sequences! This line might be too long!");
     }
 
     nseqBuf = seqCount;
@@ -175,7 +176,8 @@ int T2TBatchLoader::LoadBuf(FILE * file, bool isSorted, int step)
 
     /* sort the sequences by length */
     if (isSorted) {
-        CheckNTErrors(seqCount % step == 0, "Wrong number of sequences!");
+        CheckNTErrors(seqCount % step == 0, 
+                      "Wrong number of sequences! This line might be too long!");
         SampleNode * nodes = new SampleNode[seqCount];
         int count = 0;
         int offset = 0;

source/sample/transformer/T2TBatchLoader.h

@@ -30,7 +30,7 @@ using namespace nts;
 namespace transformer
 {
 
-#define MAX_SEQUENCE_LENGTH 1024 * 4
+#define MAX_SEQUENCE_LENGTH 1024 * 16
 
 /* node to keep batch information */
 struct BatchNode