add doubledend flat for t2t mt

source/sample/transformer/T2TTrainer.cpp

@@ -708,17 +708,17 @@ int T2TTrainer::LoadBatchLM(FILE * file,
         for(int w = 0; w < len; w++){
             batchEnc->Set3D(1.0F, s - seq, w, buf[seqOffset[s] + w]);
             paddingEnc->Set2D(1.0F, s - seq, w);
-            if (w > 0) {
+            paddingDec->Set2D(1.0F, s - seq, w);
+            if (w > 0)
                 batchDec->Set3D(1.0F, s - seq, w - 1, buf[seqOffset[s] + w]);
-                paddingDec->Set2D(1.0F, s - seq, w - 1);
-            }
-            if(w == len - 1){
-                if(isDoubledEnd)
+            
+            if (w == len - 1) {
+                if (isDoubledEnd)
                     batchDec->Set3D(1.0F, s - seq, w, buf[seqOffset[s] + w]);
                 else
                     batchDec->Set3D(1.0F, s - seq, w, buf[seqOffset[s] + w + 1]);
-                paddingDec->Set2D(1.0F, s - seq, w);
             }
+
             wCount++;
             /*fprintf(tf, "%d", buf[seqOffset[s] + w]);
             if(w < seqLen[s] - 1)
@@ -790,7 +790,8 @@ int T2TTrainer::LoadBatchMT(FILE * file,
             maxEnc = wnEnc;
 
         /* target-side sequence */
-        wnDec = seqLen[seq + sc];
+        int len = isDoubledEnd ? seqLen[seq + sc] : seqLen[seq + sc] - 1;
+        wnDec = len;
         wcDec += wnDec;
         sc += 1;
 
@@ -837,14 +838,19 @@ int T2TTrainer::LoadBatchMT(FILE * file,
 
     /* batch of the target-side sequences */
     for(int s = seq + 1; s < seq + sc; s += 2){
-        int len = seqLen[s];
+        int len = isDoubledEnd ? seqLen[s] : seqLen[s] - 1;
+        CheckNTErrors(len <= maxDec, "Something is wrong!");
         int sent = (s - seq - 1)/2;
         for(int w = 0; w < len; w++){
             paddingDec->Set2D(1.0F, sent, w);
             if(w > 0)
                 batchDec->Set3D(1.0F, sent, w - 1, buf[seqOffset[s] + w]);
-            if(w == len - 1)
-                batchDec->Set3D(1.0F, sent, w, buf[seqOffset[s] + w]);
+            if (w == len - 1) {
+                if(isDoubledEnd)
+                    batchDec->Set3D(1.0F, sent, w, buf[seqOffset[s] + w]);
+                else
+                    batchDec->Set3D(1.0F, sent, w, buf[seqOffset[s] + w + 1]);
+            }
             wCount++;
 
             if(seqs != NULL)