normalize the loss

source/sample/transformer/T2TTrainer.cpp

@@ -215,6 +215,10 @@ void T2TTrainer::Train(const char * fn, const char * validFN, const char * model
             XTensor &g = labelSmoothingP > 0 ? goldSmoothed : gold;
 
             if (doUpdate) {
+                
+                /* recale the output for normalized loss */
+                RescaleOutput(&output, &g, &padding);
+                
                 /* back-propagation */
                 net.Backward(output, g, CROSSENTROPY);
 
@@ -870,6 +874,30 @@ void T2TTrainer::PadOutput(XTensor * output, XTensor * gold, XTensor * padding)
     delete[] dimso;
     DelTensorBuf(padding2);
 }
+
+/*
+recale the output and gold tensors for normalized loss
+>> output - output tensor of the network
+>> gold - gold standard
+>> padding - padding of a batch of sentences
+*/
+void T2TTrainer::RescaleOutput(XTensor * output, XTensor * gold, XTensor * padding)
+{
+    CheckNTErrors(output->order == 3, "Wrong dimension number!");
+    CheckNTErrors(gold->order == 3, "Wrong dimension number!");
+    
+    int num = padding->GetDim(0);
+    XTensor * factor = NewTensorBuf(1, &num, padding->dataType, 1.0F, padding->devID, padding->mem);
+    
+    _ReduceSum(padding, factor, padding->order - 1);
+    
+    _ExpMe(output);
+    _DivDim(output, factor, output, 0);
+    _LogMe(output);
+    _DivDim(gold, factor, gold, 0);
+    
+    DelTensorBuf(factor);
+}
     
 /*
 perform label smoothing

source/sample/transformer/T2TTrainer.h

@@ -179,6 +179,9 @@ public:
     /* do padding on the output */
     void PadOutput(XTensor * output, XTensor * gold, XTensor * padding);
     
+    /* recale the output and gold tensors for normalized loss */
+    void RescaleOutput(XTensor * output, XTensor * gold, XTensor * padding);
+    
     /* perform label smoothing */
     void LabelSmooth(XTensor * gold, XTensor * smoothed, DTYPE p);
 };