support dense relative transformer decode and model convert

fairseq/modules/__init__.py

@@ -15,7 +15,7 @@ from .linearized_convolution import LinearizedConvolution
 from .multihead_attention import MultiheadAttention
 from .multihead_attention import MultiheadAttention
 from .scalar_bias import ScalarBias
 from .scalar_bias import ScalarBias
 from .sinusoidal_positional_embedding import SinusoidalPositionalEmbedding
 from .sinusoidal_positional_embedding import SinusoidalPositionalEmbedding
-
+from .relative_multihead_attention import RelativeMultiheadAttention
 __all__ = [
 __all__ = [
     'AdaptiveSoftmax',
     'AdaptiveSoftmax',
     'BeamableMM',
     'BeamableMM',
@@ -27,4 +27,6 @@ __all__ = [
     'MultiheadAttention',
     'MultiheadAttention',
     'ScalarBias',
     'ScalarBias',
     'SinusoidalPositionalEmbedding',
     'SinusoidalPositionalEmbedding',
+    'RelativeMultiheadAttention'
+    ''
 ]
 ]

scripts/convert_t2t_to_fairseq.py

@@ -472,6 +472,8 @@ def convert_settings(settings):
     args['max_relative_length'] = int(settings['max_relative_length'])
     args['max_relative_length'] = int(settings['max_relative_length'])
     args['arch'] = 'relative_transformer'
     args['arch'] = 'relative_transformer'
 
 
+  if use_relative_position_representation and use_dense:
+    args['arch'] = 'relative_dense_transformer'
   return argparse.Namespace(**args)
   return argparse.Namespace(**args)