Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
F
Fairseq-S2T
概览
Overview
Details
Activity
Cycle Analytics
版本库
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
问题
0
Issues
0
列表
Board
标记
里程碑
合并请求
0
Merge Requests
0
CI / CD
CI / CD
流水线
作业
日程表
图表
维基
Wiki
代码片段
Snippets
成员
Collapse sidebar
Close sidebar
活动
图像
聊天
创建新问题
作业
提交
Issue Boards
Open sidebar
xuchen
Fairseq-S2T
Commits
306dd6fc
Commit
306dd6fc
authored
Mar 16, 2021
by
xuchen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fix the bug of writing and reading the source txt
parent
ad46064b
显示空白字符变更
内嵌
并排
正在显示
4 个修改的文件
包含
17 行增加
和
7 行删除
+17
-7
examples/speech_to_text/prep_librispeech_data.py
+2
-1
examples/speech_to_text/prep_mustc_data.py
+2
-1
fairseq/data/audio/speech_to_text_dataset.py
+1
-1
fairseq/tasks/speech_to_text.py
+12
-4
没有找到文件。
examples/speech_to_text/prep_librispeech_data.py
查看文件 @
306dd6fc
...
@@ -110,7 +110,8 @@ def process(args):
...
@@ -110,7 +110,8 @@ def process(args):
# Generate config YAML
# Generate config YAML
gen_config_yaml
(
gen_config_yaml
(
out_root
,
spm_filename_prefix
+
".model"
,
specaugment_policy
=
"ld"
,
out_root
,
spm_filename_prefix
+
".model"
,
specaugment_policy
=
"ld"
,
asr_spm_filename
=
spm_filename_prefix
+
".model"
asr_spm_filename
=
spm_filename_prefix
+
".model"
,
share_src_and_tgt
=
True
)
)
# Clean up
# Clean up
shutil
.
rmtree
(
feature_root
)
shutil
.
rmtree
(
feature_root
)
...
...
examples/speech_to_text/prep_mustc_data.py
查看文件 @
306dd6fc
...
@@ -234,7 +234,8 @@ def process(args):
...
@@ -234,7 +234,8 @@ def process(args):
cur_root
/
"gcmvn.npz"
if
args
.
cmvn_type
==
"global"
cur_root
/
"gcmvn.npz"
if
args
.
cmvn_type
==
"global"
else
None
else
None
),
),
asr_spm_filename
=
asr_spm_filename
asr_spm_filename
=
asr_spm_filename
,
share_src_and_tgt
=
True
if
args
.
task
==
"asr"
else
False
)
)
# Clean up
# Clean up
shutil
.
rmtree
(
feature_root
)
shutil
.
rmtree
(
feature_root
)
...
...
fairseq/data/audio/speech_to_text_dataset.py
查看文件 @
306dd6fc
...
@@ -85,7 +85,7 @@ class S2TDataConfig(object):
...
@@ -85,7 +85,7 @@ class S2TDataConfig(object):
a dictionary with `bpe` providing the tokenizer name and
a dictionary with `bpe` providing the tokenizer name and
the other items providing the tokenizer-specific arguments.
the other items providing the tokenizer-specific arguments.
Tokenizers are defined in `fairseq.data.encoders.*`"""
Tokenizers are defined in `fairseq.data.encoders.*`"""
return
self
.
config
.
get
(
"src_bpe_tokenizer"
,
{
"bpe"
:
None
}
)
return
self
.
config
.
get
(
"src_bpe_tokenizer"
,
None
)
@property
@property
def
prepend_tgt_lang_tag
(
self
)
->
bool
:
def
prepend_tgt_lang_tag
(
self
)
->
bool
:
...
...
fairseq/tasks/speech_to_text.py
查看文件 @
306dd6fc
...
@@ -64,13 +64,17 @@ class SpeechToTextTask(LegacyFairseqTask):
...
@@ -64,13 +64,17 @@ class SpeechToTextTask(LegacyFairseqTask):
)
)
src_dict
=
None
src_dict
=
None
if
getattr
(
data_cfg
,
"asr_vocab_filename"
,
None
):
if
getattr
(
data_cfg
,
"share_src_and_tgt"
,
False
):
dict_path
=
op
.
join
(
args
.
data
,
data_cfg
.
asr_vocab_filename
)
asr_vocab_filename
=
data_cfg
.
vocab_filename
else
:
asr_vocab_filename
=
getattr
(
data_cfg
,
"asr_vocab_filename"
,
None
)
if
asr_vocab_filename
is
not
None
:
dict_path
=
op
.
join
(
args
.
data
,
asr_vocab_filename
)
if
not
op
.
isfile
(
dict_path
):
if
not
op
.
isfile
(
dict_path
):
raise
FileNotFoundError
(
f
"Dict not found: {dict_path}"
)
raise
FileNotFoundError
(
f
"Dict not found: {dict_path}"
)
src_dict
=
Dictionary
.
load
(
dict_path
)
src_dict
=
Dictionary
.
load
(
dict_path
)
logger
.
info
(
logger
.
info
(
f
"asr dictionary size ({
data_cfg.
asr_vocab_filename}): "
f
"{len(src_dict):,}"
f
"asr dictionary size ({asr_vocab_filename}): "
f
"{len(src_dict):,}"
)
)
if
getattr
(
args
,
"train_subset"
,
None
)
is
not
None
:
if
getattr
(
args
,
"train_subset"
,
None
)
is
not
None
:
...
@@ -92,10 +96,14 @@ class SpeechToTextTask(LegacyFairseqTask):
...
@@ -92,10 +96,14 @@ class SpeechToTextTask(LegacyFairseqTask):
is_train_split
=
split
.
startswith
(
"train"
)
is_train_split
=
split
.
startswith
(
"train"
)
pre_tokenizer
=
self
.
build_tokenizer
(
self
.
args
)
pre_tokenizer
=
self
.
build_tokenizer
(
self
.
args
)
bpe_tokenizer
=
self
.
build_bpe
(
self
.
args
)
bpe_tokenizer
=
self
.
build_bpe
(
self
.
args
)
if
self
.
data_cfg
.
bpe_tokenizer
!=
self
.
data_cfg
.
src_bpe_tokenizer
:
if
self
.
data_cfg
.
src_bpe_tokenizer
is
not
None
:
src_bpe_tokenizer
=
self
.
build_src_bpe
(
self
.
args
)
src_bpe_tokenizer
=
self
.
build_src_bpe
(
self
.
args
)
else
:
else
:
src_bpe_tokenizer
=
bpe_tokenizer
src_bpe_tokenizer
=
bpe_tokenizer
# if self.data_cfg.share_src_and_tgt:
# src_bpe_tokenizer = bpe_tokenizer
# else:
# src_bpe_tokenizer = None
self
.
datasets
[
split
]
=
SpeechToTextDatasetCreator
.
from_tsv
(
self
.
datasets
[
split
]
=
SpeechToTextDatasetCreator
.
from_tsv
(
self
.
args
.
data
,
self
.
args
.
data
,
self
.
data_cfg
,
self
.
data_cfg
,
...
...
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论