flypythoncom/GPT2-Chinese
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update bpe_tokenizer.py

Browse Source
This commit is contained in:
Duzeyao
2019-11-11 09:56:23 +08:00
parent c3f880e55b
commit 80ff9f0d87
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizations/bpe_tokenizer.py
View File

@@ -125,7 +125,7 @@ def get_encoder(encoder_file, bpe_file):
shotname, extension = os.path.splitext(filename) shotname, extension = os.path.splitext(filename)
if(".model" == extension) and (bpe_file == ""): if(".model" == extension) and (bpe_file == ""):
return Encoder_SP(encoder_path) return Encoder_SP(encoder_file)
else: else:
with open(encoder_file, 'r', encoding="utf-8") as f: with open(encoder_file, 'r', encoding="utf-8") as f:
encoder = json.load(f) encoder = json.load(f)