We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 16a7d31 commit c7d54f6Copy full SHA for c7d54f6
week07_seq2seq/voc.py
@@ -23,7 +23,7 @@ def __len__(self):
23
def from_lines(lines, bos="__BOS__", eos="__EOS__", sep=''):
24
flat_lines = sep.join(list(lines))
25
flat_lines = list(flat_lines.split(sep)) if sep else list(flat_lines)
26
- tokens = list(set(sep.join(flat_lines)))
+ tokens = sorted(set(sep.join(flat_lines)))
27
tokens = [t for t in tokens if t not in (bos, eos) and len(t) != 0]
28
tokens = [bos, eos] + tokens
29
return Vocab(tokens, bos, eos, sep)
0 commit comments