Export
标准化和预标记化
Install the Transformers, Datasets, and Evaluate libraries to run this notebook.
[ ]
[ ]
<class 'tokenizers.Tokenizer'>
[ ]
'hello how are u?'
[ ]
[('Hello', (0, 5)), (',', (5, 6)), ('how', (7, 10)), ('are', (11, 14)), ('you', (16, 19)), ('?', (19, 20))] [ ]
[('Hello', (0, 5)), (',', (5, 6)), ('Ġhow', (6, 10)), ('Ġare', (10, 14)), ('Ġ', (14, 15)), ('Ġyou', (15, 19)),
, ('?', (19, 20))] [ ]
[('▁Hello,', (0, 6)), ('▁how', (7, 10)), ('▁are', (11, 14)), ('▁you?', (16, 20))]