Path |
Digest |
Size |
tokenizers-0.19.1.dist-info/METADATA |
sha256=zc6oRN6a7CK6GxahW8DWG00HCHKqdhxclAO9DOicMAY
|
6889 |
tokenizers-0.19.1.dist-info/WHEEL |
sha256=5HyBAIXy6w_lmX9AQnFspp6An6SPZJwRKkYM1H8iHP0
|
95 |
tokenizers/decoders/__init__.py |
sha256=xffrQt9YxtuKLOBEkJnZZPu03ozf3sm-x1KgxF-f0DQ
|
349 |
tokenizers/decoders/__init__.pyi |
sha256=ACZDc_jw7DJsrLaW_TW7ynlnjZhFexwaaMP3_aVEmTE
|
7515 |
tokenizers/implementations/base_tokenizer.py |
sha256=nOUjtXTgDJSdIjDDnBzoWELUnzpRy6yLyVhUOL5t5xY
|
14624 |
tokenizers/implementations/bert_wordpiece.py |
sha256=jRUMSYU1C7nVEDZMKdyG3Vp7-CUongE1kGJhvOP_EHM
|
5671 |
tokenizers/implementations/byte_level_bpe.py |
sha256=iCCZSWW9nwQgfeXeOiwVEX7sLNfPsQxO7CCuEmTaHXw
|
4411 |
tokenizers/implementations/char_level_bpe.py |
sha256=AQaZFT_RK7SW1ubGsFtXOnftit_vWp0fj4-fJnzZn7c
|
5616 |
tokenizers/implementations/sentencepiece_bpe.py |
sha256=MpYmKEzitTb4RRTN7ElSaGrVZFC1PD-8P5k2g5W6144
|
3841 |
tokenizers/implementations/sentencepiece_unigram.py |
sha256=5SliPQsU_cAB1Pqt7ufL9w8fBrNoLixVC-rqF4MRcm0
|
7776 |
tokenizers/implementations/__init__.py |
sha256=9ZI2cJHPCUCZXP37GNmRHtBzIYiKWzyqky1rtcYdrPw
|
316 |
tokenizers/models/__init__.py |
sha256=qE73qycPAKcey_pS8ov1FIz10L2Ybzy1E-1KGee27Qg
|
184 |
tokenizers/models/__init__.pyi |
sha256=pn1-ztyVLn7yG3Rcc4wuXJGacDED4Y9BKnhkp5kycTE
|
17523 |
tokenizers/normalizers/__init__.py |
sha256=HdGKGl2qjqQN520YXGgBn6Ti3qMQ-grvNH9QgzeA-wY
|
837 |
tokenizers/normalizers/__init__.pyi |
sha256=SETiwcGHrU4AfgjFWEDUDhojR10vjzAbkEjScZN2_eo
|
20192 |
tokenizers/pre_tokenizers/__init__.py |
sha256=ZWLPWG2nhoTojLyHIDUPRsa1V8drqWufJVjzbRmR7UQ
|
572 |
tokenizers/pre_tokenizers/__init__.pyi |
sha256=doL9JamZTw67bXMy-Bbpc2qUgwAzqfiXTXFuxosfoKg
|
23865 |
tokenizers/processors/__init__.py |
sha256=X1OKKtr3IUeede85mI1cLF0MqmDRlr7LNFMYdEy5KMU
|
316 |
tokenizers/processors/__init__.pyi |
sha256=qMoapc_ufxbrL-ZVAkpYugLzxxuz1UBBNyoEiySjt-4
|
11699 |
tokenizers/tools/visualizer-styles.css |
sha256=g98RJucUCjYnPLwXycfcV-dAbHMzqJpnZ7NoP_cCUBk
|
5020 |
tokenizers/tools/visualizer.py |
sha256=8t51UabkxYPArq3I5Um93OT8o2jBQLLGxh7YPxaEJko
|
15028 |
tokenizers/tools/__init__.py |
sha256=mJLVrHN1FP23Cf4_EJWoYUdh8yTb5BbJFNfa8Ec5vHM
|
56 |
tokenizers/trainers/__init__.py |
sha256=v9CdtoVauwD6b1wAA6R5goSLZslBbPE5vZHM2d8sgKM
|
256 |
tokenizers/trainers/__init__.pyi |
sha256=13m2jGMVhn4dJyxkfgzR-JKvuBGvuj7fvgwJHlbnQUs
|
5538 |
tokenizers/__init__.py |
sha256=AJ5NdKOV_dZ_SOw9oFF1M4ln0fh1hod-voqzeYHS71U
|
2715 |
tokenizers/__init__.pyi |
sha256=xQVTb2uN6wfs0nLugox1qQ_woqtFUb9Y1TA6dRejTf4
|
39736 |
tokenizers/tokenizers.cp310-win_amd64.pyd |
sha256=CN40BUzucLGu8wumHW-JRDb1iom1A7bhzsiO_K08jxc
|
6065152 |
tokenizers-0.19.1.dist-info/RECORD |
— |
— |