Path |
Digest |
Size |
tokenizers-0.20.0.dist-info/METADATA |
sha256=8ShnugZ3_tcw069SjCP7UEo448UfZUJ00_QA5_1Zx-g
|
6719 |
tokenizers-0.20.0.dist-info/WHEEL |
sha256=7WQXo-DyJikNS__3YGnYu_GR-RciZxr7kr6jtbowDRw
|
127 |
tokenizers/processors/__init__.pyi |
sha256=hx767ZY8SHhxb_hiXPRxm-f_KcoR4XDx7vfK2c0lR-Q
|
11357 |
tokenizers/processors/__init__.py |
sha256=xM2DEKwKtHIumHsszM8AMkq-AlaqvBZFXWgLU8SNhOY
|
307 |
tokenizers/__init__.pyi |
sha256=39h2qoUbbcfMWgaH3hiGXSaC5BSqgZKQxvfhXKZ9tD8
|
40037 |
tokenizers/pre_tokenizers/__init__.pyi |
sha256=YYMCn3qXGA5v0UCeP-Q4--yVTaiUwRdIM-o0YfNfcZk
|
23604 |
tokenizers/pre_tokenizers/__init__.py |
sha256=wd6KYQA_RsGSQK-HeG9opTRhv4ttSRkyno2dk6az-PM
|
557 |
tokenizers/implementations/char_level_bpe.py |
sha256=Q2ZEAW0xMQHF7YCUtmplwaxbU-J0P2NK4PJGMxUb-_c
|
5466 |
tokenizers/implementations/bert_wordpiece.py |
sha256=sKCum0FKPYdSgJFJN8LDerVBoTDRSqyqSdrcm-lvQqI
|
5520 |
tokenizers/implementations/byte_level_bpe.py |
sha256=OA_jyy3EQmYTa6hnf-EKwLOFuyroqFYOJz25ysM2BUk
|
4289 |
tokenizers/implementations/sentencepiece_bpe.py |
sha256=LwrofoohnUfME2lK2lQYoyQIhP84RP0CIlHRaj0hyNs
|
3738 |
tokenizers/implementations/base_tokenizer.py |
sha256=2TFZhLupaJiMDYGJuUNmxYJv-cnR8bDHmbMzaYpFROs
|
14206 |
tokenizers/implementations/__init__.py |
sha256=VzAsplaIo7rl4AFO8Miu7ig7MfZjvonwVblZw01zR6M
|
310 |
tokenizers/implementations/sentencepiece_unigram.py |
sha256=SYiVXL8ZtqLXKpuqwnwmrfxgGotu8yAkOu7dLztEXIo
|
7580 |
tokenizers/__init__.py |
sha256=ZE5ZagUvobBScrHBQdEobhx4wqM0bsq9F9aLYkBNjYQ
|
2615 |
tokenizers/trainers/__init__.pyi |
sha256=3TwFKts4me7zQfVRcSTmtXYiP4XwcRjfAYtwqoZVtoQ
|
5382 |
tokenizers/trainers/__init__.py |
sha256=UTu22AGcp76IvpW45xLRbJWET04NxPW6NfCb2YYz0EM
|
248 |
tokenizers/models/__init__.pyi |
sha256=clPTwiyjz7FlVdEuwo_3Wa_TmQrbZhW0SGmnNylepnY
|
16929 |
tokenizers/models/__init__.py |
sha256=eJZ4HTAQZpxnKILNylWaTFqxXy-Ba6OKswWN47feeV8
|
176 |
tokenizers/normalizers/__init__.pyi |
sha256=dwfVsvg0YbeYoAaBSmKsImqL-tyxiDyHaaTFsZK4aZw
|
20897 |
tokenizers/normalizers/__init__.py |
sha256=_06w4cqRItveEgIddYaLMScgkSOkIAMIzYCesb5AA4U
|
841 |
tokenizers/tools/visualizer.py |
sha256=gi-E2NCP7FuG6ujpQOdalSTXUlaV85V6NI-ZPPTvA_4
|
14625 |
tokenizers/tools/__init__.py |
sha256=xG8caB9OHC8cbB01S5vYV14HZxhO6eWbLehsb70ppio
|
55 |
tokenizers/tools/visualizer-styles.css |
sha256=zAydq1oGWD8QEll4-eyL8Llw0B1sty_hpIE3tYxL02k
|
4850 |
tokenizers/decoders/__init__.pyi |
sha256=xsReo7OFRCiQ4bBZY9ogYb1iLJ5DTgI5elNB-Uggocs
|
7244 |
tokenizers/decoders/__init__.py |
sha256=lGp32h8qerE0F48gyZL8wGmeQVlmjVpeIsRb1SM9kf4
|
335 |
tokenizers/tokenizers.cpython-39-x86_64-linux-gnu.so |
sha256=RGtRcJ5kC0qj0oKHcX6h0mkfCAbvvprY6t9ssGs4JKc
|
8703152 |
tokenizers-0.20.0.dist-info/RECORD |
— |
— |