Path |
Digest |
Size |
tokenizers-0.19.1.dist-info/METADATA |
sha256=rCAgg9DA_ZsETxHzh_dz0hPeRKTvoj9m9kUNZe14vxc
|
6719 |
tokenizers-0.19.1.dist-info/WHEEL |
sha256=ZecsC-02CgJW1eVh0SnzSmfDXdlpfFNFEdQSIIzPs-Q
|
127 |
tokenizers/decoders/__init__.py |
sha256=lGp32h8qerE0F48gyZL8wGmeQVlmjVpeIsRb1SM9kf4
|
335 |
tokenizers/decoders/__init__.pyi |
sha256=xsReo7OFRCiQ4bBZY9ogYb1iLJ5DTgI5elNB-Uggocs
|
7244 |
tokenizers/tools/visualizer.py |
sha256=gi-E2NCP7FuG6ujpQOdalSTXUlaV85V6NI-ZPPTvA_4
|
14625 |
tokenizers/tools/__init__.py |
sha256=xG8caB9OHC8cbB01S5vYV14HZxhO6eWbLehsb70ppio
|
55 |
tokenizers/tools/visualizer-styles.css |
sha256=zAydq1oGWD8QEll4-eyL8Llw0B1sty_hpIE3tYxL02k
|
4850 |
tokenizers/pre_tokenizers/__init__.py |
sha256=wd6KYQA_RsGSQK-HeG9opTRhv4ttSRkyno2dk6az-PM
|
557 |
tokenizers/pre_tokenizers/__init__.pyi |
sha256=IhF7dZt9_9_WM2ESKwEIvN59uW_YzS2PzmWBUScysWU
|
23258 |
tokenizers/__init__.py |
sha256=ZE5ZagUvobBScrHBQdEobhx4wqM0bsq9F9aLYkBNjYQ
|
2615 |
tokenizers/__init__.pyi |
sha256=YBIWZCSN4Rs_-yKdEwhVv77bgHRE36hX9iwFrWGMJ8E
|
38536 |
tokenizers/implementations/sentencepiece_bpe.py |
sha256=LwrofoohnUfME2lK2lQYoyQIhP84RP0CIlHRaj0hyNs
|
3738 |
tokenizers/implementations/sentencepiece_unigram.py |
sha256=SYiVXL8ZtqLXKpuqwnwmrfxgGotu8yAkOu7dLztEXIo
|
7580 |
tokenizers/implementations/char_level_bpe.py |
sha256=Q2ZEAW0xMQHF7YCUtmplwaxbU-J0P2NK4PJGMxUb-_c
|
5466 |
tokenizers/implementations/byte_level_bpe.py |
sha256=OA_jyy3EQmYTa6hnf-EKwLOFuyroqFYOJz25ysM2BUk
|
4289 |
tokenizers/implementations/__init__.py |
sha256=VzAsplaIo7rl4AFO8Miu7ig7MfZjvonwVblZw01zR6M
|
310 |
tokenizers/implementations/base_tokenizer.py |
sha256=2TFZhLupaJiMDYGJuUNmxYJv-cnR8bDHmbMzaYpFROs
|
14206 |
tokenizers/implementations/bert_wordpiece.py |
sha256=sKCum0FKPYdSgJFJN8LDerVBoTDRSqyqSdrcm-lvQqI
|
5520 |
tokenizers/normalizers/__init__.py |
sha256=hKOwnqWM-IlcVv7HDWT9SYhlczevuCNDQJY05ZFxkzk
|
808 |
tokenizers/normalizers/__init__.pyi |
sha256=5SGm-u896MZht6TXMS9sWv1lCATnwNqbC2Udl5aP4dg
|
19597 |
tokenizers/models/__init__.py |
sha256=eJZ4HTAQZpxnKILNylWaTFqxXy-Ba6OKswWN47feeV8
|
176 |
tokenizers/models/__init__.pyi |
sha256=wH4M-ZZprw3UQ98fxWrF3MpivuNVY3s3pv4pGY0A_kE
|
16932 |
tokenizers/processors/__init__.py |
sha256=xM2DEKwKtHIumHsszM8AMkq-AlaqvBZFXWgLU8SNhOY
|
307 |
tokenizers/processors/__init__.pyi |
sha256=hx767ZY8SHhxb_hiXPRxm-f_KcoR4XDx7vfK2c0lR-Q
|
11357 |
tokenizers/trainers/__init__.py |
sha256=UTu22AGcp76IvpW45xLRbJWET04NxPW6NfCb2YYz0EM
|
248 |
tokenizers/trainers/__init__.pyi |
sha256=3TwFKts4me7zQfVRcSTmtXYiP4XwcRjfAYtwqoZVtoQ
|
5382 |
tokenizers/tokenizers.cpython-39-x86_64-linux-gnu.so |
sha256=k1sl24EzellCirjGve8FoHBg4YSl6h_jPJ9CdPYIj2E
|
11817040 |
tokenizers-0.19.1.dist-info/RECORD |
— |
— |