Wheelodex » Projects » tokenizers

tokenizers

View on PyPI — Reverse Dependencies (354)

0.15.2	tokenizers-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.15.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.15.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.15.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-cp39-none-win_amd64.whl
	tokenizers-0.15.2-cp39-none-win32.whl
	tokenizers-0.15.2-cp39-cp39-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-cp39-cp39-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-cp39-cp39-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-cp39-cp39-macosx_11_0_arm64.whl
	tokenizers-0.15.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.15.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.15.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.15.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-cp38-none-win_amd64.whl
	tokenizers-0.15.2-cp38-none-win32.whl
	tokenizers-0.15.2-cp38-cp38-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-cp38-cp38-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-cp38-cp38-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-cp38-cp38-macosx_11_0_arm64.whl
	tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-cp37-none-win_amd64.whl
	tokenizers-0.15.2-cp37-none-win32.whl
	tokenizers-0.15.2-cp37-cp37m-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-cp37-cp37m-macosx_11_0_arm64.whl
	tokenizers-0.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.15.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.15.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-cp313-cp313-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-cp313-cp313-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-cp313-cp313-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-cp313-cp313-macosx_11_0_arm64.whl
	tokenizers-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-cp312-none-win_amd64.whl
	tokenizers-0.15.2-cp312-none-win32.whl
	tokenizers-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-cp312-cp312-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-cp312-cp312-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-cp312-cp312-macosx_11_0_arm64.whl
	tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.15.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.15.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-cp311-none-win_amd64.whl
	tokenizers-0.15.2-cp311-none-win32.whl
	tokenizers-0.15.2-cp311-cp311-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-cp311-cp311-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-cp311-cp311-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-cp311-cp311-macosx_11_0_arm64.whl
	tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.15.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.15.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-cp310-none-win_amd64.whl
	tokenizers-0.15.2-cp310-none-win32.whl
	tokenizers-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-cp310-cp310-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-cp310-cp310-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-cp310-cp310-macosx_11_0_arm64.whl
	tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl
	tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl
	tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl

Wheel Details

Project:	tokenizers
Version:	0.15.2
Filename:	tokenizers-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Download:	[link]
Size:	3577424
MD5:	`b73c566e3af8b6c69c433feba3a28abd`
SHA256:	`0cf6b7f1d4dc59af960e6ffdc4faffe6460bbfa8dce27a58bf75755ffdb2526d`
Uploaded:	2024-02-12 02:27:26 +0000

dist-info

METADATA · WHEEL · RECORD

METADATA

Metadata-Version:	2.1
Name:	tokenizers
Version:	0.15.2
Author:	Anthony MOI <m.anthony.moi@gmail.com>
Author-Email:	Nicolas Patry <patry.nicolas[at]protonmail.com>, Anthony Moi <anthony[at]huggingface.co>
Project-Url:	Homepage, https://github.com/huggingface/tokenizers
Project-Url:	Source, https://github.com/huggingface/tokenizers
Keywords:	NLP,tokenizer,BPE,transformer,deep learning
Classifier:	Development Status :: 5 - Production/Stable
Classifier:	Intended Audience :: Developers
Classifier:	Intended Audience :: Education
Classifier:	Intended Audience :: Science/Research
Classifier:	License :: OSI Approved :: Apache Software License
Classifier:	Operating System :: OS Independent
Classifier:	Programming Language :: Python :: 3
Classifier:	Programming Language :: Python :: 3.7
Classifier:	Programming Language :: Python :: 3.8
Classifier:	Programming Language :: Python :: 3.9
Classifier:	Programming Language :: Python :: 3.10
Classifier:	Programming Language :: Python :: 3.11
Classifier:	Topic :: Scientific/Engineering :: Artificial Intelligence
Requires-Python:	>=3.7
Requires-Dist:	huggingface_hub (<1.0,>=0.16.4)
Requires-Dist:	pytest; extra == "testing"
Requires-Dist:	requests; extra == "testing"
Requires-Dist:	numpy; extra == "testing"
Requires-Dist:	datasets; extra == "testing"
Requires-Dist:	black (==22.3); extra == "testing"
Requires-Dist:	sphinx; extra == "docs"
Requires-Dist:	sphinx_rtd_theme; extra == "docs"
Requires-Dist:	setuptools_rust; extra == "docs"
Requires-Dist:	tokenizers[testing]; extra == "dev"
Provides-Extra:	testing
Provides-Extra:	docs
Provides-Extra:	dev
Description-Content-Type:	text/markdown; charset=UTF-8; variant=GFM

[Description omitted; length: 5006 characters]

WHEEL

Wheel-Version:	1.0
Generator:	maturin (1.4.0)
Root-Is-Purelib:	false
Tag:	cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64

RECORD

Path	Digest	Size
tokenizers-0.15.2.dist-info/METADATA	sha256=ksSEhu7uah79jtG-gw_GrGsyI5gmQc5jcIh-o1IwGjo	6678
tokenizers-0.15.2.dist-info/WHEEL	sha256=UaZDRGlTI9y3aimBey-_zNIrvbV38RYv9K-1MptjaL4	127
tokenizers/decoders/__init__.pyi	sha256=qF5dcAK7Hiqdj_eXCTaG_-AdC_mvLy-E96_HzovIzgI	7041
tokenizers/decoders/__init__.py	sha256=lGp32h8qerE0F48gyZL8wGmeQVlmjVpeIsRb1SM9kf4	335
tokenizers/__init__.pyi	sha256=gwfh8FGRwAAHs3sedFzRpfVDFnDoLaCtm3Aa4nG_sPs	38459
tokenizers/pre_tokenizers/__init__.pyi	sha256=KI36GgHlR3p9GtLtheqNdFmt6jm9PVzNKq4awcy1dKU	23042
tokenizers/pre_tokenizers/__init__.py	sha256=wd6KYQA_RsGSQK-HeG9opTRhv4ttSRkyno2dk6az-PM	557
tokenizers/trainers/__init__.pyi	sha256=q5gsMXnp2UBO5-gHBlajb9lhOo8FOeKBCwVGGSA23Ws	5384
tokenizers/trainers/__init__.py	sha256=UTu22AGcp76IvpW45xLRbJWET04NxPW6NfCb2YYz0EM	248
tokenizers/normalizers/__init__.pyi	sha256=pR7NwcZDyFncmyZa9Wn1gjqzopaq5HyiPkRNHYuhzZk	19585
tokenizers/normalizers/__init__.py	sha256=hKOwnqWM-IlcVv7HDWT9SYhlczevuCNDQJY05ZFxkzk	808
tokenizers/tools/visualizer.py	sha256=0KUrLhkBLhPvg3GAkvsiBokb517bMCMSN-vuYY1qmEo	14621
tokenizers/tools/visualizer-styles.css	sha256=zAydq1oGWD8QEll4-eyL8Llw0B1sty_hpIE3tYxL02k	4850
tokenizers/tools/__init__.py	sha256=xG8caB9OHC8cbB01S5vYV14HZxhO6eWbLehsb70ppio	55
tokenizers/models/__init__.pyi	sha256=2RefLTJlKOF1o0ZI6wpBHDGIRMsXvn25-26yWCYavPY	16749
tokenizers/models/__init__.py	sha256=eJZ4HTAQZpxnKILNylWaTFqxXy-Ba6OKswWN47feeV8	176
tokenizers/implementations/base_tokenizer.py	sha256=2TFZhLupaJiMDYGJuUNmxYJv-cnR8bDHmbMzaYpFROs	14206
tokenizers/implementations/char_level_bpe.py	sha256=Q2ZEAW0xMQHF7YCUtmplwaxbU-J0P2NK4PJGMxUb-_c	5466
tokenizers/implementations/sentencepiece_bpe.py	sha256=syNXoQZX1JtI8U1A9XSDcoihlF3bIGVTVN53YJ83pV4	3679
tokenizers/implementations/sentencepiece_unigram.py	sha256=iCf9NKPAxTWfNGmwx0AooA1ubN44bAgTdvjU7LhnvhM	7462
tokenizers/implementations/bert_wordpiece.py	sha256=sKCum0FKPYdSgJFJN8LDerVBoTDRSqyqSdrcm-lvQqI	5520
tokenizers/implementations/byte_level_bpe.py	sha256=OA_jyy3EQmYTa6hnf-EKwLOFuyroqFYOJz25ysM2BUk	4289
tokenizers/implementations/__init__.py	sha256=VzAsplaIo7rl4AFO8Miu7ig7MfZjvonwVblZw01zR6M	310
tokenizers/__init__.py	sha256=ZE5ZagUvobBScrHBQdEobhx4wqM0bsq9F9aLYkBNjYQ	2615
tokenizers/processors/__init__.pyi	sha256=AZXH9ZDWb0jvzp0DKC7lKjCeOwli70_ZvR3zqMKf7v4	11352
tokenizers/processors/__init__.py	sha256=xM2DEKwKtHIumHsszM8AMkq-AlaqvBZFXWgLU8SNhOY	307
tokenizers/tokenizers.cpython-39-x86_64-linux-gnu.so	sha256=h4geEZq6IeSb1s539yu2ztiMlbVh2yovyhAPqhx4NJg	11730112
tokenizers-0.15.2.dist-info/RECORD	—	—