Wheelodex » Projects » tokenizers » tokenizers-0.19.1-cp311-none-win_amd64.whl

tokenizers

View on PyPI — Reverse Dependencies (360)

0.19.1	tokenizers-0.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.19.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.19.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.19.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-cp39-none-win_amd64.whl
	tokenizers-0.19.1-cp39-none-win32.whl
	tokenizers-0.19.1-cp39-cp39-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-cp39-cp39-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-cp39-cp39-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-cp39-cp39-macosx_11_0_arm64.whl
	tokenizers-0.19.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.19.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.19.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.19.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-cp38-none-win_amd64.whl
	tokenizers-0.19.1-cp38-none-win32.whl
	tokenizers-0.19.1-cp38-cp38-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-cp38-cp38-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-cp38-cp38-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-cp38-cp38-macosx_11_0_arm64.whl
	tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-cp37-none-win_amd64.whl
	tokenizers-0.19.1-cp37-none-win32.whl
	tokenizers-0.19.1-cp37-cp37m-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-cp37-cp37m-macosx_11_0_arm64.whl
	tokenizers-0.19.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.19.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.19.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.19.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-cp312-none-win_amd64.whl
	tokenizers-0.19.1-cp312-none-win32.whl
	tokenizers-0.19.1-cp312-cp312-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-cp312-cp312-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-cp312-cp312-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-cp312-cp312-macosx_11_0_arm64.whl
	tokenizers-0.19.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.19.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.19.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.19.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-cp311-none-win_amd64.whl
	tokenizers-0.19.1-cp311-none-win32.whl
	tokenizers-0.19.1-cp311-cp311-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-cp311-cp311-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-cp311-cp311-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-cp311-cp311-macosx_11_0_arm64.whl
	tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
	tokenizers-0.19.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl
	tokenizers-0.19.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
	tokenizers-0.19.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-cp310-none-win_amd64.whl
	tokenizers-0.19.1-cp310-none-win32.whl
	tokenizers-0.19.1-cp310-cp310-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-cp310-cp310-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-cp310-cp310-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-cp310-cp310-macosx_11_0_arm64.whl
	tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl
	tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl
	tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-pp37-pypy37_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
	tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
	tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl
	tokenizers-0.19.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl
	tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl
	tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl
	tokenizers-0.19.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl

Wheel Details

Project:	tokenizers
Version:	0.19.1
Filename:	tokenizers-0.19.1-cp311-none-win_amd64.whl
Download:	[link]
Size:	2220386
MD5:	`17c0000754eb1df2fe6053f7d31f5f90`
SHA256:	`ad57d59341710b94a7d9dbea13f5c1e7d76fd8d9bcd944a7a6ab0b0da6e0cc66`
Uploaded:	2024-04-17 21:37:08 +0000

dist-info

METADATA · WHEEL · RECORD

METADATA

Metadata-Version:	2.3
Name:	tokenizers
Version:	0.19.1
Author:	Anthony MOI <m.anthony.moi@gmail.com>
Author-Email:	Nicolas Patry <patry.nicolas[at]protonmail.com>, Anthony Moi <anthony[at]huggingface.co>
Project-Url:	Homepage, https://github.com/huggingface/tokenizers
Project-Url:	Source, https://github.com/huggingface/tokenizers
Keywords:	NLP,tokenizer,BPE,transformer,deep learning
Classifier:	Development Status :: 5 - Production/Stable
Classifier:	Intended Audience :: Developers
Classifier:	Intended Audience :: Education
Classifier:	Intended Audience :: Science/Research
Classifier:	License :: OSI Approved :: Apache Software License
Classifier:	Operating System :: OS Independent
Classifier:	Programming Language :: Python :: 3
Classifier:	Programming Language :: Python :: 3.7
Classifier:	Programming Language :: Python :: 3.8
Classifier:	Programming Language :: Python :: 3.9
Classifier:	Programming Language :: Python :: 3.10
Classifier:	Programming Language :: Python :: 3.11
Classifier:	Topic :: Scientific/Engineering :: Artificial Intelligence
Requires-Python:	>=3.7
Requires-Dist:	huggingface-hub (<1.0,>=0.16.4)
Requires-Dist:	pytest; extra == "testing"
Requires-Dist:	requests; extra == "testing"
Requires-Dist:	numpy; extra == "testing"
Requires-Dist:	datasets; extra == "testing"
Requires-Dist:	black (==22.3); extra == "testing"
Requires-Dist:	ruff; extra == "testing"
Requires-Dist:	sphinx; extra == "docs"
Requires-Dist:	sphinx-rtd-theme; extra == "docs"
Requires-Dist:	setuptools-rust; extra == "docs"
Requires-Dist:	tokenizers[testing]; extra == "dev"
Provides-Extra:	testing
Provides-Extra:	docs
Provides-Extra:	dev
Description-Content-Type:	text/markdown; charset=UTF-8; variant=GFM

[Description omitted; length: 5006 characters]

WHEEL

Wheel-Version:	1.0
Generator:	maturin (1.5.1)
Root-Is-Purelib:	false
Tag:	cp311-none-win_amd64

RECORD

Path	Digest	Size
tokenizers-0.19.1.dist-info/METADATA	sha256=zc6oRN6a7CK6GxahW8DWG00HCHKqdhxclAO9DOicMAY	6889
tokenizers-0.19.1.dist-info/WHEEL	sha256=88DykRyub23IQeCxphAc1eW7PZo2npnb8m2maKWpwLI	95
tokenizers/decoders/__init__.py	sha256=xffrQt9YxtuKLOBEkJnZZPu03ozf3sm-x1KgxF-f0DQ	349
tokenizers/decoders/__init__.pyi	sha256=ACZDc_jw7DJsrLaW_TW7ynlnjZhFexwaaMP3_aVEmTE	7515
tokenizers/implementations/base_tokenizer.py	sha256=nOUjtXTgDJSdIjDDnBzoWELUnzpRy6yLyVhUOL5t5xY	14624
tokenizers/implementations/bert_wordpiece.py	sha256=jRUMSYU1C7nVEDZMKdyG3Vp7-CUongE1kGJhvOP_EHM	5671
tokenizers/implementations/byte_level_bpe.py	sha256=iCCZSWW9nwQgfeXeOiwVEX7sLNfPsQxO7CCuEmTaHXw	4411
tokenizers/implementations/char_level_bpe.py	sha256=AQaZFT_RK7SW1ubGsFtXOnftit_vWp0fj4-fJnzZn7c	5616
tokenizers/implementations/sentencepiece_bpe.py	sha256=MpYmKEzitTb4RRTN7ElSaGrVZFC1PD-8P5k2g5W6144	3841
tokenizers/implementations/sentencepiece_unigram.py	sha256=5SliPQsU_cAB1Pqt7ufL9w8fBrNoLixVC-rqF4MRcm0	7776
tokenizers/implementations/__init__.py	sha256=9ZI2cJHPCUCZXP37GNmRHtBzIYiKWzyqky1rtcYdrPw	316
tokenizers/models/__init__.py	sha256=qE73qycPAKcey_pS8ov1FIz10L2Ybzy1E-1KGee27Qg	184
tokenizers/models/__init__.pyi	sha256=pn1-ztyVLn7yG3Rcc4wuXJGacDED4Y9BKnhkp5kycTE	17523
tokenizers/normalizers/__init__.py	sha256=HdGKGl2qjqQN520YXGgBn6Ti3qMQ-grvNH9QgzeA-wY	837
tokenizers/normalizers/__init__.pyi	sha256=SETiwcGHrU4AfgjFWEDUDhojR10vjzAbkEjScZN2_eo	20192
tokenizers/pre_tokenizers/__init__.py	sha256=ZWLPWG2nhoTojLyHIDUPRsa1V8drqWufJVjzbRmR7UQ	572
tokenizers/pre_tokenizers/__init__.pyi	sha256=doL9JamZTw67bXMy-Bbpc2qUgwAzqfiXTXFuxosfoKg	23865
tokenizers/processors/__init__.py	sha256=X1OKKtr3IUeede85mI1cLF0MqmDRlr7LNFMYdEy5KMU	316
tokenizers/processors/__init__.pyi	sha256=qMoapc_ufxbrL-ZVAkpYugLzxxuz1UBBNyoEiySjt-4	11699
tokenizers/tools/visualizer-styles.css	sha256=g98RJucUCjYnPLwXycfcV-dAbHMzqJpnZ7NoP_cCUBk	5020
tokenizers/tools/visualizer.py	sha256=8t51UabkxYPArq3I5Um93OT8o2jBQLLGxh7YPxaEJko	15028
tokenizers/tools/__init__.py	sha256=mJLVrHN1FP23Cf4_EJWoYUdh8yTb5BbJFNfa8Ec5vHM	56
tokenizers/trainers/__init__.py	sha256=v9CdtoVauwD6b1wAA6R5goSLZslBbPE5vZHM2d8sgKM	256
tokenizers/trainers/__init__.pyi	sha256=13m2jGMVhn4dJyxkfgzR-JKvuBGvuj7fvgwJHlbnQUs	5538
tokenizers/__init__.py	sha256=AJ5NdKOV_dZ_SOw9oFF1M4ln0fh1hod-voqzeYHS71U	2715
tokenizers/__init__.pyi	sha256=xQVTb2uN6wfs0nLugox1qQ_woqtFUb9Y1TA6dRejTf4	39736
tokenizers/tokenizers.cp311-win_amd64.pyd	sha256=Xlj0sbvt7vfTxHanHE-fs1J7V0nDT_svj3l4159hWgY	6065152
tokenizers-0.19.1.dist-info/RECORD	—	—