[project]
name = 'tokenizers'
requires-python = '>=3.9'
authors = [
  { name = 'Nicolas Patry', email = 'patry.nicolas@protonmail.com' },
  { name = 'Anthony Moi', email = 'anthony@huggingface.co' },
]
classifiers = [
  "Development Status :: 5 - Production/Stable",
  "Intended Audience :: Developers",
  "Intended Audience :: Education",
  "Intended Audience :: Science/Research",
  "License :: OSI Approved :: Apache Software License",
  "Operating System :: OS Independent",
  "Programming Language :: Python :: 3",
  "Programming Language :: Python :: 3.7",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Topic :: Scientific/Engineering :: Artificial Intelligence",
]
keywords = ["NLP", "tokenizer", "BPE", "transformer", "deep learning"]
dynamic = ['description', 'license', 'readme', 'version']
dependencies = ["huggingface_hub>=0.16.4,<1.0"]

[project.urls]
Homepage = 'https://github.com/huggingface/tokenizers'
Source = 'https://github.com/huggingface/tokenizers'


[project.optional-dependencies]
testing = ["pytest", "requests", "numpy", "datasets", "black==22.3", "ruff"]
docs = ["sphinx", "sphinx_rtd_theme", "setuptools_rust"]
dev = ["tokenizers[testing]"]


[build-system]
requires = ["maturin>=1.0,<2.0"]
build-backend = "maturin"

[tool.maturin]
python-source = "py_src"
module-name = "tokenizers.tokenizers"
bindings = 'pyo3'
features = ["pyo3/extension-module"]

[tool.black]
line-length = 119
target-version = ['py35']

[tool.ruff]
line-length = 119
target-version = "py311"
lint.ignore = [
  # a == None in tests vs is None.
  "E711",
  # a == False in tests vs is False.
  "E712",
  # try.. import except.. pattern without using the lib.
  "F401",
  # Raw type equality is required in asserts
  "E721",
  # Import order
  "E402",
  # Fixtures unused import
  "F811",
]
