


import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
_lowerCamelCase     =  sys.version_info >= (3, 10)


def __lowercase     (      lowerCamelCase_  :      Tuple=None						,					lowerCamelCase_  :      Dict=None  ):
				return field(default_factory=lambda: default						,					metadata=lowerCamelCase_  )


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      int
				_lowerCAmelCase      :      float
				_lowerCAmelCase      :      str
				_lowerCAmelCase      :      bool


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      int      							=					42
				_lowerCAmelCase      :      str      							=					field(default="toto"       ,  metadata={"help": "help message"}   )


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      bool      							=					False
				_lowerCAmelCase      :      bool      							=					True
				_lowerCAmelCase      :      Optional[bool]      							=					None


class 			lowerCamelCase_		(			lowercase   ):





				"""simple docstring"""







				_lowerCAmelCase      :      Union[str, Any]      							=					"titi"
				_lowerCAmelCase      :      List[Any]      							=					"toto"


class 			lowerCamelCase_		(			lowercase   ):





				"""simple docstring"""







				_lowerCAmelCase      :      Any      							=					"titi"
				_lowerCAmelCase      :      Optional[Any]      							=					"toto"
				_lowerCAmelCase      :      List[str]      							=					42


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      BasicEnum      							=					"toto"



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				BasicEnum(self.foo     )


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      MixedTypeEnum      							=					"toto"



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				MixedTypeEnum(self.foo     )


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      Optional[int]      							=					None
				_lowerCAmelCase      :      Optional[float]      							=					field(default=lowercase       ,  metadata={"help": "help message"}   )
				_lowerCAmelCase      :      Optional[str]      							=					None
				_lowerCAmelCase      :      Optional[List[str]]      							=					list_field(default=[]   )
				_lowerCAmelCase      :      Optional[List[int]]      							=					list_field(default=[]   )


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      List[int]      							=					list_field(default=[]   )
				_lowerCAmelCase      :      List[int]      							=					list_field(default=[1, 2, 3]   )
				_lowerCAmelCase      :      List[str]      							=					list_field(default=["Hallo", "Bonjour", "Hello"]   )
				_lowerCAmelCase      :      List[float]      							=					list_field(default=[0.1, 0.2, 0.3]   )


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      List[int]      							=					field()
				_lowerCAmelCase      :      str      							=					field()
				_lowerCAmelCase      :      BasicEnum      							=					field()



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				BasicEnum(self.required_enum     )


@dataclass
class 			lowerCamelCase_		:





				"""simple docstring"""







				_lowerCAmelCase      :      int
				_lowerCAmelCase      :      "BasicEnum"      							=					field()
				_lowerCAmelCase      :      "Optional[bool]"      							=					None
				_lowerCAmelCase      :      "str"      							=					field(default="toto"       ,  metadata={"help": "help message"}   )
				_lowerCAmelCase      :      "List[str]"      							=					list_field(default=["Hallo", "Bonjour", "Hello"]   )


if is_python_no_less_than_3_10:


	@dataclass
	class 			lowerCamelCase_		:





					"""simple docstring"""







					_lowerCAmelCase      :      bool      							=					False
					_lowerCAmelCase      :      bool      							=					True
					_lowerCAmelCase      :      bool | None      							=					None


	@dataclass
	class 			lowerCamelCase_		:





					"""simple docstring"""







					_lowerCAmelCase      :      int | None      							=					None
					_lowerCAmelCase      :      float | None      							=					field(default=lowercase       ,  metadata={"help": "help message"}   )
					_lowerCAmelCase      :      str | None      							=					None
					_lowerCAmelCase      :      list[str] | None      							=					list_field(default=[]   )
					_lowerCAmelCase      :      list[int] | None      							=					list_field(default=[]   )


class 			lowerCamelCase_		(			unittest.TestCase   ):





				"""simple docstring"""







				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__     ):
								self.assertEqual(len(a._actions     )       ,		len(b._actions     )     )
								for x, y in zip(a._actions       ,		b._actions     ):
												SCREAMING_SNAKE_CASE__     							=				{k: v for k, v in vars(UpperCAmelCase__     ).items() if k != "container"}
												SCREAMING_SNAKE_CASE__     							=				{k: v for k, v in vars(UpperCAmelCase__     ).items() if k != "container"}

												# Choices with mixed type have custom function as "type"
												# So we need to compare results directly for equality
												if xx.get("choices"       ,		UpperCAmelCase__     ) and yy.get("choices"       ,		UpperCAmelCase__     ):
																for expected_choice in yy["choices"] + xx["choices"]:
																				self.assertEqual(xx["type"](UpperCAmelCase__     )       ,		yy["type"](UpperCAmelCase__     )     )
																del xx["type"], yy["type"]

												self.assertEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument("--foo"       ,		type=UpperCAmelCase__       ,		required=UpperCAmelCase__     )
								expected.add_argument("--bar"       ,		type=UpperCAmelCase__       ,		required=UpperCAmelCase__     )
								expected.add_argument("--baz"       ,		type=UpperCAmelCase__       ,		required=UpperCAmelCase__     )
								expected.add_argument("--flag"       ,		type=UpperCAmelCase__       ,		default=UpperCAmelCase__       ,		const=UpperCAmelCase__       ,		nargs="?"     )
								self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				["--foo", "1", "--baz", "quux", "--bar", "0.5"]
								((SCREAMING_SNAKE_CASE__      )							,            )     							=				parser.parse_args_into_dataclasses(UpperCAmelCase__       ,		look_for_args_file=UpperCAmelCase__     )
								self.assertFalse(example.flag     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument("--foo"       ,		default=42       ,		type=UpperCAmelCase__     )
								expected.add_argument("--baz"       ,		default="toto"       ,		type=UpperCAmelCase__       ,		help="help message"     )
								self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument("--foo"       ,		type=UpperCAmelCase__       ,		default=UpperCAmelCase__       ,		const=UpperCAmelCase__       ,		nargs="?"     )
								expected.add_argument("--baz"       ,		type=UpperCAmelCase__       ,		default=UpperCAmelCase__       ,		const=UpperCAmelCase__       ,		nargs="?"     )
								# A boolean no_* argument always has to come after its "default: True" regular counter-part
								# and its default must be set to False
								expected.add_argument("--no_baz"       ,		action="store_false"       ,		default=UpperCAmelCase__       ,		dest="baz"     )
								expected.add_argument("--opt"       ,		type=UpperCAmelCase__       ,		default=UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				[WithDefaultBoolExample]
								if is_python_no_less_than_3_10:
												dataclass_types.append(UpperCAmelCase__     )

								for dataclass_type in dataclass_types:
												SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )
												self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

												SCREAMING_SNAKE_CASE__     							=				parser.parse_args([]     )
												self.assertEqual(UpperCAmelCase__       ,		Namespace(foo=UpperCAmelCase__       ,		baz=UpperCAmelCase__       ,		opt=UpperCAmelCase__     )     )

												SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "--no_baz"]     )
												self.assertEqual(UpperCAmelCase__       ,		Namespace(foo=UpperCAmelCase__       ,		baz=UpperCAmelCase__       ,		opt=UpperCAmelCase__     )     )

												SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "--baz"]     )
												self.assertEqual(UpperCAmelCase__       ,		Namespace(foo=UpperCAmelCase__       ,		baz=UpperCAmelCase__       ,		opt=UpperCAmelCase__     )     )

												SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "True", "--baz", "True", "--opt", "True"]     )
												self.assertEqual(UpperCAmelCase__       ,		Namespace(foo=UpperCAmelCase__       ,		baz=UpperCAmelCase__       ,		opt=UpperCAmelCase__     )     )

												SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "False", "--baz", "False", "--opt", "False"]     )
												self.assertEqual(UpperCAmelCase__       ,		Namespace(foo=UpperCAmelCase__       ,		baz=UpperCAmelCase__       ,		opt=UpperCAmelCase__     )     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument(
								    "--foo"       ,		default="toto"       ,		choices=["titi", "toto", 42]       ,		type=make_choice_type_function(["titi", "toto", 42]     )       ,		)
								self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args([]     )
								self.assertEqual(args.foo       ,		"toto"     )
								SCREAMING_SNAKE_CASE__     							=				parser.parse_args_into_dataclasses([]     )[0]
								self.assertEqual(enum_ex.foo       ,		MixedTypeEnum.toto     )

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "titi"]     )
								self.assertEqual(args.foo       ,		"titi"     )
								SCREAMING_SNAKE_CASE__     							=				parser.parse_args_into_dataclasses(["--foo", "titi"]     )[0]
								self.assertEqual(enum_ex.foo       ,		MixedTypeEnum.titi     )

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "42"]     )
								self.assertEqual(args.foo       ,		42     )
								SCREAMING_SNAKE_CASE__     							=				parser.parse_args_into_dataclasses(["--foo", "42"]     )[0]
								self.assertEqual(enum_ex.foo       ,		MixedTypeEnum.fourtytwo     )



				def   lowerCAmelCase__		(						self     ):


								@dataclass
								class 			lowerCamelCase_		:





												"""simple docstring"""







												_lowerCAmelCase      :      Literal["titi", "toto", 42]      							=					"toto"


								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument(
								    "--foo"       ,		default="toto"       ,		choices=("titi", "toto", 42)       ,		type=make_choice_type_function(["titi", "toto", 42]     )       ,		)
								self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args([]     )
								self.assertEqual(args.foo       ,		"toto"     )

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "titi"]     )
								self.assertEqual(args.foo       ,		"titi"     )

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args(["--foo", "42"]     )
								self.assertEqual(args.foo       ,		42     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument("--foo_int"       ,		nargs="+"       ,		default=[]       ,		type=UpperCAmelCase__     )
								expected.add_argument("--bar_int"       ,		nargs="+"       ,		default=[1, 2, 3]       ,		type=UpperCAmelCase__     )
								expected.add_argument("--foo_str"       ,		nargs="+"       ,		default=["Hallo", "Bonjour", "Hello"]       ,		type=UpperCAmelCase__     )
								expected.add_argument("--foo_float"       ,		nargs="+"       ,		default=[0.1, 0.2, 0.3]       ,		type=UpperCAmelCase__     )

								self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args([]     )
								self.assertEqual(
								    UpperCAmelCase__       ,		Namespace(foo_int=[]       ,		bar_int=[1, 2, 3]       ,		foo_str=["Hallo", "Bonjour", "Hello"]       ,		foo_float=[0.1, 0.2, 0.3]     )       ,		)

								SCREAMING_SNAKE_CASE__     							=				parser.parse_args("--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7".split()     )
								self.assertEqual(UpperCAmelCase__       ,		Namespace(foo_int=[1]       ,		bar_int=[2, 3]       ,		foo_str=["a", "b", "c"]       ,		foo_float=[0.1, 0.7]     )     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument("--foo"       ,		default=UpperCAmelCase__       ,		type=UpperCAmelCase__     )
								expected.add_argument("--bar"       ,		default=UpperCAmelCase__       ,		type=UpperCAmelCase__       ,		help="help message"     )
								expected.add_argument("--baz"       ,		default=UpperCAmelCase__       ,		type=UpperCAmelCase__     )
								expected.add_argument("--ces"       ,		nargs="+"       ,		default=[]       ,		type=UpperCAmelCase__     )
								expected.add_argument("--des"       ,		nargs="+"       ,		default=[]       ,		type=UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				[OptionalExample]
								if is_python_no_less_than_3_10:
												dataclass_types.append(UpperCAmelCase__     )

								for dataclass_type in dataclass_types:
												SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

												self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )

												SCREAMING_SNAKE_CASE__     							=				parser.parse_args([]     )
												self.assertEqual(UpperCAmelCase__       ,		Namespace(foo=UpperCAmelCase__       ,		bar=UpperCAmelCase__       ,		baz=UpperCAmelCase__       ,		ces=[]       ,		des=[]     )     )

												SCREAMING_SNAKE_CASE__     							=				parser.parse_args("--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3".split()     )
												self.assertEqual(UpperCAmelCase__       ,		Namespace(foo=12       ,		bar=3.14       ,		baz="42"       ,		ces=["a", "b", "c"]       ,		des=[1, 2, 3]     )     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument("--required_list"       ,		nargs="+"       ,		type=UpperCAmelCase__       ,		required=UpperCAmelCase__     )
								expected.add_argument("--required_str"       ,		type=UpperCAmelCase__       ,		required=UpperCAmelCase__     )
								expected.add_argument(
								    "--required_enum"       ,		type=make_choice_type_function(["titi", "toto"]     )       ,		choices=["titi", "toto"]       ,		required=UpperCAmelCase__       ,		)
								self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				argparse.ArgumentParser()
								expected.add_argument("--foo"       ,		type=UpperCAmelCase__       ,		required=UpperCAmelCase__     )
								expected.add_argument(
								    "--required_enum"       ,		type=make_choice_type_function(["titi", "toto"]     )       ,		choices=["titi", "toto"]       ,		required=UpperCAmelCase__       ,		)
								expected.add_argument("--opt"       ,		type=UpperCAmelCase__       ,		default=UpperCAmelCase__     )
								expected.add_argument("--baz"       ,		default="toto"       ,		type=UpperCAmelCase__       ,		help="help message"     )
								expected.add_argument("--foo_str"       ,		nargs="+"       ,		default=["Hallo", "Bonjour", "Hello"]       ,		type=UpperCAmelCase__     )
								self.argparsersEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				{
								    "foo": 12,
								    "bar": 3.14,
								    "baz": "42",
								    "flag": True,
								}

								SCREAMING_SNAKE_CASE__     							=				parser.parse_dict(UpperCAmelCase__     )[0]
								SCREAMING_SNAKE_CASE__     							=				BasicExample(**UpperCAmelCase__     )
								self.assertEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				{
								    "foo": 12,
								    "bar": 3.14,
								    "baz": "42",
								    "flag": True,
								    "extra": 42,
								}

								self.assertRaises(UpperCAmelCase__       ,		parser.parse_dict       ,		UpperCAmelCase__       ,		allow_extra_keys=UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				{
								    "foo": 12,
								    "bar": 3.14,
								    "baz": "42",
								    "flag": True,
								}
								with tempfile.TemporaryDirectory() as tmp_dir:
												SCREAMING_SNAKE_CASE__     							=				os.path.join(UpperCAmelCase__       ,		"temp_json"     )
												os.mkdir(UpperCAmelCase__     )
												with open(temp_local_path + ".json"       ,		"w+"     ) as f:
																json.dump(UpperCAmelCase__       ,		UpperCAmelCase__     )
												SCREAMING_SNAKE_CASE__     							=				parser.parse_yaml_file(Path(temp_local_path + ".json"     )     )[0]

								SCREAMING_SNAKE_CASE__     							=				BasicExample(**UpperCAmelCase__     )
								self.assertEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				{
								    "foo": 12,
								    "bar": 3.14,
								    "baz": "42",
								    "flag": True,
								}
								with tempfile.TemporaryDirectory() as tmp_dir:
												SCREAMING_SNAKE_CASE__     							=				os.path.join(UpperCAmelCase__       ,		"temp_yaml"     )
												os.mkdir(UpperCAmelCase__     )
												with open(temp_local_path + ".yaml"       ,		"w+"     ) as f:
																yaml.dump(UpperCAmelCase__       ,		UpperCAmelCase__     )
												SCREAMING_SNAKE_CASE__     							=				parser.parse_yaml_file(Path(temp_local_path + ".yaml"     )     )[0]
								SCREAMING_SNAKE_CASE__     							=				BasicExample(**UpperCAmelCase__     )
								self.assertEqual(UpperCAmelCase__       ,		UpperCAmelCase__     )






				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				HfArgumentParser(UpperCAmelCase__     )
								self.assertIsNotNone(UpperCAmelCase__     )




"""simple docstring"""






import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class 			lowerCamelCase_		(			lowercase   ):





				"""simple docstring"""







				def __init__(						self       ,		UpperCAmelCase__ = "▁"       ,		UpperCAmelCase__ = True       ,		UpperCAmelCase__ = "<unk>"       ,		UpperCAmelCase__ = "</s>"       ,		UpperCAmelCase__ = "<pad>"       ,		):
								SCREAMING_SNAKE_CASE__     							=				{
								    "pad": {"id": 0, "token": pad_token},
								    "eos": {"id": 1, "token": eos_token},
								    "unk": {"id": 2, "token": unk_token},
								}

								SCREAMING_SNAKE_CASE__     							=				[None] * len(self.special_tokens     )
								for token_dict in self.special_tokens.values():
												SCREAMING_SNAKE_CASE__     							=				token_dict["token"]

								SCREAMING_SNAKE_CASE__     							=				Tokenizer(Unigram()     )

								SCREAMING_SNAKE_CASE__     							=				normalizers.Sequence(
								    [
								        normalizers.Nmt(),
								        normalizers.NFKC(),
								        normalizers.Replace(Regex(" {2,}"     )       ,		" "     ),
								        normalizers.Lowercase(),
								    ]     )
								SCREAMING_SNAKE_CASE__     							=				pre_tokenizers.Sequence(
								    [
								        pre_tokenizers.Metaspace(replacement=UpperCAmelCase__       ,		add_prefix_space=UpperCAmelCase__     ),
								        pre_tokenizers.Digits(individual_digits=UpperCAmelCase__     ),
								        pre_tokenizers.Punctuation(),
								    ]     )
								SCREAMING_SNAKE_CASE__     							=				decoders.Metaspace(replacement=UpperCAmelCase__       ,		add_prefix_space=UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				TemplateProcessing(
								    single=f'''$A {self.special_tokens["eos"]["token"]}'''       ,		special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])]       ,		)

								SCREAMING_SNAKE_CASE__     							=				{
								    "model": "SentencePieceUnigram",
								    "replacement": replacement,
								    "add_prefix_space": add_prefix_space,
								}

								super().__init__(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__ = 8000       ,		UpperCAmelCase__ = True       ,		):
								SCREAMING_SNAKE_CASE__     							=				trainers.UnigramTrainer(
								    vocab_size=UpperCAmelCase__       ,		special_tokens=self.special_tokens_list       ,		show_progress=UpperCAmelCase__       ,		)

								if isinstance(UpperCAmelCase__       ,		UpperCAmelCase__     ):
												SCREAMING_SNAKE_CASE__     							=				[files]
								self._tokenizer.train(UpperCAmelCase__       ,		trainer=UpperCAmelCase__     )

								self.add_unk_id()



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__ = 8000       ,		UpperCAmelCase__ = True       ,		):
								SCREAMING_SNAKE_CASE__     							=				trainers.UnigramTrainer(
								    vocab_size=UpperCAmelCase__       ,		special_tokens=self.special_tokens_list       ,		show_progress=UpperCAmelCase__       ,		)

								self._tokenizer.train_from_iterator(UpperCAmelCase__       ,		trainer=UpperCAmelCase__     )

								self.add_unk_id()






				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				json.loads(self._tokenizer.to_str()     )

								SCREAMING_SNAKE_CASE__     							=				self.special_tokens["unk"]["id"]

								SCREAMING_SNAKE_CASE__     							=				Tokenizer.from_str(json.dumps(UpperCAmelCase__     )     )
