





import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
_UpperCAmelCase        =				sys.version_info >= (3, 1_0)




def  lowerCAmelCase_ (							UpperCamelCase_=None					,		UpperCamelCase_=None  )     ->		List[Any]:
       return field(default_factory=lambda: default					,		metadata=_lowerCamelCase  )



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Optional[Any]         =   4_2
       _UpperCamelCase				:					Tuple         =   4_2
       _UpperCamelCase				:					Optional[Any]         =   4_2
       _UpperCamelCase				:					Optional[int]         =   4_2



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Optional[int]         =   4_2
       _UpperCamelCase				:					Optional[int]         =   field(default='''toto'''  ,						metadata={'''help''': '''help message'''}     )



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Optional[int]         =   False
       _UpperCamelCase				:					int         =   True
       _UpperCamelCase				:					Union[str, Any]         =   None



class 							_UpperCamelCase					(	_A     ):
       _UpperCamelCase				:					str         =   '''titi'''
       _UpperCamelCase				:					List[Any]         =   '''toto'''



class 							_UpperCamelCase					(	_A     ):
       _UpperCamelCase				:					str         =   '''titi'''
       _UpperCamelCase				:					List[Any]         =   '''toto'''
       _UpperCamelCase				:					Optional[int]         =   4_2



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					List[str]         =   '''toto'''

       def lowercase (		self: Any		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =       BasicEnum(self.foo		)



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Optional[Any]         =   '''toto'''

       def lowercase (		self: str		)       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =       MixedTypeEnum(self.foo		)



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Any         =   None
       _UpperCamelCase				:					Any         =   field(default=_A  ,						metadata={'''help''': '''help message'''}     )
       _UpperCamelCase				:					Optional[int]         =   None
       _UpperCamelCase				:					Tuple         =   list_field(default=[]     )
       _UpperCamelCase				:					str         =   list_field(default=[]     )



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Dict         =   list_field(default=[]     )
       _UpperCamelCase				:					List[Any]         =   list_field(default=[1, 2, 3]     )
       _UpperCamelCase				:					Optional[int]         =   list_field(default=['''Hallo''', '''Bonjour''', '''Hello''']     )
       _UpperCamelCase				:					Optional[int]         =   list_field(default=[0.1, 0.2, 0.3]     )



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Tuple         =   field()
       _UpperCamelCase				:					int         =   field()
       _UpperCamelCase				:					Optional[Any]         =   field()

       def lowercase (		self: List[Any]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =       BasicEnum(self.required_enum		)



@dataclass
class 							_UpperCamelCase					:
       _UpperCamelCase				:					Dict         =   4_2
       _UpperCamelCase				:					List[str]         =   field()
       _UpperCamelCase				:					Any         =   None
       _UpperCamelCase				:					Tuple         =   field(default='''toto'''  ,						metadata={'''help''': '''help message'''}     )
       _UpperCamelCase				:					Tuple         =   list_field(default=['''Hallo''', '''Bonjour''', '''Hello''']     )


if is_python_no_less_than_3_10:



     @dataclass
     class 							_UpperCamelCase					:
            _UpperCamelCase				:					List[str]         =   False
            _UpperCamelCase				:					Tuple         =   True
            _UpperCamelCase				:					List[Any]         =   None



     @dataclass
     class 							_UpperCamelCase					:
            _UpperCamelCase				:					Any         =   None
            _UpperCamelCase				:					int         =   field(default=_A  ,						metadata={'''help''': '''help message'''}     )
            _UpperCamelCase				:					Optional[Any]         =   None
            _UpperCamelCase				:					int         =   list_field(default=[]     )
            _UpperCamelCase				:					List[Any]         =   list_field(default=[]     )







class 							_UpperCamelCase					(	unittest.TestCase     ):

       def lowercase (		self: str					,   _SCREAMING_SNAKE_CASE: argparse.ArgumentParser					,   _SCREAMING_SNAKE_CASE: argparse.ArgumentParser		)       -> Optional[Any]:




              """simple docstring"""


              self.assertEqual(len(a._actions		)					,   len(b._actions		)		)
              for x, y in zip(a._actions					,   b._actions		):
                     UpperCamelCase_          =       {k: v for k, v in vars(__lowerCamelCase		).items() if k != "container"}
                     UpperCamelCase_          =       {k: v for k, v in vars(__lowerCamelCase		).items() if k != "container"}

                     # Choices with mixed type have custom function as "type"
                     # So we need to compare results directly for equality
                     if xx.get("choices"					,   __lowerCamelCase		) and yy.get("choices"					,   __lowerCamelCase		):
                            for expected_choice in yy["choices"] + xx["choices"]:
                                   self.assertEqual(xx["type"](__lowerCamelCase		)					,   yy["type"](__lowerCamelCase		)		)
                            del xx["type"], yy["type"]

                     self.assertEqual(__lowerCamelCase					,   __lowerCamelCase		)

       def lowercase (		self: Union[str, Any]		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument("--foo"					,   type=__lowerCamelCase					,   required=__lowerCamelCase		)
              expected.add_argument("--bar"					,   type=__lowerCamelCase					,   required=__lowerCamelCase		)
              expected.add_argument("--baz"					,   type=__lowerCamelCase					,   required=__lowerCamelCase		)
              expected.add_argument("--flag"					,   type=__lowerCamelCase					,   default=__lowerCamelCase					,   const=__lowerCamelCase					,   nargs="?"		)
              self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

              UpperCamelCase_          =       ["--foo", "1", "--baz", "quux", "--bar", "0.5"]
              (UpperCamelCase_ )          =       parser.parse_args_into_dataclasses(__lowerCamelCase					,   look_for_args_file=__lowerCamelCase		)
              self.assertFalse(example.flag		)

       def lowercase (		self: Optional[Any]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument("--foo"					,   default=42					,   type=__lowerCamelCase		)
              expected.add_argument("--baz"					,   default="toto"					,   type=__lowerCamelCase					,   help="help message"		)
              self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

       def lowercase (		self: Union[str, Any]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument("--foo"					,   type=__lowerCamelCase					,   default=__lowerCamelCase					,   const=__lowerCamelCase					,   nargs="?"		)
              expected.add_argument("--baz"					,   type=__lowerCamelCase					,   default=__lowerCamelCase					,   const=__lowerCamelCase					,   nargs="?"		)
              # A boolean no_* argument always has to come after its "default: True" regular counter-part
              # and its default must be set to False
              expected.add_argument("--no_baz"					,   action="store_false"					,   default=__lowerCamelCase					,   dest="baz"		)
              expected.add_argument("--opt"					,   type=__lowerCamelCase					,   default=__lowerCamelCase		)

              UpperCamelCase_          =       [WithDefaultBoolExample]
              if is_python_no_less_than_3_10:
                     dataclass_types.append(__lowerCamelCase		)

              for dataclass_type in dataclass_types:
                     UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)
                     self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

                     UpperCamelCase_          =       parser.parse_args([]		)
                     self.assertEqual(__lowerCamelCase					,   Namespace(foo=__lowerCamelCase					,   baz=__lowerCamelCase					,   opt=__lowerCamelCase		)		)

                     UpperCamelCase_          =       parser.parse_args(["--foo", "--no_baz"]		)
                     self.assertEqual(__lowerCamelCase					,   Namespace(foo=__lowerCamelCase					,   baz=__lowerCamelCase					,   opt=__lowerCamelCase		)		)

                     UpperCamelCase_          =       parser.parse_args(["--foo", "--baz"]		)
                     self.assertEqual(__lowerCamelCase					,   Namespace(foo=__lowerCamelCase					,   baz=__lowerCamelCase					,   opt=__lowerCamelCase		)		)

                     UpperCamelCase_          =       parser.parse_args(["--foo", "True", "--baz", "True", "--opt", "True"]		)
                     self.assertEqual(__lowerCamelCase					,   Namespace(foo=__lowerCamelCase					,   baz=__lowerCamelCase					,   opt=__lowerCamelCase		)		)

                     UpperCamelCase_          =       parser.parse_args(["--foo", "False", "--baz", "False", "--opt", "False"]		)
                     self.assertEqual(__lowerCamelCase					,   Namespace(foo=__lowerCamelCase					,   baz=__lowerCamelCase					,   opt=__lowerCamelCase		)		)

       def lowercase (		self: int		)       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument(
                  "--foo"					,   default="toto"					,   choices=["titi", "toto", 42]					,   type=make_choice_type_function(["titi", "toto", 42]		)					,   )
              self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

              UpperCamelCase_          =       parser.parse_args([]		)
              self.assertEqual(args.foo					,   "toto"		)
              UpperCamelCase_          =       parser.parse_args_into_dataclasses([]		)[0]
              self.assertEqual(enum_ex.foo					,   MixedTypeEnum.toto		)

              UpperCamelCase_          =       parser.parse_args(["--foo", "titi"]		)
              self.assertEqual(args.foo					,   "titi"		)
              UpperCamelCase_          =       parser.parse_args_into_dataclasses(["--foo", "titi"]		)[0]
              self.assertEqual(enum_ex.foo					,   MixedTypeEnum.titi		)

              UpperCamelCase_          =       parser.parse_args(["--foo", "42"]		)
              self.assertEqual(args.foo					,   42		)
              UpperCamelCase_          =       parser.parse_args_into_dataclasses(["--foo", "42"]		)[0]
              self.assertEqual(enum_ex.foo					,   MixedTypeEnum.fourtytwo		)

       def lowercase (		self: Optional[int]		)       -> Tuple:




              """simple docstring"""


              @dataclass
              class 							_UpperCamelCase					:
                     _UpperCamelCase				:					Optional[Any]         =   '''toto'''

              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument(
                  "--foo"					,   default="toto"					,   choices=("titi", "toto", 42)					,   type=make_choice_type_function(["titi", "toto", 42]		)					,   )
              self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

              UpperCamelCase_          =       parser.parse_args([]		)
              self.assertEqual(args.foo					,   "toto"		)

              UpperCamelCase_          =       parser.parse_args(["--foo", "titi"]		)
              self.assertEqual(args.foo					,   "titi"		)

              UpperCamelCase_          =       parser.parse_args(["--foo", "42"]		)
              self.assertEqual(args.foo					,   42		)

       def lowercase (		self: Tuple		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument("--foo_int"					,   nargs="+"					,   default=[]					,   type=__lowerCamelCase		)
              expected.add_argument("--bar_int"					,   nargs="+"					,   default=[1, 2, 3]					,   type=__lowerCamelCase		)
              expected.add_argument("--foo_str"					,   nargs="+"					,   default=["Hallo", "Bonjour", "Hello"]					,   type=__lowerCamelCase		)
              expected.add_argument("--foo_float"					,   nargs="+"					,   default=[0.1, 0.2, 0.3]					,   type=__lowerCamelCase		)

              self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

              UpperCamelCase_          =       parser.parse_args([]		)
              self.assertEqual(
                  __lowerCamelCase					,   Namespace(foo_int=[]					,   bar_int=[1, 2, 3]					,   foo_str=["Hallo", "Bonjour", "Hello"]					,   foo_float=[0.1, 0.2, 0.3]		)					,   )

              UpperCamelCase_          =       parser.parse_args("--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7".split()		)
              self.assertEqual(__lowerCamelCase					,   Namespace(foo_int=[1]					,   bar_int=[2, 3]					,   foo_str=["a", "b", "c"]					,   foo_float=[0.1, 0.7]		)		)

       def lowercase (		self: Optional[int]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument("--foo"					,   default=__lowerCamelCase					,   type=__lowerCamelCase		)
              expected.add_argument("--bar"					,   default=__lowerCamelCase					,   type=__lowerCamelCase					,   help="help message"		)
              expected.add_argument("--baz"					,   default=__lowerCamelCase					,   type=__lowerCamelCase		)
              expected.add_argument("--ces"					,   nargs="+"					,   default=[]					,   type=__lowerCamelCase		)
              expected.add_argument("--des"					,   nargs="+"					,   default=[]					,   type=__lowerCamelCase		)

              UpperCamelCase_          =       [OptionalExample]
              if is_python_no_less_than_3_10:
                     dataclass_types.append(__lowerCamelCase		)

              for dataclass_type in dataclass_types:
                     UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

                     self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

                     UpperCamelCase_          =       parser.parse_args([]		)
                     self.assertEqual(__lowerCamelCase					,   Namespace(foo=__lowerCamelCase					,   bar=__lowerCamelCase					,   baz=__lowerCamelCase					,   ces=[]					,   des=[]		)		)

                     UpperCamelCase_          =       parser.parse_args("--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3".split()		)
                     self.assertEqual(__lowerCamelCase					,   Namespace(foo=12					,   bar=3.14					,   baz="42"					,   ces=["a", "b", "c"]					,   des=[1, 2, 3]		)		)

       def lowercase (		self: Optional[Any]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument("--required_list"					,   nargs="+"					,   type=__lowerCamelCase					,   required=__lowerCamelCase		)
              expected.add_argument("--required_str"					,   type=__lowerCamelCase					,   required=__lowerCamelCase		)
              expected.add_argument(
                  "--required_enum"					,   type=make_choice_type_function(["titi", "toto"]		)					,   choices=["titi", "toto"]					,   required=__lowerCamelCase					,   )
              self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

       def lowercase (		self: Optional[Any]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       argparse.ArgumentParser()
              expected.add_argument("--foo"					,   type=__lowerCamelCase					,   required=__lowerCamelCase		)
              expected.add_argument(
                  "--required_enum"					,   type=make_choice_type_function(["titi", "toto"]		)					,   choices=["titi", "toto"]					,   required=__lowerCamelCase					,   )
              expected.add_argument("--opt"					,   type=__lowerCamelCase					,   default=__lowerCamelCase		)
              expected.add_argument("--baz"					,   default="toto"					,   type=__lowerCamelCase					,   help="help message"		)
              expected.add_argument("--foo_str"					,   nargs="+"					,   default=["Hallo", "Bonjour", "Hello"]					,   type=__lowerCamelCase		)
              self.argparsersEqual(__lowerCamelCase					,   __lowerCamelCase		)

       def lowercase (		self: int		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       {
                  "foo": 12,
                  "bar": 3.14,
                  "baz": "42",
                  "flag": True,
              }

              UpperCamelCase_          =       parser.parse_dict(__lowerCamelCase		)[0]
              UpperCamelCase_          =       BasicExample(**__lowerCamelCase		)
              self.assertEqual(__lowerCamelCase					,   __lowerCamelCase		)

       def lowercase (		self: Optional[Any]		)       -> Optional[Any]:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       {
                  "foo": 12,
                  "bar": 3.14,
                  "baz": "42",
                  "flag": True,
                  "extra": 42,
              }

              self.assertRaises(__lowerCamelCase					,   parser.parse_dict					,   __lowerCamelCase					,   allow_extra_keys=__lowerCamelCase		)

       def lowercase (		self: List[Any]		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       {
                  "foo": 12,
                  "bar": 3.14,
                  "baz": "42",
                  "flag": True,
              }
              with tempfile.TemporaryDirectory() as tmp_dir:
                     UpperCamelCase_          =       os.path.join(__lowerCamelCase					,   "temp_json"		)
                     os.mkdir(__lowerCamelCase		)
                     with open(temp_local_path + ".json"					,   "w+"		) as f:
                            json.dump(__lowerCamelCase					,   __lowerCamelCase		)
                     UpperCamelCase_          =       parser.parse_yaml_file(Path(temp_local_path + ".json"		)		)[0]

              UpperCamelCase_          =       BasicExample(**__lowerCamelCase		)
              self.assertEqual(__lowerCamelCase					,   __lowerCamelCase		)

       def lowercase (		self: List[Any]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)

              UpperCamelCase_          =       {
                  "foo": 12,
                  "bar": 3.14,
                  "baz": "42",
                  "flag": True,
              }
              with tempfile.TemporaryDirectory() as tmp_dir:
                     UpperCamelCase_          =       os.path.join(__lowerCamelCase					,   "temp_yaml"		)
                     os.mkdir(__lowerCamelCase		)
                     with open(temp_local_path + ".yaml"					,   "w+"		) as f:
                            yaml.dump(__lowerCamelCase					,   __lowerCamelCase		)
                     UpperCamelCase_          =       parser.parse_yaml_file(Path(temp_local_path + ".yaml"		)		)[0]
              UpperCamelCase_          =       BasicExample(**__lowerCamelCase		)
              self.assertEqual(__lowerCamelCase					,   __lowerCamelCase		)



       def lowercase (		self: str		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =       HfArgumentParser(__lowerCamelCase		)
              self.assertIsNotNone(__lowerCamelCase		)








import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class 							_UpperCamelCase					(	lowerCAmelCase_  ,						unittest.TestCase     ):
       _UpperCamelCase				:					Union[str, Any]         =   XLMTokenizer
       _UpperCamelCase				:					Optional[int]         =   False

       def lowercase (		self: Dict		)       -> Union[str, Any]:




              """simple docstring"""


              super().setUp()

              # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
              UpperCamelCase_          =  [
                  "l",
                  "o",
                  "w",
                  "e",
                  "r",
                  "s",
                  "t",
                  "i",
                  "d",
                  "n",
                  "w</w>",
                  "r</w>",
                  "t</w>",
                  "lo",
                  "low",
                  "er</w>",
                  "low</w>",
                  "lowest</w>",
                  "newer</w>",
                  "wider</w>",
                  "<unk>",
              ]
              UpperCamelCase_          =  dict(zip(_SCREAMING_SNAKE_CASE					,   range(len(_SCREAMING_SNAKE_CASE		)		)		)		)
              UpperCamelCase_          =  ["l o 123", "lo w 1456", "e r</w> 1789", ""]

              UpperCamelCase_          =  os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["vocab_file"]		)
              UpperCamelCase_          =  os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["merges_file"]		)
              with open(self.vocab_file					,   "w"		) as fp:
                     fp.write(json.dumps(_SCREAMING_SNAKE_CASE		)		)
              with open(self.merges_file					,   "w"		) as fp:
                     fp.write("\n".join(_SCREAMING_SNAKE_CASE		)		)

       def lowercase (		self: Optional[Any]					,   _SCREAMING_SNAKE_CASE: List[str]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =  "lower newer"
              UpperCamelCase_          =  "lower newer"
              return input_text, output_text

       def lowercase (		self: int		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  XLMTokenizer(self.vocab_file					,   self.merges_file		)

              UpperCamelCase_          =  "lower"
              UpperCamelCase_          =  ["low", "er</w>"]
              UpperCamelCase_          =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
              self.assertListEqual(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  tokens + ["<unk>"]
              UpperCamelCase_          =  [14, 15, 20]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)



       @slow
       def lowercase (		self: Optional[int]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  XLMTokenizer.from_pretrained("xlm-mlm-en-2048"		)

              UpperCamelCase_          =  tokenizer.encode("sequence builders"					,   add_special_tokens=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tokenizer.encode("multi-sequence build"					,   add_special_tokens=_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

              assert encoded_sentence == [0] + text + [1]
              assert encoded_pair == [0] + text + [1] + text_a + [1]

