






import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 							_UpperCamelCase			(a_							,		unittest.TestCase			):
      snake_case_  =							LongformerTokenizer
      snake_case_  =							True
      snake_case_  =							LongformerTokenizerFast
      snake_case_  =							True


      def  __UpperCAmelCase		(	self	)->	Tuple:
           super().setUp()

           # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
           __lowerCAmelCase								=					[
               "l",
               "o",
               "w",
               "e",
               "r",
               "s",
               "t",
               "i",
               "d",
               "n",
               "\u0120",
               "\u0120l",
               "\u0120n",
               "\u0120lo",
               "\u0120low",
               "er",
               "\u0120lowest",
               "\u0120newer",
               "\u0120wider",
               "<unk>",
           ]
           __lowerCAmelCase								=					dict(zip(__UpperCamelCase					,						range(len(__UpperCamelCase	)	)	)	)
           __lowerCAmelCase								=					["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
           __lowerCAmelCase								=					{"unk_token": "<unk>"}

           __lowerCAmelCase								=					os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["vocab_file"]	)
           __lowerCAmelCase								=					os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["merges_file"]	)
           with open(self.vocab_file					,						"w"					,						encoding="utf-8"	) as fp:
                fp.write(json.dumps(__UpperCamelCase	) + "\n"	)
           with open(self.merges_file					,						"w"					,						encoding="utf-8"	) as fp:
                fp.write("\n".join(__UpperCamelCase	)	)


      def  __UpperCAmelCase		(	self					,						**__UpperCamelCase	)->	Tuple:
           kwargs.update(self.special_tokens_map	)
           return self.tokenizer_class.from_pretrained(self.tmpdirname					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						**__UpperCamelCase	)->	List[str]:
           kwargs.update(self.special_tokens_map	)
           return self.rust_tokenizer_class.from_pretrained(self.tmpdirname					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase	)->	Union[str, Any]:
           __lowerCAmelCase								=					"lower newer"
           __lowerCAmelCase								=					"lower newer"
           return input_text, output_text


      def  __UpperCAmelCase		(	self	)->	str:
           __lowerCAmelCase								=					self.tokenizer_class(self.vocab_file					,						self.merges_file					,						**self.special_tokens_map	)
           __lowerCAmelCase								=					"lower newer"
           __lowerCAmelCase								=					["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
           __lowerCAmelCase								=					tokenizer.tokenize(__UpperCamelCase	)  # , add_prefix_space=True)
           self.assertListEqual(__UpperCamelCase					,						__UpperCamelCase	)

           __lowerCAmelCase								=					tokens + [tokenizer.unk_token]
           __lowerCAmelCase								=					[0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase	)					,						__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	Optional[int]:
           __lowerCAmelCase								=					self.get_tokenizer()

           self.assertListEqual(tokenizer.encode("Hello world!"					,						add_special_tokens=__UpperCamelCase	)					,						[0, 3_1_4_1_4, 2_3_2, 3_2_8, 2]	)
           self.assertListEqual(
               tokenizer.encode("Hello world! cécé herlolip 418"					,						add_special_tokens=__UpperCamelCase	)					,						[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]					,						)


      @slow
      def  __UpperCAmelCase		(	self	)->	List[str]:
           __lowerCAmelCase								=					self.tokenizer_class.from_pretrained("allenai/longformer-base-4096"	)

           __lowerCAmelCase								=					tokenizer.encode("sequence builders"					,						add_special_tokens=__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.encode("multi-sequence build"					,						add_special_tokens=__UpperCamelCase	)

           __lowerCAmelCase								=					tokenizer.encode(
               "sequence builders"					,						add_special_tokens=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.encode(
               "sequence builders"					,						"multi-sequence build"					,						add_special_tokens=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase	)

           __lowerCAmelCase								=					tokenizer.build_inputs_with_special_tokens(__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.build_inputs_with_special_tokens(__UpperCamelCase					,						__UpperCamelCase	)

           assert encoded_sentence == encoded_text_from_decode
           assert encoded_pair == encoded_pair_from_decode


      def  __UpperCAmelCase		(	self	)->	Tuple:
           __lowerCAmelCase								=					self.get_tokenizer()

           __lowerCAmelCase								=					"Encode this sequence."
           __lowerCAmelCase								=					tokenizer.byte_encoder[" ".encode("utf-8"	)[0]]

           # Testing encoder arguments
           __lowerCAmelCase								=					tokenizer.encode(__UpperCamelCase					,						add_special_tokens=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.convert_ids_to_tokens(encoded[0]	)[0]
           self.assertNotEqual(__UpperCamelCase					,						__UpperCamelCase	)

           __lowerCAmelCase								=					tokenizer.encode(__UpperCamelCase					,						add_special_tokens=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.convert_ids_to_tokens(encoded[0]	)[0]
           self.assertEqual(__UpperCamelCase					,						__UpperCamelCase	)

           tokenizer.add_special_tokens({"bos_token": "<s>"}	)
           __lowerCAmelCase								=					tokenizer.encode(__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.convert_ids_to_tokens(encoded[1]	)[0]
           self.assertNotEqual(__UpperCamelCase					,						__UpperCamelCase	)

           # Testing spaces after special tokens
           __lowerCAmelCase								=					"<mask>"
           tokenizer.add_special_tokens(
               {"mask_token": AddedToken(__UpperCamelCase					,						lstrip=__UpperCamelCase					,						rstrip=__UpperCamelCase	)}	)  # mask token has a left space
           __lowerCAmelCase								=					tokenizer.convert_tokens_to_ids(__UpperCamelCase	)

           __lowerCAmelCase								=					"Encode <mask> sequence"
           __lowerCAmelCase								=					"Encode <mask>sequence"

           __lowerCAmelCase								=					tokenizer.encode(__UpperCamelCase	)
           __lowerCAmelCase								=					encoded.index(__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]	)[0]
           self.assertEqual(__UpperCamelCase					,						__UpperCamelCase	)

           __lowerCAmelCase								=					tokenizer.encode(__UpperCamelCase	)
           __lowerCAmelCase								=					encoded.index(__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]	)[0]
           self.assertNotEqual(__UpperCamelCase					,						__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	Union[str, Any]:
           pass


      def  __UpperCAmelCase		(	self	)->	Tuple:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,						**__UpperCamelCase	)
                     __lowerCAmelCase								=					self.tokenizer_class.from_pretrained(__UpperCamelCase					,						**__UpperCamelCase	)
                     __lowerCAmelCase								=					"A, <mask> AllenNLP sentence."
                     __lowerCAmelCase								=					tokenizer_r.encode_plus(__UpperCamelCase					,						add_special_tokens=__UpperCamelCase					,						return_token_type_ids=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_p.encode_plus(__UpperCamelCase					,						add_special_tokens=__UpperCamelCase					,						return_token_type_ids=__UpperCamelCase	)

                     # token_type_ids should put 0 everywhere
                     self.assertEqual(sum(tokens_r["token_type_ids"]	)					,						sum(tokens_p["token_type_ids"]	)	)

                     # attention_mask should put 1 everywhere, so sum over length should be 1
                     self.assertEqual(
                         sum(tokens_r["attention_mask"]	) / len(tokens_r["attention_mask"]	)					,						sum(tokens_p["attention_mask"]	) / len(tokens_p["attention_mask"]	)					,						)

                     __lowerCAmelCase								=					tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]	)
                     __lowerCAmelCase								=					tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]	)

                     # Rust correctly handles the space before the mask while python doesnt
                     self.assertSequenceEqual(tokens_p["input_ids"]					,						[0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]	)
                     self.assertSequenceEqual(tokens_r["input_ids"]					,						[0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]	)

                     self.assertSequenceEqual(
                         __UpperCamelCase					,						["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]	)
                     self.assertSequenceEqual(
                         __UpperCamelCase					,						["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]	)


      def  __UpperCAmelCase		(	self	)->	Dict:
           for trim_offsets, add_prefix_space in itertools.product([True, False]					,						repeat=2	):
                __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                    self.tmpdirname					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)

                __lowerCAmelCase								=					json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()	)
                __lowerCAmelCase								=					json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()	)

                self.assertEqual(pre_tokenizer_state["add_prefix_space"]					,						__UpperCamelCase	)

                self.assertEqual(post_processor_state["add_prefix_space"]					,						__UpperCamelCase	)
                self.assertEqual(post_processor_state["trim_offsets"]					,						__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	Any:
           # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
           # `trim_offsets`
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                     __lowerCAmelCase								=					"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                     __lowerCAmelCase								=					F"""{text_of_1_token} {text_of_1_token}"""

                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                         __UpperCamelCase					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_r(__UpperCamelCase					,						return_offsets_mapping=__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
                     self.assertEqual(encoding.offset_mapping[0]					,						(0, len(__UpperCamelCase	))	)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,						(len(__UpperCamelCase	) + 1, len(__UpperCamelCase	) + 1 + len(__UpperCamelCase	))					,						)

                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                         __UpperCamelCase					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_r(__UpperCamelCase					,						return_offsets_mapping=__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
                     self.assertEqual(encoding.offset_mapping[0]					,						(0, len(__UpperCamelCase	))	)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,						(len(__UpperCamelCase	) + 1, len(__UpperCamelCase	) + 1 + len(__UpperCamelCase	))					,						)

                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                         __UpperCamelCase					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_r(__UpperCamelCase					,						return_offsets_mapping=__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
                     self.assertEqual(encoding.offset_mapping[0]					,						(0, len(__UpperCamelCase	))	)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,						(len(__UpperCamelCase	), len(__UpperCamelCase	) + 1 + len(__UpperCamelCase	))					,						)

                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                         __UpperCamelCase					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_r(__UpperCamelCase					,						return_offsets_mapping=__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
                     self.assertEqual(encoding.offset_mapping[0]					,						(0, len(__UpperCamelCase	))	)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,						(len(__UpperCamelCase	), len(__UpperCamelCase	) + 1 + len(__UpperCamelCase	))					,						)

                     __lowerCAmelCase								=					F""" {text}"""

                     # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                     #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                     # )
                     # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                     # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                     # self.assertEqual(
                     #     encoding.offset_mapping[1],
                     #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                     # )

                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                         __UpperCamelCase					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_r(__UpperCamelCase					,						return_offsets_mapping=__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
                     self.assertEqual(encoding.offset_mapping[0]					,						(1, 1 + len(__UpperCamelCase	))	)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,						(1 + len(__UpperCamelCase	) + 1, 1 + len(__UpperCamelCase	) + 1 + len(__UpperCamelCase	))					,						)

                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                         __UpperCamelCase					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_r(__UpperCamelCase					,						return_offsets_mapping=__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
                     self.assertEqual(encoding.offset_mapping[0]					,						(0, 1 + len(__UpperCamelCase	))	)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,						(1 + len(__UpperCamelCase	), 1 + len(__UpperCamelCase	) + 1 + len(__UpperCamelCase	))					,						)

                     __lowerCAmelCase								=					self.rust_tokenizer_class.from_pretrained(
                         __UpperCamelCase					,						use_fast=__UpperCamelCase					,						add_prefix_space=__UpperCamelCase					,						trim_offsets=__UpperCamelCase	)
                     __lowerCAmelCase								=					tokenizer_r(__UpperCamelCase					,						return_offsets_mapping=__UpperCamelCase					,						add_special_tokens=__UpperCamelCase	)
                     self.assertEqual(encoding.offset_mapping[0]					,						(0, 1 + len(__UpperCamelCase	))	)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,						(1 + len(__UpperCamelCase	), 1 + len(__UpperCamelCase	) + 1 + len(__UpperCamelCase	))					,						)














# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path

import torch

from ...utils import is_npu_available, is_xpu_available
from .config_args import ClusterConfig, default_json_config_file
from .config_utils import SubcommandHelpFormatter


lowerCamelCase			:					List[str]         =	'''Create a default config file for Accelerate with only a few flags set.'''





def       __lowerCAmelCase      (						__snake_case="no"					, __snake_case = default_json_config_file					, __snake_case = False       ):
     __lowerCAmelCase								=					Path(__snake_case       )
     path.parent.mkdir(parents=__snake_case					, exist_ok=__snake_case       )
     if path.exists():
          print(
              F"""Configuration already exists at {save_location}, will not override. Run `accelerate config` manually or pass a different `save_location`."""       )
          return False
     __lowerCAmelCase								=					mixed_precision.lower()
     if mixed_precision not in ["no", "fp16", "bf16", "fp8"]:
          raise ValueError(
              F"""`mixed_precision` should be one of 'no', 'fp16', 'bf16', or 'fp8'. Received {mixed_precision}"""       )
     __lowerCAmelCase								=					{
         "compute_environment": "LOCAL_MACHINE",
         "mixed_precision": mixed_precision,
     }
     if torch.cuda.is_available():
          __lowerCAmelCase								=					torch.cuda.device_count()
          __lowerCAmelCase								=					num_gpus
          __lowerCAmelCase								=					False
          if num_gpus > 1:
               __lowerCAmelCase								=					"MULTI_GPU"
          else:
               __lowerCAmelCase								=					"NO"
     elif is_xpu_available() and use_xpu:
          __lowerCAmelCase								=					torch.xpu.device_count()
          __lowerCAmelCase								=					num_xpus
          __lowerCAmelCase								=					False
          if num_xpus > 1:
               __lowerCAmelCase								=					"MULTI_XPU"
          else:
               __lowerCAmelCase								=					"NO"
     elif is_npu_available():
          __lowerCAmelCase								=					torch.npu.device_count()
          __lowerCAmelCase								=					num_npus
          __lowerCAmelCase								=					False
          if num_npus > 1:
               __lowerCAmelCase								=					"MULTI_NPU"
          else:
               __lowerCAmelCase								=					"NO"
     else:
          __lowerCAmelCase								=					0
          __lowerCAmelCase								=					True
          __lowerCAmelCase								=					1
          __lowerCAmelCase								=					"NO"
     __lowerCAmelCase								=					ClusterConfig(**__snake_case       )
     config.to_json_file(__snake_case       )
     return path





def       __lowerCAmelCase      (						__snake_case					, __snake_case       ):
     __lowerCAmelCase								=					parser.add_parser("default"					, parents=__snake_case					, help=__snake_case					, formatter_class=__snake_case       )
     parser.add_argument(
         "--config_file"					, default=__snake_case					, help=(
             "The path to use to store the config file. Will default to a file named default_config.yaml in the cache "
             "location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have "
             "such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed "
             "with 'huggingface'."
         )					, dest="save_location"					, )

     parser.add_argument(
         "--mixed_precision"					, choices=["no", "fp16", "bf16"]					, type=__snake_case					, help="Whether or not to use mixed precision training. "
         "Choose between FP16 and BF16 (bfloat16) training. "
         "BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later."					, default="no"					, )
     parser.set_defaults(func=__snake_case       )
     return parser





def       __lowerCAmelCase      (						__snake_case       ):
     __lowerCAmelCase								=					write_basic_config(args.mixed_precision					, args.save_location       )
     if config_file:
          print(F"""accelerate configuration saved at {config_file}"""       )






