



"""simple docstring"""
import json
import os
import shutil
import tempfile
import unittest
from multiprocessing import get_context
from pathlib import Path

import datasets
import numpy as np
from datasets import load_dataset
from parameterized import parameterized

from transformers import AutoProcessor
from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.testing_utils import require_pyctcdecode, require_torch, require_torchaudio, slow
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_pyctcdecode_available, is_torch_available

from ..wavaveca.test_feature_extraction_wavaveca import floats_list


if is_pyctcdecode_available():
					from huggingface_hub import snapshot_download
					from pyctcdecode import BeamSearchDecoderCTC

					from transformers.models.wavaveca_with_lm import WavaVecaProcessorWithLM
					from transformers.models.wavaveca_with_lm.processing_wavaveca_with_lm import WavaVecaDecoderWithLMOutput

if is_torch_available():
					from transformers import WavaVecaForCTC




@require_pyctcdecode
class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self      ):
									a_     :      Tuple 						=     "| <pad> <unk> <s> </s> a b c d e f g h i j k".split()
									a_     :      Optional[int] 						=     dict(zip(a_			,     range(len(a_      )      )      )      )

									a_     :      Dict 						=     {
									    "unk_token": "<unk>",
									    "bos_token": "<s>",
									    "eos_token": "</s>",
									}
									a_     :      Any 						=     {
									    "feature_size": 1,
									    "padding_value": 0.0,
									    "sampling_rate": 1_6_0_0_0,
									    "return_attention_mask": False,
									    "do_normalize": True,
									}

									a_     :      Dict 						=     tempfile.mkdtemp()
									a_     :      List[str] 						=     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]      )
									a_     :      List[Any] 						=     os.path.join(self.tmpdirname			,     a_      )
									with open(self.vocab_file			,     "w"			,     encoding="utf-8"      ) as fp:
											fp.write(json.dumps(a_      ) + "\n"      )

									with open(self.feature_extraction_file			,     "w"			,     encoding="utf-8"      ) as fp:
											fp.write(json.dumps(a_      ) + "\n"      )

									# load decoder from hub
									a_     :      Union[str, Any] 						=     "hf-internal-testing/ngram-beam-search-decoder"






							def snake_case_  (   self			,     **a_      ):
									a_     :      List[str] 						=     self.add_kwargs_tokens_map.copy()
									kwargs.update(a_      )
									return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname			,     **a_      )






							def snake_case_  (   self			,     **a_      ):
									return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname			,     **a_      )






							def snake_case_  (   self			,     **a_      ):
									return BeamSearchDecoderCTC.load_from_hf_hub(self.decoder_name			,     **a_      )






							def snake_case_  (   self      ):
									shutil.rmtree(self.tmpdirname      )






							def snake_case_  (   self      ):
									a_     :      int 						=     self.get_tokenizer()
									a_     :      List[str] 						=     self.get_feature_extractor()
									a_     :      List[Any] 						=     self.get_decoder()

									a_     :      Optional[Any] 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									processor.save_pretrained(self.tmpdirname      )
									a_     :      List[Any] 						=     WavaVecaProcessorWithLM.from_pretrained(self.tmpdirname      )

									# tokenizer
									self.assertEqual(processor.tokenizer.get_vocab()			,     tokenizer.get_vocab()      )
									self.assertIsInstance(processor.tokenizer			,     a_      )

									# feature extractor
									self.assertEqual(processor.feature_extractor.to_json_string()			,     feature_extractor.to_json_string()      )
									self.assertIsInstance(processor.feature_extractor			,     a_      )

									# decoder
									self.assertEqual(processor.decoder._alphabet.labels			,     decoder._alphabet.labels      )
									self.assertEqual(
									    processor.decoder.model_container[decoder._model_key]._unigram_set			,     decoder.model_container[decoder._model_key]._unigram_set			,     )
									self.assertIsInstance(processor.decoder			,     a_      )






							def snake_case_  (   self      ):
									a_     :      str 						=     WavaVecaProcessorWithLM(
									    tokenizer=self.get_tokenizer()			,     feature_extractor=self.get_feature_extractor()			,     decoder=self.get_decoder()      )
									processor.save_pretrained(self.tmpdirname      )

									# make sure that error is thrown when decoder alphabet doesn't match
									a_     :      int 						=     WavaVecaProcessorWithLM.from_pretrained(
									    self.tmpdirname			,     alpha=5.0			,     beta=3.0			,     score_boundary=-7.0			,     unk_score_offset=3      )

									# decoder
									self.assertEqual(processor.language_model.alpha			,     5.0      )
									self.assertEqual(processor.language_model.beta			,     3.0      )
									self.assertEqual(processor.language_model.score_boundary			,     -7.0      )
									self.assertEqual(processor.language_model.unk_score_offset			,     3      )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     self.get_tokenizer()
									# add token to trigger raise
									tokenizer.add_tokens(["xx"]      )
									with self.assertRaisesRegex(a_			,     "include"      ):
											WavaVecaProcessorWithLM(
											    tokenizer=a_			,     feature_extractor=self.get_feature_extractor()			,     decoder=self.get_decoder()      )






							def snake_case_  (   self      ):
									a_     :      Any 						=     self.get_feature_extractor()
									a_     :      str 						=     self.get_tokenizer()
									a_     :      Optional[int] 						=     self.get_decoder()

									a_     :      List[str] 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									a_     :      Optional[int] 						=     floats_list((3, 1_0_0_0)      )

									a_     :      List[Any] 						=     feature_extractor(a_			,     return_tensors="np"      )
									a_     :      int 						=     processor(a_			,     return_tensors="np"      )

									for key in input_feat_extract.keys():
											self.assertAlmostEqual(input_feat_extract[key].sum()			,     input_processor[key].sum()			,     delta=1e-2      )






							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     self.get_feature_extractor()
									a_     :      Optional[int] 						=     self.get_tokenizer()
									a_     :      Union[str, Any] 						=     self.get_decoder()

									a_     :      List[Any] 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									a_     :      Optional[Any] 						=     "This is a test string"

									a_     :      str 						=     processor(text=a_      )

									a_     :      Dict 						=     tokenizer(a_      )

									for key in encoded_tok.keys():
											self.assertListEqual(encoded_tok[key]			,     encoded_processor[key]      )






							def snake_case_  (   self			,     a_=(2, 1_0, 1_6)			,     a_=7_7      ):
									np.random.seed(a_      )
									return np.random.rand(*a_      )






							def snake_case_  (   self      ):
									a_     :      Dict 						=     self.get_feature_extractor()
									a_     :      Optional[Any] 						=     self.get_tokenizer()
									a_     :      Tuple 						=     self.get_decoder()

									a_     :      List[Any] 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									a_     :      str 						=     self._get_dummy_logits(shape=(1_0, 1_6)			,     seed=1_3      )

									a_     :      Any 						=     processor.decode(a_      )

									a_     :      Optional[int] 						=     decoder.decode_beams(a_      )[0]

									self.assertEqual(decoded_decoder[0]			,     decoded_processor.text      )
									self.assertEqual("</s> <s> </s>"			,     decoded_processor.text      )
									self.assertEqual(decoded_decoder[-2]			,     decoded_processor.logit_score      )
									self.assertEqual(decoded_decoder[-1]			,     decoded_processor.lm_score      )






							@parameterized.expand([[None], ["fork"], ["spawn"]]      )
							def snake_case_  (   self			,     a_      ):
									a_     :      List[str] 						=     self.get_feature_extractor()
									a_     :      Optional[Any] 						=     self.get_tokenizer()
									a_     :      Tuple 						=     self.get_decoder()

									a_     :      List[str] 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									a_     :      List[Any] 						=     self._get_dummy_logits()

									# note: pool should be instantiated *after* Wav2Vec2ProcessorWithLM.
									#       otherwise, the LM won't be available to the pool's sub-processes.
									# manual logic used to allow parameterized test for both pool=None and pool=Pool(...)
									if pool_context is None:
											a_     :      str 						=     processor.batch_decode(a_      )
									else:
											with get_context(a_      ).Pool() as pool:
													a_     :      str 						=     processor.batch_decode(a_			,     a_      )

									a_     :      Tuple 						=     list(a_      )

									with get_context("fork"      ).Pool() as p:
											a_     :      Tuple 						=     decoder.decode_beams_batch(a_			,     a_      )

									a_						,				a_						,				a_     :      Union[str, Any] 						=     [], [], []
									for beams in decoded_beams:
											texts_decoder.append(beams[0][0]      )
											logit_scores_decoder.append(beams[0][-2]      )
											lm_scores_decoder.append(beams[0][-1]      )

									self.assertListEqual(a_			,     decoded_processor.text      )
									self.assertListEqual(["<s> <s> </s>", "<s> <s> <s>"]			,     decoded_processor.text      )
									self.assertListEqual(a_			,     decoded_processor.logit_score      )
									self.assertListEqual(a_			,     decoded_processor.lm_score      )






							def snake_case_  (   self      ):
									a_     :      Any 						=     self.get_feature_extractor()
									a_     :      Tuple 						=     self.get_tokenizer()
									a_     :      int 						=     self.get_decoder()

									a_     :      Union[str, Any] 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									a_     :      List[Any] 						=     self._get_dummy_logits()

									a_     :      Union[str, Any] 						=     1_5
									a_     :      Optional[int] 						=     -20.0
									a_     :      Optional[Any] 						=     -4.0

									a_     :      Union[str, Any] 						=     processor.batch_decode(
									    a_			,     beam_width=a_			,     beam_prune_logp=a_			,     token_min_logp=a_			,     )
									a_     :      int 						=     decoded_processor_out.text

									a_     :      List[Any] 						=     list(a_      )

									with get_context("fork"      ).Pool() as pool:
											a_     :      Union[str, Any] 						=     decoder.decode_beams_batch(
											    a_			,     a_			,     beam_width=a_			,     beam_prune_logp=a_			,     token_min_logp=a_			,     )

									a_     :      List[str] 						=     [d[0][0] for d in decoded_decoder_out]
									a_     :      Optional[Any] 						=     [d[0][2] for d in decoded_decoder_out]
									a_     :      List[str] 						=     [d[0][3] for d in decoded_decoder_out]

									self.assertListEqual(a_			,     a_      )
									self.assertListEqual(["</s> <s> <s>", "<s> <s> <s>"]			,     a_      )

									self.assertTrue(np.array_equal(a_			,     decoded_processor_out.logit_score      )      )
									self.assertTrue(np.allclose([-20.054, -18.447]			,     a_			,     atol=1e-3      )      )

									self.assertTrue(np.array_equal(a_			,     decoded_processor_out.lm_score      )      )
									self.assertTrue(np.allclose([-15.554, -13.9_474]			,     a_			,     atol=1e-3      )      )






							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     self.get_feature_extractor()
									a_     :      Dict 						=     self.get_tokenizer()
									a_     :      List[Any] 						=     self.get_decoder()

									a_     :      Optional[int] 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									a_     :      Union[str, Any] 						=     self._get_dummy_logits()

									a_     :      Optional[Any] 						=     2.0
									a_     :      Tuple 						=     5.0
									a_     :      str 						=     -20.0
									a_     :      Optional[int] 						=     True

									a_     :      List[Any] 						=     processor.batch_decode(
									    a_			,     alpha=a_			,     beta=a_			,     unk_score_offset=a_			,     lm_score_boundary=a_			,     )
									a_     :      str 						=     decoded_processor_out.text

									a_     :      List[str] 						=     list(a_      )
									decoder.reset_params(
									    alpha=a_			,     beta=a_			,     unk_score_offset=a_			,     lm_score_boundary=a_			,     )

									with get_context("fork"      ).Pool() as pool:
											a_     :      List[str] 						=     decoder.decode_beams_batch(
											    a_			,     a_			,     )

									a_     :      Optional[int] 						=     [d[0][0] for d in decoded_decoder_out]

									self.assertListEqual(a_			,     a_      )
									self.assertListEqual(["<s> </s> <s> </s> </s>", "</s> </s> <s> </s> </s>"]			,     a_      )
									a_     :      Union[str, Any] 						=     processor.decoder.model_container[processor.decoder._model_key]
									self.assertEqual(lm_model.alpha			,     2.0      )
									self.assertEqual(lm_model.beta			,     5.0      )
									self.assertEqual(lm_model.unk_score_offset			,     -20.0      )
									self.assertEqual(lm_model.score_boundary			,     a_      )






							def snake_case_  (   self      ):
									a_     :      Tuple 						=     WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"      )

									a_     :      str 						=     processor.decoder.model_container[processor.decoder._model_key]
									a_     :      str 						=     Path(language_model._kenlm_model.path.decode("utf-8"      )      ).parent.parent.absolute()

									a_     :      Dict 						=     os.listdir(a_      )
									a_     :      List[str] 						=     ["alphabet.json", "language_model"]

									downloaded_decoder_files.sort()
									expected_decoder_files.sort()

									# test that only decoder relevant files from
									# https://huggingface.co/hf-internal-testing/processor_with_lm/tree/main
									# are downloaded and none of the rest (e.g. README.md, ...)
									self.assertListEqual(a_			,     a_      )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     snapshot_download("hf-internal-testing/processor_with_lm"      )

									a_     :      Optional[int] 						=     WavaVecaProcessorWithLM.from_pretrained(a_      )

									a_     :      Union[str, Any] 						=     processor.decoder.model_container[processor.decoder._model_key]
									a_     :      str 						=     Path(language_model._kenlm_model.path.decode("utf-8"      )      ).parent.parent.absolute()

									a_     :      Tuple 						=     os.listdir(a_      )
									a_     :      Union[str, Any] 						=     os.listdir(a_      )

									local_decoder_files.sort()
									expected_decoder_files.sort()

									# test that both decoder form hub and local files in cache are the same
									self.assertListEqual(a_			,     a_      )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"      )
									a_     :      Union[str, Any] 						=     AutoProcessor.from_pretrained("hf-internal-testing/processor_with_lm"      )

									a_     :      str 						=     floats_list((3, 1_0_0_0)      )

									a_     :      Optional[int] 						=     processor_wavaveca(a_			,     return_tensors="np"      )
									a_     :      int 						=     processor_auto(a_			,     return_tensors="np"      )

									for key in input_wavaveca.keys():
											self.assertAlmostEqual(input_wavaveca[key].sum()			,     input_auto[key].sum()			,     delta=1e-2      )

									a_     :      int 						=     self._get_dummy_logits()

									a_     :      Optional[int] 						=     processor_wavaveca.batch_decode(a_      )
									a_     :      str 						=     processor_auto.batch_decode(a_      )

									self.assertListEqual(decoded_wavaveca.text			,     decoded_auto.text      )






							def snake_case_  (   self      ):
									a_     :      Any 						=     self.get_feature_extractor()
									a_     :      Any 						=     self.get_tokenizer()
									a_     :      Union[str, Any] 						=     self.get_decoder()

									a_     :      Tuple 						=     WavaVecaProcessorWithLM(tokenizer=a_			,     feature_extractor=a_			,     decoder=a_      )

									self.assertListEqual(
									    processor.model_input_names			,     feature_extractor.model_input_names			,     msg="`processor` and `feature_extractor` model input names do not match"			,     )






							@staticmethod
							def snake_case_  (   a_			,     a_      ):
									a_     :      Union[str, Any] 						=     [d[key] for d in offsets]
									return retrieved_list






							def snake_case_  (   self      ):
									a_     :      int 						=     WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"      )
									a_     :      List[Any] 						=     self._get_dummy_logits()[0]

									a_     :      Optional[Any] 						=     processor.decode(a_			,     output_word_offsets=a_      )
									# check Wav2Vec2CTCTokenizerOutput keys for word
									self.assertEqual(len(outputs.keys()      )			,     4      )
									self.assertTrue("text" in outputs      )
									self.assertTrue("word_offsets" in outputs      )
									self.assertTrue(isinstance(a_			,     a_      )      )

									self.assertEqual(" ".join(self.get_from_offsets(outputs["word_offsets"]			,     "word"      )      )			,     outputs.text      )
									self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]			,     "word"      )			,     ["<s>", "<s>", "</s>"]      )
									self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]			,     "start_offset"      )			,     [0, 2, 4]      )
									self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]			,     "end_offset"      )			,     [1, 3, 5]      )






							def snake_case_  (   self      ):
									a_     :      Optional[int] 						=     WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"      )
									a_     :      int 						=     self._get_dummy_logits()

									a_     :      Tuple 						=     processor.batch_decode(a_			,     output_word_offsets=a_      )

									# check Wav2Vec2CTCTokenizerOutput keys for word
									self.assertEqual(len(outputs.keys()      )			,     4      )
									self.assertTrue("text" in outputs      )
									self.assertTrue("word_offsets" in outputs      )
									self.assertTrue(isinstance(a_			,     a_      )      )

									self.assertListEqual(
									    [" ".join(self.get_from_offsets(a_			,     "word"      )      ) for o in outputs["word_offsets"]]			,     outputs.text      )
									self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]			,     "word"      )			,     ["<s>", "<s>", "</s>"]      )
									self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]			,     "start_offset"      )			,     [0, 2, 4]      )
									self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]			,     "end_offset"      )			,     [1, 3, 5]      )






							@slow
							@require_torch
							@require_torchaudio
							def snake_case_  (   self      ):
									import torch

									a_     :      str 						=     load_dataset("common_voice"			,     "en"			,     split="train"			,     streaming=a_      )
									a_     :      Tuple 						=     ds.cast_column("audio"			,     datasets.Audio(sampling_rate=1_6_0_0_0      )      )
									a_     :      str 						=     iter(a_      )
									a_     :      Union[str, Any] 						=     next(a_      )

									a_     :      Union[str, Any] 						=     AutoProcessor.from_pretrained("patrickvonplaten/wav2vec2-base-100h-with-lm"      )
									a_     :      List[Any] 						=     WavaVecaForCTC.from_pretrained("patrickvonplaten/wav2vec2-base-100h-with-lm"      )

									# compare to filename `common_voice_en_100038.mp3` of dataset viewer on https://huggingface.co/datasets/common_voice/viewer/en/train
									a_     :      Optional[int] 						=     processor(sample["audio"]["array"]			,     return_tensors="pt"      ).input_values

									with torch.no_grad():
											a_     :      List[str] 						=     model(a_      ).logits.cpu().numpy()

									a_     :      Union[str, Any] 						=     processor.decode(logits[0]			,     output_word_offsets=a_      )

									a_     :      str 						=     model.config.inputs_to_logits_ratio / processor.feature_extractor.sampling_rate
									a_     :      Any 						=     [
									    {
									        "start_time": d["start_offset"] * time_offset,
									        "end_time": d["end_offset"] * time_offset,
									        "word": d["word"],
									    }
									    for d in output["word_offsets"]
									]

									a_     :      List[str] 						=     "WHY DOES MILISANDRA LOOK LIKE SHE WANTS TO CONSUME JOHN SNOW ON THE RIVER AT THE WALL"

									# output words
									self.assertEqual(" ".join(self.get_from_offsets(a_			,     "word"      )      )			,     a_      )
									self.assertEqual(" ".join(self.get_from_offsets(a_			,     "word"      )      )			,     output.text      )

									# output times
									a_     :      int 						=     torch.tensor(self.get_from_offsets(a_			,     "start_time"      )      )
									a_     :      Dict 						=     torch.tensor(self.get_from_offsets(a_			,     "end_time"      )      )

									# fmt: off
									a_     :      Union[str, Any] 						=     torch.tensor([1.4_199, 1.6_599, 2.2_599, 3.0, 3.24, 3.5_999, 3.7_999, 4.0_999, 4.26, 4.94, 5.28, 5.6_599, 5.78, 5.94, 6.32, 6.5_399, 6.6_599]      )
									a_     :      Any 						=     torch.tensor([1.5_399, 1.8_999, 2.9, 3.16, 3.5_399, 3.72, 4.0_199, 4.1_799, 4.76, 5.1_599, 5.5_599, 5.6_999, 5.86, 6.1_999, 6.38, 6.6_199, 6.94]      )
									# fmt: on

									self.assertTrue(torch.allclose(a_			,     a_			,     atol=0.01      )      )
									self.assertTrue(torch.allclose(a_			,     a_			,     atol=0.01      )      )




"""simple docstring"""
import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_				= get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")




@require_sentencepiece
@require_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					GPTSwaTokenizer
							__lowerCAmelCase    					=					False
							__lowerCAmelCase    					=					True
							__lowerCAmelCase    					=					False






							def snake_case_  (   self      ):
									super().setUp()

									# We have a SentencePiece fixture for testing
									a_     :      Optional[int] 						=     GPTSwaTokenizer(a_			,     eos_token="<unk>"			,     bos_token="<unk>"			,     pad_token="<unk>"      )

									tokenizer.save_pretrained(self.tmpdirname      )






							def snake_case_  (   self			,     a_      ):
									a_     :      Union[str, Any] 						=     "This is a test"
									a_     :      Tuple 						=     "This is a test"
									return input_text, output_text






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     "<s>"
									a_     :      Any 						=     1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(a_      )			,     a_      )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(a_      )			,     a_      )






							def snake_case_  (   self      ):
									a_     :      int 						=     list(self.get_tokenizer().get_vocab().keys()      )

									self.assertEqual(vocab_keys[0]			,     "<unk>"      )
									self.assertEqual(vocab_keys[1]			,     "<s>"      )
									self.assertEqual(vocab_keys[-1]			,     "j"      )
									self.assertEqual(len(a_      )			,     2_0_0_0      )






							def snake_case_  (   self      ):
									self.assertEqual(self.get_tokenizer().vocab_size			,     2_0_0_0      )






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     GPTSwaTokenizer(a_      )

									a_     :      str 						=     tokenizer.tokenize("This is a test"      )
									self.assertListEqual(a_			,     ["▁This", "▁is", "▁a", "▁t", "est"]      )

									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2]      )

									a_     :      str 						=     tokenizer.tokenize("I was born in 92000, and this is falsé."      )
									# fmt: off
									self.assertListEqual(
									    a_			,     ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]			,     )
									# fmt: on

									a_     :      List[str] 						=     tokenizer.convert_tokens_to_ids(a_      )
									self.assertListEqual(
									    a_			,     [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0]			,     )

									a_     :      Optional[int] 						=     tokenizer.convert_ids_to_tokens(a_      )
									# fmt: off
									self.assertListEqual(
									    a_			,     ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]      )
									# fmt: on






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     GPTSwaTokenizer(a_      )
									a_     :      List[Any] 						=     ["This is a test", "I was born in 92000, and this is falsé."]
									a_     :      Optional[Any] 						=     [
									    [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2],
									    [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0],
									]

									# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
									for text, expected_ids in zip(a_			,     a_      ):
											self.assertListEqual(tokenizer.encode_fast(a_      )			,     a_      )

									# Test that decode_fast returns the input text
									for text, token_ids in zip(a_			,     a_      ):
											self.assertEqual(tokenizer.decode_fast(a_      )			,     a_      )






							@slow
							def snake_case_  (   self      ):
									a_     :      Dict 						=     [
									    "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
									    "Hey there, how are you doing this fine day?",
									    "This is a text with a trailing spaces followed by a dot     .",
									    "Häj sväjs lillebrör! =)",
									    "Det är inget fel på Mr. Cool",
									]

									# fmt: off
									a_     :      Union[str, Any] 						=     {"input_ids": [[6_3_4_2_3, 5, 6_8_1_1, 1_4_9_5_4, 2_8_2, 8_1_6, 3_8_2_1, 6_3_4_6_6, 6_3_4_2_5, 6_3_4_6_2, 1_8, 6_3_9_7_8, 6_7_8, 3_0_1, 1_3_2_0, 6_3_4_2_3, 6_3_4_5_5, 6_3_4_5_8, 1_8, 6_3_9_8_2, 4_2_4_6, 3_9_4_0, 1_9_0_1, 4_7_7_8_9, 5_5_4_7, 1_8_9_9_4], [1_9_6_3_0, 1_1_0_0, 6_3_4_4_6, 1_3_4_2, 6_3_3, 5_4_4, 4_4_8_8, 5_9_3, 5_1_0_2, 2_4_1_6, 6_3_4_9_5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_6_5_2, 4_2_8, 2_6_8, 1_9_3_6, 5_1_5, 2_6_8, 5_8_5_9_3, 2_2_4_1_3, 9_1_0_6, 5_4_6, 2_6_8, 3_3_2_1_3, 6_3_9_7_9, 6_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5_1_3_0, 6_3_4_5_0, 9_2_4, 6_3_4_4_9, 2_2_4_9, 4_0_6_2, 1_5_5_8, 3_1_8, 6_3_5_0_4, 2_1_4_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_0_9, 3_7_7, 2_8_2_7, 2_5_5_9, 3_3_2, 6_5_7_5, 6_3_4_4_3, 2_6_8_0_1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
									# fmt: on
									self.tokenizer_integration_test_util(
									    expected_encoding=a_			,     model_name="AI-Sweden/gpt-sw3-126m"			,     sequences=a_			,     )