"""simple docstring"""


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a				=logging.get_logger(__name__)

a				={'vocab_file': 'spiece.model'}

a				={
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

a				={
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
a				=0
a				=1
a				=2
a				=3
a				=4






class   __UpperCAmelCase		(      __lowerCAmelCase      ):
				A__ :  str    				=			VOCAB_FILES_NAMES
				A__ :  Optional[Any]    				=			PRETRAINED_VOCAB_FILES_MAP
				A__ :  Optional[int]    				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A__ :  Dict    				=			'''left'''

				def __init__( self      ,					_lowerCamelCase      ,					_lowerCamelCase=False      ,					_lowerCamelCase=True      ,					_lowerCamelCase=False      ,					_lowerCamelCase="<s>"      ,					_lowerCamelCase="</s>"      ,					_lowerCamelCase="<unk>"      ,					_lowerCamelCase="<sep>"      ,					_lowerCamelCase="<pad>"      ,					_lowerCamelCase="<cls>"      ,					_lowerCamelCase="<mask>"      ,					_lowerCamelCase=["<eop>", "<eod>"]      ,					_lowerCamelCase = None      ,					**_lowerCamelCase      ,					):
										# Mask token behave like a normal word, i.e. include the space before it
										lowerCamelCase__   =AddedToken(_lowerCamelCase      ,					lstrip=_lowerCamelCase      ,					rstrip=_lowerCamelCase ) if isinstance(_lowerCamelCase      ,					_lowerCamelCase ) else mask_token

										lowerCamelCase__   ={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=_lowerCamelCase      ,					remove_space=_lowerCamelCase      ,					keep_accents=_lowerCamelCase      ,					bos_token=_lowerCamelCase      ,					eos_token=_lowerCamelCase      ,					unk_token=_lowerCamelCase      ,					sep_token=_lowerCamelCase      ,					pad_token=_lowerCamelCase      ,					cls_token=_lowerCamelCase      ,					mask_token=_lowerCamelCase      ,					additional_special_tokens=_lowerCamelCase      ,					sp_model_kwargs=self.sp_model_kwargs      ,					**_lowerCamelCase      ,					)

										lowerCamelCase__   =3

										lowerCamelCase__   =do_lower_case
										lowerCamelCase__   =remove_space
										lowerCamelCase__   =keep_accents
										lowerCamelCase__   =vocab_file

										lowerCamelCase__   =spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(_lowerCamelCase )

				@property
				def 						_a      ( self ):
										return len(self.sp_model )

				def 						_a      ( self ):
										lowerCamelCase__   ={self.convert_ids_to_tokens(_lowerCamelCase ): i for i in range(self.vocab_size )}
										vocab.update(self.added_tokens_encoder )
										return vocab

				def __getstate__( self ):
										lowerCamelCase__   =self.__dict__.copy()
										lowerCamelCase__   =None
										return state

				def __setstate__( self      ,					_lowerCamelCase ):
										lowerCamelCase__   =d

										# for backward compatibility
										if not hasattr(self      ,					"sp_model_kwargs" ):
																lowerCamelCase__   ={}

										lowerCamelCase__   =spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(self.vocab_file )

				def 						_a      ( self      ,					_lowerCamelCase ):
										if self.remove_space:
																lowerCamelCase__   =" ".join(inputs.strip().split() )
										else:
																lowerCamelCase__   =inputs
										lowerCamelCase__   =outputs.replace("``"      ,					"\"" ).replace("''"      ,					"\"" )

										if not self.keep_accents:
																lowerCamelCase__   =unicodedata.normalize("NFKD"      ,					_lowerCamelCase )
																lowerCamelCase__   ="".join([c for c in outputs if not unicodedata.combining(_lowerCamelCase )] )
										if self.do_lower_case:
																lowerCamelCase__   =outputs.lower()

										return outputs

				def 						_a      ( self      ,					_lowerCamelCase ):
										lowerCamelCase__   =self.preprocess_text(_lowerCamelCase )
										lowerCamelCase__   =self.sp_model.encode(_lowerCamelCase      ,					out_type=_lowerCamelCase )
										lowerCamelCase__   =[]
										for piece in pieces:
																if len(_lowerCamelCase ) > 1 and piece[-1] == str("," ) and piece[-2].isdigit():
																						lowerCamelCase__   =self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCamelCase      ,					"" ) )
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																												if len(cur_pieces[0] ) == 1:
																																		lowerCamelCase__   =cur_pieces[1:]
																												else:
																																		lowerCamelCase__   =cur_pieces[0][1:]
																						cur_pieces.append(piece[-1] )
																						new_pieces.extend(_lowerCamelCase )
																else:
																						new_pieces.append(_lowerCamelCase )

										return new_pieces

				def 						_a      ( self      ,					_lowerCamelCase ):
										return self.sp_model.PieceToId(_lowerCamelCase )

				def 						_a      ( self      ,					_lowerCamelCase ):
										return self.sp_model.IdToPiece(_lowerCamelCase )

				def 						_a      ( self      ,					_lowerCamelCase ):
										lowerCamelCase__   ="".join(_lowerCamelCase ).replace(_lowerCamelCase      ,					" " ).strip()
										return out_string

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = False      ,					_lowerCamelCase = None      ,					_lowerCamelCase = True      ,					**_lowerCamelCase      ,					):
										lowerCamelCase__   =kwargs.pop("use_source_tokenizer"      ,					_lowerCamelCase )

										lowerCamelCase__   =self.convert_ids_to_tokens(_lowerCamelCase      ,					skip_special_tokens=_lowerCamelCase )

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										lowerCamelCase__   =[]
										lowerCamelCase__   =[]
										for token in filtered_tokens:
																if skip_special_tokens and token in self.all_special_ids:
																						continue
																if token in self.added_tokens_encoder:
																						if current_sub_text:
																												sub_texts.append(self.convert_tokens_to_string(_lowerCamelCase ) )
																												lowerCamelCase__   =[]
																						sub_texts.append(_lowerCamelCase )
																else:
																						current_sub_text.append(_lowerCamelCase )
										if current_sub_text:
																sub_texts.append(self.convert_tokens_to_string(_lowerCamelCase ) )

										# Mimic the behavior of the Rust tokenizer:
										# By default, there are no spaces between special tokens
										lowerCamelCase__   ="".join(_lowerCamelCase )

										lowerCamelCase__   =(
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
																lowerCamelCase__   =self.clean_up_tokenization(_lowerCamelCase )
																return clean_text
										else:
																return text

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[self.cls_token_id]
										if token_ids_a is None:
																return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None      ,					_lowerCamelCase = False ):

										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowerCamelCase      ,					token_ids_a=_lowerCamelCase      ,					already_has_special_tokens=_lowerCamelCase )

										if token_ids_a is not None:
																return ([0] * len(_lowerCamelCase )) + [1] + ([0] * len(_lowerCamelCase )) + [1, 1]
										return ([0] * len(_lowerCamelCase )) + [1, 1]

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[2]

										if token_ids_a is None:
																return len(token_ids_a + sep ) * [0] + cls_segment_id
										return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id






				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										if not os.path.isdir(_lowerCamelCase ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
																return
										lowerCamelCase__   =os.path.join(
										    _lowerCamelCase      ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(_lowerCamelCase ) and os.path.isfile(self.vocab_file ):
																copyfile(self.vocab_file      ,					_lowerCamelCase )
										elif not os.path.isfile(self.vocab_file ):
																with open(_lowerCamelCase      ,					"wb" ) as fi:
																						lowerCamelCase__   =self.sp_model.serialized_model_proto()
																						fi.write(_lowerCamelCase )

										return (out_vocab_file,)



"""simple docstring"""


import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer






class   __UpperCAmelCase		(      __lowerCAmelCase      ):
				A__ :  List[Any]    				=			'''Wav2Vec2FeatureExtractor'''
				A__ :  Any    				=			'''AutoTokenizer'''

				def __init__( self      ,					_lowerCamelCase      ,					_lowerCamelCase ):
										super().__init__(_lowerCamelCase      ,					_lowerCamelCase )
										lowerCamelCase__   =self.feature_extractor
										lowerCamelCase__   =False

				@classmethod
				def 						_a      ( cls      ,					_lowerCamelCase      ,					**_lowerCamelCase ):
										try:
																return super().from_pretrained(_lowerCamelCase      ,					**_lowerCamelCase )
										except OSError:
																warnings.warn(
																    F'''Loading a tokenizer inside {cls.__name__} from a config that does not'''
																    " include a `tokenizer_class` attribute is deprecated and will be "
																    "removed in v5. Please add `'tokenizer_class': 'Wav2Vec2CTCTokenizer'`"
																    " attribute to either your `config.json` or `tokenizer_config.json` "
																    "file to suppress this warning: "      ,					_lowerCamelCase      ,					)

																lowerCamelCase__   =WavaVecaFeatureExtractor.from_pretrained(_lowerCamelCase      ,					**_lowerCamelCase )
																lowerCamelCase__   =WavaVecaCTCTokenizer.from_pretrained(_lowerCamelCase      ,					**_lowerCamelCase )

																return cls(feature_extractor=_lowerCamelCase      ,					tokenizer=_lowerCamelCase )

				def __call__( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										# For backward compatibility
										if self._in_target_context_manager:
																return self.current_processor(*_lowerCamelCase      ,					**_lowerCamelCase )

										if "raw_speech" in kwargs:
																warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead." )
																lowerCamelCase__   =kwargs.pop("raw_speech" )
										else:
																lowerCamelCase__   =kwargs.pop("audio"      ,					_lowerCamelCase )
										lowerCamelCase__   =kwargs.pop("sampling_rate"      ,					_lowerCamelCase )
										lowerCamelCase__   =kwargs.pop("text"      ,					_lowerCamelCase )
										if len(_lowerCamelCase ) > 0:
																lowerCamelCase__   =args[0]
																lowerCamelCase__   =args[1:]

										if audio is None and text is None:
																raise ValueError("You need to specify either an `audio` or `text` input to process." )

										if audio is not None:
																lowerCamelCase__   =self.feature_extractor(_lowerCamelCase      ,					*_lowerCamelCase      ,					sampling_rate=_lowerCamelCase      ,					**_lowerCamelCase )
										if text is not None:
																lowerCamelCase__   =self.tokenizer(_lowerCamelCase      ,					**_lowerCamelCase )

										if text is None:
																return inputs
										elif audio is None:
																return encodings
										else:
																lowerCamelCase__   =encodings["input_ids"]
																return inputs

				def 						_a      ( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										# For backward compatibility
										if self._in_target_context_manager:
																return self.current_processor.pad(*_lowerCamelCase      ,					**_lowerCamelCase )

										lowerCamelCase__   =kwargs.pop("input_features"      ,					_lowerCamelCase )
										lowerCamelCase__   =kwargs.pop("labels"      ,					_lowerCamelCase )
										if len(_lowerCamelCase ) > 0:
																lowerCamelCase__   =args[0]
																lowerCamelCase__   =args[1:]

										if input_features is not None:
																lowerCamelCase__   =self.feature_extractor.pad(_lowerCamelCase      ,					*_lowerCamelCase      ,					**_lowerCamelCase )
										if labels is not None:
																lowerCamelCase__   =self.tokenizer.pad(_lowerCamelCase      ,					**_lowerCamelCase )

										if labels is None:
																return input_features
										elif input_features is None:
																return labels
										else:
																lowerCamelCase__   =labels["input_ids"]
																return input_features

				def 						_a      ( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										return self.tokenizer.batch_decode(*_lowerCamelCase      ,					**_lowerCamelCase )

				def 						_a      ( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										return self.tokenizer.decode(*_lowerCamelCase      ,					**_lowerCamelCase )






				@contextmanager
				def 						_a      ( self ):
										warnings.warn(
										    "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
										    "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
										    "your audio inputs, or in a separate call." )
										lowerCamelCase__   =True
										lowerCamelCase__   =self.tokenizer
										yield
										lowerCamelCase__   =self.feature_extractor
										lowerCamelCase__   =False


