



'''simple docstring'''






# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


lowerCAmelCase:      Optional[Any]		 =		'3'

print('Python version:', sys.version)
print('transformers version:', transformers.__version__)

try:
     import torch

     print('Torch version:', torch.__version__)
     print('Cuda available:', torch.cuda.is_available())
     print('Cuda version:', torch.version.cuda)
     print('CuDNN version:', torch.backends.cudnn.version())
     print('Number of GPUs available:', torch.cuda.device_count())
     print('NCCL version:', torch.cuda.nccl.version())
except ImportError:
     print('Torch version:', None)

try:
     import deepspeed

     print('DeepSpeed version:', deepspeed.__version__)
except ImportError:
     print('DeepSpeed version:', None)

try:
     import tensorflow as tf

     print('TensorFlow version:', tf.__version__)
     print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))
     print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))
except ImportError:
     print('TensorFlow version:', None)




'''simple docstring'''






import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase:      List[Any]		 =		logging.get_logger(__name__)

lowerCAmelCase:      Dict		 =		{
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

lowerCAmelCase:      List[Any]		 =		{
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

lowerCAmelCase:      Optional[Any]		 =		{
    'vinai/phobert-base': 2_5_6,
    'vinai/phobert-large': 2_5_6,
}
def    lowerCamelCase__ (		_A       ):
  a		:      List[Any]       =  set()
  a		:      Union[str, Any]       =  word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)       )
    a		:      Tuple       =  char

  a		:      Any       =  set(_A       )
  return pairs

class 							a__(      lowerCamelCase__			):
    lowercase__				=					VOCAB_FILES_NAMES
    lowercase__				=					PRETRAINED_VOCAB_FILES_MAP
    lowercase__				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







    def __init__(   self				:						Optional[Any]		,    __snake_case				:						Union[str, Any]		,    __snake_case				:						int		,    __snake_case				:						Any="<s>"		,    __snake_case				:						List[Any]="</s>"		,    __snake_case				:						Union[str, Any]="</s>"		,    __snake_case				:						List[Any]="<s>"		,    __snake_case				:						Optional[Any]="<unk>"		,    __snake_case				:						Union[str, Any]="<pad>"		,    __snake_case				:						List[Any]="<mask>"		,    **__snake_case				:						int		,    ):
      super().__init__(
          bos_token=__snake_case		,    eos_token=__snake_case		,    unk_token=__snake_case		,    sep_token=__snake_case		,    cls_token=__snake_case		,    pad_token=__snake_case		,    mask_token=__snake_case		,    **__snake_case		,    )

      a		:      Optional[int]       =  vocab_file
      a		:      Optional[Any]       =  merges_file

      a		:      int       =  {}
      a		:      List[str]       =  0
      a		:      Union[str, Any]       =  1
      a		:      Optional[Any]       =  2
      a		:      List[Any]       =  3

      self.add_from_file(__snake_case			)

      a		:      Dict       =  {v: k for k, v in self.encoder.items()}

      with open(__snake_case		,    encoding='utf-8'			) as merges_handle:
        a		:      Optional[int]       =  merges_handle.read().split('\n'			)[:-1]
      a		:      Any       =  [tuple(merge.split()[:-1]			) for merge in merges]
      a		:      List[Any]       =  dict(zip(__snake_case		,    range(len(__snake_case			)			)			)			)
      a		:      List[str]       =  {}







    def     lowercase_ (   self				:						Tuple		,    __snake_case				:						List[int]		,    __snake_case				:						Optional[List[int]] = None			):

      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      a		:      Union[str, Any]       =  [self.cls_token_id]
      a		:      Any       =  [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep







    def     lowercase_ (   self				:						Optional[int]		,    __snake_case				:						List[int]		,    __snake_case				:						Optional[List[int]] = None		,    __snake_case				:						bool = False			):

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__snake_case		,    token_ids_a=__snake_case		,    already_has_special_tokens=__snake_case			)

      if token_ids_a is None:
        return [1] + ([0] * len(__snake_case			)) + [1]
      return [1] + ([0] * len(__snake_case			)) + [1, 1] + ([0] * len(__snake_case			)) + [1]







    def     lowercase_ (   self				:						int		,    __snake_case				:						List[int]		,    __snake_case				:						Optional[List[int]] = None			):
      a		:      Dict       =  [self.sep_token_id]
      a		:      Optional[int]       =  [self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep			) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]







    @property
    def     lowercase_ (   self				:						Optional[int]			):
      return len(self.encoder			)







    def     lowercase_ (   self				:						Dict			):
      return dict(self.encoder		,    **self.added_tokens_encoder			)







    def     lowercase_ (   self				:						Optional[int]		,    __snake_case				:						Optional[Any]			):
      if token in self.cache:
        return self.cache[token]
      a		:      int       =  tuple(__snake_case			)
      a		:      int       =  tuple(list(word[:-1]			) + [word[-1] + '</w>']			)
      a		:      Dict       =  get_pairs(__snake_case			)

      if not pairs:
        return token

      while True:
        a		:      str       =  min(__snake_case		,    key=lambda __snake_case			: self.bpe_ranks.get(__snake_case		,    float('inf'			)			)			)
        if bigram not in self.bpe_ranks:
          break
        a     ,			a		:      Tuple       =  bigram
        a		:      int       =  []
        a		:      Optional[int]       =  0
        while i < len(__snake_case			):
          try:
            a		:      List[str]       =  word.index(__snake_case		,    __snake_case			)
          except ValueError:
            new_word.extend(word[i:]			)
            break
          else:
            new_word.extend(word[i:j]			)
            a		:      Dict       =  j

          if word[i] == first and i < len(__snake_case			) - 1 and word[i + 1] == second:
            new_word.append(first + second			)
            i += 2
          else:
            new_word.append(word[i]			)
            i += 1
        a		:      Optional[Any]       =  tuple(__snake_case			)
        a		:      Tuple       =  new_word
        if len(__snake_case			) == 1:
          break
        else:
          a		:      Union[str, Any]       =  get_pairs(__snake_case			)
      a		:      List[Any]       =  '@@ '.join(__snake_case			)
      a		:      int       =  word[:-4]
      a		:      Optional[int]       =  word
      return word







    def     lowercase_ (   self				:						Dict		,    __snake_case				:						List[str]			):
      a		:      Optional[int]       =  []

      a		:      Optional[Any]       =  re.findall(r'\S+\n?'		,    __snake_case			)

      for token in words:
        split_tokens.extend(list(self.bpe(__snake_case			).split(' '			)			)			)
      return split_tokens







    def     lowercase_ (   self				:						List[Any]		,    __snake_case				:						Dict			):
      return self.encoder.get(__snake_case		,    self.encoder.get(self.unk_token			)			)







    def     lowercase_ (   self				:						Dict		,    __snake_case				:						Dict			):
      return self.decoder.get(__snake_case		,    self.unk_token			)







    def     lowercase_ (   self				:						List[Any]		,    __snake_case				:						Any			):
      a		:      Union[str, Any]       =  ' '.join(__snake_case			).replace('@@ '		,    ''			).strip()
      return out_string







    def     lowercase_ (   self				:						str		,    __snake_case				:						str		,    __snake_case				:						Optional[str] = None			):
      if not os.path.isdir(__snake_case			):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
        return
      a		:      Dict       =  os.path.join(
          __snake_case		,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
      a		:      int       =  os.path.join(
          __snake_case		,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']			)

      if os.path.abspath(self.vocab_file			) != os.path.abspath(__snake_case			):
        copyfile(self.vocab_file		,    __snake_case			)

      if os.path.abspath(self.merges_file			) != os.path.abspath(__snake_case			):
        copyfile(self.merges_file		,    __snake_case			)

      return out_vocab_file, out_merge_file







    def     lowercase_ (   self				:						Tuple		,    __snake_case				:						List[str]			):
      if isinstance(__snake_case		,    __snake_case			):
        try:
          with open(__snake_case		,    'r'		,    encoding='utf-8'			) as fd:
            self.add_from_file(__snake_case			)
        except FileNotFoundError as fnfe:
          raise fnfe
        except UnicodeError:
          raise Exception(F"""Incorrect encoding detected in {f}, please rebuild the dataset"""			)
        return

      a		:      List[str]       =  f.readlines()
      for lineTmp in lines:
        a		:      Any       =  lineTmp.strip()
        a		:      Dict       =  line.rfind(' '			)
        if idx == -1:
          raise ValueError('Incorrect dictionary format, expected \'<token> <cnt>\''			)
        a		:      str       =  line[:idx]
        a		:      Optional[int]       =  len(self.encoder			)