




'''simple docstring'''





import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
_lowerCAmelCase						:   Union[str, Any]		=   logging.get_logger(__name__)



def 						lowerCAmelCase		(		_lowerCAmelCase							:   Any						,							_lowerCAmelCase							:   int						,							_lowerCAmelCase							:   str	):




 """simple docstring"""







 UpperCAmelCase__							 =						os.path.abspath(_snake_case	)
 logger.info(F'''Converting TensorFlow checkpoint from {tf_path}'''	)
 # Load weights from TF model
 UpperCAmelCase__							 =						tf.train.list_variables(_snake_case	)
 UpperCAmelCase__							 =						[]
 UpperCAmelCase__							 =						[]
 UpperCAmelCase__							 =						[]
 for full_name, shape in init_vars:
  # logger.info(f"Loading TF weight {name} with shape {shape}")
  UpperCAmelCase__							 =						full_name.split("/"	)
  if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
   logger.info(F'''Skipping non-model layer {full_name}'''	)
   continue
  if "optimizer" in full_name:
   logger.info(F'''Skipping optimization layer {full_name}'''	)
   continue
  if name[0] == "model":
   # ignore initial 'model'
   UpperCAmelCase__							 =						name[1:]
  # figure out how many levels deep the name is
  UpperCAmelCase__							 =						0
  for _name in name:
   if _name.startswith("layer_with_weights"	):
    depth += 1
   else:
    break
  layer_depth.append(_snake_case	)
  # read data
  UpperCAmelCase__							 =						tf.train.load_variable(_snake_case						,							_snake_case	)
  names.append("/".join(_snake_case	)	)
  arrays.append(_snake_case	)
 logger.info(F'''Read a total of {len(_snake_case	):,} layers'''	)

 # Sanity check
 if len(set(_snake_case	)	) != 1:
  raise ValueError(F'''Found layer names with different depths (layer depth {list(set(_snake_case	)	)})'''	)
 UpperCAmelCase__							 =						list(set(_snake_case	)	)[0]
 if layer_depth != 1:
  raise ValueError(
      "The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP"
      " heads."	)

 # convert layers
 logger.info("Converting weights..."	)
 for full_name, array in zip(_snake_case						,							_snake_case	):
  UpperCAmelCase__							 =						full_name.split("/"	)
  UpperCAmelCase__							 =						model
  UpperCAmelCase__							 =						[]
  for i, m_name in enumerate(_snake_case	):
   if m_name == ".ATTRIBUTES":
    # variable names end with .ATTRIBUTES/VARIABLE_VALUE
    break
   if m_name.startswith("layer_with_weights"	):
    UpperCAmelCase__							 =						int(m_name.split("-"	)[-1]	)
    if layer_num <= 2:
     # embedding layers
     # layer_num 0: word_embeddings
     # layer_num 1: position_embeddings
     # layer_num 2: token_type_embeddings
     continue
    elif layer_num == 3:
     # embedding LayerNorm
     trace.extend(["embeddings", "LayerNorm"]	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"embeddings"	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"LayerNorm"	)
    elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
     # encoder layers
     trace.extend(["encoder", "layer", str(layer_num - 4	)]	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"encoder"	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"layer"	)
     UpperCAmelCase__							 =						pointer[layer_num - 4]
    elif layer_num == config.num_hidden_layers + 4:
     # pooler layer
     trace.extend(["pooler", "dense"]	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"pooler"	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"dense"	)
   elif m_name == "embeddings":
    trace.append("embeddings"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"embeddings"	)
    if layer_num == 0:
     trace.append("word_embeddings"	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"word_embeddings"	)
    elif layer_num == 1:
     trace.append("position_embeddings"	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"position_embeddings"	)
    elif layer_num == 2:
     trace.append("token_type_embeddings"	)
     UpperCAmelCase__							 =						getattr(_snake_case						,							"token_type_embeddings"	)
    else:
     raise ValueError(F'''Unknown embedding layer with name {full_name}'''	)
    trace.append("weight"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"weight"	)
   elif m_name == "_attention_layer":
    # self-attention layer
    trace.extend(["attention", "self"]	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"attention"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"self"	)
   elif m_name == "_attention_layer_norm":
    # output attention norm
    trace.extend(["attention", "output", "LayerNorm"]	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"attention"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"output"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"LayerNorm"	)
   elif m_name == "_attention_output_dense":
    # output attention dense
    trace.extend(["attention", "output", "dense"]	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"attention"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"output"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"dense"	)
   elif m_name == "_output_dense":
    # output dense
    trace.extend(["output", "dense"]	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"output"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"dense"	)
   elif m_name == "_output_layer_norm":
    # output dense
    trace.extend(["output", "LayerNorm"]	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"output"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"LayerNorm"	)
   elif m_name == "_key_dense":
    # attention key
    trace.append("key"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"key"	)
   elif m_name == "_query_dense":
    # attention query
    trace.append("query"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"query"	)
   elif m_name == "_value_dense":
    # attention value
    trace.append("value"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"value"	)
   elif m_name == "_intermediate_dense":
    # attention intermediate dense
    trace.extend(["intermediate", "dense"]	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"intermediate"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"dense"	)
   elif m_name == "_output_layer_norm":
    # output layer norm
    trace.append("output"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"output"	)
   # weights & biases
   elif m_name in ["bias", "beta"]:
    trace.append("bias"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"bias"	)
   elif m_name in ["kernel", "gamma"]:
    trace.append("weight"	)
    UpperCAmelCase__							 =						getattr(_snake_case						,							"weight"	)
   else:
    logger.warning(F'''Ignored {m_name}'''	)
        # for certain layers reshape is necessary
  UpperCAmelCase__							 =						".".join(_snake_case	)
  if re.match(R"(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)"						,							_snake_case	) or re.match(
      R"(\S+)\.attention\.output\.dense\.weight"						,							_snake_case	):
   UpperCAmelCase__							 =						array.reshape(pointer.data.shape	)
  if "kernel" in full_name:
   UpperCAmelCase__							 =						array.transpose()
  if pointer.shape == array.shape:
   UpperCAmelCase__							 =						torch.from_numpy(_snake_case	)
  else:
   raise ValueError(
       F'''Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:'''
       F''' {array.shape}'''	)
  logger.info(F'''Successfully set variable {full_name} to PyTorch layer {trace}'''	)
 return model



def 						lowerCAmelCase		(		_lowerCAmelCase							:   Tuple						,							_lowerCAmelCase							:   Any						,							_lowerCAmelCase							:   Dict	):




 """simple docstring"""







 logger.info(F'''Loading model based on config from {config_path}...'''	)
 UpperCAmelCase__							 =						BertConfig.from_json_file(_snake_case	)
 UpperCAmelCase__							 =						BertModel(_snake_case	)

 # Load weights from checkpoint
 logger.info(F'''Loading weights from checkpoint {tf_checkpoint_path}...'''	)
 load_tfa_weights_in_bert(_snake_case						,							_snake_case						,							_snake_case	)

 # Save pytorch-model
 logger.info(F'''Saving PyTorch model to {pytorch_dump_path}...'''	)
 torch.save(model.state_dict()						,							_snake_case	)


if __name__ == "__main__":
  _lowerCAmelCase						:   Tuple		=   argparse.ArgumentParser()
  parser.add_argument(
      "--tf_checkpoint_path", type=str, required=True, help="Path to the TensorFlow 2.x checkpoint path."
  )
  parser.add_argument(
      "--bert_config_file",
      type=str,
      required=True,
      help="The config json file corresponding to the BERT model. This specifies the model architecture.",
  )
  parser.add_argument(
      "--pytorch_dump_path",
      type=str,
      required=True,
      help="Path to the output PyTorch model (must include filename).",
  )
  _lowerCAmelCase						:   Union[str, Any]		=   parser.parse_args()
  convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)











import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


_lowerCAmelCase						:   str		=   logging.get_logger(__name__)

_lowerCAmelCase						:   List[Any]		=   {"tokenizer_file": "tokenizer.json"}

_lowerCAmelCase						:   Tuple		=   {
    "tokenizer_file": {
        "bigscience/tokenizer": "https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json",
        "bigscience/bloom-560m": "https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json",
        "bigscience/bloom-1b1": "https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json",
        "bigscience/bloom-1b7": "https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json",
        "bigscience/bloom-3b": "https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json",
        "bigscience/bloom-7b1": "https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json",
        "bigscience/bloom": "https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json",
    },
}
class 			_UpperCamelCase      (				lowerCAmelCase		):
      UpperCAmelCase_        =		VOCAB_FILES_NAMES
      UpperCAmelCase_        =		PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_        =		["""input_ids""", """attention_mask"""]
      UpperCAmelCase_        =		None
      def __init__(			self   :List[Any] ,      lowerCamelCase   :Union[str, Any]=None ,      lowerCamelCase   :Optional[Any]=None ,      lowerCamelCase   :List[Any]=None ,      lowerCamelCase   :Union[str, Any]="<unk>" ,      lowerCamelCase   :Union[str, Any]="<s>" ,      lowerCamelCase   :Union[str, Any]="</s>" ,      lowerCamelCase   :str="<pad>" ,      lowerCamelCase   :Any=False ,      lowerCamelCase   :Optional[Any]=False ,      **lowerCamelCase   :Optional[Any] ,      )     ->			Any:
       super().__init__(
           lowerCamelCase ,      lowerCamelCase ,      tokenizer_file=lowerCamelCase ,      unk_token=lowerCamelCase ,      bos_token=lowerCamelCase ,      eos_token=lowerCamelCase ,      pad_token=lowerCamelCase ,      add_prefix_space=lowerCamelCase ,      clean_up_tokenization_spaces=lowerCamelCase ,      **lowerCamelCase ,      )
       UpperCAmelCase__							 =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
       if pre_tok_state.get("add_prefix_space" ,      lowerCamelCase      ) != add_prefix_space:
        UpperCAmelCase__							 =						getattr(lowerCamelCase ,      pre_tok_state.pop("type"      )      )
        UpperCAmelCase__							 =						add_prefix_space
        UpperCAmelCase__							 =						pre_tok_class(**lowerCamelCase      )

       UpperCAmelCase__							 =						add_prefix_space
      def 	UpperCAmelCase_						(			self   :int ,      *lowerCamelCase   :Any ,      **lowerCamelCase   :int      )     ->			BatchEncoding:
       UpperCAmelCase__							 =						kwargs.get("is_split_into_words" ,      lowerCamelCase      )
       if not (self.add_prefix_space or not is_split_into_words):
        raise Exception(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
            " pretokenized inputs."      )

       return super()._batch_encode_plus(*lowerCamelCase ,      **lowerCamelCase      )
      def 	UpperCAmelCase_						(			self   :Any ,      *lowerCamelCase   :List[Any] ,      **lowerCamelCase   :Union[str, Any]      )     ->			BatchEncoding:
       UpperCAmelCase__							 =						kwargs.get("is_split_into_words" ,      lowerCamelCase      )

       if not (self.add_prefix_space or not is_split_into_words):
        raise Exception(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
            " pretokenized inputs."      )

       return super()._encode_plus(*lowerCamelCase ,      **lowerCamelCase      )
      def 	UpperCAmelCase_						(			self   :Tuple ,      lowerCamelCase   :str ,      lowerCamelCase   :Optional[str] = None      )     ->			Tuple[str]:
       UpperCAmelCase__							 =						self._tokenizer.model.save(lowerCamelCase ,      name=lowerCamelCase      )
       return tuple(lowerCamelCase      )







      def 	UpperCAmelCase_						(			self   :List[str] ,      lowerCamelCase   :"Conversation"      )     ->			List[int]:
       UpperCAmelCase__							 =						[]
       for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      ) + [self.eos_token_id]      )

       if len(lowerCamelCase      ) > self.model_max_length:
        UpperCAmelCase__							 =						input_ids[-self.model_max_length :]
       return input_ids





