






"""simple docstring"""






from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
     from ..models.auto import AutoTokenizer




class 							lowerCAmelCase__       :






      '''simple docstring'''

      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[int]		,			lowercase_  :						List[str]):




          '''simple docstring'''


          raise NotImplementedError()



      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          raise NotImplementedError()




class 							lowerCAmelCase__       (    UpperCAmelCase__				):






      '''simple docstring'''

      def __init__(		self  :						Dict		,			lowercase_  :						"AutoTokenizer"		,			lowercase_  :						bool = False		,			**lowercase_  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[str]							=	tokenizer
          SCREAMING_SNAKE_CASE_   :						str							=	skip_prompt
          SCREAMING_SNAKE_CASE_   :						List[str]							=	decode_kwargs

          # variables used in the streaming process
          SCREAMING_SNAKE_CASE_   :						Dict							=	[]
          SCREAMING_SNAKE_CASE_   :						int							=	0
          SCREAMING_SNAKE_CASE_   :						Tuple							=	True
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple		,			lowercase_  :						Tuple):




          '''simple docstring'''


          if len(value.shape) > 1 and value.shape[0] > 1:
              raise ValueError('''TextStreamer only supports batch size 1''')
          elif len(value.shape) > 1:
              SCREAMING_SNAKE_CASE_   :						List[str]							=	value[0]

          if self.skip_prompt and self.next_tokens_are_prompt:
              SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	False
              return

          # Add the new token to the cache and decodes the entire thing.
          self.token_cache.extend(value.tolist())
          SCREAMING_SNAKE_CASE_   :						Any							=	self.tokenizer.decode(self.token_cache		,			**self.decode_kwargs)

          # After the symbol for a new line, we flush the cache.
          if text.endswith('''\n'''):
              SCREAMING_SNAKE_CASE_   :						Any							=	text[self.print_len :]
              SCREAMING_SNAKE_CASE_   :						Any							=	[]
              SCREAMING_SNAKE_CASE_   :						Tuple							=	0
          # If the last token is a CJK character, we print the characters.
          elif len(lowercase_) > 0 and self._is_chinese_char(ord(text[-1])):
              SCREAMING_SNAKE_CASE_   :						Optional[int]							=	text[self.print_len :]
              self.print_len += len(lowercase_)
          # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
          # which may change with the subsequent token -- there are probably smarter ways to do this!)
          else:
              SCREAMING_SNAKE_CASE_   :						str							=	text[self.print_len : text.rfind(''' ''') + 1]
              self.print_len += len(lowercase_)

          self.on_finalized_text(lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          if len(self.token_cache) > 0:
              SCREAMING_SNAKE_CASE_   :						str							=	self.tokenizer.decode(self.token_cache		,			**self.decode_kwargs)
              SCREAMING_SNAKE_CASE_   :						str							=	text[self.print_len :]
              SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	[]
              SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	0
          else:
              SCREAMING_SNAKE_CASE_   :						Dict							=	''''''

          SCREAMING_SNAKE_CASE_   :						str							=	True
          self.on_finalized_text(lowercase_		,			stream_end=lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]		,			lowercase_  :						str		,			lowercase_  :						bool = False):




          '''simple docstring'''


          print(lowercase_		,			flush=lowercase_		,			end='''''' if not stream_end else None)



      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]		,			lowercase_  :						Tuple):




          '''simple docstring'''


          if (
              (cp >= 0X4E00 and cp <= 0X9FFF)
              or (cp >= 0X3400 and cp <= 0X4DBF)  #
              or (cp >= 0X2_0000 and cp <= 0X2_A6DF)  #
              or (cp >= 0X2_A700 and cp <= 0X2_B73F)  #
              or (cp >= 0X2_B740 and cp <= 0X2_B81F)  #
              or (cp >= 0X2_B820 and cp <= 0X2_CEAF)  #
              or (cp >= 0XF900 and cp <= 0XFAFF)
              or (cp >= 0X2_F800 and cp <= 0X2_FA1F)  #
          ):  #
              return True

          return False




class 							lowerCAmelCase__       (    UpperCAmelCase__				):






      '''simple docstring'''

      def __init__(		self  :						str		,			lowercase_  :						"AutoTokenizer"		,			lowercase_  :						bool = False		,			lowercase_  :						Optional[float] = None		,			**lowercase_  :						str):




          '''simple docstring'''


          super().__init__(lowercase_		,			lowercase_		,			**lowercase_)
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	Queue()
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	None
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	timeout
      def _SCREAMING_SNAKE_CASE				(		self  :						Any		,			lowercase_  :						str		,			lowercase_  :						bool = False):




          '''simple docstring'''


          self.text_queue.put(lowercase_		,			timeout=self.timeout)
          if stream_end:
              self.text_queue.put(self.stop_signal		,			timeout=self.timeout)
      def __iter__(		self  :						Optional[int]):




          '''simple docstring'''


          return self



      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	self.text_queue.get(timeout=self.timeout)
          if value == self.stop_signal:
              raise StopIteration()
          else:
              return value










"""simple docstring"""






import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


UpperCAmelCase_     :				int         =				{
    """cola""": 2,
    """mnli""": 3,
    """mrpc""": 2,
    """sst-2""": 2,
    """sts-b""": 1,
    """qqp""": 2,
    """qnli""": 2,
    """rte""": 2,
    """wnli""": 2,
}


logging.set_verbosity_info()



def       _A    (__a					,       __a					,       __a					,       __a=None							)				-> int:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						str							=	XLNetConfig.from_json_file(__a							)

    SCREAMING_SNAKE_CASE_   :						List[Any]							=	finetuning_task.lower() if finetuning_task is not None else ''''''
    if finetuning_task in GLUE_TASKS_NUM_LABELS:
        print(f'Building PyTorch XLNetForSequenceClassification model from configuration: {config}'							)
        SCREAMING_SNAKE_CASE_   :						List[Any]							=	finetuning_task
        SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	GLUE_TASKS_NUM_LABELS[finetuning_task]
        SCREAMING_SNAKE_CASE_   :						Dict							=	XLNetForSequenceClassification(__a							)
    elif "squad" in finetuning_task:
        SCREAMING_SNAKE_CASE_   :						str							=	finetuning_task
        SCREAMING_SNAKE_CASE_   :						Dict							=	XLNetForQuestionAnswering(__a							)
    else:
        SCREAMING_SNAKE_CASE_   :						List[str]							=	XLNetLMHeadModel(__a							)

    # Load weights from tf checkpoint
    load_tf_weights_in_xlnet(__a					,       __a					,       __a							)

    # Save pytorch-model
    SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	os.path.join(__a					,       __a							)
    SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	os.path.join(__a					,       __a							)
    print(f'Save PyTorch model to {os.path.abspath(__a							)}'							)
    torch.save(model.state_dict()					,       __a							)
    print(f'Save configuration file to {os.path.abspath(__a							)}'							)
    with open(__a					,       '''w'''					,       encoding='''utf-8'''							) as f:
        f.write(config.to_json_string()							)


if __name__ == "__main__":
  UpperCAmelCase_     :				str         =				argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      """--tf_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
  )
  parser.add_argument(
      """--xlnet_config_file""",
      default=None,
      type=str,
      required=True,
      help=(
          """The config json file corresponding to the pre-trained XLNet model. \n"""
          """This specifies the model architecture."""
      ),
  )
  parser.add_argument(
      """--pytorch_dump_folder_path""",
      default=None,
      type=str,
      required=True,
      help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
  )
  parser.add_argument(
      """--finetuning_task""",
      default=None,
      type=str,
      help="""Name of a task on which the XLNet TensorFlow model was fine-tuned""",
  )
  UpperCAmelCase_     :				int         =				parser.parse_args()
  print(args)

  convert_xlnet_checkpoint_to_pytorch(
      args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
  )


