




import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast







@dataclass
class     A(		datasets.BuilderConfig  ):



 '''simple docstring'''


 UpperCamelCase		  =		None







class     A(		datasets.ArrowBasedBuilder  ):



 '''simple docstring'''


 UpperCamelCase		  =		PandasConfig

 def 							a__					(							self   :    Optional[int]    )  ->      str:

       """simple docstring"""






       return datasets.DatasetInfo(features=self.config.features    )

 def 							a__					(							self   :    Dict		,				A_   :    int    )  ->      str:

       """simple docstring"""






       if not self.config.data_files:
             raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
       lowerCamelCase_							=       dl_manager.download_and_extract(self.config.data_files    )
       if isinstance(A_		,				(str, list, tuple)    ):
             lowerCamelCase_							=       data_files
             if isinstance(A_		,				A_    ):
                   lowerCamelCase_							=       [files]
             # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
             lowerCamelCase_							=       [dl_manager.iter_files(A_    ) for file in files]
             return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,				gen_kwargs={'files': files}    )]
       lowerCamelCase_							=       []
       for split_name, files in data_files.items():
             if isinstance(A_		,				A_    ):
                   lowerCamelCase_							=       [files]
             # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
             lowerCamelCase_							=       [dl_manager.iter_files(A_    ) for file in files]
             splits.append(datasets.SplitGenerator(name=A_		,				gen_kwargs={'files': files}    )    )
       return splits

 def 							a__					(							self   :    int		,				A_   :    pa.Table    )  ->      pa.Table:

       """simple docstring"""






       if self.config.features is not None:
             # more expensive cast to support nested features with keys in a different order
             # allows str <-> int/float or str to Audio for example
             lowerCamelCase_							=       table_cast(A_		,				self.config.features.arrow_schema    )
       return pa_table





 def 							a__					(							self   :    str		,				A_   :    Optional[Any]    )  ->      str:

       """simple docstring"""






       for i, file in enumerate(itertools.chain.from_iterable(A_    )    ):
             with open(A_		,				'rb'    ) as f:
                   lowerCamelCase_							=       pa.Table.from_pandas(pd.read_pickle(A_    )    )
                   yield i, self._cast_table(A_    )





import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


__A							={
    '''cola''': 2,
    '''mnli''': 3,
    '''mrpc''': 2,
    '''sst-2''': 2,
    '''sts-b''': 1,
    '''qqp''': 2,
    '''qnli''': 2,
    '''rte''': 2,
    '''wnli''': 2,
}


logging.set_verbosity_info()





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__=None       ):
				# Initialise PyTorch model
				lowerCamelCase_         =     XLNetConfig.from_json_file(lowerCamelCase__       )

				lowerCamelCase_         =     finetuning_task.lower() if finetuning_task is not None else ""
				if finetuning_task in GLUE_TASKS_NUM_LABELS:
								print(F'Building PyTorch XLNetForSequenceClassification model from configuration: {config}'       )
								lowerCamelCase_         =     finetuning_task
								lowerCamelCase_         =     GLUE_TASKS_NUM_LABELS[finetuning_task]
								lowerCamelCase_         =     XLNetForSequenceClassification(lowerCamelCase__       )
				elif "squad" in finetuning_task:
								lowerCamelCase_         =     finetuning_task
								lowerCamelCase_         =     XLNetForQuestionAnswering(lowerCamelCase__       )
				else:
								lowerCamelCase_         =     XLNetLMHeadModel(lowerCamelCase__       )

				# Load weights from tf checkpoint
				load_tf_weights_in_xlnet(lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__       )

				# Save pytorch-model
				lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
				lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					lowerCamelCase__       )
				print(F'Save PyTorch model to {os.path.abspath(lowerCamelCase__       )}'       )
				torch.save(model.state_dict()      ,					lowerCamelCase__       )
				print(F'Save configuration file to {os.path.abspath(lowerCamelCase__       )}'       )
				with open(lowerCamelCase__      ,					"w"      ,					encoding="utf-8"       ) as f:
								f.write(config.to_json_string()       )


if __name__ == "__main__":
			__A							=argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
			)
			parser.add_argument(
			    '''--xlnet_config_file''',
			    default=None,
			    type=str,
			    required=True,
			    help=(
			        '''The config json file corresponding to the pre-trained XLNet model. \n'''
			        '''This specifies the model architecture.'''
			    ),
			)
			parser.add_argument(
			    '''--pytorch_dump_folder_path''',
			    default=None,
			    type=str,
			    required=True,
			    help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
			)
			parser.add_argument(
			    '''--finetuning_task''',
			    default=None,
			    type=str,
			    help='''Name of a task on which the XLNet TensorFlow model was fine-tuned''',
			)
			__A							=parser.parse_args()
			print(args)

			convert_xlnet_checkpoint_to_pytorch(
			    args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
			)


