
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def      UpperCamelCase   (       __lowerCamelCase	:					int							,   __lowerCamelCase	:					List[Any]       ):
       assert isinstance(__lowerCamelCase							,   __lowerCamelCase       )
       assert dataset.num_rows == 4
       assert dataset.num_columns == 1
       assert dataset.column_names == ["text"]
       for feature, expected_dtype in expected_features.items():
              assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"							,   [False, True]       )
def      UpperCamelCase   (       __lowerCamelCase	:					str							,   __lowerCamelCase	:					Tuple							,   __lowerCamelCase	:					int       ):
       snake_case			:  Optional[Any]     							= tmp_path / "cache"
       snake_case			:  Optional[int]     							= {"text": "string"}
       with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
              snake_case			:  Tuple     							= TextDatasetReader(__lowerCamelCase							,   cache_dir=__lowerCamelCase							,   keep_in_memory=__lowerCamelCase       ).read()
       _check_text_dataset(__lowerCamelCase							,   __lowerCamelCase       )


@pytest.mark.parametrize(
    "features"							,   [
        None,
        {"text": "string"},
        {"text": "int32"},
        {"text": "float32"},
    ]							,   )
def      UpperCamelCase   (       __lowerCamelCase	:					Union[str, Any]							,   __lowerCamelCase	:					Optional[Any]							,   __lowerCamelCase	:					str       ):
       snake_case			:  List[str]     							= tmp_path / "cache"
       snake_case			:  Optional[Any]     							= {"text": "string"}
       snake_case			:  int     							= features.copy() if features else default_expected_features
       snake_case			:  Optional[Any]     							= (
           Features({feature: Value(__lowerCamelCase       ) for feature, dtype in features.items()}       ) if features is not None else None
       )
       snake_case			:  int     							= TextDatasetReader(__lowerCamelCase							,   features=__lowerCamelCase							,   cache_dir=__lowerCamelCase       ).read()
       _check_text_dataset(__lowerCamelCase							,   __lowerCamelCase       )


@pytest.mark.parametrize("split"							,   [None, NamedSplit("train"       ), "train", "test"]       )
def      UpperCamelCase   (       __lowerCamelCase	:					List[str]							,   __lowerCamelCase	:					Any							,   __lowerCamelCase	:					Union[str, Any]       ):
       snake_case			:  str     							= tmp_path / "cache"
       snake_case			:  Tuple     							= {"text": "string"}
       snake_case			:  Optional[Any]     							= TextDatasetReader(__lowerCamelCase							,   cache_dir=__lowerCamelCase							,   split=__lowerCamelCase       ).read()
       _check_text_dataset(__lowerCamelCase							,   __lowerCamelCase       )
       assert dataset.split == split if split else "train"


@pytest.mark.parametrize("path_type"							,   [str, list]       )
def      UpperCamelCase   (       __lowerCamelCase	:					Optional[Any]							,   __lowerCamelCase	:					Any							,   __lowerCamelCase	:					Dict       ):
       if issubclass(__lowerCamelCase							,   __lowerCamelCase       ):
              snake_case			:  Optional[Any]     							= text_path
       elif issubclass(__lowerCamelCase							,   __lowerCamelCase       ):
              snake_case			:  Any     							= [text_path]
       snake_case			:  Optional[int]     							= tmp_path / "cache"
       snake_case			:  Tuple     							= {"text": "string"}
       snake_case			:  Tuple     							= TextDatasetReader(__lowerCamelCase							,   cache_dir=__lowerCamelCase       ).read()
       _check_text_dataset(__lowerCamelCase							,   __lowerCamelCase       )


def      UpperCamelCase   (       __lowerCamelCase	:					Any							,   __lowerCamelCase	:					Optional[Any]							,   __lowerCamelCase	:					int=("train",)       ):
       assert isinstance(__lowerCamelCase							,   __lowerCamelCase       )
       for split in splits:
              snake_case			:  str     							= dataset_dict[split]
              assert dataset.num_rows == 4
              assert dataset.num_columns == 1
              assert dataset.column_names == ["text"]
              for feature, expected_dtype in expected_features.items():
                     assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"							,   [False, True]       )
def      UpperCamelCase   (       __lowerCamelCase	:					Optional[Any]							,   __lowerCamelCase	:					str							,   __lowerCamelCase	:					int       ):
       snake_case			:  Optional[Any]     							= tmp_path / "cache"
       snake_case			:  int     							= {"text": "string"}
       with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
              snake_case			:  Tuple     							= TextDatasetReader({"train": text_path}							,   cache_dir=__lowerCamelCase							,   keep_in_memory=__lowerCamelCase       ).read()
       _check_text_datasetdict(__lowerCamelCase							,   __lowerCamelCase       )


@pytest.mark.parametrize(
    "features"							,   [
        None,
        {"text": "string"},
        {"text": "int32"},
        {"text": "float32"},
    ]							,   )
def      UpperCamelCase   (       __lowerCamelCase	:					Dict							,   __lowerCamelCase	:					Dict							,   __lowerCamelCase	:					int       ):
       snake_case			:  Union[str, Any]     							= tmp_path / "cache"
       # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
       snake_case			:  int     							= {"text": "string"}
       snake_case			:  int     							= features.copy() if features else default_expected_features
       snake_case			:  List[str]     							= (
           Features({feature: Value(__lowerCamelCase       ) for feature, dtype in features.items()}       ) if features is not None else None
       )
       snake_case			:  str     							= TextDatasetReader({"train": text_path}							,   features=__lowerCamelCase							,   cache_dir=__lowerCamelCase       ).read()
       _check_text_datasetdict(__lowerCamelCase							,   __lowerCamelCase       )




@pytest.mark.parametrize("split"							,   [None, NamedSplit("train"       ), "train", "test"]       )
def      UpperCamelCase   (       __lowerCamelCase	:					Optional[Any]							,   __lowerCamelCase	:					int							,   __lowerCamelCase	:					List[Any]       ):
       if split:
              snake_case			:  int     							= {split: text_path}
       else:
              snake_case			:  Union[str, Any]     							= "train"
              snake_case			:  Dict     							= {"train": text_path, "test": text_path}
       snake_case			:  List[Any]     							= tmp_path / "cache"
       snake_case			:  str     							= {"text": "string"}
       snake_case			:  Optional[int]     							= TextDatasetReader(__lowerCamelCase							,   cache_dir=__lowerCamelCase       ).read()
       _check_text_datasetdict(__lowerCamelCase							,   __lowerCamelCase							,   splits=list(path.keys()       )       )
       assert all(dataset[split].split == split for split in path.keys()       )







import argparse

import torch

from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from transformers.utils import logging


logging.set_verbosity_info()


def      UpperCamelCase   (       __lowerCamelCase	:					List[str]							,   __lowerCamelCase	:					Union[str, Any]							,   __lowerCamelCase	:					Union[str, Any]       ):
       # Initialise PyTorch model
       snake_case			:  Optional[int]     							= BertConfig.from_json_file(__lowerCamelCase       )
       print(f"""Building PyTorch model from configuration: {config}"""       )
       snake_case			:  Any     							= BertForPreTraining(__lowerCamelCase       )

       # Load weights from tf checkpoint
       load_tf_weights_in_bert(__lowerCamelCase							,   __lowerCamelCase							,   __lowerCamelCase       )

       # Save pytorch-model
       print(f"""Save PyTorch model to {pytorch_dump_path}"""       )
       torch.save(model.state_dict()							,   __lowerCamelCase       )


if __name__ == "__main__":
   __lowerCamelCase				=			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       """--tf_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
   )
   parser.add_argument(
       """--bert_config_file""",
       default=None,
       type=str,
       required=True,
       help=(
           """The config json file corresponding to the pre-trained BERT model. \n"""
           """This specifies the model architecture."""
       ),
   )
   parser.add_argument(
       """--pytorch_dump_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
   )
   __lowerCamelCase				=			parser.parse_args()
   convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)





