





import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							):
    UpperCamelCase__:	int					=     10
    UpperCamelCase__:	Any					=     datasets.Features(
        {
            "tokens": datasets.Sequence(datasets.Value("string")),
            "labels": datasets.Sequence(datasets.ClassLabel(names=["negative", "positive"])),
            "answers": datasets.Sequence(
                {
                    "text": datasets.Value("string"),
                    "answer_start": datasets.Value("int32"),
                }),
            "id": datasets.Value("int64"),
        })
    UpperCamelCase__:	List[str]					=     datasets.Dataset.from_dict(
        {
            "tokens": [["foo"] * 5] * n,
            "labels": [[1] * 5] * n,
            "answers": [{"answer_start": [97], "text": ["1976"]}] * 10,
            "id": list(range(A_)),
        }    ,features=A_    ,)
    return dataset



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	Optional[Any]					=     str(tmp_path_factory.mktemp("data") / "file.arrow")
    dataset.map(cache_file_name=A_)
    return filename


# FILE_CONTENT + files


A__:    List[Any]  	=	'''\
    Text data.
    Second line of data.'''



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Dict					=     tmp_path_factory.mktemp("data") / "file.txt"
    UpperCamelCase__:	Tuple					=     FILE_CONTENT
    with open(A_    ,"w") as f:
        f.write(A_)
    return filename



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    import bza

    UpperCamelCase__:	Optional[int]					=     tmp_path_factory.mktemp("data") / "file.txt.bz2"
    UpperCamelCase__:	Tuple					=     bytes(A_    ,"utf-8")
    with bza.open(A_    ,"wb") as f:
        f.write(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    import gzip

    UpperCamelCase__:	Union[str, Any]					=     str(tmp_path_factory.mktemp("data") / "file.txt.gz")
    UpperCamelCase__:	Optional[int]					=     bytes(A_    ,"utf-8")
    with gzip.open(A_    ,"wb") as f:
        f.write(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    if datasets.config.LZ4_AVAILABLE:
        import lza.frame

        UpperCamelCase__:	List[Any]					=     tmp_path_factory.mktemp("data") / "file.txt.lz4"
        UpperCamelCase__:	Dict					=     bytes(A_    ,"utf-8")
        with lza.frame.open(A_    ,"wb") as f:
            f.write(A_)
        return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    if datasets.config.PY7ZR_AVAILABLE:
        import pyazr

        UpperCamelCase__:	Optional[int]					=     tmp_path_factory.mktemp("data") / "file.txt.7z"
        with pyazr.SevenZipFile(A_    ,"w") as archive:
            archive.write(A_    ,arcname=os.path.basename(A_))
        return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    import tarfile

    UpperCamelCase__:	Any					=     tmp_path_factory.mktemp("data") / "file.txt.tar"
    with tarfile.TarFile(A_    ,"w") as f:
        f.add(A_    ,arcname=os.path.basename(A_))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    import lzma

    UpperCamelCase__:	int					=     tmp_path_factory.mktemp("data") / "file.txt.xz"
    UpperCamelCase__:	List[str]					=     bytes(A_    ,"utf-8")
    with lzma.open(A_    ,"wb") as f:
        f.write(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    import zipfile

    UpperCamelCase__:	Tuple					=     tmp_path_factory.mktemp("data") / "file.txt.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.basename(A_))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    if datasets.config.ZSTANDARD_AVAILABLE:
        import zstandard as zstd

        UpperCamelCase__:	Union[str, Any]					=     tmp_path_factory.mktemp("data") / "file.txt.zst"
        UpperCamelCase__:	List[str]					=     bytes(A_    ,"utf-8")
        with zstd.open(A_    ,"wb") as f:
            f.write(A_)
        return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Any					=     tmp_path_factory.mktemp("data") / "file.xml"
    UpperCamelCase__:	Dict					=     textwrap.dedent(
        "\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>")
    with open(A_    ,"w") as f:
        f.write(A_)
    return filename


A__:    Optional[Any]  	=	[
    {'''col_1''': '''0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''3''', '''col_2''': 3, '''col_3''': 3.0},
]
A__:    str  	=	[
    {'''col_1''': '''4''', '''col_2''': 4, '''col_3''': 4.0},
    {'''col_1''': '''5''', '''col_2''': 5, '''col_3''': 5.0},
]
A__:    Dict  	=	{
    '''col_1''': ['''0''', '''1''', '''2''', '''3'''],
    '''col_2''': [0, 1, 2, 3],
    '''col_3''': [0.0, 1.0, 2.0, 3.0],
}

A__:    Tuple  	=	[
    {'''col_3''': 0.0, '''col_1''': '''0''', '''col_2''': 0},
    {'''col_3''': 1.0, '''col_1''': '''1''', '''col_2''': 1},
]

A__:    str  	=	[
    {'''col_1''': '''s0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''s1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''s2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''s3''', '''col_2''': 3, '''col_3''': 3.0},
]



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							):
    return DATA_DICT_OF_LISTS



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Any					=     datasets.Dataset.from_dict(A_)
    UpperCamelCase__:	str					=     str(tmp_path_factory.mktemp("data") / "dataset.arrow")
    dataset.map(cache_file_name=A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Tuple					=     str(tmp_path_factory.mktemp("data") / "dataset.sqlite")
    with contextlib.closing(sqlitea.connect(A_)) as con:
        UpperCamelCase__:	int					=     con.cursor()
        cur.execute("CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)")
        for item in DATA:
            cur.execute("INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"    ,tuple(item.values()))
        con.commit()
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Any					=     str(tmp_path_factory.mktemp("data") / "dataset.csv")
    with open(A_    ,"w"    ,newline="") as f:
        UpperCamelCase__:	List[str]					=     csv.DictWriter(A_    ,fieldnames=["col_1", "col_2", "col_3"])
        writer.writeheader()
        for item in DATA:
            writer.writerow(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Optional[Any]					=     str(tmp_path_factory.mktemp("data") / "dataset2.csv")
    with open(A_    ,"w"    ,newline="") as f:
        UpperCamelCase__:	Tuple					=     csv.DictWriter(A_    ,fieldnames=["col_1", "col_2", "col_3"])
        writer.writeheader()
        for item in DATA:
            writer.writerow(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    import bza

    UpperCamelCase__:	str					=     tmp_path_factory.mktemp("data") / "dataset.csv.bz2"
    with open(A_    ,"rb") as f:
        UpperCamelCase__:	Dict					=     f.read()
    # data = bytes(FILE_CONTENT, "utf-8")
    with bza.open(A_    ,"wb") as f:
        f.write(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	Any					=     tmp_path_factory.mktemp("data") / "dataset.csv.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.basename(A_))
        f.write(A_    ,arcname=os.path.basename(A_))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	Dict					=     tmp_path_factory.mktemp("data") / "dataset.csv.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.basename(csv_path.replace(".csv"    ,".CSV")))
        f.write(A_    ,arcname=os.path.basename(csva_path.replace(".csv"    ,".CSV")))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	Optional[int]					=     tmp_path_factory.mktemp("data") / "dataset_with_dir.csv.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.join("main_dir"    ,os.path.basename(A_)))
        f.write(A_    ,arcname=os.path.join("main_dir"    ,os.path.basename(A_)))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Union[str, Any]					=     str(tmp_path_factory.mktemp("data") / "dataset.parquet")
    UpperCamelCase__:	str					=     pa.schema(
        {
            "col_1": pa.string(),
            "col_2": pa.intaa(),
            "col_3": pa.floataa(),
        })
    with open(A_    ,"wb") as f:
        UpperCamelCase__:	int					=     pq.ParquetWriter(A_    ,schema=A_)
        UpperCamelCase__:	List[str]					=     pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(A_))] for k in DATA[0]}    ,schema=A_)
        writer.write_table(A_)
        writer.close()
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	List[Any]					=     str(tmp_path_factory.mktemp("data") / "dataset.json")
    UpperCamelCase__:	Union[str, Any]					=     {"data": DATA}
    with open(A_    ,"w") as f:
        json.dump(A_    ,A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Tuple					=     str(tmp_path_factory.mktemp("data") / "dataset.json")
    UpperCamelCase__:	Union[str, Any]					=     {"data": DATA_DICT_OF_LISTS}
    with open(A_    ,"w") as f:
        json.dump(A_    ,A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Any					=     str(tmp_path_factory.mktemp("data") / "dataset.jsonl")
    with open(A_    ,"w") as f:
        for item in DATA:
            f.write(json.dumps(A_) + "\n")
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	List[str]					=     str(tmp_path_factory.mktemp("data") / "dataset2.jsonl")
    with open(A_    ,"w") as f:
        for item in DATA:
            f.write(json.dumps(A_) + "\n")
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Dict					=     str(tmp_path_factory.mktemp("data") / "dataset_312.jsonl")
    with open(A_    ,"w") as f:
        for item in DATA_312:
            f.write(json.dumps(A_) + "\n")
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Tuple					=     str(tmp_path_factory.mktemp("data") / "dataset-str.jsonl")
    with open(A_    ,"w") as f:
        for item in DATA_STR:
            f.write(json.dumps(A_) + "\n")
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    import gzip

    UpperCamelCase__:	Dict					=     str(tmp_path_factory.mktemp("data") / "dataset.txt.gz")
    with open(A_    ,"rb") as orig_file:
        with gzip.open(A_    ,"wb") as zipped_file:
            zipped_file.writelines(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    import gzip

    UpperCamelCase__:	Dict					=     str(tmp_path_factory.mktemp("data") / "dataset.jsonl.gz")
    with open(A_    ,"rb") as orig_file:
        with gzip.open(A_    ,"wb") as zipped_file:
            zipped_file.writelines(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	int					=     tmp_path_factory.mktemp("data") / "dataset.jsonl.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.basename(A_))
        f.write(A_    ,arcname=os.path.basename(A_))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_    ,A_):
    UpperCamelCase__:	Any					=     tmp_path_factory.mktemp("data") / "dataset_nested.jsonl.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.join("nested"    ,os.path.basename(A_)))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	List[str]					=     tmp_path_factory.mktemp("data") / "dataset_with_dir.jsonl.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.join("main_dir"    ,os.path.basename(A_)))
        f.write(A_    ,arcname=os.path.join("main_dir"    ,os.path.basename(A_)))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	Any					=     tmp_path_factory.mktemp("data") / "dataset.jsonl.tar"
    with tarfile.TarFile(A_    ,"w") as f:
        f.add(A_    ,arcname=os.path.basename(A_))
        f.add(A_    ,arcname=os.path.basename(A_))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_    ,A_):
    UpperCamelCase__:	Dict					=     tmp_path_factory.mktemp("data") / "dataset_nested.jsonl.tar"
    with tarfile.TarFile(A_    ,"w") as f:
        f.add(A_    ,arcname=os.path.join("nested"    ,os.path.basename(A_)))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	int					=     ["0", "1", "2", "3"]
    UpperCamelCase__:	Optional[Any]					=     str(tmp_path_factory.mktemp("data") / "dataset.txt")
    with open(A_    ,"w") as f:
        for item in data:
            f.write(item + "\n")
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	int					=     ["0", "1", "2", "3"]
    UpperCamelCase__:	int					=     str(tmp_path_factory.mktemp("data") / "dataset2.txt")
    with open(A_    ,"w") as f:
        for item in data:
            f.write(item + "\n")
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	List[str]					=     ["0", "1", "2", "3"]
    UpperCamelCase__:	int					=     tmp_path_factory.mktemp("data") / "dataset.abc"
    with open(A_    ,"w") as f:
        for item in data:
            f.write(item + "\n")
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	Dict					=     tmp_path_factory.mktemp("data") / "dataset.text.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.basename(A_))
        f.write(A_    ,arcname=os.path.basename(A_))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	Union[str, Any]					=     tmp_path_factory.mktemp("data") / "dataset_with_dir.text.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.join("main_dir"    ,os.path.basename(A_)))
        f.write(A_    ,arcname=os.path.join("main_dir"    ,os.path.basename(A_)))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	Optional[Any]					=     tmp_path_factory.mktemp("data") / "dataset.ext.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.basename("unsupported.ext"))
        f.write(A_    ,arcname=os.path.basename("unsupported_2.ext"))
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	List[str]					=     "\n".join(["First", "Second\u2029with Unicode new line", "Third"])
    UpperCamelCase__:	Any					=     str(tmp_path_factory.mktemp("data") / "dataset_with_unicode_new_lines.txt")
    with open(A_    ,"w"    ,encoding="utf-8") as f:
        f.write(A_)
    return path



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							):
    return os.path.join("tests"    ,"features"    ,"data"    ,"test_image_rgb.jpg")



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							):
    return os.path.join("tests"    ,"features"    ,"data"    ,"test_audio_44100.wav")



@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_    ,A_):
    UpperCamelCase__:	str					=     tmp_path_factory.mktemp("data") / "dataset.img.zip"
    with zipfile.ZipFile(A_    ,"w") as f:
        f.write(A_    ,arcname=os.path.basename(A_))
        f.write(A_    ,arcname=os.path.basename(A_).replace(".jpg"    ,"2.jpg"))
    return path




@pytest.fixture(scope="session")
def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Dict					=     tmp_path_factory.mktemp("data_dir")

    (data_dir / "subdir").mkdir()
    with open(data_dir / "subdir" / "train.txt"    ,"w") as f:
        f.write("foo\n" * 10)
    with open(data_dir / "subdir" / "test.txt"    ,"w") as f:
        f.write("bar\n" * 10)
    # hidden file
    with open(data_dir / "subdir" / ".test.txt"    ,"w") as f:
        f.write("bar\n" * 10)

    # hidden directory
    (data_dir / ".subdir").mkdir()
    with open(data_dir / ".subdir" / "train.txt"    ,"w") as f:
        f.write("foo\n" * 10)
    with open(data_dir / ".subdir" / "test.txt"    ,"w") as f:
        f.write("bar\n" * 10)

    return data_dir








import argparse

import torch

from transformers import OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()



def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    # Construct model
    if openai_config_file == "":
        UpperCamelCase__:	Optional[Any]					=     OpenAIGPTConfig()
    else:
        UpperCamelCase__:	List[Any]					=     OpenAIGPTConfig.from_json_file(A_)
    UpperCamelCase__:	Optional[Any]					=     OpenAIGPTModel(A_)

    # Load weights from numpy
    load_tf_weights_in_openai_gpt(A_    ,A_    ,A_)

    # Save pytorch-model
    UpperCamelCase__:	Tuple					=     pytorch_dump_folder_path + "/" + WEIGHTS_NAME
    UpperCamelCase__:	List[Any]					=     pytorch_dump_folder_path + "/" + CONFIG_NAME
    print(F"Save PyTorch model to {pytorch_weights_dump_path}")
    torch.save(model.state_dict()    ,A_)
    print(F"Save configuration file to {pytorch_config_dump_path}")
    with open(A_    ,"w"    ,encoding="utf-8") as f:
        f.write(config.to_json_string())


if __name__ == "__main__":
   A__:    List[str]  	=	argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '''--openai_checkpoint_folder_path''',
       default=None,
       type=str,
       required=True,
       help='''Path to the TensorFlow checkpoint path.''',
   )
   parser.add_argument(
       '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
   )
   parser.add_argument(
       '''--openai_config_file''',
       default='''''',
       type=str,
       help=(
           '''An optional config json file corresponding to the pre-trained OpenAI model. \n'''
           '''This specifies the model architecture.'''
       ),
   )
   A__:    Union[str, Any]  	=	parser.parse_args()
   convert_openai_checkpoint_to_pytorch(
       args.openai_checkpoint_folder_path, args.openai_config_file, args.pytorch_dump_folder_path
   )

