



import argparse
import json
from collections import OrderedDict
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    SegformerConfig,
    SegformerForImageClassification,
    SegformerForSemanticSegmentation,
    SegformerImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
_lowercase						  =		logging.get_logger(__name__)



def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       List[Any]=False	)->  List[Any]:
    A__		 =					OrderedDict()
    for key, value in state_dict.items():
        if encoder_only and not key.startswith('''head'''	):
            A__		 =					'''segformer.encoder.''' + key
        if key.startswith('''backbone'''	):
            A__		 =					key.replace('''backbone''' ,       '''segformer.encoder'''	)
        if "patch_embed" in key:
            # replace for example patch_embed1 by patch_embeddings.0
            A__		 =					key[key.find('''patch_embed'''	) + len('''patch_embed'''	)]
            A__		 =					key.replace(f"patch_embed{idx}" ,       f"patch_embeddings.{int(UpperCamelCase_	)-1}"	)
        if "norm" in key:
            A__		 =					key.replace('''norm''' ,       '''layer_norm'''	)
        if "segformer.encoder.layer_norm" in key:
            # replace for example layer_norm1 by layer_norm.0
            A__		 =					key[key.find('''segformer.encoder.layer_norm'''	) + len('''segformer.encoder.layer_norm'''	)]
            A__		 =					key.replace(f"layer_norm{idx}" ,       f"layer_norm.{int(UpperCamelCase_	)-1}"	)
        if "layer_norm1" in key:
            A__		 =					key.replace('''layer_norm1''' ,       '''layer_norm_1'''	)
        if "layer_norm2" in key:
            A__		 =					key.replace('''layer_norm2''' ,       '''layer_norm_2'''	)
        if "block" in key:
            # replace for example block1 by block.0
            A__		 =					key[key.find('''block'''	) + len('''block'''	)]
            A__		 =					key.replace(f"block{idx}" ,       f"block.{int(UpperCamelCase_	)-1}"	)
        if "attn.q" in key:
            A__		 =					key.replace('''attn.q''' ,       '''attention.self.query'''	)
        if "attn.proj" in key:
            A__		 =					key.replace('''attn.proj''' ,       '''attention.output.dense'''	)
        if "attn" in key:
            A__		 =					key.replace('''attn''' ,       '''attention.self'''	)
        if "fc1" in key:
            A__		 =					key.replace('''fc1''' ,       '''dense1'''	)
        if "fc2" in key:
            A__		 =					key.replace('''fc2''' ,       '''dense2'''	)
        if "linear_pred" in key:
            A__		 =					key.replace('''linear_pred''' ,       '''classifier'''	)
        if "linear_fuse" in key:
            A__		 =					key.replace('''linear_fuse.conv''' ,       '''linear_fuse'''	)
            A__		 =					key.replace('''linear_fuse.bn''' ,       '''batch_norm'''	)
        if "linear_c" in key:
            # replace for example linear_c4 by linear_c.3
            A__		 =					key[key.find('''linear_c'''	) + len('''linear_c'''	)]
            A__		 =					key.replace(f"linear_c{idx}" ,       f"linear_c.{int(UpperCamelCase_	)-1}"	)
        if key.startswith('''head'''	):
            A__		 =					key.replace('''head''' ,       '''classifier'''	)
        A__		 =					value

    return new_state_dict



def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict ,       UpperCamelCase_				:       List[Any]	)->  Optional[Any]:
    # for each of the encoder blocks:
    for i in range(config.num_encoder_blocks	):
        for j in range(config.depths[i]	):
            # read in weights + bias of keys and values (which is a single matrix in the original implementation)
            A__		 =					state_dict.pop(f"segformer.encoder.block.{i}.{j}.attention.self.kv.weight"	)
            A__		 =					state_dict.pop(f"segformer.encoder.block.{i}.{j}.attention.self.kv.bias"	)
            # next, add keys and values (in that order) to the state dict
            A__		 =					kv_weight[
                : config.hidden_sizes[i], :
            ]
            A__		 =					kv_bias[: config.hidden_sizes[i]]
            A__		 =					kv_weight[
                config.hidden_sizes[i] :, :
            ]
            A__		 =					kv_bias[
                config.hidden_sizes[i] :
            ]



def 		lowerCAmelCase__    (					)->  str:
    A__		 =					'''http://images.cocodataset.org/val2017/000000039769.jpg'''
    A__		 =					Image.open(requests.get(UpperCamelCase_ ,       stream=UpperCamelCase_	).raw	)

    return image



@torch.no_grad()
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    A__		 =					SegformerConfig()
    A__		 =					False

    # set attributes based on model_name
    A__		 =					'''huggingface/label-files'''
    if "segformer" in model_name:
        A__		 =					model_name[len('''segformer.'''	) : len('''segformer.'''	) + 2]
        if "ade" in model_name:
            A__		 =					1_5_0
            A__		 =					'''ade20k-id2label.json'''
            A__		 =					(1, 1_5_0, 1_2_8, 1_2_8)
        elif "city" in model_name:
            A__		 =					1_9
            A__		 =					'''cityscapes-id2label.json'''
            A__		 =					(1, 1_9, 1_2_8, 1_2_8)
        else:
            raise ValueError(f"Model {model_name} not supported"	)
    elif "mit" in model_name:
        A__		 =					True
        A__		 =					model_name[4:6]
        A__		 =					1_0_0_0
        A__		 =					'''imagenet-1k-id2label.json'''
        A__		 =					(1, 1_0_0_0)
    else:
        raise ValueError(f"Model {model_name} not supported"	)

    # set config attributes
    A__		 =					json.load(open(hf_hub_download(UpperCamelCase_ ,       UpperCamelCase_ ,       repo_type='''dataset'''	) ,       '''r'''	)	)
    A__		 =					{int(UpperCamelCase_	): v for k, v in idalabel.items()}
    A__		 =					idalabel
    A__		 =					{v: k for k, v in idalabel.items()}
    if size == "b0":
        pass
    elif size == "b1":
        A__		 =					[6_4, 1_2_8, 3_2_0, 5_1_2]
        A__		 =					2_5_6
    elif size == "b2":
        A__		 =					[6_4, 1_2_8, 3_2_0, 5_1_2]
        A__		 =					7_6_8
        A__		 =					[3, 4, 6, 3]
    elif size == "b3":
        A__		 =					[6_4, 1_2_8, 3_2_0, 5_1_2]
        A__		 =					7_6_8
        A__		 =					[3, 4, 1_8, 3]
    elif size == "b4":
        A__		 =					[6_4, 1_2_8, 3_2_0, 5_1_2]
        A__		 =					7_6_8
        A__		 =					[3, 8, 2_7, 3]
    elif size == "b5":
        A__		 =					[6_4, 1_2_8, 3_2_0, 5_1_2]
        A__		 =					7_6_8
        A__		 =					[3, 6, 4_0, 3]
    else:
        raise ValueError(f"Size {size} not supported"	)

    # load image processor (only resize + normalize)
    A__		 =					SegformerImageProcessor(
        image_scale=(5_1_2, 5_1_2) ,       keep_ratio=UpperCamelCase_ ,       align=UpperCamelCase_ ,       do_random_crop=UpperCamelCase_	)

    # prepare image
    A__		 =					prepare_img()
    A__		 =					image_processor(images=UpperCamelCase_ ,       return_tensors='''pt'''	).pixel_values

    logger.info(f"Converting model {model_name}..."	)

    # load original state dict
    if encoder_only:
        A__		 =					torch.load(UpperCamelCase_ ,       map_location=torch.device('''cpu'''	)	)
    else:
        A__		 =					torch.load(UpperCamelCase_ ,       map_location=torch.device('''cpu'''	)	)['''state_dict''']

    # rename keys
    A__		 =					rename_keys(UpperCamelCase_ ,       encoder_only=UpperCamelCase_	)
    if not encoder_only:
        del state_dict["decode_head.conv_seg.weight"]
        del state_dict["decode_head.conv_seg.bias"]

    # key and value matrices need special treatment
    read_in_k_v(UpperCamelCase_ ,       UpperCamelCase_	)

    # create HuggingFace model and load state dict
    if encoder_only:
        A__		 =					False
        A__		 =					SegformerForImageClassification(UpperCamelCase_	)
    else:
        A__		 =					SegformerForSemanticSegmentation(UpperCamelCase_	)
    model.load_state_dict(UpperCamelCase_	)
    model.eval()

    # forward pass
    A__		 =					model(UpperCamelCase_	)
    A__		 =					outputs.logits

    # set expected_slice based on model name
    # ADE20k checkpoints
    if model_name == "segformer.b0.512x512.ade.160k":
        A__		 =					torch.tensor(
            [
                [[-4.6310, -5.5232, -6.2356], [-5.1921, -6.1444, -6.5996], [-5.4424, -6.2790, -6.7574]],
                [[-12.1391, -13.3122, -13.9554], [-12.8732, -13.9352, -14.3563], [-12.9438, -13.8226, -14.2513]],
                [[-12.5134, -13.4686, -14.4915], [-12.8669, -14.4343, -14.7758], [-13.2523, -14.5819, -15.0694]],
            ]	)
    elif model_name == "segformer.b1.512x512.ade.160k":
        A__		 =					torch.tensor(
            [
                [[-7.5820, -8.7231, -8.3215], [-8.0600, -10.3529, -10.0304], [-7.5208, -9.4103, -9.6239]],
                [[-12.6918, -13.8994, -13.7137], [-13.3196, -15.7523, -15.4789], [-12.9343, -14.8757, -14.9689]],
                [[-11.1911, -11.9421, -11.3243], [-11.3342, -13.6839, -13.3581], [-10.3909, -12.1832, -12.4858]],
            ]	)
    elif model_name == "segformer.b2.512x512.ade.160k":
        A__		 =					torch.tensor(
            [
                [[-11.8173, -14.3850, -16.3128], [-14.5648, -16.5804, -18.6568], [-14.7223, -15.7387, -18.4218]],
                [[-15.7290, -17.9171, -19.4423], [-18.3105, -19.9448, -21.4661], [-17.9296, -18.6497, -20.7910]],
                [[-15.0783, -17.0336, -18.2789], [-16.8771, -18.6870, -20.1612], [-16.2454, -17.1426, -19.5055]],
            ]	)
    elif model_name == "segformer.b3.512x512.ade.160k":
        A__		 =					torch.tensor(
            [
                [[-9.0878, -10.2081, -10.1891], [-9.3144, -10.7941, -10.9843], [-9.2294, -10.3855, -10.5704]],
                [[-12.2316, -13.9068, -13.6102], [-12.9161, -14.3702, -14.3235], [-12.5233, -13.7174, -13.7932]],
                [[-14.6275, -15.2490, -14.9727], [-14.3400, -15.9687, -16.2827], [-14.1484, -15.4033, -15.8937]],
            ]	)
    elif model_name == "segformer.b4.512x512.ade.160k":
        A__		 =					torch.tensor(
            [
                [[-12.3144, -13.2447, -14.0802], [-13.3614, -14.5816, -15.6117], [-13.3340, -14.4433, -16.2219]],
                [[-19.2781, -20.4128, -20.7506], [-20.6153, -21.6566, -22.0998], [-19.9800, -21.0430, -22.1494]],
                [[-18.8739, -19.7804, -21.1834], [-20.1233, -21.6765, -23.2944], [-20.0315, -21.2641, -23.6944]],
            ]	)
    elif model_name == "segformer.b5.640x640.ade.160k":
        A__		 =					torch.tensor(
            [
                [[-9.5524, -12.0835, -11.7348], [-10.5229, -13.6446, -14.5662], [-9.5842, -12.8851, -13.9414]],
                [[-15.3432, -17.5323, -17.0818], [-16.3330, -18.9255, -19.2101], [-15.1340, -17.7848, -18.3971]],
                [[-12.6072, -14.9486, -14.6631], [-13.7629, -17.0907, -17.7745], [-12.7899, -16.1695, -17.1671]],
            ]	)
    # Cityscapes checkpoints
    elif model_name == "segformer.b0.1024x1024.city.160k":
        A__		 =					torch.tensor(
            [
                [[-11.9295, -13.4057, -14.8106], [-13.3431, -14.8179, -15.3781], [-14.2836, -15.5942, -16.1588]],
                [[-11.4906, -12.8067, -13.6564], [-13.1189, -14.0500, -14.1543], [-13.8748, -14.5136, -14.8789]],
                [[0.5374, 0.1067, -0.4742], [0.1141, -0.2255, -0.7099], [-0.3000, -0.5924, -1.3105]],
            ]	)
    elif model_name == "segformer.b0.512x1024.city.160k":
        A__		 =					torch.tensor(
            [
                [[-7.8217, -9.8767, -10.1717], [-9.4438, -10.9058, -11.4047], [-9.7939, -12.3495, -12.1079]],
                [[-7.1514, -9.5336, -10.0860], [-9.7776, -11.6822, -11.8439], [-10.1411, -12.7655, -12.8972]],
                [[0.3021, 0.0805, -0.2310], [-0.0328, -0.1605, -0.2714], [-0.1408, -0.5477, -0.6976]],
            ]	)
    elif model_name == "segformer.b0.640x1280.city.160k":
        A__		 =					torch.tensor(
            [
                [
                    [-1.1372E01, -1.2787E01, -1.3477E01],
                    [-1.2536E01, -1.4194E01, -1.4409E01],
                    [-1.3217E01, -1.4888E01, -1.5327E01],
                ],
                [
                    [-1.4791E01, -1.7122E01, -1.8277E01],
                    [-1.7163E01, -1.9192E01, -1.9533E01],
                    [-1.7897E01, -1.9991E01, -2.0315E01],
                ],
                [
                    [7.6723E-01, 4.1921E-01, -7.7878E-02],
                    [4.7772E-01, 9.5557E-03, -2.8082E-01],
                    [3.6032E-01, -2.4826E-01, -5.1168E-01],
                ],
            ]	)
    elif model_name == "segformer.b0.768x768.city.160k":
        A__		 =					torch.tensor(
            [
                [[-9.4959, -11.3087, -11.7479], [-11.0025, -12.6540, -12.3319], [-11.4064, -13.0487, -12.9905]],
                [[-9.8905, -11.3084, -12.0854], [-11.1726, -12.7698, -12.9583], [-11.5985, -13.3278, -14.1774]],
                [[0.2213, 0.0192, -0.2466], [-0.1731, -0.4213, -0.4874], [-0.3126, -0.6541, -1.1389]],
            ]	)
    elif model_name == "segformer.b1.1024x1024.city.160k":
        A__		 =					torch.tensor(
            [
                [[-13.5748, -13.9111, -12.6500], [-14.3500, -15.3683, -14.2328], [-14.7532, -16.0424, -15.6087]],
                [[-17.1651, -15.8725, -12.9653], [-17.2580, -17.3718, -14.8223], [-16.6058, -16.8783, -16.7452]],
                [[-3.6456, -3.0209, -1.4203], [-3.0797, -3.1959, -2.0000], [-1.8757, -1.9217, -1.6997]],
            ]	)
    elif model_name == "segformer.b2.1024x1024.city.160k":
        A__		 =					torch.tensor(
            [
                [[-16.0976, -16.4856, -17.3962], [-16.6234, -19.0342, -19.7685], [-16.0900, -18.0661, -19.1180]],
                [[-18.4750, -18.8488, -19.5074], [-19.4030, -22.1570, -22.5977], [-19.1191, -20.8486, -22.3783]],
                [[-4.5178, -5.5037, -6.5109], [-5.0884, -7.2174, -8.0334], [-4.4156, -5.8117, -7.2970]],
            ]	)
    elif model_name == "segformer.b3.1024x1024.city.160k":
        A__		 =					torch.tensor(
            [
                [[-14.2081, -14.4732, -14.1977], [-14.5867, -16.4423, -16.6356], [-13.4441, -14.9685, -16.8696]],
                [[-14.4576, -14.7073, -15.0451], [-15.0816, -17.6237, -17.9873], [-14.4213, -16.0199, -18.5992]],
                [[-4.7349, -4.9588, -5.0966], [-4.3210, -6.9325, -7.2591], [-3.4312, -4.7484, -7.1917]],
            ]	)
    elif model_name == "segformer.b4.1024x1024.city.160k":
        A__		 =					torch.tensor(
            [
                [[-11.7737, -11.9526, -11.3273], [-13.6692, -14.4574, -13.8878], [-13.8937, -14.6924, -15.9345]],
                [[-14.6706, -14.5330, -14.1306], [-16.1502, -16.8180, -16.4269], [-16.8338, -17.8939, -20.1746]],
                [[1.0491, 0.8289, 1.0310], [1.1044, 0.5219, 0.8055], [1.0899, 0.6926, 0.5590]],
            ]	)
    elif model_name == "segformer.b5.1024x1024.city.160k":
        A__		 =					torch.tensor(
            [
                [[-12.5641, -13.4777, -13.0684], [-13.9587, -15.8983, -16.6557], [-13.3109, -15.7350, -16.3141]],
                [[-14.7074, -15.4352, -14.5944], [-16.6353, -18.1663, -18.6120], [-15.1702, -18.0329, -18.1547]],
                [[-1.7990, -2.0951, -1.7784], [-2.6397, -3.8245, -3.9686], [-1.5264, -2.8126, -2.9316]],
            ]	)
    else:
        A__		 =					logits.argmax(-1	).item()
        print('''Predicted class:''' ,       model.config.idalabel[predicted_class_idx]	)

    # verify logits
    if not encoder_only:
        assert logits.shape == expected_shape
        assert torch.allclose(logits[0, :3, :3, :3] ,       UpperCamelCase_ ,       atol=1E-2	)

    # finally, save model and image processor
    logger.info(f"Saving PyTorch model and image processor to {pytorch_dump_folder_path}..."	)
    Path(UpperCamelCase_	).mkdir(exist_ok=UpperCamelCase_	)
    model.save_pretrained(UpperCamelCase_	)
    image_processor.save_pretrained(UpperCamelCase_	)


if __name__ == "__main__":
   _lowercase						  =		argparse.ArgumentParser()

   parser.add_argument(
       "--model_name",
       default="segformer.b0.512x512.ade.160k",
       type=str,
       help="Name of the model you'd like to convert.",
   )
   parser.add_argument(
       "--checkpoint_path", default=None, type=str, help="Path to the original PyTorch checkpoint (.pth file)."
   )
   parser.add_argument(
       "--pytorch_dump_folder_path", default=None, type=str, help="Path to the folder to output PyTorch model."
   )
   _lowercase						  =		parser.parse_args()
   convert_segformer_checkpoint(args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path)





import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    A__		 =					1_0
    A__		 =					datasets.Features(
        {
            '''tokens''': datasets.Sequence(datasets.Value('''string'''	)	),
            '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']	)	),
            '''answers''': datasets.Sequence(
                {
                    '''text''': datasets.Value('''string'''	),
                    '''answer_start''': datasets.Value('''int32'''	),
                }	),
            '''id''': datasets.Value('''int64'''	),
        }	)
    A__		 =					datasets.Dataset.from_dict(
        {
            '''tokens''': [['''foo'''] * 5] * n,
            '''labels''': [[1] * 5] * n,
            '''answers''': [{'''answer_start''': [9_7], '''text''': ['''1976''']}] * 1_0,
            '''id''': list(range(UpperCamelCase_	)	),
        } ,       features=UpperCamelCase_ ,       )
    return dataset



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return filename


# FILE_CONTENT + files


_lowercase						  =		"\\n    Text data.\n    Second line of data."



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt'''
    A__		 =					FILE_CONTENT
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.bz2'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  int:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''file.txt.gz'''	)
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with gzip.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Any:
    if datasets.config.LZ4_AVAILABLE:
        import lza.frame

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.lz4'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with lza.frame.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Tuple	)->  int:
    if datasets.config.PY7ZR_AVAILABLE:
        import pyazr

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.7z'''
        with pyazr.SevenZipFile(UpperCamelCase_ ,       '''w'''	) as archive:
            archive.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       int	)->  Optional[Any]:
    import tarfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    import lzma

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.xz'''
    A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
    with lzma.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       List[str]	)->  List[str]:
    import zipfile

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Tuple	)->  str:
    if datasets.config.ZSTANDARD_AVAILABLE:
        import zstandard as zstd

        A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.txt.zst'''
        A__		 =					bytes(UpperCamelCase_ ,       '''utf-8'''	)
        with zstd.open(UpperCamelCase_ ,       '''wb'''	) as f:
            f.write(UpperCamelCase_	)
        return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  int:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''file.xml'''
    A__		 =					textwrap.dedent(
        '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_	)
    return filename


_lowercase						  =		[
    {"col_1": "0", "col_2": 0, "col_3": 0.0},
    {"col_1": "1", "col_2": 1, "col_3": 1.0},
    {"col_1": "2", "col_2": 2, "col_3": 2.0},
    {"col_1": "3", "col_2": 3, "col_3": 3.0},
]
_lowercase						  =		[
    {"col_1": "4", "col_2": 4, "col_3": 4.0},
    {"col_1": "5", "col_2": 5, "col_3": 5.0},
]
_lowercase						  =		{
    "col_1": ["0", "1", "2", "3"],
    "col_2": [0, 1, 2, 3],
    "col_3": [0.0, 1.0, 2.0, 3.0],
}

_lowercase						  =		[
    {"col_3": 0.0, "col_1": "0", "col_2": 0},
    {"col_3": 1.0, "col_1": "1", "col_2": 1},
]

_lowercase						  =		[
    {"col_1": "s0", "col_2": 0, "col_3": 0.0},
    {"col_1": "s1", "col_2": 1, "col_3": 1.0},
    {"col_1": "s2", "col_2": 2, "col_3": 2.0},
    {"col_1": "s3", "col_2": 3, "col_3": 3.0},
]



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  str:
    return DATA_DICT_OF_LISTS



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any]	)->  List[str]:
    A__		 =					datasets.Dataset.from_dict(UpperCamelCase_	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.arrow'''	)
    dataset.map(cache_file_name=UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[str]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.sqlite'''	)
    with contextlib.closing(sqlitea.connect(UpperCamelCase_	)	) as con:
        A__		 =					con.cursor()
        cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''	)
        for item in DATA:
            cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)''' ,       tuple(item.values()	)	)
        con.commit()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.csv'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       newline=''''''	) as f:
        A__		 =					csv.DictWriter(UpperCamelCase_ ,       fieldnames=['''col_1''', '''col_2''', '''col_3''']	)
        writer.writeheader()
        for item in DATA:
            writer.writerow(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       Optional[int]	)->  List[str]:
    import bza

    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.bz2'''
    with open(UpperCamelCase_ ,       '''rb'''	) as f:
        A__		 =					f.read()
    # data = bytes(FILE_CONTENT, "utf-8")
    with bza.open(UpperCamelCase_ ,       '''wb'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Any	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csv_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(csva_path.replace('''.csv''' ,       '''.CSV'''	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str ,       UpperCamelCase_				:       str ,       UpperCamelCase_				:       Optional[Any]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.csv.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.parquet'''	)
    A__		 =					pa.schema(
        {
            '''col_1''': pa.string(),
            '''col_2''': pa.intaa(),
            '''col_3''': pa.floataa(),
        }	)
    with open(UpperCamelCase_ ,       '''wb'''	) as f:
        A__		 =					pq.ParquetWriter(UpperCamelCase_ ,       schema=UpperCamelCase_	)
        A__		 =					pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCamelCase_	)	)] for k in DATA[0]} ,       schema=UpperCamelCase_	)
        writer.write_table(UpperCamelCase_	)
        writer.close()
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  Optional[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.json'''	)
    A__		 =					{'''data''': DATA_DICT_OF_LISTS}
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        json.dump(UpperCamelCase_ ,       UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  List[Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Optional[int]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_312.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_312:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Union[str, Any]:
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset-str.jsonl'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in DATA_STR:
            f.write(json.dumps(UpperCamelCase_	) + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       str	)->  List[str]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[Any]:
    import gzip

    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.gz'''	)
    with open(UpperCamelCase_ ,       '''rb'''	) as orig_file:
        with gzip.open(UpperCamelCase_ ,       '''wb'''	) as zipped_file:
            zipped_file.writelines(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  Optional[int]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Union[str, Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.jsonl.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       str	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.add(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int] ,       UpperCamelCase_				:       int ,       UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       List[str]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_nested.jsonl.tar'''
    with tarfile.TarFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.add(UpperCamelCase_ ,       arcname=os.path.join('''nested''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any	)->  List[str]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       int	)->  Dict:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset2.txt'''	)
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any]	)->  Union[str, Any]:
    A__		 =					['''0''', '''1''', '''2''', '''3''']
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.abc'''
    with open(UpperCamelCase_ ,       '''w'''	) as f:
        for item in data:
            f.write(item + '''\n'''	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[str] ,       UpperCamelCase_				:       Any ,       UpperCamelCase_				:       Optional[Any]	)->  List[str]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Any ,       UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       Optional[int]	)->  Tuple:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset_with_dir.text.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.join('''main_dir''' ,       os.path.basename(UpperCamelCase_	)	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any] ,       UpperCamelCase_				:       Tuple ,       UpperCamelCase_				:       Tuple	)->  Any:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.ext.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported.ext'''	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename('''unsupported_2.ext'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Dict	)->  Any:
    A__		 =					'''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']	)
    A__		 =					str(tmp_path_factory.mktemp('''data'''	) / '''dataset_with_unicode_new_lines.txt'''	)
    with open(UpperCamelCase_ ,       '''w''' ,       encoding='''utf-8'''	) as f:
        f.write(UpperCamelCase_	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Tuple:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_image_rgb.jpg'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					)->  Dict:
    return os.path.join('''tests''' ,       '''features''' ,       '''data''' ,       '''test_audio_44100.wav'''	)



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Union[str, Any] ,       UpperCamelCase_				:       Union[str, Any]	)->  List[Any]:
    A__		 =					tmp_path_factory.mktemp('''data'''	) / '''dataset.img.zip'''
    with zipfile.ZipFile(UpperCamelCase_ ,       '''w'''	) as f:
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	)	)
        f.write(UpperCamelCase_ ,       arcname=os.path.basename(UpperCamelCase_	).replace('''.jpg''' ,       '''2.jpg'''	)	)
    return path



@pytest.fixture(scope='''session'''	)
def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[Any]	)->  str:
    A__		 =					tmp_path_factory.mktemp('''data_dir'''	)

    (data_dir / "subdir").mkdir()
    with open(data_dir / '''subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)
    # hidden file
    with open(data_dir / '''subdir''' / '''.test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    # hidden directory
    (data_dir / ".subdir").mkdir()
    with open(data_dir / '''.subdir''' / '''train.txt''' ,       '''w'''	) as f:
        f.write('''foo\n''' * 1_0	)
    with open(data_dir / '''.subdir''' / '''test.txt''' ,       '''w'''	) as f:
        f.write('''bar\n''' * 1_0	)

    return data_dir
