


'''simple docstring'''






import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					Any:
							lowerCAmelCase__      =			10
							lowerCAmelCase__      =			datasets.Features(
							    {
							        '''tokens''': datasets.Sequence(datasets.Value('''string''' ) ),
							        '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive'''] ) ),
							        '''answers''': datasets.Sequence(
							            {
							                '''text''': datasets.Value('''string''' ),
							                '''answer_start''': datasets.Value('''int32''' ),
							            } ),
							        '''id''': datasets.Value('''int64''' ),
							    } )
							lowerCAmelCase__      =			datasets.Dataset.from_dict(
							    {
							        '''tokens''': [['''foo'''] * 5] * n,
							        '''labels''': [[1] * 5] * n,
							        '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
							        '''id''': list(range(A ) ),
							    }				,					features=A				,					)
							return dataset


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					List[Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''file.arrow''' )
							dataset.map(cache_file_name=A )
							return filename


# FILE_CONTENT + files


__UpperCAmelCase					       =							'''\
    Text data.
    Second line of data.'''


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt'''
							lowerCAmelCase__      =			FILE_CONTENT
							with open(A				,					'''w''' ) as f:
														f.write(A )
							return filename


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							import bza

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.bz2'''
							lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
							with bza.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Tuple:
							import gzip

							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''file.txt.gz''' )
							lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
							with gzip.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					List[str]:
							if datasets.config.LZ4_AVAILABLE:
														import lza.frame

														lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.lz4'''
														lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
														with lza.frame.open(A				,					'''wb''' ) as f:
																					f.write(A )
														return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					int:
							if datasets.config.PY7ZR_AVAILABLE:
														import pyazr

														lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.7z'''
														with pyazr.SevenZipFile(A				,					'''w''' ) as archive:
																					archive.write(A				,					arcname=os.path.basename(A ) )
														return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					str:
							import tarfile

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.tar'''
							with tarfile.TarFile(A				,					'''w''' ) as f:
														f.add(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[int]:
							import lzma

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.xz'''
							lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
							with lzma.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					List[str]:
							import zipfile

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							if datasets.config.ZSTANDARD_AVAILABLE:
														import zstandard as zstd

														lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.zst'''
														lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
														with zstd.open(A				,					'''wb''' ) as f:
																					f.write(A )
														return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.xml'''
							lowerCAmelCase__      =			textwrap.dedent(
							    '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>''' )
							with open(A				,					'''w''' ) as f:
														f.write(A )
							return filename


__UpperCAmelCase					       =							[
    {'''col_1''': '''0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''3''', '''col_2''': 3, '''col_3''': 3.0},
]
__UpperCAmelCase					       =							[
    {'''col_1''': '''4''', '''col_2''': 4, '''col_3''': 4.0},
    {'''col_1''': '''5''', '''col_2''': 5, '''col_3''': 5.0},
]
__UpperCAmelCase					       =							{
    '''col_1''': ['''0''', '''1''', '''2''', '''3'''],
    '''col_2''': [0, 1, 2, 3],
    '''col_3''': [0.0, 1.0, 2.0, 3.0],
}

__UpperCAmelCase					       =							[
    {'''col_3''': 0.0, '''col_1''': '''0''', '''col_2''': 0},
    {'''col_3''': 1.0, '''col_1''': '''1''', '''col_2''': 1},
]

__UpperCAmelCase					       =							[
    {'''col_1''': '''s0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''s1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''s2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''s3''', '''col_2''': 3, '''col_3''': 3.0},
]


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					Union[str, Any]:
							return DATA_DICT_OF_LISTS


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[Any]:
							lowerCAmelCase__      =			datasets.Dataset.from_dict(A )
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.arrow''' )
							dataset.map(cache_file_name=A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.sqlite''' )
							with contextlib.closing(sqlitea.connect(A ) ) as con:
														lowerCAmelCase__      =			con.cursor()
														cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)''' )
														for item in DATA:
																					cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''				,					tuple(item.values() ) )
														con.commit()
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.csv''' )
							with open(A				,					'''w'''				,					newline='''''' ) as f:
														lowerCAmelCase__      =			csv.DictWriter(A				,					fieldnames=['''col_1''', '''col_2''', '''col_3'''] )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[int]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset2.csv''' )
							with open(A				,					'''w'''				,					newline='''''' ) as f:
														lowerCAmelCase__      =			csv.DictWriter(A				,					fieldnames=['''col_1''', '''col_2''', '''col_3'''] )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					Optional[int]:
							import bza

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.csv.bz2'''
							with open(A				,					'''rb''' ) as f:
														lowerCAmelCase__      =			f.read()
							# data = bytes(FILE_CONTENT, "utf-8")
							with bza.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[str]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.csv.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Dict:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.csv.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(csv_path.replace('''.csv'''				,					'''.CSV''' ) ) )
														f.write(A				,					arcname=os.path.basename(csva_path.replace('''.csv'''				,					'''.CSV''' ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_with_dir.csv.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.parquet''' )
							lowerCAmelCase__      =			pa.schema(
							    {
							        '''col_1''': pa.string(),
							        '''col_2''': pa.intaa(),
							        '''col_3''': pa.floataa(),
							    } )
							with open(A				,					'''wb''' ) as f:
														lowerCAmelCase__      =			pq.ParquetWriter(A				,					schema=A )
														lowerCAmelCase__      =			pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(A ) )] for k in DATA[0]}				,					schema=A )
														writer.write_table(A )
														writer.close()
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.json''' )
							lowerCAmelCase__      =			{'''data''': DATA}
							with open(A				,					'''w''' ) as f:
														json.dump(A				,					A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.json''' )
							lowerCAmelCase__      =			{'''data''': DATA_DICT_OF_LISTS}
							with open(A				,					'''w''' ) as f:
														json.dump(A				,					A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset2.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					List[str]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset_312.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA_312:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					List[Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset-str.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA_STR:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					List[str]:
							import gzip

							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.txt.gz''' )
							with open(A				,					'''rb''' ) as orig_file:
														with gzip.open(A				,					'''wb''' ) as zipped_file:
																					zipped_file.writelines(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					Dict:
							import gzip

							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl.gz''' )
							with open(A				,					'''rb''' ) as orig_file:
														with gzip.open(A				,					'''wb''' ) as zipped_file:
																					zipped_file.writelines(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					str:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A				,					A )		->					Optional[Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_nested.jsonl.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''nested'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_with_dir.jsonl.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[str]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl.tar'''
							with tarfile.TarFile(A				,					'''w''' ) as f:
														f.add(A				,					arcname=os.path.basename(A ) )
														f.add(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_nested.jsonl.tar'''
							with tarfile.TarFile(A				,					'''w''' ) as f:
														f.add(A				,					arcname=os.path.join('''nested'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							lowerCAmelCase__      =			['''0''', '''1''', '''2''', '''3''']
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.txt''' )
							with open(A				,					'''w''' ) as f:
														for item in data:
																					f.write(item + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							lowerCAmelCase__      =			['''0''', '''1''', '''2''', '''3''']
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset2.txt''' )
							with open(A				,					'''w''' ) as f:
														for item in data:
																					f.write(item + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							lowerCAmelCase__      =			['''0''', '''1''', '''2''', '''3''']
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.abc'''
							with open(A				,					'''w''' ) as f:
														for item in data:
																					f.write(item + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.text.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Union[str, Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_with_dir.text.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.ext.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename('''unsupported.ext''' ) )
														f.write(A				,					arcname=os.path.basename('''unsupported_2.ext''' ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Tuple:
							lowerCAmelCase__      =			'''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third'''] )
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset_with_unicode_new_lines.txt''' )
							with open(A				,					'''w'''				,					encoding='''utf-8''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					List[str]:
							return os.path.join('''tests'''				,					'''features'''				,					'''data'''				,					'''test_image_rgb.jpg''' )


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					Any:
							return os.path.join('''tests'''				,					'''features'''				,					'''data'''				,					'''test_audio_44100.wav''' )


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					Union[str, Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.img.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ).replace('''.jpg'''				,					'''2.jpg''' ) )
							return path






@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					int:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data_dir''' )

							(data_dir / "subdir").mkdir()
							with open(data_dir / '''subdir''' / '''train.txt'''				,					'''w''' ) as f:
														f.write('''foo\n''' * 10 )
							with open(data_dir / '''subdir''' / '''test.txt'''				,					'''w''' ) as f:
														f.write('''bar\n''' * 10 )
							# hidden file
							with open(data_dir / '''subdir''' / '''.test.txt'''				,					'''w''' ) as f:
														f.write('''bar\n''' * 10 )

							# hidden directory
							(data_dir / ".subdir").mkdir()
							with open(data_dir / '''.subdir''' / '''train.txt'''				,					'''w''' ) as f:
														f.write('''foo\n''' * 10 )
							with open(data_dir / '''.subdir''' / '''test.txt'''				,					'''w''' ) as f:
														f.write('''bar\n''' * 10 )

							return data_dir



'''simple docstring'''






import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    WavaVecaConformerConfig,
    WavaVecaConformerForCTC,
    WavaVecaConformerForPreTraining,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{
    '''post_extract_proj''': '''feature_projection.projection''',
    '''encoder.pos_conv.0''': '''encoder.pos_conv_embed.conv''',
    '''self_attn.linear_k''': '''encoder.layers.*.self_attn.linear_k''',
    '''self_attn.linear_v''': '''encoder.layers.*.self_attn.linear_v''',
    '''self_attn.linear_q''': '''encoder.layers.*.self_attn.linear_q''',
    '''self_attn.pos_bias_u''': '''encoder.layers.*.self_attn.pos_bias_u''',
    '''self_attn.pos_bias_v''': '''encoder.layers.*.self_attn.pos_bias_v''',
    '''self_attn.linear_out''': '''encoder.layers.*.self_attn.linear_out''',
    '''self_attn.linear_pos''': '''encoder.layers.*.self_attn.linear_pos''',
    '''self_attn.rotary_emb''': '''encoder.embed_positions''',
    '''self_attn_layer_norm''': '''encoder.layers.*.self_attn_layer_norm''',
    '''conv_module.pointwise_conv1''': '''encoder.layers.*.conv_module.pointwise_conv1''',
    '''conv_module.pointwise_conv2''': '''encoder.layers.*.conv_module.pointwise_conv2''',
    '''conv_module.depthwise_conv''': '''encoder.layers.*.conv_module.depthwise_conv''',
    '''conv_module.batch_norm''': '''encoder.layers.*.conv_module.batch_norm''',
    '''conv_module.layer_norm''': '''encoder.layers.*.conv_module.layer_norm''',
    '''ffn1.w_1''': '''encoder.layers.*.ffn1.intermediate_dense''',
    '''ffn1.w_2''': '''encoder.layers.*.ffn1.output_dense''',
    '''ffn1.layer_norm''': '''encoder.layers.*.ffn1_layer_norm''',
    '''ffn2.w_1''': '''encoder.layers.*.ffn2.intermediate_dense''',
    '''ffn2.w_2''': '''encoder.layers.*.ffn2.output_dense''',
    '''ffn2.layer_norm''': '''encoder.layers.*.ffn2_layer_norm''',
    '''final_layer_norm''': '''encoder.layers.*.final_layer_norm''',
    '''encoder.layer_norm''': '''encoder.layer_norm''',
    '''w2v_model.layer_norm''': '''feature_projection.layer_norm''',
    '''quantizer.weight_proj''': '''quantizer.weight_proj''',
    '''quantizer.vars''': '''quantizer.codevectors''',
    '''project_q''': '''project_q''',
    '''final_proj''': '''project_hid''',
    '''w2v_encoder.proj''': '''lm_head''',
    '''mask_emb''': '''masked_spec_embed''',
}
__UpperCAmelCase					       =							[
    '''lm_head''',
    '''quantizer.weight_proj''',
    '''quantizer.codevectors''',
    '''project_q''',
    '''project_hid''',
]


def       _snake_case     (  A				,					A				,					A				,					A				,					A )		->					Optional[Any]:
							for attribute in key.split('''.''' ):
														lowerCAmelCase__      =			getattr(A				,					A )

							if weight_type is not None:
														lowerCAmelCase__      =			getattr(A				,					A ).shape
							else:
														lowerCAmelCase__      =			hf_pointer.shape

							if hf_shape != value.shape:
														raise ValueError(
														    F"""Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be"""
														    F""" {value.shape} for {full_name}""" )

							if weight_type == "weight":
														lowerCAmelCase__      =			value
							elif weight_type == "weight_g":
														lowerCAmelCase__      =			value
							elif weight_type == "weight_v":
														lowerCAmelCase__      =			value
							elif weight_type == "bias":
														lowerCAmelCase__      =			value
							elif weight_type == "running_mean":
														lowerCAmelCase__      =			value
							elif weight_type == "running_var":
														lowerCAmelCase__      =			value
							elif weight_type == "num_batches_tracked":
														lowerCAmelCase__      =			value
							elif weight_type == "inv_freq":
														lowerCAmelCase__      =			value
							else:
														lowerCAmelCase__      =			value

							logger.info(F"""{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}.""" )


def       _snake_case     (  A				,					A				,					A )		->					Any:
							lowerCAmelCase__      =			[]
							lowerCAmelCase__      =			fairseq_model.state_dict()

							lowerCAmelCase__      =			hf_model.wavaveca_conformer.feature_extractor

							for name, value in fairseq_dict.items():
														lowerCAmelCase__      =			False
														if "conv_layers" in name:
																					load_conv_layer(
																					    A				,					A				,					A				,					A				,					hf_model.config.feat_extract_norm == '''group'''				,					)
																					lowerCAmelCase__      =			True
														else:
																					for key, mapped_key in MAPPING.items():
																												lowerCAmelCase__      =			'''wav2vec2_conformer.''' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
																												if key in name or key.split('''w2v_model.''' )[-1] == name.split('''.''' )[0]:
																																			lowerCAmelCase__      =			True
																																			if "*" in mapped_key:
																																										lowerCAmelCase__      =			name.split(A )[0].split('''.''' )[-2]
																																										lowerCAmelCase__      =			mapped_key.replace('''*'''				,					A )
																																			if "pos_bias_u" in name:
																																										lowerCAmelCase__      =			None
																																			elif "pos_bias_v" in name:
																																										lowerCAmelCase__      =			None
																																			elif "weight_g" in name:
																																										lowerCAmelCase__      =			'''weight_g'''
																																			elif "weight_v" in name:
																																										lowerCAmelCase__      =			'''weight_v'''
																																			elif "bias" in name:
																																										lowerCAmelCase__      =			'''bias'''
																																			elif "weight" in name:
																																										# TODO: don't match quantizer.weight_proj
																																										lowerCAmelCase__      =			'''weight'''
																																			elif "running_mean" in name:
																																										lowerCAmelCase__      =			'''running_mean'''
																																			elif "inv_freq" in name:
																																										lowerCAmelCase__      =			'''inv_freq'''
																																			elif "running_var" in name:
																																										lowerCAmelCase__      =			'''running_var'''
																																			elif "num_batches_tracked" in name:
																																										lowerCAmelCase__      =			'''num_batches_tracked'''
																																			else:
																																										lowerCAmelCase__      =			None
																																			set_recursively(A				,					A				,					A				,					A				,					A )
																												continue
														if not is_used:
																					unused_weights.append(A )

							logger.warning(F"""Unused weights: {unused_weights}""" )


def       _snake_case     (  A				,					A				,					A				,					A				,					A )		->					Tuple:
							lowerCAmelCase__      =			full_name.split('''conv_layers.''' )[-1]
							lowerCAmelCase__      =			name.split('''.''' )
							lowerCAmelCase__      =			int(items[0] )
							lowerCAmelCase__      =			int(items[1] )

							if type_id == 0:
														if "bias" in name:
																					if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
																												raise ValueError(
																												    F"""{full_name} has size {value.shape}, but"""
																												    F""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.""" )
																					lowerCAmelCase__      =			value
																					logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}.""" )
														elif "weight" in name:
																					if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
																												raise ValueError(
																												    F"""{full_name} has size {value.shape}, but"""
																												    F""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.""" )
																					lowerCAmelCase__      =			value
																					logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}.""" )
							elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
														if "bias" in name:
																					if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
																												raise ValueError(
																												    F"""{full_name} has size {value.shape}, but"""
																												    F""" {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found.""" )
																					lowerCAmelCase__      =			value
																					logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.""" )
														elif "weight" in name:
																					if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
																												raise ValueError(
																												    F"""{full_name} has size {value.shape}, but"""
																												    F""" {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found.""" )
																					lowerCAmelCase__      =			value
																					logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.""" )
							else:
														unused_weights.append(A )


@torch.no_grad()
def       _snake_case     (  A				,					A				,					A=None				,					A=None				,					A=True )		->					Optional[int]:
							if config_path is not None:
														lowerCAmelCase__      =			WavaVecaConformerConfig.from_pretrained(A				,					hidden_act='''swish''' )
							else:
														lowerCAmelCase__      =			WavaVecaConformerConfig()

							if "rope" in checkpoint_path:
														lowerCAmelCase__      =			'''rotary'''

							if is_finetuned:
														if dict_path:
																					lowerCAmelCase__      =			Dictionary.load(A )

																					# important change bos & pad token id since CTC symbol is <pad> and
																					# not <s> as in fairseq
																					lowerCAmelCase__      =			target_dict.pad_index
																					lowerCAmelCase__      =			target_dict.bos_index
																					lowerCAmelCase__      =			target_dict.eos_index
																					lowerCAmelCase__      =			len(target_dict.symbols )
																					lowerCAmelCase__      =			os.path.join(A				,					'''vocab.json''' )
																					if not os.path.isdir(A ):
																												logger.error('''--pytorch_dump_folder_path ({}) should be a directory'''.format(A ) )
																												return
																					os.makedirs(A				,					exist_ok=A )
																					lowerCAmelCase__      =			target_dict.indices

																					# fairseq has the <pad> and <s> switched
																					lowerCAmelCase__      =			0
																					lowerCAmelCase__      =			1
																					with open(A				,					'''w'''				,					encoding='''utf-8''' ) as vocab_handle:
																												json.dump(A				,					A )
																					lowerCAmelCase__      =			WavaVecaCTCTokenizer(
																					    A				,					unk_token=target_dict.unk_word				,					pad_token=target_dict.pad_word				,					bos_token=target_dict.bos_word				,					eos_token=target_dict.eos_word				,					word_delimiter_token='''|'''				,					do_lower_case=A				,					)
																					lowerCAmelCase__      =			True if config.feat_extract_norm == '''layer''' else False
																					lowerCAmelCase__      =			WavaVecaFeatureExtractor(
																					    feature_size=1				,					sampling_rate=16000				,					padding_value=0				,					do_normalize=A				,					return_attention_mask=A				,					)
																					lowerCAmelCase__      =			WavaVecaProcessor(feature_extractor=A				,					tokenizer=A )
																					processor.save_pretrained(A )

														lowerCAmelCase__      =			WavaVecaConformerForCTC(A )
							else:
														lowerCAmelCase__      =			WavaVecaConformerForPreTraining(A )

							if is_finetuned:
														lowerCAmelCase__     ,      lowerCAmelCase__     ,      lowerCAmelCase__      =			fairseq.checkpoint_utils.load_model_ensemble_and_task(
														    [checkpoint_path]				,					arg_overrides={'''data''': '''/'''.join(dict_path.split('''/''' )[:-1] )} )
							else:
														lowerCAmelCase__      =			argparse.Namespace(task='''audio_pretraining''' )
														lowerCAmelCase__      =			fairseq.tasks.setup_task(A )

														lowerCAmelCase__     ,      lowerCAmelCase__     ,      lowerCAmelCase__      =			fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]				,					task=A )

							lowerCAmelCase__      =			model[0].eval()

							recursively_load_weights(A				,					A				,					not is_finetuned )

							hf_wavavec.save_pretrained(A )


if __name__ == "__main__":
		__UpperCAmelCase					       =							argparse.ArgumentParser()
		parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
		parser.add_argument('''--checkpoint_path''', default=None, type=str, help='''Path to fairseq checkpoint''')
		parser.add_argument('''--dict_path''', default=None, type=str, help='''Path to dict of fine-tuned model''')
		parser.add_argument('''--config_path''', default=None, type=str, help='''Path to hf config.json of model to convert''')
		parser.add_argument(
		    '''--not_finetuned''', action='''store_true''', help='''Whether the model to convert is a fine-tuned model or not'''
		)
		__UpperCAmelCase					       =							parser.parse_args()
		convert_wavaveca_conformer_checkpoint(
		    args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
		)