

import json
import os
from datetime import date
from pathlib import Path

from tabulate import DataRow, TableFormat, tabulate


lowerCAmelCase_			      =      TableFormat(
    lineabove=None,
    linebelowheader=None,
    linebetweenrows=None,
    linebelow=None,
    headerrow=DataRow('''''', '''|''', '''|'''),
    datarow=DataRow('''''', '''|''', '''|'''),
    padding=1,
    with_header_hide=None,
)


lowerCAmelCase_			      =      []
lowerCAmelCase_			      =      []

lowerCAmelCase_			      =      {'''type''': '''section''', '''text''': {'''type''': '''plain_text''', '''text''': '''No failed tests! 🤗''', '''emoji''': True}}

lowerCAmelCase_			      =      [
    {
        '''type''': '''header''',
        '''text''': {
            '''type''': '''plain_text''',
            '''text''': f"""🤗 Accelerate nightly {os.environ.get("TEST_TYPE", "")} test results""",
            '''emoji''': True,
        },
    }
]

lowerCAmelCase_			      =      0
for log in Path().glob('''*.log'''):
					lowerCAmelCase_			      =      0
					with open(log, '''r''') as f:
										for line in f:
															lowerCAmelCase_			      =      json.loads(line)
															if line.get('''nodeid''', '''''') != "":
																				lowerCAmelCase_			      =      line['''nodeid''']
																				if line.get('''duration''', None) is not None:
																									lowerCAmelCase_			      =      f"""{line["duration"]:.4f}"""
																									if line.get('''outcome''', '''''') == "failed":
																														section_num_failed += 1
																														failed.append([test, duration, log.name.split('''_''')[0]])
																														total_num_failed += 1
					group_info.append([str(log), section_num_failed, failed])
					lowerCAmelCase_			      =      []
					log.unlink()

lowerCAmelCase_			      =      ''''''
lowerCAmelCase_			      =      []
if total_num_failed > 0:
					for name, num_failed, failed_tests in group_info:
										if num_failed > 0:
															if num_failed == 1:
																				message += f"*{name[1:]}: {num_failed} failed test*\n"
															else:
																				message += f"*{name[1:]}: {num_failed} failed tests*\n"
															lowerCAmelCase_			      =      []
															lowerCAmelCase_			      =      {}
															for test in failed_tests:
																				lowerCAmelCase_			      =      test[0].split('''::''')
																				lowerCAmelCase_			      =      data[0].split('''/''')[-1]
																				if data[0] not in filesafailed:
																									lowerCAmelCase_			      =      [data[1:]]
																				else:
																									filesafailed[data[0]] += [data[1:]]
																				failed_table.append(data)

															lowerCAmelCase_			      =      [test[0] for test in failed_table]
															lowerCAmelCase_			      =      list(set(files))
															# Count number of instances in failed_tests
															lowerCAmelCase_			      =      []
															for file in individual_files:
																				table.append([file, len(filesafailed[file])])

															lowerCAmelCase_			      =      tabulate(
															    table,
															    headers=['''Test Location''', '''Num Failed'''],
															    tablefmt=hf_table_format,
															    stralign='''right''',
															)
															message += f"\n```\n{failed_table}\n```"
															all_filesafailed.append(filesafailed)
					if len(message) > 30_00:
										lowerCAmelCase_			      =      '''Too many failed tests, please see the full report in the Action results.'''
										lowerCAmelCase_			      =      len(err) + 10
										lowerCAmelCase_			      =      message[: 30_00 - offset] + f"""\n...\n```\n{err}"""
					print(f"""### {message}""")
else:
					lowerCAmelCase_			      =      '''No failed tests! 🤗'''
					print(f"""## {message}""")
					payload.append(no_error_payload)

if os.environ.get('''TEST_TYPE''', '''''') != "":
					from slack_sdk import WebClient

					lowerCAmelCase_			      =      WebClient(token=os.environ['''SLACK_API_TOKEN'''])
					if message != "No failed tests! 🤗":
										lowerCAmelCase_			      =      {
										    '''type''': '''section''',
										    '''text''': {
										        '''type''': '''mrkdwn''',
										        '''text''': message,
										    },
										}
										payload.append(md_report)
										lowerCAmelCase_			      =      {
										    '''type''': '''section''',
										    '''text''': {
										        '''type''': '''mrkdwn''',
										        '''text''': '''*For more details:*''',
										    },
										    '''accessory''': {
										        '''type''': '''button''',
										        '''text''': {
										            '''type''': '''plain_text''',
										            '''text''': '''Check Action results''',
										            '''emoji''': True,
										        },
										        '''url''': f"""https://github.com/{os.environ["GITHUB_REPOSITORY"]}/actions/runs/{os.environ["GITHUB_RUN_ID"]}""",
										    },
										}
										payload.append(action_button)
										lowerCAmelCase_			      =      {
										    '''type''': '''context''',
										    '''elements''': [
										        {
										            '''type''': '''plain_text''',
										            '''text''': f"""Nightly {os.environ.get("TEST_TYPE")} test results for {date.today()}""",
										        }
										    ],
										}
										payload.append(date_report)
					lowerCAmelCase_			      =      client.chat_postMessage(channel='''#accelerate-ci-daily''', text=message, blocks=payload)
					lowerCAmelCase_			      =      response.data['''ts''']
					for failed_file in all_filesafailed:
										for test_location, test_failures in failed_file.items():
															# Keep only the first instance of the test name
															lowerCAmelCase_			      =      ''''''
															for i, row in enumerate(test_failures):
																				if row[0] != test_class:
																									lowerCAmelCase_			      =      row[0]
																				else:
																									lowerCAmelCase_			      =      ''''''

															lowerCAmelCase_			      =      {
															    '''type''': '''section''',
															    '''text''': {
															        '''type''': '''mrkdwn''',
															        '''text''': f"""Test location: {test_location}\n```\n{tabulate(test_failures, headers=["Class", "Test"], tablefmt=hf_table_format, stralign="right")}\n```""",
															    },
															}

															client.chat_postMessage(
															    channel='''#accelerate-ci-daily''',
															    thread_ts=ts,
															    blocks=[payload],
															)




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Dict								=     GPTSanJapaneseTokenizer
				A_   :     Optional[int]								=     False
				A_   :     List[str]								=     {'do_clean_text': False, 'add_prefix_space': False}


				def __UpperCamelCase						(							self      :  Tuple       )		->     Any:
											super().setUp()

											# fmt: off
											A      						=    ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
											# fmt: on
											A      						=    {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
											A      						=    {'unk_token': '<unk>'}

											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['vocab_file']       )
											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['emoji_file']       )
											with open(self.vocab_file						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
											with open(self.emoji_file						, 'w'       ) as emoji_writer:
																		emoji_writer.write(json.dumps(__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  Any						, **__UpperCamelCase      :  Any       )		->     Tuple:
											kwargs.update(self.special_tokens_map       )
											return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Dict       )		->     List[str]:
											A      						=    'こんにちは、世界。 \nこんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。 \nこんばんは、世界。😀'
											return input_text, output_text


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Optional[Any]       )		->     List[str]:
											A						,   A      						=    self.get_input_output_texts(__UpperCamelCase       )
											A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase						, clean_up_tokenization_spaces=__UpperCamelCase       )
											return text, ids


				def __UpperCamelCase						(							self      :  Any       )		->     int:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     int:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、世界。　こんばんは、㔺界。'
											A      						=    ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
											A      						=    tokenizer.tokenize(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids without special tokens
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids with special tokens
											A      						=    tokens + [tokenizer.unk_token]
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[str]:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
											A      						=    'こんにちは、、、、世界。こんばんは、、、、世界。'
											A      						=    tokenizer.encode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。こんばんは、世界。😀'
											A      						=    tokenizer.encode(prefix_text + input_text       )
											A      						=    tokenizer.encode(''						, prefix_text=prefix_text + input_text       )
											A      						=    tokenizer.encode(__UpperCamelCase						, prefix_text=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'

											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2
											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2

											A      						=    [1] + [0] * (len_prefix + len_text + 1)
											A      						=    [1] * (len_prefix + len_text + 1) + [0]
											A      						=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

											A      						=    tokenizer(prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(''						, prefix_text=prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(__UpperCamelCase						, prefix_text=__UpperCamelCase       ).token_type_ids
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    tokenizer.encode('あンいワ'       )
											A      						=    tokenizer.encode(''						, prefix_text='あンいワ'       )
											A      						=    tokenizer.encode('いワ'						, prefix_text='あン'       )

											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(x_token_a[1]						, x_token_a[-1]       )  # SEG token
											self.assertEqual(x_token_a[1]						, x_token_a[3]       )  # SEG token


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
											A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase       )
											A      						=    tokenizer.batch_encode_plus(__UpperCamelCase						, padding=__UpperCamelCase       )

											# fmt: off
											A      						=    [[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
											A      						=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
											A      						=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
											# fmt: on
											self.assertListEqual(x_token.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.attention_mask						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.attention_mask						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											# Intentionally convert some words to accommodate character fluctuations unique to Japanese
											pass


				def __UpperCamelCase						(							self      :  Dict       )		->     int:
											# tokenizer has no padding token
											pass