"""simple docstring"""


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class        a				(		lowerCAmelCase_	,							unittest.TestCase   ):
    _snake_case     :  Optional[Any]     							=							BertJapaneseTokenizer
    _snake_case     :  Optional[Any]     							=							False
    _snake_case     :  Dict     							=							True


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      super().setUp()

      _UpperCAmelCase						 =      [
          """[UNK]""",
          """[CLS]""",
          """[SEP]""",
          """こんにちは""",
          """こん""",
          """にちは""",
          """ばんは""",
          """##こん""",
          """##にちは""",
          """##ばんは""",
          """世界""",
          """##世界""",
          """、""",
          """##、""",
          """。""",
          """##。""",
      ]

      _UpperCAmelCase						 =      os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES["""vocab_file"""]					)
      with open(self.vocab_file   ,				"""w"""   ,				encoding="""utf-8"""					) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)


    def      lowerCAmelCase_  (					self	:  Any   ,				__lowerCAmelCase	:  Tuple					):
      _UpperCAmelCase						 =      """こんにちは、世界。 \nこんばんは、世界。"""
      _UpperCAmelCase						 =      """こんにちは 、 世界 。 こんばんは 、 世界 。"""
      return input_text, output_text


    def      lowerCAmelCase_  (					self	:  List[str]   ,				__lowerCAmelCase	:  List[str]					):
      _UpperCAmelCase ,				_UpperCAmelCase						 =      self.get_input_output_texts(__lowerCAmelCase					)
      _UpperCAmelCase						 =      tokenizer.encode(__lowerCAmelCase   ,				add_special_tokens=__lowerCAmelCase					)
      _UpperCAmelCase						 =      tokenizer.decode(__lowerCAmelCase   ,				clean_up_tokenization_spaces=__lowerCAmelCase					)
      return text, ids


    def      lowerCAmelCase_  (					self	:  int					):
      pass  # TODO add if relevant


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      pass  # TODO add if relevant


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      pass  # TODO add if relevant


    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      self.tokenizer_class(self.vocab_file					)

      _UpperCAmelCase						 =      tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""					)
      self.assertListEqual(__lowerCAmelCase   ,				["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase					)   ,				[3, 12, 10, 14, 4, 9, 12, 10, 14]					)


    def      lowerCAmelCase_  (					self	:  Any					):
      _UpperCAmelCase						 =      self.tokenizer_class(self.vocab_file   ,				word_tokenizer_type="""mecab"""					)
      self.assertIsNotNone(__lowerCAmelCase					)

      _UpperCAmelCase						 =      """こんにちは、世界。\nこんばんは、世界。"""
      _UpperCAmelCase						 =      tokenizer.tokenize(__lowerCAmelCase					)
      self.assertListEqual(__lowerCAmelCase   ,				["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase					)   ,				[3, 12, 10, 14, 4, 9, 12, 10, 14]					)

      _UpperCAmelCase						 =      os.path.join(self.tmpdirname   ,				"""tokenizer.bin"""					)
      with open(__lowerCAmelCase   ,				"""wb"""					) as handle:
        pickle.dump(__lowerCAmelCase   ,				__lowerCAmelCase					)

      with open(__lowerCAmelCase   ,				"""rb"""					) as handle:
        _UpperCAmelCase						 =      pickle.load(__lowerCAmelCase					)

      _UpperCAmelCase						 =      tokenizer_new.tokenize(__lowerCAmelCase					)

      self.assertListEqual(__lowerCAmelCase   ,				__lowerCAmelCase					)


    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      MecabTokenizer(mecab_dic="""ipadic"""					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,				)


    def      lowerCAmelCase_  (					self	:  List[str]					):
      try:
        _UpperCAmelCase						 =      MecabTokenizer(mecab_dic="""unidic_lite"""					)
      except ModuleNotFoundError:
        return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,				)


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      try:
        _UpperCAmelCase						 =      MecabTokenizer(mecab_dic="""unidic"""					)
      except ModuleNotFoundError:
        return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,				)


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      _UpperCAmelCase						 =      MecabTokenizer(do_lower_case=__lowerCAmelCase   ,				mecab_dic="""ipadic"""					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,				)


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      try:
        _UpperCAmelCase						 =      MecabTokenizer(
            do_lower_case=__lowerCAmelCase   ,				normalize_text=__lowerCAmelCase   ,				mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""					)
      except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,				)


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      _UpperCAmelCase						 =      MecabTokenizer(normalize_text=__lowerCAmelCase   ,				mecab_dic="""ipadic"""					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]   ,				)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      _UpperCAmelCase						 =      self.tokenizer_class(self.vocab_file   ,				word_tokenizer_type="""sudachi"""					)
      self.assertIsNotNone(__lowerCAmelCase					)

      _UpperCAmelCase						 =      """こんにちは、世界。\nこんばんは、世界。"""
      _UpperCAmelCase						 =      tokenizer.tokenize(__lowerCAmelCase					)
      self.assertListEqual(__lowerCAmelCase   ,				["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase					)   ,				[3, 12, 10, 14, 4, 9, 12, 10, 14]					)

      _UpperCAmelCase						 =      os.path.join(self.tmpdirname   ,				"""tokenizer.bin"""					)
      with open(__lowerCAmelCase   ,				"""wb"""					) as handle:
        pickle.dump(__lowerCAmelCase   ,				__lowerCAmelCase					)

      with open(__lowerCAmelCase   ,				"""rb"""					) as handle:
        _UpperCAmelCase						 =      pickle.load(__lowerCAmelCase					)

      _UpperCAmelCase						 =      tokenizer_new.tokenize(__lowerCAmelCase					)

      self.assertListEqual(__lowerCAmelCase   ,				__lowerCAmelCase					)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  Any					):
      _UpperCAmelCase						 =      SudachiTokenizer(sudachi_dict_type="""core"""					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]   ,				)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      SudachiTokenizer(sudachi_dict_type="""core"""   ,				sudachi_split_mode="""A"""					)

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""					)   ,				["""外国""", """人""", """参政""", """権"""]					)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  Any					):
      _UpperCAmelCase						 =      SudachiTokenizer(sudachi_dict_type="""core"""   ,				sudachi_split_mode="""B"""					)

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""					)   ,				["""外国人""", """参政権"""]					)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  int					):
      _UpperCAmelCase						 =      SudachiTokenizer(sudachi_dict_type="""core"""   ,				sudachi_split_mode="""C"""					)

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""					)   ,				["""外国人参政権"""]					)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      SudachiTokenizer(do_lower_case=__lowerCAmelCase   ,				sudachi_dict_type="""core"""					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]   ,				)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      SudachiTokenizer(normalize_text=__lowerCAmelCase   ,				sudachi_dict_type="""core"""					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]   ,				)


    @require_sudachi
    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      SudachiTokenizer(trim_whitespace=__lowerCAmelCase   ,				sudachi_dict_type="""core"""					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,				)


    @require_jumanpp
    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      self.tokenizer_class(self.vocab_file   ,				word_tokenizer_type="""jumanpp"""					)
      self.assertIsNotNone(__lowerCAmelCase					)

      _UpperCAmelCase						 =      """こんにちは、世界。\nこんばんは、世界。"""
      _UpperCAmelCase						 =      tokenizer.tokenize(__lowerCAmelCase					)
      self.assertListEqual(__lowerCAmelCase   ,				["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase					)   ,				[3, 12, 10, 14, 4, 9, 12, 10, 14]					)

      _UpperCAmelCase						 =      os.path.join(self.tmpdirname   ,				"""tokenizer.bin"""					)
      with open(__lowerCAmelCase   ,				"""wb"""					) as handle:
        pickle.dump(__lowerCAmelCase   ,				__lowerCAmelCase					)

      with open(__lowerCAmelCase   ,				"""rb"""					) as handle:
        _UpperCAmelCase						 =      pickle.load(__lowerCAmelCase					)

      _UpperCAmelCase						 =      tokenizer_new.tokenize(__lowerCAmelCase					)

      self.assertListEqual(__lowerCAmelCase   ,				__lowerCAmelCase					)


    @require_jumanpp
    def      lowerCAmelCase_  (					self	:  Union[str, Any]					):
      _UpperCAmelCase						 =      JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,				)


    @require_jumanpp
    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      JumanppTokenizer(do_lower_case=__lowerCAmelCase					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,				)


    @require_jumanpp
    def      lowerCAmelCase_  (					self	:  int					):
      _UpperCAmelCase						 =      JumanppTokenizer(normalize_text=__lowerCAmelCase					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,				)


    @require_jumanpp
    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      JumanppTokenizer(trim_whitespace=__lowerCAmelCase					)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """					)   ,				["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]   ,				)


    @require_jumanpp
    def      lowerCAmelCase_  (					self	:  Tuple					):
      _UpperCAmelCase						 =      JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""					)   ,				["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]   ,				)


    def      lowerCAmelCase_  (					self	:  int					):
      _UpperCAmelCase						 =      ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

      _UpperCAmelCase						 =      {}
      for i, token in enumerate(__lowerCAmelCase					):
        _UpperCAmelCase						 =      i
      _UpperCAmelCase						 =      WordpieceTokenizer(vocab=__lowerCAmelCase   ,				unk_token="""[UNK]"""					)

      self.assertListEqual(tokenizer.tokenize(""""""					)   ,				[]					)

      self.assertListEqual(tokenizer.tokenize("""こんにちは"""					)   ,				["""こんにちは"""]					)

      self.assertListEqual(tokenizer.tokenize("""こんばんは"""					)   ,				["""こん""", """##ばんは"""]					)

      self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""					)   ,				["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]					)


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      _UpperCAmelCase						 =      BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""					)
      _UpperCAmelCase						 =      tokenizer.subword_tokenizer

      _UpperCAmelCase						 =      subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""					)
      self.assertListEqual(__lowerCAmelCase   ,				["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]					)

      _UpperCAmelCase						 =      subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""					)
      self.assertListEqual(__lowerCAmelCase   ,				["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]					)


    def      lowerCAmelCase_  (					self	:  Any					):
      _UpperCAmelCase						 =      self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""					)

      _UpperCAmelCase						 =      tokenizer.encode("""ありがとう。"""   ,				add_special_tokens=__lowerCAmelCase					)
      _UpperCAmelCase						 =      tokenizer.encode("""どういたしまして。"""   ,				add_special_tokens=__lowerCAmelCase					)

      _UpperCAmelCase						 =      tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase					)
      _UpperCAmelCase						 =      tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase   ,				__lowerCAmelCase					)

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class        a				(		lowerCAmelCase_	,							unittest.TestCase   ):
    _snake_case     :  Optional[int]     							=							BertJapaneseTokenizer
    _snake_case     :  Union[str, Any]     							=							False


    def      lowerCAmelCase_  (					self	:  List[str]					):
      super().setUp()

      _UpperCAmelCase						 =      ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

      _UpperCAmelCase						 =      os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES["""vocab_file"""]					)
      with open(self.vocab_file   ,				"""w"""   ,				encoding="""utf-8"""					) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)


    def      lowerCAmelCase_  (					self	:  str   ,				**__lowerCAmelCase	:  Tuple					):
      return BertJapaneseTokenizer.from_pretrained(self.tmpdirname   ,				subword_tokenizer_type="""character"""   ,				**__lowerCAmelCase					)


    def      lowerCAmelCase_  (					self	:  Any   ,				__lowerCAmelCase	:  List[str]					):
      _UpperCAmelCase						 =      """こんにちは、世界。 \nこんばんは、世界。"""
      _UpperCAmelCase						 =      """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
      return input_text, output_text


    def      lowerCAmelCase_  (					self	:  str					):
      pass  # TODO add if relevant


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      pass  # TODO add if relevant


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      pass  # TODO add if relevant


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      _UpperCAmelCase						 =      self.tokenizer_class(self.vocab_file   ,				subword_tokenizer_type="""character"""					)

      _UpperCAmelCase						 =      tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""					)
      self.assertListEqual(
          __lowerCAmelCase   ,				["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]					)
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__lowerCAmelCase					)   ,				[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]					)


    def      lowerCAmelCase_  (					self	:  str					):
      _UpperCAmelCase						 =      ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

      _UpperCAmelCase						 =      {}
      for i, token in enumerate(__lowerCAmelCase					):
        _UpperCAmelCase						 =      i
      _UpperCAmelCase						 =      CharacterTokenizer(vocab=__lowerCAmelCase   ,				unk_token="""[UNK]"""					)

      self.assertListEqual(tokenizer.tokenize(""""""					)   ,				[]					)

      self.assertListEqual(tokenizer.tokenize("""こんにちは"""					)   ,				["""こ""", """ん""", """に""", """ち""", """は"""]					)

      self.assertListEqual(tokenizer.tokenize("""こんにちほ"""					)   ,				["""こ""", """ん""", """に""", """ち""", """[UNK]"""]					)


    def      lowerCAmelCase_  (					self	:  Optional[Any]					):
      _UpperCAmelCase						 =      self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""					)

      _UpperCAmelCase						 =      tokenizer.encode("""ありがとう。"""   ,				add_special_tokens=__lowerCAmelCase					)
      _UpperCAmelCase						 =      tokenizer.encode("""どういたしまして。"""   ,				add_special_tokens=__lowerCAmelCase					)

      _UpperCAmelCase						 =      tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase					)
      _UpperCAmelCase						 =      tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase   ,				__lowerCAmelCase					)

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class        a				(		unittest.TestCase   ):


    def      lowerCAmelCase_  (					self	:  Union[str, Any]					):
      _UpperCAmelCase						 =      """cl-tohoku/bert-base-japanese"""
      _UpperCAmelCase						 =      AutoTokenizer.from_pretrained(__lowerCAmelCase					)
      self.assertIsInstance(__lowerCAmelCase   ,				__lowerCAmelCase					)



class        a				(		unittest.TestCase   ):


    def      lowerCAmelCase_  (					self	:  str					):
      _UpperCAmelCase						 =      """cl-tohoku/bert-base-japanese"""
      with self.assertLogs("""transformers"""   ,				level="""WARNING"""					) as cm:
        BertTokenizer.from_pretrained(__lowerCAmelCase					)
        self.assertTrue(
            cm.records[0].message.startswith(
                """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                """ is called from."""					)					)
      _UpperCAmelCase						 =      """bert-base-cased"""
      with self.assertLogs("""transformers"""   ,				level="""WARNING"""					) as cm:
        BertJapaneseTokenizer.from_pretrained(__lowerCAmelCase					)
        self.assertTrue(
            cm.records[0].message.startswith(
                """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                """ is called from."""					)					)

"""simple docstring"""


from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def     __UpperCAmelCase					(			lowercase   ,lowercase		):







  """simple docstring"""
  _UpperCAmelCase						 =      []
  for part_id in partition_order:
    _UpperCAmelCase						 =      df.where(f'''SPARK_PARTITION_ID() = {part_id}'''		).collect()
    for row_idx, row in enumerate(lowercase		):
      expected_row_ids_and_row_dicts.append((f'''{part_id}_{row_idx}''', row.asDict())		)
  return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def     __UpperCAmelCase					(			):







  """simple docstring"""
  _UpperCAmelCase						 =      pyspark.sql.SparkSession.builder.master("""local[*]"""		).appName("""pyspark"""		).getOrCreate()
  _UpperCAmelCase						 =      spark.range(1_00		).repartition(1		)
  _UpperCAmelCase						 =      Spark(lowercase		)
  # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
  # that each partition can hold 2 rows.
  spark_builder._repartition_df_if_needed(max_shard_size=16		)
  # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
  assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def     __UpperCAmelCase					(			):







  """simple docstring"""
  _UpperCAmelCase						 =      pyspark.sql.SparkSession.builder.master("""local[*]"""		).appName("""pyspark"""		).getOrCreate()
  _UpperCAmelCase						 =      spark.range(10		).repartition(2		)
  _UpperCAmelCase						 =      [1, 0]
  _UpperCAmelCase						 =      _generate_iterable_examples(lowercase   ,lowercase		)  # Reverse the partitions.
  _UpperCAmelCase						 =      _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase   ,lowercase		)

  for i, (row_id, row_dict) in enumerate(generate_fn()		):
    _UpperCAmelCase ,				_UpperCAmelCase						 =      expected_row_ids_and_row_dicts[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def     __UpperCAmelCase					(			):







  """simple docstring"""
  _UpperCAmelCase						 =      pyspark.sql.SparkSession.builder.master("""local[*]"""		).appName("""pyspark"""		).getOrCreate()
  _UpperCAmelCase						 =      spark.range(10		).repartition(1		)
  _UpperCAmelCase						 =      SparkExamplesIterable(lowercase		)
  assert it.n_shards == 1
  for i, (row_id, row_dict) in enumerate(lowercase		):
    assert row_id == f'''0_{i}'''
    assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def     __UpperCAmelCase					(			):







  """simple docstring"""
  _UpperCAmelCase						 =      pyspark.sql.SparkSession.builder.master("""local[*]"""		).appName("""pyspark"""		).getOrCreate()
  _UpperCAmelCase						 =      spark.range(30		).repartition(3		)
  # Mock the generator so that shuffle reverses the partition indices.
  with patch("""numpy.random.Generator"""		) as generator_mock:
    _UpperCAmelCase						 =      lambda lowercase		: x.reverse()
    _UpperCAmelCase						 =      _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase   ,[2, 1, 0]		)

    _UpperCAmelCase						 =      SparkExamplesIterable(lowercase		).shuffle_data_sources(lowercase		)
    assert shuffled_it.n_shards == 3
    for i, (row_id, row_dict) in enumerate(lowercase		):
      _UpperCAmelCase ,				_UpperCAmelCase						 =      expected_row_ids_and_row_dicts[i]
      assert row_id == expected_row_id
      assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def     __UpperCAmelCase					(			):







  """simple docstring"""
  _UpperCAmelCase						 =      pyspark.sql.SparkSession.builder.master("""local[*]"""		).appName("""pyspark"""		).getOrCreate()
  _UpperCAmelCase						 =      spark.range(20		).repartition(4		)

  # Partitions 0 and 2
  _UpperCAmelCase						 =      SparkExamplesIterable(lowercase		).shard_data_sources(worker_id=0   ,num_workers=2		)
  assert shard_it_a.n_shards == 2
  _UpperCAmelCase						 =      _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase   ,[0, 2]		)
  for i, (row_id, row_dict) in enumerate(lowercase		):
    _UpperCAmelCase ,				_UpperCAmelCase						 =      expected_row_ids_and_row_dicts_a[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict

  # Partitions 1 and 3
  _UpperCAmelCase						 =      SparkExamplesIterable(lowercase		).shard_data_sources(worker_id=1   ,num_workers=2		)
  assert shard_it_a.n_shards == 2
  _UpperCAmelCase						 =      _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase   ,[1, 3]		)
  for i, (row_id, row_dict) in enumerate(lowercase		):
    _UpperCAmelCase ,				_UpperCAmelCase						 =      expected_row_ids_and_row_dicts_a[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def     __UpperCAmelCase					(			):







  """simple docstring"""
  _UpperCAmelCase						 =      pyspark.sql.SparkSession.builder.master("""local[*]"""		).appName("""pyspark"""		).getOrCreate()
  _UpperCAmelCase						 =      spark.range(1_00		).repartition(1		)
  _UpperCAmelCase						 =      Spark(lowercase		)
  # Choose a small max_shard_size for maximum partitioning.
  spark_builder._repartition_df_if_needed(max_shard_size=1		)
  # The new number of partitions should not be greater than the number of rows.
  assert spark_builder.df.rdd.getNumPartitions() == 1_00
