class_docstr: Test case for SentencePieceProcessor
class_docstr_tok: ["Test","case","for","SentencePieceProcessor"]
code: def test_new_api(self):
      sp = spm.SentencePieceProcessor(
          model_file=os.path.join("test", "test_model.model")
      )
      text = "hello world"
      text2 = "Tokyo"
      ids = self.sp_.EncodeAsIds(text)
      ids2 = self.sp_.EncodeAsIds(text2)
      pieces = self.sp_.EncodeAsPieces(text)
      pieces2 = self.sp_.EncodeAsPieces(text2)
      sprotos = self.sp_.EncodeAsSerializedProto(text)
      sproto2 = self.sp_.EncodeAsSerializedProto(text2)
      iprotos = self.sp_.EncodeAsImmutableProto(text)
      iprotos2 = self.sp_.EncodeAsImmutableProto(text2)
  
      self.assertEqual(sp.encode(text, out_type=int), ids)
      self.assertEqual(sp.encode(text, out_type=str), pieces)
      self.assertEqual(
          sp.encode(text, out_type="serialized_proto"), sprotos
      )
      self.assertEqual(
          sp.encode(text, out_type="immutable_proto"), iprotos
      )
  
      self.assertEqual(sp.encode([text], out_type=int), [ids])
      self.assertEqual(sp.encode([text], out_type=str), [pieces])
      self.assertEqual(
          sp.encode([text], out_type="serialized_proto"), [sprotos]
      )
      self.assertEqual(
          sp.encode([text], out_type="immutable_proto"), [iprotos]
      )
  
      self.assertEqual(len(iprotos.pieces), len(pieces))
      self.assertEqual(len(iprotos.pieces), len(ids))
      self.assertEqual(iprotos.text, text)
  
      self.assertEqual(len(iprotos2.pieces), len(pieces2))
      self.assertEqual(len(iprotos2.pieces), len(ids2))
      self.assertEqual(iprotos2.text, text2)
  
      for i in range(len(iprotos.pieces)):
          self.assertEqual(ids[i], iprotos.pieces[i].id)
          self.assertEqual(pieces[i], iprotos.pieces[i].piece)
  
      for i, piece in enumerate(iprotos.pieces):
          self.assertEqual(ids[i], piece.id)
          self.assertEqual(pieces[i], piece.piece)
  
      for i in range(len(iprotos2.pieces)):
          self.assertEqual(ids2[i], iprotos2.pieces[i].id)
          self.assertEqual(pieces2[i], iprotos2.pieces[i].piece)
  
      for i, piece in enumerate(iprotos2.pieces):
          self.assertEqual(ids2[i], piece.id)
          self.assertEqual(pieces2[i], piece.piece)
  
      detok_ids = self.sp_.DecodeIds(ids)
      detok_pieces = self.sp_.DecodePieces(pieces)
      self.assertEqual(sp.decode(ids), detok_ids)
      self.assertEqual(sp.decode(pieces), detok_pieces)
      self.assertEqual(sp.decode([]), "")
      self.assertEqual(sp.decode([[]]), [""])
  
      # add_bos, add_eos, reverse
      self.assertEqual(
          [sp.bos_id()] + ids, sp.encode(text, add_bos=True)
      )
      self.assertEqual(
          ids + [sp.eos_id()], sp.encode(text, add_eos=True)
      )
      self.assertEqual(
          ids + [sp.eos_id()], sp.EncodeAsIds(text, add_eos=True)
      )
      rids = ids[:]
      rids.reverse()
  
      self.assertEqual(rids, sp.encode(text, reverse=True))
      self.assertEqual(rids, sp.EncodeAsIds(text, reverse=True))
  
      # different shape.
      self.assertEqual([ids, ids2], sp.encode([text, text2]))
      self.assertEqual(
          [pieces, pieces2], sp.encode([text, text2], out_type=str)
      )
      self.assertEqual([text, text2], sp.decode([ids, ids2]))
      self.assertEqual([text, text2], sp.decode([pieces, pieces2]))
  
      pieces = list(reversed(self.sp_.EncodeAsPieces(text)))
      self.assertEqual(
          pieces, sp.encode(text, reverse=True, out_type=str)
      )
  
      # emit unk piece
      unk_char = "藤"
      pieces = self.sp_.EncodeAsIds(unk_char, emit_unk_piece=True)
      pieces2 = self.sp_.encode(
          unk_char, out_type=int, emit_unk_piece=True
      )
      self.assertEqual(pieces[1], sp.unk_id())
      self.assertEqual(pieces2[1], sp.unk_id())
      self.assertEqual(pieces, pieces2)
  
      pieces = self.sp_.EncodeAsPieces(unk_char, emit_unk_piece=True)
      pieces2 = self.sp_.encode(
          unk_char, out_type=str, emit_unk_piece=True
      )
      self.assertEqual(pieces[1], "<unk>")
      self.assertEqual(pieces2[1], "<unk>")
      self.assertEqual(pieces, pieces2)
  
      pieces = self.sp_.EncodeAsPieces(unk_char, emit_unk_piece=False)
      pieces2 = self.sp_.encode(
          unk_char, out_type=str, emit_unk_piece=False
      )
      self.assertEqual(pieces[1], unk_char)
      self.assertEqual(pieces2[1], unk_char)
      self.assertEqual(pieces, pieces2)
  
imports: from collections import defaultdict
  import io
  import os
  import pickle
  import sys
  import unittest
  import sentencepiece
label_desc: Use image node_id 11 for calling the TestSentencepieceProcessor obj's underlying member method code with example usage: obj.test_new_api() without return types