code: def test_vocab(
      fairseq_model, gluon_tokenizer, check_all_tokens=False
  ):
      print("testing vocab")
      fairseq_vocab = fairseq_model.task.dictionary
      gluon_vocab = gluon_tokenizer.vocab
      assert len(fairseq_vocab) == len(gluon_vocab)
  
      # assert all_tokens
      # roberta with gpt2 bytebpe bpe does not provide all tokens directly
      if check_all_tokens:
          for i in range(len(fairseq_vocab)):
              assert (
                  fairseq_vocab[i] == gluon_vocab.all_tokens[i]
              ), "{}, {}, {}".format(
                  i, fairseq_vocab[i], gluon_vocab.all_tokens[i]
              )
  
      # assert special tokens
      for special_tokens in ["unk", "pad", "eos", "bos"]:
          assert getattr(
              fairseq_vocab, special_tokens + "_index"
          ) == getattr(gluon_vocab, special_tokens + "_id")
          assert getattr(
              fairseq_vocab, special_tokens + "_word"
          ) == getattr(gluon_vocab, special_tokens + "_token")
          # <mask> is the last one
          assert (
              fairseq_vocab[-1]
              == gluon_vocab.all_tokens[-1]
              == "<mask>"
          )
  
      sentence = (
          "Hello, y'all! How are you Ⅷ 😁 😁 😁 ?"
          + "GluonNLP is great！！！!!!"
          + "GluonNLP-Amazon-Haibin-Leonard-Sheng-Shuai-Xingjian...../:!@# 'abc'"
      )
      # assert encode
      fs_tokens = fairseq_model.encode(sentence)
      gl_tokens = gluon_tokenizer.encode(sentence, int)
      # Notice: we may append bos and eos
      # manuually after tokenizing sentences
      assert fs_tokens.numpy().tolist()[1:-1] == gl_tokens
  
      # assert decode
      fs_sentence = fairseq_model.decode(fs_tokens)
      gl_sentence = gluon_tokenizer.decode(gl_tokens)
      assert fs_sentence == gl_sentence
  
imports: import os
  import re
  import sys
  import json
  import shutil
  import logging
  import argparse
  import mxnet
  import numpy
  from numpy.testing import assert_allclose
  import torch
  from gluonnlp.data.vocab import Vocab
  from gluonnlp.utils.misc import sha1sum, logging_config, naming_convention
  from fairseq.models.roberta import RobertaModel
  from gluonnlp.models.roberta import RobertaModel, RobertaForMLM
  from gluonnlp.data.tokenizers import HuggingFaceByteBPETokenizer
label_desc: Use image node_id 3 for calling a global function with example usage: test_vocab(fairseq_model, gluon_tokenizer, check_all_tokens) without return types