class_docstr: Find samples that contain tokens unsupported by your tokenizer.
  
  Parameters
  ----------
  tokenizer: t.Any , default: None
      Tokenizer from the HuggingFace transformers library to use for tokenization. If None,
      AutoTokenizer.from_pretrained('bert-base-uncased') will be used.
  group_singleton_words: bool, default: False
      If True, group all words that appear only once in the data into the "Other" category in the display.
  n_most_common : int , default: 5
      Number of most common words with unknown tokens to show in the display.
  n_samples: int, default: 1_000_000
      number of samples to use for this check.
  random_state : int, default: 42
      random seed for all check internals.
  {max_text_length_for_display_param:1*indent}
class_docstr_tok: ["Find","samples","that","contain","tokens","unsupported","by","your","tokenizer",".","Parameters","--","--","--","--","--","tokenizer",":","t.Any",",","default",":","None","Tokenizer","from","the","HuggingFace","transformers","library","to","use","for","tokenization",".","If","None",",","AutoTokenizer.from_pretrained","(","'bert-base-uncased","'",")","will","be","used",".","group_singleton_words",":","bool",",","default",":","False","If","True",",","group","all","words","that","appear","only","once","in","the","data","into","the","``","Other","''","category","in","the","display",".","n_most_common",":","int",",","default",":","5","Number","of","most","common","words","with","unknown","tokens","to","show","in","the","display",".","n_samples",":","int",",","default",":","1_000_000","number","of","samples","to","use","for","this","check",".","random_state",":","int",",","default",":","42","random","seed","for","all","check","internals",".","{","max_text_length_for_display_param:1","*","indent","}"]
docstr: Find words with unknown tokens in samples.
docstr_tok: ["Find","words","with","unknown","tokens","in","samples","."]
code: def find_unknown_words(self, samples, indices):
      """Find words with unknown tokens in samples."""
      all_unknown_tokens = []
      unknown_token_indexes = {}
      total_tokens = 0
  
      if self._use_fast_method:
          non_text_token_ids = self._get_non_text_token_ids()
  
          # Batch tokenization
          # ------------------
          # Needed to avoid warning when used after loading a hub dataset
          # We divert the printing to stdout (done by the rust code within the HuggingFace tokenizer)
          # into this filter, that will filter out any print containing the str 'huggingface/tokenizers'
          # This warning printout is activated when running this check after loading a HuggingFace dataset,
          # and is irrelevant to us because we're not forking the process.
          # see: https://github.com/huggingface/transformers/issues/5486
          with contextlib.redirect_stdout(PrintFilter(sys.stdout)):
              tokenized_samples = self.tokenizer(
                  list(samples),
                  return_offsets_mapping=True,
                  is_split_into_words=False,
                  truncation=False,
              )
  
          for idx, (tokens, offsets_mapping, sample) in zip(
              indices,
              zip(
                  tokenized_samples["input_ids"],
                  tokenized_samples["offset_mapping"],
                  samples,
              ),
          ):
              for token_id, offset_mapping in zip(
                  tokens, offsets_mapping
              ):
                  if token_id == self.tokenizer.unk_token_id:
                      start, end = offset_mapping
                      token = sample[start:end]
                      all_unknown_tokens.append(token)
                      unknown_token_indexes.setdefault(
                          token, []
                      ).append(idx)
                  if token_id not in non_text_token_ids:
                      total_tokens += 1
      else:
          # Tokenization for each word
          # --------------------------
          # Choose tokenizer based on availability of nltk
          if nltk.download("punkt", quiet=True):
              tokenize = nltk.word_tokenize
          else:
              warnings.warn(
                  "nltk punkt is not available, using str.split instead to identify individual words. "
                  "Please check your internet connection."
              )
              tokenize = str.split
  
          # Tokenize samples and count unknown words
          words_array = [tokenize(sample) for sample in samples]
          for idx, words in zip(indices, words_array):
              total_tokens += len(words)
              for word in words:
                  tokens = self.tokenizer.tokenize(word)
                  if any(
                      self.tokenizer.convert_tokens_to_ids(token)
                      == self.tokenizer.unk_token_id
                      for token in tokens
                  ):
                      all_unknown_tokens.append(word)
                      unknown_token_indexes.setdefault(word, []).append(
                          idx
                      )
  
      return (
          Counter(all_unknown_tokens),
          total_tokens,
          unknown_token_indexes,
      )
  
imports: import contextlib
  import copy
  import sys
  import typing
  import warnings
  from collections import Counter
  import nltk
  import plotly.graph_objects
  from deepchecks.core import CheckResult, ConditionCategory, ConditionResult
  from deepchecks.core.errors import DeepchecksProcessError, DeepchecksValueError
  from deepchecks.nlp import Context, SingleDatasetCheck
  from deepchecks.nlp._shared_docs import docstrings
  from deepchecks.nlp.text_data import TextData
  from deepchecks.utils.numbers import round_sig
  from deepchecks.utils.strings import format_list, format_percent, truncate_string
label_desc: Use image node_id 5 for calling the UnknownTokens obj's underlying member method code with example usage: obj.find_unknown_words(samples, indices) and returns: Counter,  total_tokens,  unknown_token_indexes