code: def validate_token(sentence, i_end, file):
      global error_count
      base_name = basename(file)
      nodes = sentence.strip().split("\n")
      i_start = i_end - len(nodes) + 1
      sent_id = None
      for i, node in enumerate(nodes):
          if node.startswith("#"):
              if node.startswith("# sent_id = "):
                  sent_id = node[12:]
              continue
          tokens = node.split("\t")
          line_number = i_start + i
          if len(tokens) != 2:
              warn(
                  message="Each row must has 2 tokens",
                  error_type="Format invalid-num-tokens",
                  file=base_name,
                  line_number=line_number,
                  level=2,
              )
          token = tokens[0]
          tag = tokens[1]
          if token.startswith(" ") or token.endswith(" "):
              warn(
                  message=f"token should not start and end with spaces: {token}",
                  error_type="Format token-error",
                  file=base_name,
                  line_number=line_number,
                  level=3,
              )
  
          if tag not in ["B-W", "I-W"]:
              warn(
                  message=f"Tag must be B-W or I-W, found {tag}",
                  error_type="Format tag-error",
                  file=base_name,
                  line_number=line_number,
                  level=3,
                  sent_id=sent_id,
              )
  
      content_nodes = [
          node for node in nodes if not node.startswith("#")
      ]
      content_tokens = [node.split("\t")[0] for node in content_nodes]
      text = " ".join(content_tokens)
      tokenized_tokens = tokenize(text)
      if tokenized_tokens != content_tokens:
          if len(text) > 53:
              message = "tokenized error for text: " + text[:50] + "..."
          else:
              message = "tokenized error for text: " + text
          message += "\n"
          message += "  Actual:" + str(content_tokens)
          message += "\n"
          message += "Expected:" + str(tokenized_tokens)
          warn(
              message=message,
              error_type="Format tokenize-error",
              file=base_name,
              line_number=i_start,
              level=3,
              sent_id=sent_id,
          )
  
file_path: underthesea/underthesea/corpus/validate_corpus.py
imports: import sys
  from os import listdir
  from os.path import join, basename
  import charset_normalizer
  from underthesea.feature_engineering.text import Text
  from underthesea.file_utils import DATASETS_FOLDER
  from underthesea.pipeline.word_tokenize import tokenize
label_desc: Use image node_id 7 for calling a global function with example usage: validate_token(sentence, i_end, file) without return types