class_docstr: The tokenizer for YaLM, which was trained on Russian and English text.
  Source: https://github.com/yandex/YaLM-100B
class_docstr_tok: ["The","tokenizer","for","YaLM",",","which","was","trained","on","Russian","and","English","text",".","Source",":","https",":","\/\/github.com\/yandex\/YaLM-100B"]
code: def tokenize(
      self, request: TokenizationRequest
  ) -> TokenizationRequestResult:
      cache_key = asdict(request)
  
      try:
  
          def do_it():
              token_ids = self.tokenizer.tokenize(request.text)
              if request.truncation:
                  token_ids = token_ids[: request.max_length]
              return {
                  "tokens": token_ids
                  if request.encode
                  else self.tokenizer.convert_ids_to_tokens(token_ids)
              }
              # TODO(1522): Check if we can reenable this to remove "▁"
              # return {"tokens": token_ids if request.encode else self.tokenizer.convert_ids_to_string(token_ids)}
  
          result, cached = self.cache.get(
              cache_key, wrap_request_time(do_it)
          )
      except Exception as e:
          error: str = f"YaLM Tokenizer error: {e}"
          return TokenizationRequestResult(
              success=False,
              cached=False,
              error=error,
              text="",
              tokens=[],
          )
  
      return TokenizationRequestResult(
          success=True,
          cached=cached,
          text=request.text,
          tokens=[
              TokenizationToken(value) for value in result["tokens"]
          ],
          request_time=result["request_time"],
      )
  
file_path: H2O/h2o_hf/helm/src/helm/proxy/clients/yalm_tokenizer_client.py
imports: from dataclasses import asdict
  from helm.common.cache import Cache, CacheConfig
  from helm.common.request import Request, RequestResult
  from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult, DecodeRequest, DecodeRequestResult, TokenizationToken
  from .client import Client, wrap_request_time
  from .yalm_tokenizer.yalm_tokenizer import YaLMTokenizer
label_desc: Use image node_id 3 for calling the YaLMTokenizerClient obj's underlying member method code with example usage: obj.tokenize(request) and returns: TokenizationRequestResult, dict, TokenizationRequestResult