tokenizer.py

import torch
from transformers import AutoTokenizer

class BathSalt1DaedalusPhi3Tokenizer(AutoTokenizer):
    def __init__(self, config):
        super().__init__(config)
        self.config = config

    def encode(self, text, return_tensors="pt"):
        encoding = super().encode(text, return_tensors=return_tensors)
        return encoding