feat: optimize max tokens

handler.py

@@ -7,7 +7,7 @@ from transformers import AutoTokenizer
 INSTRUCTION = "rewrite: "
 generation_config = {
     "max_new_tokens": 16,
-    "use_cache": False,
+    "use_cache": True,
     "temperature": 0.6,
     "do_sample": True,
     "top_p": 0.95,
@@ -21,7 +21,7 @@ class EndpointHandler:
         self.model = OVModelForSeq2SeqLM.from_pretrained(
             path, use_cache=True, use_io_binding=False
         )
-        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.tokenizer = AutoTokenizer.from_pretrained(path, use_fast=True)
 
     def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
         """
@@ -37,6 +37,8 @@ class EndpointHandler:
             ["{} {}".format(INSTRUCTION, inputs)],
             padding=False,
             return_tensors="pt",
+            max_length=20,
+            truncation=True,
         )
 
         outputs = self.model.generate(**inputs, **parameters)