instantiate a new grammar object for each call

app.py

@@ -15,12 +15,6 @@ if __name__ == "__main__":
         tokenizer.pad_token_id = tokenizer.eos_token_id
         model.config.pad_token_id = model.config.eos_token_id
 
-    # Load json grammar
-    with open("json_minimal.ebnf", "r") as file:
-        grammar_str = file.read()
-    grammar = IncrementalGrammarConstraint(grammar_str, "root", tokenizer)
-    grammar_processor = GrammarConstrainedLogitsProcessor(grammar)
-
     # Define your color-coding labels; if prob > x, then label = y; Sorted in descending probability order!
     probs_to_label = [
         (0.1, "p >= 10%"),
@@ -40,8 +34,15 @@ if __name__ == "__main__":
         Given the prompt (text), return a list of tuples (decoded_token, label)
         """
         inputs = tokenizer([prompt], return_tensors="pt")
+
+        # Load json grammar and create a GrammarConstrainedLogitsProcessor for each call
+        with open("json_minimal.ebnf", "r") as file:
+            grammar_str = file.read()
+        grammar = IncrementalGrammarConstraint(grammar_str, "root", tokenizer)
+        grammar_processor = GrammarConstrainedLogitsProcessor(grammar)
+
         outputs = model.generate(
-            **inputs, max_new_tokens=20, return_dict_in_generate=True, output_scores=True, logits_processor=[grammar_processor]
+            **inputs, max_new_tokens=50, repetition_penalty=1.1, return_dict_in_generate=True, output_scores=True, logits_processor=[grammar_processor]
         )
         # Important: don't forget to set `normalize_logits=True` to obtain normalized probabilities (i.e. sum(p) = 1)
         transition_scores = model.compute_transition_scores(outputs.sequences, outputs.scores, normalize_logits=True)