commit

app.py

@@ -19,7 +19,8 @@ from instruct_pipeline import InstructionTextGenerationPipeline
 
 from training.generate import load_model_tokenizer_for_generate
 from ctransformers import AutoModelForCausalLM
-
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+import ctransformers
 # from training.generate import InstructionTextGenerationPipeline, load_model_tokenizer_for_generate
 # from googletrans import Translator
 # translator = Translator()
@@ -65,8 +66,9 @@ match model_type:
         # generate_text = InstructionTextGenerationPipeline(model=model, tokenizer=tokenizer)
         # llm = HuggingFacePipeline(pipeline=generate_text)
 
-        llm = AutoModelForCausalLM.from_pretrained(model_path, model_type='dolly-v2')
+        # llm = AutoModelForCausalLM.from_pretrained(model_path, model_type='dolly-v2')
 
+        llm = CTransformers(model_path, callbacks=[StreamingStdOutCallbackHandler()])
 
         # llm = HuggingFacePipeline(
         #     pipeline=InstructionTextGenerationPipeline(