update

main.py

@@ -6,14 +6,38 @@ from transformers import pipeline
 
 app = FastAPI()
 
-pipe_flan = pipeline("text2text-generation", model="google/flan-t5-small")
-
-@app.get("/query_gorilla")
+@app.post("/query_gorilla")
 def query_gorilla(input):
-    return {"output": "Test Result"}
+    return input
 
 app.mount("/", StaticFiles(directory="static", html=True), name="static")
 
 @app.get("/")
 def index() -> FileResponse:
     return FileResponse(path="/app/static/index.html", media_type="text/html")
+
+TODO = '''
+print('Device setup')
+device : str = "cuda:0" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+
+print('Model and tokenizer setup')
+model_id : str = "gorilla-llm/gorilla-openfunctions-v1"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True)
+
+print('Move model to device')
+model.to(device)
+
+print('Pipeline setup')
+pipe = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+    max_new_tokens=128,
+    batch_size=16,
+    torch_dtype=torch_dtype,
+    device=device,
+)
+'''
+

static/index.html

@@ -15,14 +15,30 @@
 Prompt:<br/>
 <textarea
 	id="text-gen-input"
-	type="text">Sample</textarea>
+	type="text">
+
+{
+  query: "Call me an Uber ride type \"Plus\" in Berkeley at zipcode 94704 in 10 minutes",
+  function: [
+    {
+        "name": "Uber Carpool",
+        "api_name": "uber.ride",
+        "description": "Find suitable ride for customers given the location, type of ride, and the amount of time the customer is willing to wait as parameters",
+        "parameters":  [
+            {"name": "loc", "description": "Location of the starting place of the Uber ride"},
+            {"name": "type", "enum": ["plus", "comfort", "black"], "description": "Types of Uber ride user is ordering"},
+            {"name": "time", "description": "The amount of time in minutes the customer is willing to wait"}
+        ]
+    }
+]
+
+</textarea>
 	<br/>
 <button id="text-gen-submit">Submit</button>
 
 <p/>
 Result:<br/>
           <p class="text-gen-output">
-					axyz
 					</p>
 					
         </form>

static/script.js

@@ -1,7 +1,13 @@
 const textGenForm = document.querySelector('.text-gen-form');
 
 const translateText = async (text) => {
-    const inferResponse = await fetch(`query_gorilla?input=${text}`);
+    const inferResponse = await fetch(
+		  `query_gorilla`,
+			{
+				method: 'POST',
+				body: text
+			}
+		);
     const inferJson = await inferResponse.json();
 
     return inferJson.output;