bugfix of text input handling

examples/LynxScribe Image RAG

@@ -1,19 +1,5 @@
 {
   "edges": [
-    {
-      "id": "LynxScribe Image Describer 1 LynxScribe Image RAG Builder 1",
-      "source": "LynxScribe Image Describer 1",
-      "sourceHandle": "output",
-      "target": "LynxScribe Image RAG Builder 1",
-      "targetHandle": "image_describer"
-    },
-    {
-      "id": "LynxScribe RAG Vector Store 1 LynxScribe Image RAG Builder 1",
-      "source": "LynxScribe RAG Vector Store 1",
-      "sourceHandle": "output",
-      "target": "LynxScribe Image RAG Builder 1",
-      "targetHandle": "rag_graph"
-    },
     {
       "id": "GCP Image Loader 1 LynxScribe Image RAG Builder 1",
       "source": "GCP Image Loader 1",
@@ -34,79 +20,31 @@
       "sourceHandle": "output",
       "target": "LynxScribe Image RAG Query 1",
       "targetHandle": "text"
+    },
+    {
+      "id": "LynxScribe Image Describer 1 LynxScribe Image RAG Builder 1",
+      "source": "LynxScribe Image Describer 1",
+      "sourceHandle": "output",
+      "target": "LynxScribe Image RAG Builder 1",
+      "targetHandle": "image_describer"
+    },
+    {
+      "id": "LynxScribe RAG Vector Store 1 LynxScribe Image RAG Builder 1",
+      "source": "LynxScribe RAG Vector Store 1",
+      "sourceHandle": "output",
+      "target": "LynxScribe Image RAG Builder 1",
+      "targetHandle": "rag_graph"
+    },
+    {
+      "id": "LynxScribe Image RAG Query 1 View image 1",
+      "source": "LynxScribe Image RAG Query 1",
+      "sourceHandle": "output",
+      "target": "View image 1",
+      "targetHandle": "embedding_similarities"
     }
   ],
   "env": "LynxScribe",
   "nodes": [
-    {
-      "data": {
-        "__execution_delay": 0.0,
-        "collapsed": false,
-        "display": null,
-        "error": null,
-        "meta": {
-          "inputs": {},
-          "name": "LynxScribe Image Describer",
-          "outputs": {
-            "output": {
-              "name": "output",
-              "position": "top",
-              "type": {
-                "type": "None"
-              }
-            }
-          },
-          "params": {
-            "llm_interface": {
-              "default": "openai",
-              "name": "llm_interface",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "llm_prompt_name": {
-              "default": "cot_picture_descriptor",
-              "name": "llm_prompt_name",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "llm_prompt_path": {
-              "default": "/Users/mszel/git/lynxscribe-demos/component_tutorials/04_image_search/image_description_prompts.yaml",
-              "name": "llm_prompt_path",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "llm_visual_model": {
-              "default": "gpt-4o",
-              "name": "llm_visual_model",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            }
-          },
-          "type": "basic"
-        },
-        "params": {
-          "llm_interface": "openai",
-          "llm_prompt_name": "cot_picture_descriptor",
-          "llm_prompt_path": "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
-          "llm_visual_model": "gpt-4o"
-        },
-        "status": "done",
-        "title": "LynxScribe Image Describer"
-      },
-      "dragHandle": ".bg-primary",
-      "height": 358.0,
-      "id": "LynxScribe Image Describer 1",
-      "position": {
-        "x": 97.54029108623294,
-        "y": 622.6506477264763
-      },
-      "type": "basic",
-      "width": 376.0
-    },
     {
       "data": {
         "__execution_delay": 0.0,
@@ -160,87 +98,6 @@
       "type": "basic",
       "width": 282.0
     },
-    {
-      "data": {
-        "__execution_delay": 0.0,
-        "collapsed": null,
-        "display": null,
-        "error": null,
-        "meta": {
-          "inputs": {},
-          "name": "LynxScribe RAG Vector Store",
-          "outputs": {
-            "output": {
-              "name": "output",
-              "position": "top",
-              "type": {
-                "type": "None"
-              }
-            }
-          },
-          "params": {
-            "collection_name": {
-              "default": "lynx",
-              "name": "collection_name",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "name": {
-              "default": "faiss",
-              "name": "name",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "num_dimensions": {
-              "default": 3072.0,
-              "name": "num_dimensions",
-              "type": {
-                "type": "<class 'int'>"
-              }
-            },
-            "text_embedder_interface": {
-              "default": "openai",
-              "name": "text_embedder_interface",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "text_embedder_model_name_or_path": {
-              "default": "text-embedding-3-large",
-              "name": "text_embedder_model_name_or_path",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            }
-          },
-          "position": {
-            "x": 807.0,
-            "y": 315.0
-          },
-          "type": "basic"
-        },
-        "params": {
-          "collection_name": "lynx",
-          "name": "faiss",
-          "num_dimensions": 3072.0,
-          "text_embedder_interface": "openai",
-          "text_embedder_model_name_or_path": "text-embedding-3-large"
-        },
-        "status": "active",
-        "title": "LynxScribe RAG Vector Store"
-      },
-      "dragHandle": ".bg-primary",
-      "height": 435.0,
-      "id": "LynxScribe RAG Vector Store 1",
-      "position": {
-        "x": 507.56541832959726,
-        "y": 625.9615546166448
-      },
-      "type": "basic",
-      "width": 283.0
-    },
     {
       "data": {
         "__execution_delay": 0.0,
@@ -344,7 +201,7 @@
           "type": "basic"
         },
         "params": {
-          "chat": "Show me two cyclists!"
+          "chat": "Show me a picture about meditation!"
         },
         "status": "done",
         "title": "Input chat"
@@ -422,6 +279,219 @@
       },
       "type": "basic",
       "width": 200.0
+    },
+    {
+      "data": {
+        "display": null,
+        "error": null,
+        "meta": {
+          "inputs": {},
+          "name": "LynxScribe Image Describer",
+          "outputs": {
+            "output": {
+              "name": "output",
+              "position": "top",
+              "type": {
+                "type": "None"
+              }
+            }
+          },
+          "params": {
+            "api_key_name": {
+              "default": "OPENAI_API_KEY",
+              "name": "api_key_name",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "llm_interface": {
+              "default": "openai",
+              "name": "llm_interface",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "llm_prompt_name": {
+              "default": "cot_picture_descriptor",
+              "name": "llm_prompt_name",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "llm_prompt_path": {
+              "default": "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
+              "name": "llm_prompt_path",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "llm_visual_model": {
+              "default": "gpt-4o",
+              "name": "llm_visual_model",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            }
+          },
+          "position": {
+            "x": 879.0,
+            "y": 686.0
+          },
+          "type": "basic"
+        },
+        "params": {
+          "api_key_name": "OPENAI_API_KEY",
+          "llm_interface": "openai",
+          "llm_prompt_name": "cot_picture_descriptor",
+          "llm_prompt_path": "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
+          "llm_visual_model": "gpt-4o"
+        },
+        "status": "done",
+        "title": "LynxScribe Image Describer"
+      },
+      "dragHandle": ".bg-primary",
+      "height": 457.0,
+      "id": "LynxScribe Image Describer 1",
+      "position": {
+        "x": 41.706162883849856,
+        "y": 609.2665142696401
+      },
+      "type": "basic",
+      "width": 357.0
+    },
+    {
+      "data": {
+        "display": null,
+        "error": null,
+        "meta": {
+          "inputs": {},
+          "name": "LynxScribe RAG Vector Store",
+          "outputs": {
+            "output": {
+              "name": "output",
+              "position": "top",
+              "type": {
+                "type": "None"
+              }
+            }
+          },
+          "params": {
+            "api_key_name": {
+              "default": "OPENAI_API_KEY",
+              "name": "api_key_name",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "collection_name": {
+              "default": "lynx",
+              "name": "collection_name",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "name": {
+              "default": "faiss",
+              "name": "name",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "num_dimensions": {
+              "default": 3072.0,
+              "name": "num_dimensions",
+              "type": {
+                "type": "<class 'int'>"
+              }
+            },
+            "text_embedder_interface": {
+              "default": "openai",
+              "name": "text_embedder_interface",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "text_embedder_model_name_or_path": {
+              "default": "text-embedding-3-large",
+              "name": "text_embedder_model_name_or_path",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            }
+          },
+          "position": {
+            "x": 1010.0,
+            "y": 713.0
+          },
+          "type": "basic"
+        },
+        "params": {
+          "api_key_name": "OPENAI_API_KEY",
+          "collection_name": "lynx",
+          "name": "faiss",
+          "num_dimensions": 3072.0,
+          "text_embedder_interface": "openai",
+          "text_embedder_model_name_or_path": "text-embedding-3-large"
+        },
+        "status": "active",
+        "title": "LynxScribe RAG Vector Store"
+      },
+      "dragHandle": ".bg-primary",
+      "height": 515.0,
+      "id": "LynxScribe RAG Vector Store 1",
+      "position": {
+        "x": 470.84700620705013,
+        "y": 609.2665142696401
+      },
+      "type": "basic",
+      "width": 350.0
+    },
+    {
+      "data": {
+        "__execution_delay": null,
+        "collapsed": false,
+        "display": null,
+        "error": null,
+        "meta": {
+          "inputs": {
+            "embedding_similarities": {
+              "name": "embedding_similarities",
+              "position": "left",
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              }
+            }
+          },
+          "name": "View image",
+          "outputs": {},
+          "params": {
+            "idx": {
+              "default": 0.0,
+              "name": "idx",
+              "type": {
+                "type": "<class 'int'>"
+              }
+            }
+          },
+          "position": {
+            "x": 1418.0,
+            "y": 272.0
+          },
+          "type": "table_view"
+        },
+        "params": {},
+        "status": "done",
+        "title": "View image"
+      },
+      "dragHandle": ".bg-primary",
+      "height": 342.0,
+      "id": "View image 1",
+      "position": {
+        "x": 1424.7240705678887,
+        "y": -202.17696978464585
+      },
+      "type": "table_view",
+      "width": 652.0
     }
   ]
 }

lynxkite-lynxscribe/src/lynxkite_lynxscribe/lynxscribe_ops.py

@@ -7,6 +7,7 @@ from copy import deepcopy
 import asyncio
 import pandas as pd
 import os
+import joblib
 
 from lynxscribe.core.llm.base import get_llm_engine
 from lynxscribe.core.vector_store.base import get_vector_store
@@ -36,6 +37,8 @@ from lynxkite.core.executors import one_by_one
 
 ENV = "LynxScribe"
 one_by_one.register(ENV)
+os.makedirs("../joblib-cache", exist_ok=True)
+mem = joblib.Memory("../joblib-cache")
 op = ops.op_registration(ENV)
 output_on_top = ops.output_position(output="top")
 
@@ -49,7 +52,6 @@ def gcp_image_loader(
     """
     Gives back the list of URLs of all the images in the GCP storage.
     """
-
     client = storage.Client()
     bucket = client.bucket(gcp_bucket)
     blobs = bucket.list_blobs(prefix=prefix)
@@ -63,6 +65,7 @@ def gcp_image_loader(
 
 @output_on_top
 @op("LynxScribe RAG Vector Store")
+# @mem.cache
 def ls_rag_graph(
     *,
     name: str = "faiss",
@@ -101,6 +104,7 @@ def ls_rag_graph(
 
 @output_on_top
 @op("LynxScribe Image Describer")
+# @mem.cache
 def ls_image_describer(
     *,
     llm_interface: str = "openai",
@@ -132,6 +136,7 @@ def ls_image_describer(
 
 @ops.input_position(image_describer="bottom", rag_graph="bottom")
 @op("LynxScribe Image RAG Builder")
+# @mem.cache
 async def ls_image_rag_builder(
     image_urls,
     image_describer,
@@ -253,9 +258,12 @@ def ls_save_rag_graph(
 @ops.input_position(rag_graph="bottom")
 @op("LynxScribe Image RAG Query")
 async def search_context(rag_graph, text, *, top_k=3):
+    message = text["text"]
+    rag_graph = rag_graph[0]["knowledge_base"]
+
     # get all similarities
     emb_similarities = await rag_graph.search_context(
-        text, max_results=top_k, unique_metadata_key="image_url"
+        message, max_results=top_k, unique_metadata_key="image_url"
     )
 
     # get the image urls, scores and descriptions
@@ -273,6 +281,15 @@ async def search_context(rag_graph, text, *, top_k=3):
     return {"embedding_similarities": result_list}
 
 
+@op("View image", view="image")
+def view_image(embedding_similarities):
+    """
+    Plotting the selected image.
+    """
+    embedding_similarities = embedding_similarities["embedding_similarities"]
+    return embedding_similarities[0]["image_url"]
+
+
 @output_on_top
 @op("Vector store")
 def vector_store(*, name="chromadb", collection_name="lynx"):