Update the LynxScribe ops with OpenAI API change.

lynxkite-app/.gitignore

@@ -0,0 +1,3 @@
+/src/lynxkite/app/web_assets
+!/src/lynxkite/app/web_assets/__init__.py
+!/src/lynxkite/app/web_assets/assets/__init__.py

lynxkite-app/data/LynxScribe demo

@@ -14,24 +14,24 @@
         "meta": {
           "outputs": {
             "output": {
-              "name": "output",
               "type": {
                 "type": "None"
               },
-              "position": "right"
+              "position": "right",
+              "name": "output"
             }
           },
-          "name": "Input chat",
           "inputs": {},
           "params": {
             "chat": {
-              "name": "chat",
-              "default": null,
               "type": {
                 "type": "<class 'str'>"
-              }
+              },
+              "name": "chat",
+              "default": null
             }
           },
+          "name": "Input chat",
           "type": "basic"
         }
       },
@@ -39,8 +39,8 @@
         "x": -493.5496596237119,
         "y": 20.90123252513356
       },
-      "parentId": null,
       "height": 186.0,
+      "parentId": null,
       "width": 259.0
     },
     {
@@ -52,13 +52,13 @@
         "display": {
           "dataframes": {
             "df": {
+              "columns": [
+                "answer"
+              ],
               "data": [
                 [
-                  "The Chief Innovation Officer (CIO) of Lynx Analytics is Gabor Benedek. He is responsible for defining the methodological roadmap for the company's predictive analytics consulting and product development departments. If you need more specific information about his role, feel free to ask!\n\nPlease visit <a href='https://www.lynxanalytics.com/team-leadership' target='_blank'>https://www.lynxanalytics.com/team-leadership</a> for further information."
+                  "The Chief Innovation Officer (CIO) of Lynx Analytics is Gabor Benedek. He is responsible for defining the methodological roadmap for Lynx's predictive analytics consulting and product development departments. If you need more specific information or have follow-up questions, feel free to ask!\n\nPlease visit https://www.lynxanalytics.com/team-leadership for further information."
                 ]
-              ],
-              "columns": [
-                "answer"
               ]
             }
           }
@@ -72,10 +72,10 @@
           "inputs": {
             "input": {
               "position": "left",
+              "name": "input",
               "type": {
                 "type": "<class 'inspect._empty'>"
-              },
-              "name": "input"
+              }
             }
           }
         }
@@ -84,9 +84,9 @@
         "x": 722.1450069267316,
         "y": -785.6076562320527
       },
+      "height": 950.0,
       "parentId": null,
-      "height": 200.0,
-      "width": 200.0
+      "width": 1256.0
     },
     {
       "id": "LLM 1",
@@ -99,8 +99,9 @@
         "display": null,
         "error": null,
         "meta": {
-          "name": "LLM",
           "inputs": {},
+          "type": "basic",
+          "name": "LLM",
           "params": {
             "name": {
               "name": "name",
@@ -112,14 +113,13 @@
           },
           "outputs": {
             "output": {
-              "position": "top",
+              "name": "output",
               "type": {
                 "type": "None"
               },
-              "name": "output"
+              "position": "top"
             }
-          },
-          "type": "basic"
+          }
         }
       },
       "position": {
@@ -142,32 +142,32 @@
         "display": null,
         "error": null,
         "meta": {
+          "type": "basic",
           "name": "Scenario selector",
           "inputs": {},
+          "outputs": {
+            "output": {
+              "type": {
+                "type": "None"
+              },
+              "position": "top",
+              "name": "output"
+            }
+          },
           "params": {
             "scenario_file": {
-              "default": null,
               "type": {
                 "type": "<class 'str'>"
               },
+              "default": null,
               "name": "scenario_file"
             },
             "node_types": {
-              "name": "node_types",
-              "default": "intent_cluster",
               "type": {
                 "type": "<class 'str'>"
-              }
-            }
-          },
-          "type": "basic",
-          "outputs": {
-            "output": {
-              "type": {
-                "type": "None"
               },
-              "name": "output",
-              "position": "top"
+              "name": "node_types",
+              "default": "intent_cluster"
             }
           }
         }
@@ -176,8 +176,8 @@
         "x": -549.1300345090008,
         "y": 1086.4852248156676
       },
-      "height": 200.0,
       "width": 200.0,
+      "height": 200.0,
       "parentId": null
     },
     {
@@ -192,30 +192,15 @@
         "error": null,
         "meta": {
           "name": "Chat API",
-          "inputs": {
-            "chatbot": {
-              "position": "bottom",
-              "name": "chatbot",
-              "type": {
-                "type": "<class 'inspect._empty'>"
-              }
-            },
-            "knowledge_base": {
-              "type": {
-                "type": "<class 'inspect._empty'>"
-              },
-              "position": "bottom",
-              "name": "knowledge_base"
-            },
-            "chat_processor": {
-              "position": "bottom",
-              "name": "chat_processor",
+          "outputs": {
+            "output": {
+              "position": "top",
+              "name": "output",
               "type": {
-                "type": "<class 'inspect._empty'>"
+                "type": "None"
               }
             }
           },
-          "type": "basic",
           "params": {
             "model": {
               "type": {
@@ -225,15 +210,30 @@
               "name": "model"
             }
           },
-          "outputs": {
-            "output": {
-              "position": "top",
+          "inputs": {
+            "knowledge_base": {
               "type": {
-                "type": "None"
+                "type": "<class 'inspect._empty'>"
               },
-              "name": "output"
+              "name": "knowledge_base",
+              "position": "bottom"
+            },
+            "chatbot": {
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              },
+              "position": "bottom",
+              "name": "chatbot"
+            },
+            "chat_processor": {
+              "position": "bottom",
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              },
+              "name": "chat_processor"
             }
-          }
+          },
+          "type": "basic"
         }
       },
       "position": {
@@ -251,55 +251,55 @@
         "title": "Knowledge base",
         "params": {
           "nodes_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/nodes.pickle",
-          "template_cluster_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/tempclusters.pickle",
-          "edges_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/edges.pickle"
+          "edges_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/edges.pickle",
+          "template_cluster_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/tempclusters.pickle"
         },
         "display": null,
         "error": null,
         "meta": {
           "params": {
-            "edges_path": {
-              "name": "edges_path",
-              "default": "edges.pickle",
+            "nodes_path": {
+              "name": "nodes_path",
+              "default": "nodes.pickle",
               "type": {
                 "type": "<class 'str'>"
               }
             },
-            "template_cluster_path": {
-              "name": "template_cluster_path",
-              "default": "tempclusters.pickle",
+            "edges_path": {
               "type": {
                 "type": "<class 'str'>"
-              }
+              },
+              "name": "edges_path",
+              "default": "edges.pickle"
             },
-            "nodes_path": {
-              "name": "nodes_path",
-              "default": "nodes.pickle",
+            "template_cluster_path": {
+              "name": "template_cluster_path",
+              "default": "tempclusters.pickle",
               "type": {
                 "type": "<class 'str'>"
               }
             }
           },
           "name": "Knowledge base",
+          "inputs": {},
+          "type": "basic",
           "outputs": {
             "output": {
+              "name": "output",
               "type": {
                 "type": "None"
               },
-              "position": "top",
-              "name": "output"
+              "position": "top"
             }
-          },
-          "inputs": {},
-          "type": "basic"
+          }
         }
       },
       "position": {
         "x": 598.8683124946176,
         "y": 609.9499973808545
       },
-      "parentId": null,
       "height": 320.0,
+      "parentId": null,
       "width": 336.0
     },
     {
@@ -308,47 +308,22 @@
       "data": {
         "title": "RAG chatbot",
         "params": {
+          "limits_by_type": "{\"information\": [2, 3], \"summary\": [2, 3]}",
           "negative_answer": "I'm sorry, but the data I've been trained on does not contain any information related to your question.",
           "max_results": "5",
-          "limits_by_type": "{\"information\": [2, 3], \"summary\": [2, 3]}",
           "strict_limits": true
         },
         "display": null,
         "error": null,
         "meta": {
-          "outputs": {
-            "output": {
-              "position": "top",
-              "type": {
-                "type": "None"
-              },
-              "name": "output"
-            }
-          },
-          "inputs": {
-            "llm": {
-              "type": {
-                "type": "<class 'inspect._empty'>"
-              },
-              "position": "bottom",
-              "name": "llm"
-            },
-            "rag_graph": {
-              "name": "rag_graph",
-              "position": "bottom",
+          "params": {
+            "negative_answer": {
+              "default": "I'm sorry, but the data I've been trained on does not contain any information related to your question.",
+              "name": "negative_answer",
               "type": {
-                "type": "<class 'inspect._empty'>"
+                "type": "<class 'str'>"
               }
             },
-            "scenario_selector": {
-              "position": "bottom",
-              "type": {
-                "type": "<class 'inspect._empty'>"
-              },
-              "name": "scenario_selector"
-            }
-          },
-          "params": {
             "max_results": {
               "name": "max_results",
               "type": {
@@ -356,30 +331,55 @@
               },
               "default": 5.0
             },
-            "negative_answer": {
-              "type": {
-                "type": "<class 'str'>"
-              },
-              "name": "negative_answer",
-              "default": "I'm sorry, but the data I've been trained on does not contain any information related to your question."
-            },
             "strict_limits": {
               "default": true,
+              "name": "strict_limits",
               "type": {
                 "type": "<class 'bool'>"
-              },
-              "name": "strict_limits"
+              }
             },
             "limits_by_type": {
-              "default": "{}",
               "name": "limits_by_type",
+              "default": "{}",
               "type": {
                 "type": "<class 'str'>"
               }
             }
           },
+          "outputs": {
+            "output": {
+              "type": {
+                "type": "None"
+              },
+              "position": "top",
+              "name": "output"
+            }
+          },
+          "type": "basic",
           "name": "RAG chatbot",
-          "type": "basic"
+          "inputs": {
+            "rag_graph": {
+              "name": "rag_graph",
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              },
+              "position": "bottom"
+            },
+            "scenario_selector": {
+              "name": "scenario_selector",
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              },
+              "position": "bottom"
+            },
+            "llm": {
+              "name": "llm",
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              },
+              "position": "bottom"
+            }
+          }
         },
         "beingResized": false
       },
@@ -387,8 +387,8 @@
         "x": -533.1301830766971,
         "y": 547.294980747757
       },
-      "parentId": null,
       "width": 339.0,
+      "parentId": null,
       "height": 399.0
     },
     {
@@ -401,8 +401,23 @@
         "error": null,
         "meta": {
           "name": "RAG graph",
+          "inputs": {
+            "vector_store": {
+              "name": "vector_store",
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              },
+              "position": "bottom"
+            },
+            "text_embedder": {
+              "type": {
+                "type": "<class 'inspect._empty'>"
+              },
+              "position": "bottom",
+              "name": "text_embedder"
+            }
+          },
           "params": {},
-          "type": "basic",
           "outputs": {
             "output": {
               "position": "top",
@@ -412,22 +427,7 @@
               }
             }
           },
-          "inputs": {
-            "text_embedder": {
-              "position": "bottom",
-              "name": "text_embedder",
-              "type": {
-                "type": "<class 'inspect._empty'>"
-              }
-            },
-            "vector_store": {
-              "name": "vector_store",
-              "position": "bottom",
-              "type": {
-                "type": "<class 'inspect._empty'>"
-              }
-            }
-          }
+          "type": "basic"
         }
       },
       "position": {
@@ -449,25 +449,25 @@
         },
         "display": null,
         "error": null,
+        "beingResized": false,
         "meta": {
-          "name": "Vector store",
           "params": {
-            "name": {
-              "name": "name",
-              "default": "chromadb",
+            "collection_name": {
+              "default": "lynx",
               "type": {
                 "type": "<class 'str'>"
-              }
+              },
+              "name": "collection_name"
             },
-            "collection_name": {
-              "default": "lynx",
-              "name": "collection_name",
+            "name": {
+              "default": "chromadb",
+              "name": "name",
               "type": {
                 "type": "<class 'str'>"
               }
             }
           },
-          "inputs": {},
+          "name": "Vector store",
           "outputs": {
             "output": {
               "name": "output",
@@ -477,17 +477,17 @@
               "position": "top"
             }
           },
-          "type": "basic"
-        },
-        "beingResized": false
+          "type": "basic",
+          "inputs": {}
+        }
       },
       "position": {
         "x": -1053.794625339574,
         "y": 1347.7711940497127
       },
-      "width": 275.0,
+      "height": 227.0,
       "parentId": null,
-      "height": 227.0
+      "width": 275.0
     },
     {
       "id": "Text embedder 2",
@@ -500,33 +500,33 @@
         "display": null,
         "error": null,
         "meta": {
-          "name": "Text embedder",
+          "type": "basic",
           "params": {
             "model": {
-              "default": "text-embedding-ada-002",
               "type": {
                 "type": "<class 'str'>"
               },
-              "name": "model"
+              "name": "model",
+              "default": "text-embedding-ada-002"
             }
           },
-          "outputs": {
-            "output": {
-              "type": {
-                "type": "None"
-              },
-              "name": "output",
-              "position": "top"
-            }
-          },
-          "type": "basic",
           "inputs": {
             "llm": {
               "name": "llm",
+              "position": "bottom",
               "type": {
                 "type": "<class 'inspect._empty'>"
+              }
+            }
+          },
+          "name": "Text embedder",
+          "outputs": {
+            "output": {
+              "name": "output",
+              "type": {
+                "type": "None"
               },
-              "position": "bottom"
+              "position": "top"
             }
           }
         }
@@ -535,9 +535,9 @@
         "x": -719.98604638686,
         "y": 1343.5978526690794
       },
+      "width": 200.0,
       "parentId": null,
-      "height": 200.0,
-      "width": 200.0
+      "height": 200.0
     },
     {
       "id": "LLM 2",
@@ -550,15 +550,8 @@
         "display": null,
         "error": null,
         "meta": {
-          "outputs": {
-            "output": {
-              "type": {
-                "type": "None"
-              },
-              "name": "output",
-              "position": "top"
-            }
-          },
+          "inputs": {},
+          "type": "basic",
           "params": {
             "name": {
               "name": "name",
@@ -568,9 +561,16 @@
               }
             }
           },
-          "inputs": {},
-          "name": "LLM",
-          "type": "basic"
+          "outputs": {
+            "output": {
+              "name": "output",
+              "type": {
+                "type": "None"
+              },
+              "position": "top"
+            }
+          },
+          "name": "LLM"
         }
       },
       "position": {
@@ -592,27 +592,27 @@
         "display": null,
         "error": null,
         "meta": {
+          "name": "Truncate history",
           "inputs": {},
+          "type": "basic",
           "params": {
             "max_tokens": {
-              "default": 10000.0,
               "type": {
                 "type": "<class 'int'>"
               },
+              "default": 10000.0,
               "name": "max_tokens"
             }
           },
           "outputs": {
             "output": {
-              "position": "top",
-              "name": "output",
               "type": {
                 "type": "None"
-              }
+              },
+              "name": "output",
+              "position": "top"
             }
-          },
-          "type": "basic",
-          "name": "Truncate history"
+          }
         }
       },
       "position": {
@@ -620,8 +620,8 @@
         "y": 1044.7639853229612
       },
       "height": 200.0,
-      "width": 200.0,
-      "parentId": null
+      "parentId": null,
+      "width": 200.0
     },
     {
       "id": "Chat processor 1",
@@ -631,30 +631,30 @@
         "params": {},
         "display": null,
         "error": null,
-        "__execution_delay": null,
         "collapsed": true,
+        "__execution_delay": null,
         "meta": {
-          "outputs": {
-            "output": {
-              "type": {
-                "type": "None"
-              },
-              "position": "top",
-              "name": "output"
-            }
-          },
-          "type": "basic",
           "inputs": {
             "processor": {
+              "position": "bottom",
               "type": {
                 "type": "<class 'inspect._empty'>"
               },
-              "position": "bottom",
               "name": "processor"
             }
           },
           "params": {},
-          "name": "Chat processor"
+          "name": "Chat processor",
+          "outputs": {
+            "output": {
+              "position": "top",
+              "name": "output",
+              "type": {
+                "type": "None"
+              }
+            }
+          },
+          "type": "basic"
         }
       },
       "position": {
@@ -662,8 +662,8 @@
         "y": 778.546274223181
       },
       "width": 200.0,
-      "height": 200.0,
-      "parentId": null
+      "parentId": null,
+      "height": 200.0
     },
     {
       "id": "Mask 1",
@@ -671,55 +671,55 @@
       "data": {
         "title": "Mask",
         "params": {
-          "regex": "([a-z0-9!#$%&'*+\\/=?^_`{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)",
           "exceptions": "[email protected],[email protected],[email protected],[email protected]",
-          "name": "email",
-          "mask_pattern": "masked_email_address_{}"
+          "regex": "([a-z0-9!#$%&'*+\\/=?^_`{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)",
+          "mask_pattern": "masked_email_address_{}",
+          "name": "email"
         },
         "display": null,
         "error": null,
         "meta": {
-          "inputs": {},
           "outputs": {
             "output": {
-              "name": "output",
+              "position": "top",
               "type": {
                 "type": "None"
               },
-              "position": "top"
+              "name": "output"
             }
           },
-          "type": "basic",
           "params": {
-            "mask_pattern": {
+            "name": {
               "default": "",
+              "name": "name",
               "type": {
                 "type": "<class 'str'>"
-              },
-              "name": "mask_pattern"
+              }
             },
-            "regex": {
-              "name": "regex",
+            "exceptions": {
+              "name": "exceptions",
               "default": "",
               "type": {
                 "type": "<class 'str'>"
               }
             },
-            "name": {
-              "name": "name",
+            "mask_pattern": {
               "default": "",
               "type": {
                 "type": "<class 'str'>"
-              }
+              },
+              "name": "mask_pattern"
             },
-            "exceptions": {
-              "name": "exceptions",
-              "default": "",
+            "regex": {
               "type": {
                 "type": "<class 'str'>"
-              }
+              },
+              "name": "regex",
+              "default": ""
             }
           },
+          "type": "basic",
+          "inputs": {},
           "name": "Mask"
         }
       },
@@ -727,8 +727,8 @@
         "x": 233.69759202223884,
         "y": 1041.6145468043276
       },
-      "parentId": null,
       "width": 200.0,
+      "parentId": null,
       "height": 200.0
     },
     {
@@ -738,64 +738,64 @@
         "title": "Mask",
         "params": {
           "mask_pattern": "masked_credit_card_number_{}",
-          "name": "credit_card",
+          "exceptions": "",
           "regex": "((?:(?:\\\\d{4}[- ]?){3}\\\\d{4}|\\\\d{15,16}))(?![\\\\d])",
-          "exceptions": ""
+          "name": "credit_card"
         },
         "display": null,
         "error": null,
         "meta": {
-          "inputs": {},
+          "name": "Mask",
+          "outputs": {
+            "output": {
+              "name": "output",
+              "type": {
+                "type": "None"
+              },
+              "position": "top"
+            }
+          },
           "type": "basic",
+          "inputs": {},
           "params": {
-            "exceptions": {
-              "name": "exceptions",
+            "mask_pattern": {
               "default": "",
               "type": {
                 "type": "<class 'str'>"
-              }
+              },
+              "name": "mask_pattern"
             },
-            "mask_pattern": {
-              "name": "mask_pattern",
+            "name": {
               "default": "",
+              "name": "name",
               "type": {
                 "type": "<class 'str'>"
               }
             },
-            "regex": {
-              "name": "regex",
+            "exceptions": {
+              "default": "",
               "type": {
                 "type": "<class 'str'>"
               },
-              "default": ""
+              "name": "exceptions"
             },
-            "name": {
+            "regex": {
+              "default": "",
               "type": {
                 "type": "<class 'str'>"
               },
-              "default": "",
-              "name": "name"
-            }
-          },
-          "outputs": {
-            "output": {
-              "position": "top",
-              "name": "output",
-              "type": {
-                "type": "None"
-              }
+              "name": "regex"
             }
-          },
-          "name": "Mask"
+          }
         }
       },
       "position": {
         "x": 513.2761671440603,
         "y": 1034.8547191984255
       },
-      "width": 200.0,
+      "height": 200.0,
       "parentId": null,
-      "height": 200.0
+      "width": 200.0
     },
     {
       "id": "Test Chat API 2",
@@ -803,58 +803,58 @@
       "data": {
         "title": "Test Chat API",
         "params": {
-          "show_details": true
+          "show_details": false
         },
         "display": null,
         "error": null,
-        "__execution_delay": 0.0,
-        "collapsed": false,
         "meta": {
-          "params": {
-            "show_details": {
-              "name": "show_details",
+          "outputs": {
+            "output": {
+              "position": "right",
               "type": {
-                "type": "<class 'bool'>"
+                "type": "None"
               },
-              "default": false
+              "name": "output"
             }
           },
+          "type": "basic",
           "inputs": {
-            "message": {
-              "name": "message",
+            "chat_api": {
+              "name": "chat_api",
               "type": {
                 "type": "<class 'inspect._empty'>"
               },
-              "position": "left"
+              "position": "bottom"
             },
-            "chat_api": {
-              "position": "bottom",
-              "name": "chat_api",
+            "message": {
               "type": {
                 "type": "<class 'inspect._empty'>"
-              }
+              },
+              "name": "message",
+              "position": "left"
             }
           },
           "name": "Test Chat API",
-          "outputs": {
-            "output": {
-              "position": "right",
-              "name": "output",
+          "params": {
+            "show_details": {
               "type": {
-                "type": "None"
-              }
+                "type": "<class 'bool'>"
+              },
+              "name": "show_details",
+              "default": false
             }
-          },
-          "type": "basic"
-        }
+          }
+        },
+        "__execution_delay": 0.0,
+        "collapsed": false
       },
       "position": {
-        "x": -11.653333595036415,
-        "y": -0.9210389517918429
+        "x": -57.377776548056346,
+        "y": -16.924593985348814
       },
+      "width": 376.0,
       "parentId": null,
-      "width": 200.0,
-      "height": 200.0
+      "height": 225.0
     }
   ],
   "edges": [

lynxkite-app/src/lynxkite/app/main.py

@@ -20,7 +20,6 @@ def detect_plugins():
         print("No modules found in lynxkite_plugins. Be sure to install some plugins.")
         return {}
 
-    print(list(pkgutil.iter_modules(lynxkite_plugins.__path__)))
     plugins = {}
     for _, name, _ in pkgutil.iter_modules(lynxkite_plugins.__path__):
         name = f"lynxkite_plugins.{name}"

lynxkite-core/build/lib/lynxkite/executors/one_by_one.py

@@ -1,175 +0,0 @@
-"""A LynxKite executor that assumes most operations operate on their input one by one."""
-
-from .. import ops
-from .. import workspace
-import orjson
-import pandas as pd
-import pydantic
-import traceback
-import inspect
-import typing
-
-
-class Context(ops.BaseConfig):
-    """Passed to operation functions as "_ctx" if they have such a parameter."""
-
-    node: workspace.WorkspaceNode
-    last_result: typing.Any = None
-
-
-class Output(ops.BaseConfig):
-    """Return this to send values to specific outputs of a node."""
-
-    output_handle: str
-    value: dict
-
-
-def df_to_list(df):
-    return df.to_dict(orient="records")
-
-
-def has_ctx(op):
-    sig = inspect.signature(op.func)
-    return "_ctx" in sig.parameters
-
-
-CACHES = {}
-
-
-def register(env: str, cache: bool = True):
-    """Registers the one-by-one executor."""
-    if cache:
-        CACHES[env] = {}
-        cache = CACHES[env]
-    else:
-        cache = None
-    ops.EXECUTORS[env] = lambda ws: execute(ws, ops.CATALOGS[env], cache=cache)
-
-
-def get_stages(ws, catalog):
-    """Inputs on top/bottom are batch inputs. We decompose the graph into a DAG of components along these edges."""
-    nodes = {n.id: n for n in ws.nodes}
-    batch_inputs = {}
-    inputs = {}
-    for edge in ws.edges:
-        inputs.setdefault(edge.target, []).append(edge.source)
-        node = nodes[edge.target]
-        op = catalog[node.data.title]
-        i = op.inputs[edge.targetHandle]
-        if i.position in "top or bottom":
-            batch_inputs.setdefault(edge.target, []).append(edge.source)
-    stages = []
-    for bt, bss in batch_inputs.items():
-        upstream = set(bss)
-        new = set(bss)
-        while new:
-            n = new.pop()
-            for i in inputs.get(n, []):
-                if i not in upstream:
-                    upstream.add(i)
-                    new.add(i)
-        stages.append(upstream)
-    stages.sort(key=lambda s: len(s))
-    stages.append(set(nodes))
-    return stages
-
-
-def _default_serializer(obj):
-    if isinstance(obj, pydantic.BaseModel):
-        return obj.dict()
-    return {"__nonserializable__": id(obj)}
-
-
-def make_cache_key(obj):
-    return orjson.dumps(obj, default=_default_serializer)
-
-
-EXECUTOR_OUTPUT_CACHE = {}
-
-
-async def await_if_needed(obj):
-    if inspect.isawaitable(obj):
-        return await obj
-    return obj
-
-
-async def execute(ws, catalog, cache=None):
-    nodes = {n.id: n for n in ws.nodes}
-    contexts = {n.id: Context(node=n) for n in ws.nodes}
-    edges = {n.id: [] for n in ws.nodes}
-    for e in ws.edges:
-        edges[e.source].append(e)
-    tasks = {}
-    NO_INPUT = object()  # Marker for initial tasks.
-    for node in ws.nodes:
-        node.data.error = None
-        op = catalog.get(node.data.title)
-        if op is None:
-            node.data.error = f'Operation "{node.data.title}" not found.'
-            continue
-        # Start tasks for nodes that have no non-batch inputs.
-        if all([i.position in "top or bottom" for i in op.inputs.values()]):
-            tasks[node.id] = [NO_INPUT]
-    batch_inputs = {}
-    # Run the rest until we run out of tasks.
-    stages = get_stages(ws, catalog)
-    for stage in stages:
-        next_stage = {}
-        while tasks:
-            n, ts = tasks.popitem()
-            if n not in stage:
-                next_stage.setdefault(n, []).extend(ts)
-                continue
-            node = nodes[n]
-            data = node.data
-            op = catalog[data.title]
-            params = {**data.params}
-            if has_ctx(op):
-                params["_ctx"] = contexts[node.id]
-            results = []
-            for task in ts:
-                try:
-                    inputs = []
-                    for i in op.inputs.values():
-                        if i.position in "top or bottom":
-                            assert (n, i.name) in batch_inputs, f"{i.name} is missing"
-                            inputs.append(batch_inputs[(n, i.name)])
-                        else:
-                            inputs.append(task)
-                    if cache is not None:
-                        key = make_cache_key((inputs, params))
-                        if key not in cache:
-                            cache[key] = await await_if_needed(op(*inputs, **params))
-                        result = cache[key]
-                    else:
-                        result = await await_if_needed(op(*inputs, **params))
-                except Exception as e:
-                    traceback.print_exc()
-                    data.error = str(e)
-                    break
-                contexts[node.id].last_result = result
-                # Returned lists and DataFrames are considered multiple tasks.
-                if isinstance(result, pd.DataFrame):
-                    result = df_to_list(result)
-                elif not isinstance(result, list):
-                    result = [result]
-                results.extend(result)
-            else:  # Finished all tasks without errors.
-                if (
-                    op.type == "visualization"
-                    or op.type == "table_view"
-                    or op.type == "image"
-                ):
-                    data.display = results[0]
-                for edge in edges[node.id]:
-                    t = nodes[edge.target]
-                    op = catalog[t.data.title]
-                    i = op.inputs[edge.targetHandle]
-                    if i.position in "top or bottom":
-                        batch_inputs.setdefault(
-                            (edge.target, edge.targetHandle), []
-                        ).extend(results)
-                    else:
-                        tasks.setdefault(edge.target, []).extend(results)
-        tasks = next_stage
-    return contexts

lynxkite-core/build/lib/lynxkite/ops.py

@@ -1,224 +0,0 @@
-"""API for implementing LynxKite operations."""
-
-from __future__ import annotations
-import enum
-import functools
-import inspect
-import pydantic
-import typing
-from typing_extensions import Annotated
-
-CATALOGS = {}
-EXECUTORS = {}
-
-typeof = type  # We have some arguments called "type".
-
-
-def type_to_json(t):
-    if isinstance(t, type) and issubclass(t, enum.Enum):
-        return {"enum": list(t.__members__.keys())}
-    if getattr(t, "__metadata__", None):
-        return t.__metadata__[-1]
-    return {"type": str(t)}
-
-
-Type = Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)]
-LongStr = Annotated[str, {"format": "textarea"}]
-PathStr = Annotated[str, {"format": "path"}]
-CollapsedStr = Annotated[str, {"format": "collapsed"}]
-NodeAttribute = Annotated[str, {"format": "node attribute"}]
-EdgeAttribute = Annotated[str, {"format": "edge attribute"}]
-
-
-class BaseConfig(pydantic.BaseModel):
-    model_config = pydantic.ConfigDict(
-        arbitrary_types_allowed=True,
-    )
-
-
-class Parameter(BaseConfig):
-    """Defines a parameter for an operation."""
-
-    name: str
-    default: typing.Any
-    type: Type = None
-
-    @staticmethod
-    def options(name, options, default=None):
-        e = enum.Enum(f"OptionsFor_{name}", options)
-        return Parameter.basic(name, e[default or options[0]], e)
-
-    @staticmethod
-    def collapsed(name, default, type=None):
-        return Parameter.basic(name, default, CollapsedStr)
-
-    @staticmethod
-    def basic(name, default=None, type=None):
-        if default is inspect._empty:
-            default = None
-        if type is None or type is inspect._empty:
-            type = typeof(default) if default is not None else None
-        return Parameter(name=name, default=default, type=type)
-
-
-class Input(BaseConfig):
-    name: str
-    type: Type
-    position: str = "left"
-
-
-class Output(BaseConfig):
-    name: str
-    type: Type
-    position: str = "right"
-
-
-MULTI_INPUT = Input(name="multi", type="*")
-
-
-def basic_inputs(*names):
-    return {name: Input(name=name, type=None) for name in names}
-
-
-def basic_outputs(*names):
-    return {name: Output(name=name, type=None) for name in names}
-
-
-class Op(BaseConfig):
-    func: typing.Callable = pydantic.Field(exclude=True)
-    name: str
-    params: dict[str, Parameter]
-    inputs: dict[str, Input]
-    outputs: dict[str, Output]
-    type: str = "basic"  # The UI to use for this operation.
-
-    def __call__(self, *inputs, **params):
-        # Convert parameters.
-        for p in params:
-            if p in self.params:
-                if self.params[p].type == int:
-                    params[p] = int(params[p])
-                elif self.params[p].type == float:
-                    params[p] = float(params[p])
-                elif isinstance(self.params[p].type, enum.EnumMeta):
-                    params[p] = self.params[p].type[params[p]]
-        res = self.func(*inputs, **params)
-        return res
-
-
-def op(env: str, name: str, *, view="basic", outputs=None):
-    """Decorator for defining an operation."""
-
-    def decorator(func):
-        sig = inspect.signature(func)
-        # Positional arguments are inputs.
-        inputs = {
-            name: Input(name=name, type=param.annotation)
-            for name, param in sig.parameters.items()
-            if param.kind != param.KEYWORD_ONLY
-        }
-        params = {}
-        for n, param in sig.parameters.items():
-            if param.kind == param.KEYWORD_ONLY and not n.startswith("_"):
-                params[n] = Parameter.basic(n, param.default, param.annotation)
-        if outputs:
-            _outputs = {name: Output(name=name, type=None) for name in outputs}
-        else:
-            _outputs = (
-                {"output": Output(name="output", type=None)} if view == "basic" else {}
-            )
-        op = Op(
-            func=func,
-            name=name,
-            params=params,
-            inputs=inputs,
-            outputs=_outputs,
-            type=view,
-        )
-        CATALOGS.setdefault(env, {})
-        CATALOGS[env][name] = op
-        func.__op__ = op
-        return func
-
-    return decorator
-
-
-def input_position(**kwargs):
-    """Decorator for specifying unusual positions for the inputs."""
-
-    def decorator(func):
-        op = func.__op__
-        for k, v in kwargs.items():
-            op.inputs[k].position = v
-        return func
-
-    return decorator
-
-
-def output_position(**kwargs):
-    """Decorator for specifying unusual positions for the outputs."""
-
-    def decorator(func):
-        op = func.__op__
-        for k, v in kwargs.items():
-            op.outputs[k].position = v
-        return func
-
-    return decorator
-
-
-def no_op(*args, **kwargs):
-    if args:
-        return args[0]
-    return None
-
-
-def register_passive_op(env: str, name: str, inputs=[], outputs=["output"], params=[]):
-    """A passive operation has no associated code."""
-    op = Op(
-        func=no_op,
-        name=name,
-        params={p.name: p for p in params},
-        inputs=dict(
-            (i, Input(name=i, type=None)) if isinstance(i, str) else (i.name, i)
-            for i in inputs
-        ),
-        outputs=dict(
-            (o, Output(name=o, type=None)) if isinstance(o, str) else (o.name, o)
-            for o in outputs
-        ),
-    )
-    CATALOGS.setdefault(env, {})
-    CATALOGS[env][name] = op
-    return op
-
-
-def register_executor(env: str):
-    """Decorator for registering an executor."""
-
-    def decorator(func):
-        EXECUTORS[env] = func
-        return func
-
-    return decorator
-
-
-def op_registration(env: str):
-    return functools.partial(op, env)
-
-
-def passive_op_registration(env: str):
-    return functools.partial(register_passive_op, env)
-
-
-def register_area(env, name, params=[]):
-    """A node that represents an area. It can contain other nodes, but does not restrict movement in any way."""
-    op = Op(
-        func=no_op,
-        name=name,
-        params={p.name: p for p in params},
-        inputs={},
-        outputs={},
-        type="area",
-    )
-    CATALOGS[env][name] = op

lynxkite-core/build/lib/lynxkite/workspace.py

@@ -1,96 +0,0 @@
-"""For working with LynxKite workspaces."""
-
-from typing import Optional
-import dataclasses
-import os
-import pydantic
-import tempfile
-from . import ops
-
-
-class BaseConfig(pydantic.BaseModel):
-    model_config = pydantic.ConfigDict(
-        extra="allow",
-    )
-
-
-class Position(BaseConfig):
-    x: float
-    y: float
-
-
-class WorkspaceNodeData(BaseConfig):
-    title: str
-    params: dict
-    display: Optional[object] = None
-    error: Optional[str] = None
-    # Also contains a "meta" field when going out.
-    # This is ignored when coming back from the frontend.
-
-
-class WorkspaceNode(BaseConfig):
-    id: str
-    type: str
-    data: WorkspaceNodeData
-    position: Position
-
-
-class WorkspaceEdge(BaseConfig):
-    id: str
-    source: str
-    target: str
-    sourceHandle: str
-    targetHandle: str
-
-
-class Workspace(BaseConfig):
-    env: str = ""
-    nodes: list[WorkspaceNode] = dataclasses.field(default_factory=list)
-    edges: list[WorkspaceEdge] = dataclasses.field(default_factory=list)
-
-
-async def execute(ws: Workspace):
-    if ws.env in ops.EXECUTORS:
-        await ops.EXECUTORS[ws.env](ws)
-
-
-def save(ws: Workspace, path: str):
-    j = ws.model_dump_json(indent=2)
-    dirname, basename = os.path.split(path)
-    # Create temp file in the same directory to make sure it's on the same filesystem.
-    with tempfile.NamedTemporaryFile(
-        "w", prefix=f".{basename}.", dir=dirname, delete=False
-    ) as f:
-        temp_name = f.name
-        f.write(j)
-    os.replace(temp_name, path)
-
-
-def load(path: str):
-    with open(path) as f:
-        j = f.read()
-    ws = Workspace.model_validate_json(j)
-    # Metadata is added after loading. This way code changes take effect on old boxes too.
-    _update_metadata(ws)
-    return ws
-
-
-def _update_metadata(ws):
-    catalog = ops.CATALOGS.get(ws.env, {})
-    nodes = {node.id: node for node in ws.nodes}
-    done = set()
-    while len(done) < len(nodes):
-        for node in ws.nodes:
-            if node.id in done:
-                continue
-            data = node.data
-            op = catalog.get(data.title)
-            if op:
-                data.meta = op
-                node.type = op.type
-                if data.error == "Unknown operation.":
-                    data.error = None
-            else:
-                data.error = "Unknown operation."
-            done.add(node.id)
-    return ws

lynxkite-lynxscribe/pyproject.toml

@@ -6,6 +6,7 @@ readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
     "lynxkite-core",
+    "lynxscribe[openai] @ git+ssh://[email protected]/biggraph/lynxscribe@main",
 ]
 
 [tool.uv.sources]

lynxkite-lynxscribe/src/lynxkite_plugins/lynxscribe/lynxscribe_ops.py

@@ -14,7 +14,8 @@ from lynxscribe.components.chat.processors import (
     MaskTemplate,
     TruncateHistory,
 )
-from lynxscribe.components.chat.api import ChatAPI, ChatAPIRequest, ChatAPIResponse
+from lynxscribe.components.chat.api import ChatAPI
+from lynxscribe.core.models.prompts import ChatCompletionPrompt
 
 from lynxkite.core import ops
 import json
@@ -155,16 +156,15 @@ def mask(*, name="", regex="", exceptions="", mask_pattern=""):
 @op("Test Chat API")
 async def test_chat_api(message, chat_api, *, show_details=False):
     chat_api = chat_api[0]["chat_api"]
-    request = ChatAPIRequest(
-        session_id="b43215a0-428f-11ef-9454-0242ac120002",
-        question=message["text"],
-        history=[],
+    request = ChatCompletionPrompt(
+        model="",
+        messages=[{"role": "user", "content": message["text"]}],
     )
     response = await chat_api.answer(request)
     if show_details:
         return {**response.__dict__}
     else:
-        return {"answer": response.answer}
+        return {"answer": response.choices[0].message.content}
 
 
 @op("Input chat")
@@ -237,12 +237,7 @@ async def get_chat_api(ws):
 
 async def stream_chat_api_response(request):
     chat_api = await get_chat_api(request["model"])
-    chat_api_request = ChatAPIRequest(
-        session_id=request.get("session_id", "00000000-0000-0000-0000-000000000000"),
-        history=request["messages"][:-1],
-        question=request["messages"][-1]["content"],
-    )
-    response = await chat_api.answer(chat_api_request)
+    response = await chat_api.answer(request)
     response = response.model_dump()
     yield json.dumps(
         {