Split LynxKite into several Python packages.

.gitignore

@@ -1,4 +1,3 @@
-# Editor directories and files
 .vscode/*
 !.vscode/extensions.json
 .idea
@@ -9,3 +8,6 @@
 *.sln
 *.sw?
 __pycache__
+node_modules
+dist
+*.egg-info

LICENSE.txt

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

README.md

@@ -7,25 +7,10 @@ original LynxKite. The primary goals of this rewrite are:
 - More extensible backend. Make it easy to add new LynxKite boxes. Make it easy to use our frontend for other purposes,
   configuring and executing other pipelines.
 
-## Installation
+## Structure
 
-To run the backend:
-
-```bash
-PYTHONPATH=. uv run pydantic2ts --module server.workspace --output ./web/src/apiTypes.ts --json2ts-cmd "npm exec --prefix web json2ts"
-uv run fastapi run server/main.py --reload
-```
-
-To run the frontend:
-
-```bash
-cd web
-npm i
-npm run dev
-```
-
-To run a chat UI for LynxScribe workspaces:
-
-```bash
-WEBUI_AUTH=false OPENAI_API_BASE_URL=http://localhost:8000/api/service/server.lynxscribe_ops uvx open-webui serve
-```
+- `lynxkite-ops`: Core types and utilities. Depend on this lightweight package if you are writing LynxKite plugins.
+- `lynxkite-app`: The LynxKite web application. Install some plugins then run this to use LynxKite.
+- `lynxkite-graph-analytics`: Graph analytics plugin. The classical LynxKite experience!
+- `lynxkite-pillow`: A simple example plugin.
+- `lynxkite-lynxscribe`: A plugin for building and running LynxScribe applications.

lynxkite-app/MANIFEST.in

@@ -0,0 +1,2 @@
+graft web
+prune web/node_modules

lynxkite-app/README.md

@@ -0,0 +1,25 @@
+# LynxKite MM
+
+This is an experimental rewrite of [LynxKite](https://github.com/lynxkite/lynxkite). It is not compatible with the
+original LynxKite. The primary goals of this rewrite are:
+
+- Target GPU clusters instead of Hadoop clusters. We use Python instead of Scala, RAPIDS instead of Apache Spark.
+- More extensible backend. Make it easy to add new LynxKite boxes. Make it easy to use our frontend for other purposes,
+  configuring and executing other pipelines.
+
+## Development
+
+To run the backend:
+
+```bash
+PYTHONPATH=. uv run pydantic2ts --module server.workspace --output ./web/src/apiTypes.ts --json2ts-cmd "npm exec --prefix web json2ts"
+uv run fastapi run server/main.py --reload
+```
+
+To run the frontend:
+
+```bash
+cd web
+npm i
+npm run dev
+```

{data → lynxkite-app/data}/LynxScribe demo

@@ -11,17 +11,9 @@
         },
         "display": null,
         "error": null,
+        "collapsed": null,
+        "__execution_delay": 0.0,
         "meta": {
-          "params": {
-            "chat": {
-              "name": "chat",
-              "default": null,
-              "type": {
-                "type": "<class 'str'>"
-              }
-            }
-          },
-          "inputs": {},
           "type": "basic",
           "outputs": {
             "output": {
@@ -32,10 +24,18 @@
               }
             }
           },
+          "params": {
+            "chat": {
+              "default": null,
+              "type": {
+                "type": "<class 'str'>"
+              },
+              "name": "chat"
+            }
+          },
+          "inputs": {},
           "name": "Input chat"
-        },
-        "__execution_delay": 0.0,
-        "collapsed": null
+        }
       },
       "position": {
         "x": 0.0,
@@ -48,75 +48,10 @@
       "data": {
         "title": "View",
         "params": {},
-        "display": {
-          "dataframes": {
-            "df": {
-              "columns": [
-                "answer",
-                "sources",
-                "confidence_score",
-                "confidence_score_description",
-                "meta_outputs"
-              ],
-              "data": [
-                [
-                  "The Chief Innovation Officer (CIO) of Lynx Analytics is Gabor Benedek. He is responsible for defining the methodological roadmap for Lynx's predictive analytics consulting and product development departments. If you need more information about Gabor or his role, feel free to ask!\n\nPlease visit https://www.lynxanalytics.com/team-leadership for further information.",
-                  [
-                    {
-                      "id": "i_541",
-                      "body": "Lynx Analytics is a big data company founded in 2010 by Gyorgy Lajtai and a team from INSEAD. The company aims to solve real-world problems using big data graphs. The team is composed of individuals with diverse backgrounds and experiences spanning various industries. The company has a Chief Innovation Officer, a COO, a CTO, a Senior Board Advisor, a Finance Director, and a Director of People. They are responsible for defining the methodological roadmap, implementing solutions, managing finances, and leading HR services and talent management as the company expands globally.\n\n Part of the article: \n\nAs Lynx Analytics CFO between 2016 and 2021, Dan managed the companys capital raising initiatives and set up its financial management functions and team from the ground up.\n\nHe remains involved with the company as a shareholder and in an advisory capacity.\n\nLi Lee, Finance Director\nLi oversees the whole finance spectrum at Lynx Analytics.\n\nThe dynamic environment, exposure to investors, and continuous innovation requirements are aspects of the job she particularly enjoys.\n\nLi likes to find solutions to new business needs and to implement automated internal controls & procedures to support the companys growth.\n\nLi is a financial professional with more than 15 years of experience in financial accounting & reporting and finance-related matters.\nBefore joining Lynx, she held different Finance positions at accounting & audit firms as well as MNCs in various sectors.\n\nEdina Choo, Director of People\nAs the Director of People at Lynx, Edina provides strategic leadership and management across the various HR services provided to our global talent.\n\nShe is responsible for building a proactive human resources team that drives recruitment, engagement, talent management, people and leadership development and shaping the people culture within Lynx as we expand into USA, Europe and various parts of Asia.\n\nEdina is passionate about harnessing and maximising the potential of each individual that joins our team!",
-                      "confidence_score": 0.9249657988548279,
-                      "title": null,
-                      "description": null,
-                      "type": "information"
-                    },
-                    {
-                      "id": "i_151",
-                      "body": "Lynx Analytics is a company founded by a team from INSEAD in 2010, with a focus on solving real-world problems using big data graphs. The company's CEO and co-founder, Gyorgy Lajtai, has experience in CRM challenges and marketing automation. The company's Chief Innovation Officer and co-founder, Gabor Benedek, is responsible for defining the methodological roadmap of Lynx's predictive analytics consulting and product development departments. The team also includes Miklos Sarvary, Chung Ng, Cintia O. Tavella Gomez, Paul Berriman, and Zsolt Katona, who all have extensive experience in various industries and academia.\n\n Part of the article: \n\nCintia has lived and worked in Asia for 15 years, is originally from Buenos Aires (Argentina), grew up in Spain and currently lives in Seattle.\n\nIn her free time, she loves to learn languages, train for the next triathlon, practice yoga and\n\nmeditation.\n\nPaul Berriman\n\nNon Executive Director at Spark New Zealand, Non Executive Director at Rain Networks, South Africa\n\nUntil January 2021 Paul was Group Chief Technology Officer of the HKT Trust, where he was responsible for leading the groups product and technology roadmap and strategic development,including cross industry propositions, and embedding analytics into everything HKT does.\nPrior to this he was Managing Director of management consultancy Arthur D. Little in Hong Kong and he has held roles in Reuters and several major Hong Kong service providers.\n\nIn 2008 he was listed as one of the Global Telecoms Business Magazinestop100 most influential persons in telecoms.\n\nHe is Chairman of a Global TD-LTE Initiative (GTI) Working Group, a board member of the Next Generation Mobile Networks Alliance (NGMN Ltd.) and an independent non-executive director of Spark New Zealand (f.k.a.\n\nTelecom Corporation of New Zealand) having joined the Board in December 2011.\n\nHe is also a non-executive director of mobile operator, Rain Ltd., in South Africa.\n\nHe was also a member of OFTAs Telecommunications Standards Advisory Committee for many years.",
-                      "confidence_score": 0.9231036603450775,
-                      "title": null,
-                      "description": null,
-                      "type": "information"
-                    },
-                    {
-                      "id": "s_546",
-                      "body": "Lynx Analytics is a big data company co-founded by Gyorgy Lajtai and a team from INSEAD. Miklos Sarvary, a Professor at Columbia Business School, and Chung Ng, a SVP at Group CTO Office of the HKT Trust, are among the company's new executives. Meanwhile, Cintia O. Tavella Gomez is the Senior Director of Strategic Partnerships at Expedia Group, and Paul Berriman was the Group Chief Technology Officer of the HKT Trust until January 2021. Additionally, Zsolt Katona is the faculty director of the Fisher Center for Business Analytics at the Haas School.",
-                      "confidence_score": 0.9230074882507324,
-                      "title": null,
-                      "description": null,
-                      "type": "summary"
-                    },
-                    {
-                      "id": "i_182",
-                      "body": "Lynx Analytics is a big data graph company founded in 2010 by a team from INSEAD. The company aims to solve real-world problems using big data graphs. The team includes experienced professionals, including co-founders Gyorgy Lajtai and Gabor Benedek, as well as Amit Akhelikar, Chema Lizano, Dan Florea, Adrienne Lee, Li Lee, Edina Choo, Francois de Repentigny, Marton Szel, and Daniel Darabos. The team has expertise in AI startups, telecom, technology strategy, M&A, financial accounting, HR, marketing, data science, and engineering.\n\n Part of the article: \n\nAs Lynx Analytics CFO between 2016 and 2021, Dan managed the companys capital raising initiatives and set up its financial management functions and team from the ground up.\n\nHe remains involved with the company as a shareholder and in an advisory capacity.\n\nLi Lee\n\nFinance Director\n\nLi oversees the whole finance spectrum at Lynx Analytics.\n\nThe dynamic environment, exposure to investors, and continuous innovation requirements are aspects of the job she particularly enjoys.\n\nLi likes to find solutions to new business needs and to implement automated internal controls & procedures to support the companys growth.\n\nLi is a financial professional with more than 15 years of experience in financial accounting & reporting and finance-related matters.\nBefore joining Lynx, she held different Finance positions at accounting & audit firms as well as MNCs in various sectors.\n\nEdina Choo\n\nDirector of People\n\nAs the Director of People at Lynx, Edina provides strategic leadership and management across the various HR services provided to our global talent.\n\nShe is responsible for building a proactive human resources team that drives recruitment, engagement, talent management, people and leadership development and shaping the people culture within Lynx as we expand into USA, Europe and various parts of Asia.\n\nEdina is passionate about harnessing and maximising the potential of each individual that joins our team!",
-                      "confidence_score": 0.9217821955680847,
-                      "title": null,
-                      "description": null,
-                      "type": "information"
-                    },
-                    {
-                      "id": "s_459",
-                      "body": "Lynx Analytics is a big data company founded in 2010 by Gyorgy Lajtai and a team from INSEAD. The company aims to solve real-world problems using big data graphs. The team is composed of individuals with diverse backgrounds and experiences spanning various industries. The company has a Chief Innovation Officer, a COO, a CTO, a Senior Board Advisor, a Finance Director, and a Director of People. They are responsible for defining the methodological roadmap, implementing solutions, managing finances, and leading HR services and talent management as the company expands globally.",
-                      "confidence_score": 0.9208503663539886,
-                      "title": null,
-                      "description": null,
-                      "type": "summary"
-                    }
-                  ],
-                  0.9249657988548279,
-                  "Score is always between 0 and 1, the total is the maximum of individual source scores.",
-                  {}
-                ]
-              ]
-            }
-          }
-        },
+        "display": null,
         "error": null,
         "meta": {
-          "params": {},
           "name": "View",
-          "outputs": {},
-          "type": "table_view",
           "inputs": {
             "input": {
               "position": "left",
@@ -125,7 +60,10 @@
                 "type": "<class 'inspect._empty'>"
               }
             }
-          }
+          },
+          "params": {},
+          "outputs": {},
+          "type": "table_view"
         }
       },
       "position": {
@@ -143,31 +81,31 @@
         },
         "display": null,
         "error": null,
+        "collapsed": null,
         "meta": {
-          "type": "basic",
           "name": "LLM",
           "inputs": {},
+          "type": "basic",
           "params": {
             "name": {
-              "name": "name",
+              "default": "openai",
               "type": {
                 "type": "<class 'str'>"
               },
-              "default": "openai"
+              "name": "name"
             }
           },
           "outputs": {
             "output": {
+              "name": "output",
               "type": {
                 "type": "None"
               },
-              "name": "output",
               "position": "top"
             }
           }
         },
-        "__execution_delay": 0.0,
-        "collapsed": null
+        "__execution_delay": 0.0
       },
       "position": {
         "x": 0.0,
@@ -185,38 +123,38 @@
         },
         "display": null,
         "error": null,
+        "__execution_delay": 0.0,
         "meta": {
           "outputs": {
             "output": {
+              "position": "top",
+              "name": "output",
               "type": {
                 "type": "None"
-              },
-              "name": "output",
-              "position": "top"
+              }
             }
           },
-          "type": "basic",
+          "inputs": {},
           "name": "Scenario selector",
           "params": {
             "node_types": {
+              "name": "node_types",
+              "default": "intent_cluster",
               "type": {
                 "type": "<class 'str'>"
-              },
-              "default": "intent_cluster",
-              "name": "node_types"
+              }
             },
             "scenario_file": {
-              "name": "scenario_file",
-              "default": null,
               "type": {
                 "type": "<class 'str'>"
-              }
+              },
+              "default": null,
+              "name": "scenario_file"
             }
           },
-          "inputs": {}
+          "type": "basic"
         },
-        "collapsed": null,
-        "__execution_delay": 0.0
+        "collapsed": null
       },
       "position": {
         "x": 0.0,
@@ -233,53 +171,53 @@
         },
         "display": null,
         "error": null,
-        "collapsed": null,
         "meta": {
-          "name": "Chat API",
-          "params": {
-            "model": {
-              "default": "gpt-4o-mini",
-              "type": {
-                "type": "<class 'str'>"
-              },
-              "name": "model"
-            }
-          },
           "inputs": {
             "chat_processor": {
               "position": "bottom",
-              "name": "chat_processor",
               "type": {
                 "type": "<class 'inspect._empty'>"
-              }
+              },
+              "name": "chat_processor"
             },
-            "chatbot": {
+            "knowledge_base": {
+              "name": "knowledge_base",
               "position": "bottom",
               "type": {
                 "type": "<class 'inspect._empty'>"
-              },
-              "name": "chatbot"
+              }
             },
-            "knowledge_base": {
-              "name": "knowledge_base",
+            "chatbot": {
+              "name": "chatbot",
               "type": {
                 "type": "<class 'inspect._empty'>"
               },
               "position": "bottom"
             }
           },
+          "type": "basic",
+          "name": "Chat API",
+          "params": {
+            "model": {
+              "default": "gpt-4o-mini",
+              "type": {
+                "type": "<class 'str'>"
+              },
+              "name": "model"
+            }
+          },
           "outputs": {
             "output": {
               "name": "output",
+              "position": "top",
               "type": {
                 "type": "None"
-              },
-              "position": "top"
+              }
             }
-          },
-          "type": "basic"
+          }
         },
-        "__execution_delay": 0.0
+        "__execution_delay": 0.0,
+        "collapsed": null
       },
       "position": {
         "x": 0.0,
@@ -292,47 +230,47 @@
       "data": {
         "title": "Knowledge base",
         "params": {
-          "nodes_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/nodes.pickle",
           "template_cluster_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/tempclusters.pickle",
-          "edges_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/edges.pickle"
+          "edges_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/edges.pickle",
+          "nodes_path": "/home/darabos/nvme/lynxscribe/examples/chat_api/data/lynx/nodes.pickle"
         },
         "display": null,
         "error": null,
         "meta": {
-          "type": "basic",
-          "outputs": {
-            "output": {
-              "position": "top",
-              "name": "output",
+          "params": {
+            "template_cluster_path": {
+              "name": "template_cluster_path",
+              "default": "tempclusters.pickle",
               "type": {
-                "type": "None"
+                "type": "<class 'str'>"
               }
-            }
-          },
-          "name": "Knowledge base",
-          "params": {
+            },
             "edges_path": {
-              "name": "edges_path",
               "default": "edges.pickle",
+              "name": "edges_path",
               "type": {
                 "type": "<class 'str'>"
               }
             },
             "nodes_path": {
+              "default": "nodes.pickle",
               "name": "nodes_path",
               "type": {
                 "type": "<class 'str'>"
-              },
-              "default": "nodes.pickle"
-            },
-            "template_cluster_path": {
-              "default": "tempclusters.pickle",
+              }
+            }
+          },
+          "type": "basic",
+          "outputs": {
+            "output": {
               "type": {
-                "type": "<class 'str'>"
+                "type": "None"
               },
-              "name": "template_cluster_path"
+              "position": "top",
+              "name": "output"
             }
           },
+          "name": "Knowledge base",
           "inputs": {}
         }
       },
@@ -347,15 +285,46 @@
       "data": {
         "title": "RAG chatbot",
         "params": {
-          "negative_answer": "I'm sorry, but the data I've been trained on does not contain any information related to your question.",
-          "strict_limits": true,
           "limits_by_type": "{\"information\": [2, 3], \"summary\": [2, 3]}",
-          "max_results": "5"
+          "negative_answer": "I'm sorry, but the data I've been trained on does not contain any information related to your question.",
+          "max_results": "5",
+          "strict_limits": true
         },
         "display": null,
         "error": null,
         "meta": {
+          "params": {
+            "limits_by_type": {
+              "name": "limits_by_type",
+              "default": "{}",
+              "type": {
+                "type": "<class 'str'>"
+              }
+            },
+            "max_results": {
+              "default": 5.0,
+              "name": "max_results",
+              "type": {
+                "type": "<class 'int'>"
+              }
+            },
+            "strict_limits": {
+              "type": {
+                "type": "<class 'bool'>"
+              },
+              "default": true,
+              "name": "strict_limits"
+            },
+            "negative_answer": {
+              "type": {
+                "type": "<class 'str'>"
+              },
+              "name": "negative_answer",
+              "default": "I'm sorry, but the data I've been trained on does not contain any information related to your question."
+            }
+          },
           "name": "RAG chatbot",
+          "type": "basic",
           "outputs": {
             "output": {
               "type": {
@@ -367,58 +336,27 @@
           },
           "inputs": {
             "rag_graph": {
-              "name": "rag_graph",
               "position": "bottom",
+              "name": "rag_graph",
               "type": {
                 "type": "<class 'inspect._empty'>"
               }
             },
-            "llm": {
-              "name": "llm",
+            "scenario_selector": {
               "position": "bottom",
+              "name": "scenario_selector",
               "type": {
                 "type": "<class 'inspect._empty'>"
               }
             },
-            "scenario_selector": {
+            "llm": {
               "type": {
                 "type": "<class 'inspect._empty'>"
               },
-              "name": "scenario_selector",
-              "position": "bottom"
-            }
-          },
-          "params": {
-            "limits_by_type": {
-              "default": "{}",
-              "name": "limits_by_type",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "max_results": {
-              "type": {
-                "type": "<class 'int'>"
-              },
-              "name": "max_results",
-              "default": 5.0
-            },
-            "negative_answer": {
-              "name": "negative_answer",
-              "default": "I'm sorry, but the data I've been trained on does not contain any information related to your question.",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
-            "strict_limits": {
-              "default": true,
-              "name": "strict_limits",
-              "type": {
-                "type": "<class 'bool'>"
-              }
+              "position": "bottom",
+              "name": "llm"
             }
-          },
-          "type": "basic"
+          }
         },
         "beingResized": false
       },
@@ -436,37 +374,37 @@
         "display": null,
         "error": null,
         "collapsed": true,
-        "__execution_delay": null,
         "meta": {
+          "name": "RAG graph",
+          "params": {},
+          "outputs": {
+            "output": {
+              "type": {
+                "type": "None"
+              },
+              "position": "top",
+              "name": "output"
+            }
+          },
+          "type": "basic",
           "inputs": {
             "vector_store": {
+              "name": "vector_store",
               "type": {
                 "type": "<class 'inspect._empty'>"
               },
-              "name": "vector_store",
               "position": "bottom"
             },
             "text_embedder": {
-              "name": "text_embedder",
               "position": "bottom",
               "type": {
                 "type": "<class 'inspect._empty'>"
-              }
-            }
-          },
-          "name": "RAG graph",
-          "params": {},
-          "outputs": {
-            "output": {
-              "type": {
-                "type": "None"
               },
-              "position": "top",
-              "name": "output"
+              "name": "text_embedder"
             }
-          },
-          "type": "basic"
-        }
+          }
+        },
+        "__execution_delay": null
       },
       "position": {
         "x": 0.0,
@@ -479,32 +417,31 @@
       "data": {
         "title": "Vector store",
         "params": {
-          "name": "chromadb",
-          "collection_name": "lynx"
+          "collection_name": "lynx",
+          "name": "chromadb"
         },
         "display": null,
         "error": null,
-        "beingResized": false,
         "meta": {
           "name": "Vector store",
+          "type": "basic",
+          "inputs": {},
           "params": {
-            "name": {
-              "default": "chromadb",
-              "name": "name",
-              "type": {
-                "type": "<class 'str'>"
-              }
-            },
             "collection_name": {
               "type": {
                 "type": "<class 'str'>"
               },
               "default": "lynx",
               "name": "collection_name"
+            },
+            "name": {
+              "name": "name",
+              "default": "chromadb",
+              "type": {
+                "type": "<class 'str'>"
+              }
             }
           },
-          "inputs": {},
-          "type": "basic",
           "outputs": {
             "output": {
               "position": "top",
@@ -514,7 +451,8 @@
               }
             }
           }
-        }
+        },
+        "beingResized": false
       },
       "position": {
         "x": 0.0,
@@ -534,35 +472,35 @@
         "collapsed": null,
         "__execution_delay": 0.0,
         "meta": {
-          "inputs": {
-            "llm": {
-              "position": "bottom",
-              "name": "llm",
+          "outputs": {
+            "output": {
+              "position": "top",
               "type": {
-                "type": "<class 'inspect._empty'>"
-              }
+                "type": "None"
+              },
+              "name": "output"
             }
           },
+          "name": "Text embedder",
           "params": {
             "model": {
-              "default": "text-embedding-ada-002",
               "name": "model",
+              "default": "text-embedding-ada-002",
               "type": {
                 "type": "<class 'str'>"
               }
             }
           },
-          "name": "Text embedder",
-          "type": "basic",
-          "outputs": {
-            "output": {
-              "position": "top",
-              "name": "output",
+          "inputs": {
+            "llm": {
+              "position": "bottom",
+              "name": "llm",
               "type": {
-                "type": "None"
+                "type": "<class 'inspect._empty'>"
               }
             }
-          }
+          },
+          "type": "basic"
         }
       },
       "position": {
@@ -581,26 +519,26 @@
         "display": null,
         "error": null,
         "meta": {
-          "name": "LLM",
           "inputs": {},
           "params": {
             "name": {
               "type": {
                 "type": "<class 'str'>"
               },
-              "default": "openai",
-              "name": "name"
+              "name": "name",
+              "default": "openai"
             }
           },
           "outputs": {
             "output": {
-              "position": "top",
               "name": "output",
+              "position": "top",
               "type": {
                 "type": "None"
               }
             }
           },
+          "name": "LLM",
           "type": "basic"
         }
       },
@@ -620,7 +558,8 @@
         "display": null,
         "error": null,
         "meta": {
-          "inputs": {},
+          "name": "Truncate history",
+          "type": "basic",
           "outputs": {
             "output": {
               "type": {
@@ -630,17 +569,16 @@
               "name": "output"
             }
           },
-          "type": "basic",
-          "name": "Truncate history",
           "params": {
             "max_tokens": {
               "type": {
                 "type": "<class 'int'>"
               },
-              "name": "max_tokens",
-              "default": 10000.0
+              "default": 10000.0,
+              "name": "max_tokens"
             }
-          }
+          },
+          "inputs": {}
         }
       },
       "position": {
@@ -657,30 +595,30 @@
         "display": null,
         "error": null,
         "meta": {
+          "params": {},
+          "name": "Chat processor",
           "inputs": {
             "processor": {
+              "position": "bottom",
               "name": "processor",
               "type": {
                 "type": "<class 'inspect._empty'>"
-              },
-              "position": "bottom"
+              }
             }
           },
-          "params": {},
           "outputs": {
             "output": {
-              "name": "output",
+              "position": "top",
               "type": {
                 "type": "None"
               },
-              "position": "top"
+              "name": "output"
             }
           },
-          "type": "basic",
-          "name": "Chat processor"
+          "type": "basic"
         },
-        "__execution_delay": null,
-        "collapsed": true
+        "collapsed": true,
+        "__execution_delay": null
       },
       "position": {
         "x": 0.0,
@@ -693,56 +631,56 @@
       "data": {
         "title": "Mask",
         "params": {
-          "name": "email",
           "mask_pattern": "masked_email_address_{}",
           "regex": "([a-z0-9!#$%&'*+\\/=?^_`{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)",
+          "name": "email",
           "exceptions": "[email protected],[email protected],[email protected],[email protected]"
         },
         "display": null,
         "error": null,
         "meta": {
+          "outputs": {
+            "output": {
+              "type": {
+                "type": "None"
+              },
+              "position": "top",
+              "name": "output"
+            }
+          },
+          "inputs": {},
+          "type": "basic",
+          "name": "Mask",
           "params": {
-            "exceptions": {
+            "mask_pattern": {
               "type": {
                 "type": "<class 'str'>"
               },
-              "default": "",
-              "name": "exceptions"
+              "name": "mask_pattern",
+              "default": ""
             },
-            "regex": {
-              "name": "regex",
+            "exceptions": {
               "default": "",
+              "name": "exceptions",
               "type": {
                 "type": "<class 'str'>"
               }
             },
             "name": {
+              "name": "name",
               "type": {
                 "type": "<class 'str'>"
               },
-              "default": "",
-              "name": "name"
+              "default": ""
             },
-            "mask_pattern": {
-              "name": "mask_pattern",
-              "default": "",
+            "regex": {
               "type": {
                 "type": "<class 'str'>"
-              }
-            }
-          },
-          "name": "Mask",
-          "inputs": {},
-          "outputs": {
-            "output": {
-              "name": "output",
-              "position": "top",
-              "type": {
-                "type": "None"
-              }
+              },
+              "name": "regex",
+              "default": ""
             }
-          },
-          "type": "basic"
+          }
         }
       },
       "position": {
@@ -757,55 +695,55 @@
         "title": "Mask",
         "params": {
           "regex": "((?:(?:\\\\d{4}[- ]?){3}\\\\d{4}|\\\\d{15,16}))(?![\\\\d])",
-          "mask_pattern": "masked_credit_card_number_{}",
           "exceptions": "",
-          "name": "credit_card"
+          "name": "credit_card",
+          "mask_pattern": "masked_credit_card_number_{}"
         },
         "display": null,
         "error": null,
         "meta": {
-          "type": "basic",
-          "outputs": {
-            "output": {
-              "position": "top",
-              "type": {
-                "type": "None"
-              },
-              "name": "output"
-            }
-          },
-          "name": "Mask",
           "params": {
-            "mask_pattern": {
+            "name": {
+              "name": "name",
               "default": "",
               "type": {
                 "type": "<class 'str'>"
-              },
-              "name": "mask_pattern"
+              }
             },
-            "exceptions": {
+            "regex": {
+              "name": "regex",
               "default": "",
               "type": {
                 "type": "<class 'str'>"
-              },
-              "name": "exceptions"
+              }
             },
-            "name": {
+            "mask_pattern": {
               "default": "",
-              "name": "name",
+              "name": "mask_pattern",
               "type": {
                 "type": "<class 'str'>"
               }
             },
-            "regex": {
+            "exceptions": {
+              "default": "",
+              "name": "exceptions",
               "type": {
                 "type": "<class 'str'>"
-              },
-              "name": "regex",
-              "default": ""
+              }
             }
           },
-          "inputs": {}
+          "inputs": {},
+          "outputs": {
+            "output": {
+              "name": "output",
+              "position": "top",
+              "type": {
+                "type": "None"
+              }
+            }
+          },
+          "type": "basic",
+          "name": "Mask"
         }
       },
       "position": {
@@ -823,27 +761,28 @@
         },
         "display": null,
         "error": null,
-        "collapsed": false,
+        "__execution_delay": 0.0,
         "meta": {
+          "type": "basic",
           "outputs": {
             "output": {
               "type": {
                 "type": "None"
               },
-              "name": "output",
-              "position": "right"
+              "position": "right",
+              "name": "output"
             }
           },
+          "name": "Test Chat API",
           "params": {
             "show_details": {
-              "default": false,
-              "name": "show_details",
               "type": {
                 "type": "<class 'bool'>"
-              }
+              },
+              "default": false,
+              "name": "show_details"
             }
           },
-          "type": "basic",
           "inputs": {
             "message": {
               "position": "left",
@@ -853,16 +792,15 @@
               "name": "message"
             },
             "chat_api": {
-              "name": "chat_api",
-              "position": "bottom",
               "type": {
                 "type": "<class 'inspect._empty'>"
-              }
+              },
+              "name": "chat_api",
+              "position": "bottom"
             }
-          },
-          "name": "Test Chat API"
+          }
         },
-        "__execution_delay": 0.0
+        "collapsed": false
       },
       "position": {
         "x": 0.0,

lynxkite-app/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "lynxkite"
+version = "0.1.0"
+description = "The LynxKite application, with web server and UI"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi[standard]>=0.115.6",
+    "lynxkite-ops",
+    "orjson>=3.10.13",
+    "pycrdt-websocket>=0.15.3",
+    "pydantic-to-typescript>=2.0.0",
+    "sse-starlette>=2.2.1",
+]
+
+[tool.uv.sources]
+lynxkite-ops = { path = "../lynxkite-ops" }
+
+[build-system]
+requires = ["setuptools", "wheel", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+namespaces = true
+where = ["src"]
+
+[tool.setuptools.package-data]
+"lynxkite.web_assets" = ["*"]
+"lynxkite.web_assets.assets" = ["*"]
+
+[tool.setuptools]
+py-modules = ["build_frontend"]
+include-package-data = true
+
+[tool.setuptools.cmdclass]
+build_py = "build_frontend.build_py"
+
+[project.scripts]
+lynxkite = "lynxkite.__main__:main"

lynxkite-app/src/build_frontend.py

@@ -0,0 +1,26 @@
+"""Customized build process for setuptools."""
+
+import subprocess
+from setuptools.command.build_py import build_py as _build_py
+from pathlib import Path
+import shutil
+
+
+class build_py(_build_py):
+    def run(self):
+        print("\n\nBuilding frontend...", __file__)
+        here = Path(__file__).parent.parent
+        frontend_dir = here / "web"
+        package_dir = here / "src" / "lynxkite" / "web_assets"
+        subprocess.check_call(["npm", "install"], cwd=frontend_dir)
+        subprocess.check_call(["npm", "run", "build"], cwd=frontend_dir)
+        print("files in", frontend_dir / "dist")
+        for file in (frontend_dir / "dist").iterdir():
+            print(file)
+        # shutil.rmtree(package_dir)
+        shutil.copytree(frontend_dir / "dist", package_dir, dirs_exist_ok=True)
+        # (frontend_dir / "dist").rename(package_dir)
+        print("files in", package_dir)
+        for file in package_dir.iterdir():
+            print(file)
+        super().run()

lynxkite-app/src/lynxkite/__main__.py

@@ -0,0 +1,13 @@
+import uvicorn
+from .main import app  # noqa: F401
+import os
+
+
+def main():
+    port = int(os.environ.get("PORT", "8000"))
+    reload = bool(os.environ.get("LYNXKITE_RELOAD", ""))
+    uvicorn.run("lynxkite.main:app", host="0.0.0.0", port=port, reload=reload)
+
+
+if __name__ == "__main__":
+    main()

{server → lynxkite-app/src/lynxkite}/main.py

@@ -6,9 +6,11 @@ import fastapi
 import importlib
 import pathlib
 import pkgutil
+from fastapi.staticfiles import StaticFiles
+import starlette
+from lynxkite import ops
+from lynxkite import workspace
 from . import crdt
-from . import ops
-from . import workspace
 
 here = pathlib.Path(__file__).parent
 lynxkite_modules = {}
@@ -102,3 +104,23 @@ async def service_post(req: fastapi.Request, module_path: str):
     """Executors can provide extra HTTP APIs through the /api/service endpoint."""
     module = lynxkite_modules[module_path.split("/")[0]]
     return await module.api_service_post(req)
+
+
+class SPAStaticFiles(StaticFiles):
+    """Route everything to index.html. https://stackoverflow.com/a/73552966/3318517"""
+
+    async def get_response(self, path: str, scope):
+        try:
+            return await super().get_response(path, scope)
+        except (
+            fastapi.HTTPException,
+            starlette.exceptions.HTTPException,
+        ) as ex:
+            if ex.status_code == 404:
+                return await super().get_response(".", scope)
+            else:
+                raise ex
+
+
+static_dir = SPAStaticFiles(packages=[("lynxkite", "web_assets")], html=True)
+app.mount("/", static_dir, name="web_assets")

lynxkite-app/src/lynxkite/web_assets/__init__.py

@@ -0,0 +1 @@
+"""The build process (uv build) puts the frontend build artifacts here."""

lynxkite-app/src/lynxkite/web_assets/assets/__init__.py

@@ -0,0 +1 @@
+"""More frontend build artifacts."""

{web → lynxkite-app/web}/package.json

@@ -4,10 +4,10 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
-    "dev": "vite",
-    "build": "tsc -b && vite build",
-    "lint": "eslint .",
-    "preview": "vite preview"
+    "dev": "npx vite",
+    "build": "npx tsc -b && npx vite build",
+    "lint": "npx eslint .",
+    "preview": "npx vite preview"
   },
   "dependencies": {
     "@esbuild/linux-x64": "^0.24.0",

{web → lynxkite-app/web}/src/workspace/Workspace.tsx

@@ -57,7 +57,7 @@ function LynxKiteFlow() {
     const state = syncedStore({ workspace: {} as Workspace });
     setState(state);
     const doc = getYjsDoc(state);
-    const wsProvider = new WebsocketProvider("ws://localhost:5173/ws/crdt", path!, doc);
+    const wsProvider = new WebsocketProvider(`ws://${location.host}/ws/crdt`, path!, doc);
     const onChange = (_update: any, origin: any, _doc: any, _tr: any) => {
       if (origin === wsProvider) {
         // An update from the CRDT. Apply it to the local state.

{web → lynxkite-app/web}/src/workspace/nodes/NodeParameter.tsx

@@ -1,10 +1,17 @@
 const BOOLEAN = "<class 'bool'>";
 
-function ParamName({ name }) {
+function ParamName({ name }: { name: string }) {
   return <span className="param-name bg-base-200">{name.replace(/_/g, ' ')}</span>;
 }
 
-export default function NodeParameter({ name, value, meta, onChange }) {
+interface NodeParameterProps {
+  name: string;
+  value: any;
+  meta: any;
+  onChange: (value: any, options?: { delay: number }) => void;
+}
+
+export default function NodeParameter({ name, value, meta, onChange }: NodeParameterProps) {
   return (
     <label className="param">
       {meta?.type?.format === 'collapsed' ? <>
@@ -25,7 +32,7 @@ export default function NodeParameter({ name, value, meta, onChange }) {
           value={value || meta.type.enum[0]}
           onChange={(evt) => onChange(evt.currentTarget.value)}
         >
-          {meta.type.enum.map(option =>
+          {meta.type.enum.map((option: string) =>
             <option key={option} value={option}>{option}</option>
           )}
         </select>

{web → lynxkite-app/web}/src/workspace/nodes/NodeWithVisualization.tsx

@@ -1,6 +1,6 @@
 import React, { useEffect } from 'react';
 import NodeWithParams from './NodeWithParams';
-import * as echarts from 'echarts';
+const echarts = await import('echarts');
 
 const NodeWithVisualization = (props: any) => {
   const chartsRef = React.useRef<HTMLDivElement>(null);

{web → lynxkite-app/web}/tailwind.config.js

@@ -7,6 +7,7 @@ export default {
   },
   plugins: [require('daisyui')],
   daisyui: {
+    logs: false,
     themes: [
       {
         lynxkite: {