Merge pull request #33 from biggraph/darabos-react

README.md

@@ -1,11 +1,11 @@
 # LynxKite 2024
 
-This is an experimental rewrite of [LynxKite](https://github.com/lynxkite/lynxkite).
-It is not compatible with the original LynxKite. The primary goals of this rewrite are:
-- Target GPU clusters instead of Hadoop clusters.
-  We use Python instead of Scala, RAPIDS instead of Apache Spark.
-- More extensible backend. Make it easy to add new LynxKite boxes.
-  Make it easy to use our frontend for other purposes, configuring and executing other pipelines.
+This is an experimental rewrite of [LynxKite](https://github.com/lynxkite/lynxkite). It is not compatible with the
+original LynxKite. The primary goals of this rewrite are:
+
+- Target GPU clusters instead of Hadoop clusters. We use Python instead of Scala, RAPIDS instead of Apache Spark.
+- More extensible backend. Make it easy to add new LynxKite boxes. Make it easy to use our frontend for other purposes,
+  configuring and executing other pipelines.
 
 Current status: **PROTOTYPE**
 
@@ -15,6 +15,7 @@ To run the backend:
 
 ```bash
 pip install -r requirements.txt
+PYTHONPATH=. pydantic2ts --module server.workspace --output ./web/src/apiTypes.ts
 uvicorn server.main:app --reload
 ```
 

requirements.txt

@@ -4,6 +4,7 @@ networkx
 numpy
 orjson
 pandas
+pydantic-to-typescript
 scipy
 uvicorn[standard]
 pycrdt

server/crdt.py

@@ -1,42 +1,57 @@
-'''CRDT is used to synchronize workspace state for backend and frontend(s).'''
+"""CRDT is used to synchronize workspace state for backend and frontend(s)."""
+
 import asyncio
 import contextlib
+import enum
 import fastapi
 import os.path
 import pycrdt
 import pycrdt_websocket
-
 import pycrdt_websocket.ystore
+import uvicorn
+import builtins
 
 router = fastapi.APIRouter()
 
+
 def ws_exception_handler(exception, log):
-    print('exception', exception)
-    log.exception(exception)
+    if isinstance(exception, builtins.ExceptionGroup):
+        for ex in exception.exceptions:
+            if not isinstance(ex, uvicorn.protocols.utils.ClientDisconnected):
+                log.exception(ex)
+    else:
+        log.exception(exception)
     return True
 
+
 class WebsocketServer(pycrdt_websocket.WebsocketServer):
     async def init_room(self, name):
-        ystore = pycrdt_websocket.ystore.FileYStore(f'crdt_data/{name}.crdt')
+        ystore = pycrdt_websocket.ystore.FileYStore(f"crdt_data/{name}.crdt")
         ydoc = pycrdt.Doc()
-        ydoc['workspace'] = ws = pycrdt.Map()
+        ydoc["workspace"] = ws = pycrdt.Map()
         # Replay updates from the store.
         try:
-            for update, timestamp in [(item[0], item[-1]) async for item in ystore.read()]:
+            for update, timestamp in [
+                (item[0], item[-1]) async for item in ystore.read()
+            ]:
                 ydoc.apply_update(update)
         except pycrdt_websocket.ystore.YDocNotFound:
             pass
-        if 'nodes' not in ws:
-            ws['nodes'] = pycrdt.Array()
-        if 'edges' not in ws:
-            ws['edges'] = pycrdt.Array()
-        if 'env' not in ws:
-            ws['env'] = 'unset'
+        if "nodes" not in ws:
+            ws["nodes"] = pycrdt.Array()
+        if "edges" not in ws:
+            ws["edges"] = pycrdt.Array()
+        if "env" not in ws:
+            ws["env"] = "unset"
             try_to_load_workspace(ws, name)
-        room = pycrdt_websocket.YRoom(ystore=ystore, ydoc=ydoc)
+        room = pycrdt_websocket.YRoom(
+            ystore=ystore, ydoc=ydoc, exception_handler=ws_exception_handler
+        )
         room.ws = ws
+
         def on_change(changes):
-            asyncio.create_task(workspace_changed(changes, ws))
+            asyncio.create_task(workspace_changed(name, changes, ws))
+
         ws.observe_deep(on_change)
         return room
 
@@ -47,19 +62,21 @@ class WebsocketServer(pycrdt_websocket.WebsocketServer):
         await self.start_room(room)
         return room
 
-websocket_server = WebsocketServer(exception_handler=ws_exception_handler, auto_clean_rooms=False)
-asgi_server = pycrdt_websocket.ASGIServer(websocket_server)
 
 last_ws_input = None
+
+
 def clean_input(ws_pyd):
     for node in ws_pyd.nodes:
         node.data.display = None
+        node.data.error = None
         node.position.x = 0
         node.position.y = 0
         if node.model_extra:
             for key in list(node.model_extra.keys()):
                 delattr(node, key)
 
+
 def crdt_update(crdt_obj, python_obj, boxes=set()):
     if isinstance(python_obj, dict):
         for key, value in python_obj.items():
@@ -73,6 +90,8 @@ def crdt_update(crdt_obj, python_obj, boxes=set()):
                 if crdt_obj.get(key) is None:
                     crdt_obj[key] = pycrdt.Array()
                 crdt_update(crdt_obj[key], value, boxes)
+            elif isinstance(value, enum.Enum):
+                crdt_obj[key] = str(value)
             else:
                 crdt_obj[key] = value
     elif isinstance(python_obj, list):
@@ -91,41 +110,82 @@ def crdt_update(crdt_obj, python_obj, boxes=set()):
                 else:
                     crdt_obj[i] = value
     else:
-        raise ValueError('Invalid type:', python_obj)
+        raise ValueError("Invalid type:", python_obj)
 
 
 def try_to_load_workspace(ws, name):
     from . import workspace
-    json_path = f'data/{name}'
+
+    json_path = f"data/{name}"
     if os.path.exists(json_path):
         ws_pyd = workspace.load(json_path)
-        crdt_update(ws, ws_pyd.model_dump(), boxes={'display'})
+        crdt_update(ws, ws_pyd.model_dump(), boxes={"display"})
+
 
-async def workspace_changed(e, ws_crdt):
-    global last_ws_input
+last_known_versions = {}
+delayed_executions = {}
+
+
+async def workspace_changed(name, changes, ws_crdt):
     from . import workspace
+
     ws_pyd = workspace.Workspace.model_validate(ws_crdt.to_py())
+    # Do not trigger execution for superficial changes.
+    # This is a quick solution until we build proper caching.
     clean_input(ws_pyd)
-    if ws_pyd == last_ws_input:
+    if ws_pyd == last_known_versions.get(name):
         return
-    last_ws_input = ws_pyd.model_copy(deep=True)
+    last_known_versions[name] = ws_pyd.model_copy(deep=True)
+    # Frontend changes that result from typing are delayed to avoid
+    # rerunning the workspace for every keystroke.
+    if name in delayed_executions:
+        delayed_executions[name].cancel()
+    delay = min(
+        getattr(change, "keys", {}).get("__execution_delay", {}).get("newValue", 0)
+        for change in changes
+    )
+    if delay:
+        task = asyncio.create_task(execute(ws_crdt, ws_pyd, delay))
+        delayed_executions[name] = task
+    else:
+        await execute(ws_crdt, ws_pyd)
+
+
+async def execute(ws_crdt, ws_pyd, delay=0):
+    from . import workspace
+
+    if delay:
+        try:
+            await asyncio.sleep(delay)
+        except asyncio.CancelledError:
+            return
     await workspace.execute(ws_pyd)
-    for nc, np in zip(ws_crdt['nodes'], ws_pyd.nodes):
-        if 'data' not in nc:
-            nc['data'] = pycrdt.Map()
-        # Display is added as an opaque Box.
-        nc['data']['display'] = np.data.display
-        nc['data']['error'] = np.data.error
+    with ws_crdt.doc.transaction():
+        for nc, np in zip(ws_crdt["nodes"], ws_pyd.nodes):
+            if "data" not in nc:
+                nc["data"] = pycrdt.Map()
+            # Display is added as an opaque Box.
+            nc["data"]["display"] = np.data.display
+            nc["data"]["error"] = np.data.error
+
 
 @contextlib.asynccontextmanager
 async def lifespan(app):
+    global websocket_server
+    websocket_server = WebsocketServer(
+        auto_clean_rooms=False,
+    )
     async with websocket_server:
         yield
+    print("closing websocket server")
+
 
 def sanitize_path(path):
     return os.path.relpath(os.path.normpath(os.path.join("/", path)), "/")
 
+
 @router.websocket("/ws/crdt/{room_name}")
 async def crdt_websocket(websocket: fastapi.WebSocket, room_name: str):
     room_name = sanitize_path(room_name)
-    await asgi_server({'path': room_name}, websocket._receive, websocket._send)
+    server = pycrdt_websocket.ASGIServer(websocket_server)
+    await server({"path": room_name}, websocket._receive, websocket._send)

server/executors/one_by_one.py

@@ -7,142 +7,166 @@ import traceback
 import inspect
 import typing
 
+
 class Context(ops.BaseConfig):
-  '''Passed to operation functions as "_ctx" if they have such a parameter.'''
-  node: workspace.WorkspaceNode
-  last_result: typing.Any = None
+    """Passed to operation functions as "_ctx" if they have such a parameter."""
+
+    node: workspace.WorkspaceNode
+    last_result: typing.Any = None
+
 
 class Output(ops.BaseConfig):
-  '''Return this to send values to specific outputs of a node.'''
-  output_handle: str
-  value: dict
+    """Return this to send values to specific outputs of a node."""
+
+    output_handle: str
+    value: dict
 
 
 def df_to_list(df):
-  return df.to_dict(orient='records')
+    return df.to_dict(orient="records")
+
 
 def has_ctx(op):
-  sig = inspect.signature(op.func)
-  return '_ctx' in sig.parameters
+    sig = inspect.signature(op.func)
+    return "_ctx" in sig.parameters
+
 
 CACHES = {}
 
+
 def register(env: str, cache: bool = True):
-  '''Registers the one-by-one executor.'''
-  if cache:
-    CACHES[env] = {}
-    cache = CACHES[env]
-  else:
-    cache = None
-  ops.EXECUTORS[env] = lambda ws: execute(ws, ops.CATALOGS[env], cache=cache)
+    """Registers the one-by-one executor."""
+    if cache:
+        CACHES[env] = {}
+        cache = CACHES[env]
+    else:
+        cache = None
+    ops.EXECUTORS[env] = lambda ws: execute(ws, ops.CATALOGS[env], cache=cache)
+
 
 def get_stages(ws, catalog):
-  '''Inputs on top/bottom are batch inputs. We decompose the graph into a DAG of components along these edges.'''
-  nodes = {n.id: n for n in ws.nodes}
-  batch_inputs = {}
-  inputs = {}
-  for edge in ws.edges:
-    inputs.setdefault(edge.target, []).append(edge.source)
-    node = nodes[edge.target]
-    op = catalog[node.data.title]
-    i = op.inputs[edge.targetHandle]
-    if i.position in 'top or bottom':
-      batch_inputs.setdefault(edge.target, []).append(edge.source)
-  stages = []
-  for bt, bss in batch_inputs.items():
-    upstream = set(bss)
-    new = set(bss)
-    while new:
-      n = new.pop()
-      for i in inputs.get(n, []):
-        if i not in upstream:
-          upstream.add(i)
-          new.add(i)
-    stages.append(upstream)
-  stages.sort(key=lambda s: len(s))
-  stages.append(set(nodes))
-  return stages
+    """Inputs on top/bottom are batch inputs. We decompose the graph into a DAG of components along these edges."""
+    nodes = {n.id: n for n in ws.nodes}
+    batch_inputs = {}
+    inputs = {}
+    for edge in ws.edges:
+        inputs.setdefault(edge.target, []).append(edge.source)
+        node = nodes[edge.target]
+        op = catalog[node.data.title]
+        i = op.inputs[edge.targetHandle]
+        if i.position in "top or bottom":
+            batch_inputs.setdefault(edge.target, []).append(edge.source)
+    stages = []
+    for bt, bss in batch_inputs.items():
+        upstream = set(bss)
+        new = set(bss)
+        while new:
+            n = new.pop()
+            for i in inputs.get(n, []):
+                if i not in upstream:
+                    upstream.add(i)
+                    new.add(i)
+        stages.append(upstream)
+    stages.sort(key=lambda s: len(s))
+    stages.append(set(nodes))
+    return stages
 
 
 def _default_serializer(obj):
-  if isinstance(obj, pydantic.BaseModel):
-    return obj.dict()
-  return {"__nonserializable__": id(obj)}
+    if isinstance(obj, pydantic.BaseModel):
+        return obj.dict()
+    return {"__nonserializable__": id(obj)}
+
 
 def make_cache_key(obj):
-  return orjson.dumps(obj, default=_default_serializer)
+    return orjson.dumps(obj, default=_default_serializer)
+
 
 EXECUTOR_OUTPUT_CACHE = {}
 
+
 async def await_if_needed(obj):
-  if inspect.isawaitable(obj):
-    return await obj
-  return obj
+    if inspect.isawaitable(obj):
+        return await obj
+    return obj
+
 
 async def execute(ws, catalog, cache=None):
-  nodes = {n.id: n for n in ws.nodes}
-  contexts = {n.id: Context(node=n) for n in ws.nodes}
-  edges = {n.id: [] for n in ws.nodes}
-  for e in ws.edges:
-    edges[e.source].append(e)
-  tasks = {}
-  NO_INPUT = object() # Marker for initial tasks.
-  for node in ws.nodes:
-    node.data.error = None
-    op = catalog[node.data.title]
-    # Start tasks for nodes that have no non-batch inputs.
-    if all([i.position in 'top or bottom' for i in op.inputs.values()]):
-      tasks[node.id] = [NO_INPUT]
-  batch_inputs = {}
-  # Run the rest until we run out of tasks.
-  stages = get_stages(ws, catalog)
-  for stage in stages:
-    next_stage = {}
-    while tasks:
-      n, ts = tasks.popitem()
-      if n not in stage:
-        next_stage.setdefault(n, []).extend(ts)
-        continue
-      node = nodes[n]
-      data = node.data
-      op = catalog[data.title]
-      params = {**data.params}
-      if has_ctx(op):
-        params['_ctx'] = contexts[node.id]
-      results = []
-      for task in ts:
-        try:
-          inputs = [
-            batch_inputs[(n, i.name)] if i.position in 'top or bottom' else task
-            for i in op.inputs.values()]
-          if cache is not None:
-            key = make_cache_key((inputs, params))
-            if key not in cache:
-              cache[key] = await await_if_needed(op(*inputs, **params))
-            result = cache[key]
-          else:
-            result = await await_if_needed(op(*inputs, **params))
-        except Exception as e:
-          traceback.print_exc()
-          data.error = str(e)
-          break
-        contexts[node.id].last_result = result
-        # Returned lists and DataFrames are considered multiple tasks.
-        if isinstance(result, pd.DataFrame):
-          result = df_to_list(result)
-        elif not isinstance(result, list):
-          result = [result]
-        results.extend(result)
-      else: # Finished all tasks without errors.
-        if op.type == 'visualization' or op.type == 'table_view' or op.type == 'image':
-          data.display = results[0]
-        for edge in edges[node.id]:
-          t = nodes[edge.target]
-          op = catalog[t.data.title]
-          i = op.inputs[edge.targetHandle]
-          if i.position in 'top or bottom':
-            batch_inputs.setdefault((edge.target, edge.targetHandle), []).extend(results)
-          else:
-            tasks.setdefault(edge.target, []).extend(results)
-    tasks = next_stage
-  return contexts
+    nodes = {n.id: n for n in ws.nodes}
+    contexts = {n.id: Context(node=n) for n in ws.nodes}
+    edges = {n.id: [] for n in ws.nodes}
+    for e in ws.edges:
+        edges[e.source].append(e)
+    tasks = {}
+    NO_INPUT = object()  # Marker for initial tasks.
+    for node in ws.nodes:
+        node.data.error = None
+        op = catalog.get(node.data.title)
+        if op is None:
+            node.data.error = f'Operation "{node.data.title}" not found.'
+            continue
+        # Start tasks for nodes that have no non-batch inputs.
+        if all([i.position in "top or bottom" for i in op.inputs.values()]):
+            tasks[node.id] = [NO_INPUT]
+    batch_inputs = {}
+    # Run the rest until we run out of tasks.
+    stages = get_stages(ws, catalog)
+    for stage in stages:
+        next_stage = {}
+        while tasks:
+            n, ts = tasks.popitem()
+            if n not in stage:
+                next_stage.setdefault(n, []).extend(ts)
+                continue
+            node = nodes[n]
+            data = node.data
+            op = catalog[data.title]
+            params = {**data.params}
+            if has_ctx(op):
+                params["_ctx"] = contexts[node.id]
+            results = []
+            for task in ts:
+                try:
+                    inputs = [
+                        batch_inputs[(n, i.name)]
+                        if i.position in "top or bottom"
+                        else task
+                        for i in op.inputs.values()
+                    ]
+                    if cache is not None:
+                        key = make_cache_key((inputs, params))
+                        if key not in cache:
+                            cache[key] = await await_if_needed(op(*inputs, **params))
+                        result = cache[key]
+                    else:
+                        result = await await_if_needed(op(*inputs, **params))
+                except Exception as e:
+                    traceback.print_exc()
+                    data.error = str(e)
+                    break
+                contexts[node.id].last_result = result
+                # Returned lists and DataFrames are considered multiple tasks.
+                if isinstance(result, pd.DataFrame):
+                    result = df_to_list(result)
+                elif not isinstance(result, list):
+                    result = [result]
+                results.extend(result)
+            else:  # Finished all tasks without errors.
+                if (
+                    op.type == "visualization"
+                    or op.type == "table_view"
+                    or op.type == "image"
+                ):
+                    data.display = results[0]
+                for edge in edges[node.id]:
+                    t = nodes[edge.target]
+                    op = catalog[t.data.title]
+                    i = op.inputs[edge.targetHandle]
+                    if i.position in "top or bottom":
+                        batch_inputs.setdefault(
+                            (edge.target, edge.targetHandle), []
+                        ).extend(results)
+                    else:
+                        tasks.setdefault(edge.target, []).extend(results)
+        tasks = next_stage
+    return contexts

server/llm_ops.py

@@ -1,4 +1,5 @@
-'''For specifying an LLM agent logic flow.'''
+"""For specifying an LLM agent logic flow."""
+
 from . import ops
 import chromadb
 import enum
@@ -14,177 +15,205 @@ embedding_client = openai.OpenAI(base_url="http://localhost:7997/")
 jinja = jinja2.Environment()
 chroma_client = chromadb.Client()
 LLM_CACHE = {}
-ENV = 'LLM logic'
+ENV = "LLM logic"
 one_by_one.register(ENV)
 op = ops.op_registration(ENV)
 
+
 def chat(*args, **kwargs):
-  key = json.dumps({'method': 'chat', 'args': args, 'kwargs': kwargs})
-  if key not in LLM_CACHE:
-    completion = chat_client.chat.completions.create(*args, **kwargs)
-    LLM_CACHE[key] = [c.message.content for c in completion.choices]
-  return LLM_CACHE[key]
+    key = json.dumps({"method": "chat", "args": args, "kwargs": kwargs})
+    if key not in LLM_CACHE:
+        completion = chat_client.chat.completions.create(*args, **kwargs)
+        LLM_CACHE[key] = [c.message.content for c in completion.choices]
+    return LLM_CACHE[key]
+
 
 def embedding(*args, **kwargs):
-  key = json.dumps({'method': 'embedding', 'args': args, 'kwargs': kwargs})
-  if key not in LLM_CACHE:
-    res = embedding_client.embeddings.create(*args, **kwargs)
-    [data] = res.data
-    LLM_CACHE[key] = data.embedding
-  return LLM_CACHE[key]
+    key = json.dumps({"method": "embedding", "args": args, "kwargs": kwargs})
+    if key not in LLM_CACHE:
+        res = embedding_client.embeddings.create(*args, **kwargs)
+        [data] = res.data
+        LLM_CACHE[key] = data.embedding
+    return LLM_CACHE[key]
+
 
 @op("Input CSV")
 def input_csv(*, filename: ops.PathStr, key: str):
-  return pd.read_csv(filename).rename(columns={key: 'text'})
+    return pd.read_csv(filename).rename(columns={key: "text"})
+
 
 @op("Input document")
 def input_document(*, filename: ops.PathStr):
-  with open(filename) as f:
-    return {'text': f.read()}
+    with open(filename) as f:
+        return {"text": f.read()}
+
 
 @op("Input chat")
 def input_chat(*, chat: str):
-  return {'text': chat}
+    return {"text": chat}
+
 
 @op("Split document")
-def split_document(input, *, delimiter: str = '\\n\\n'):
-  delimiter = delimiter.encode().decode('unicode_escape')
-  chunks = input['text'].split(delimiter)
-  return pd.DataFrame(chunks, columns=['text'])
+def split_document(input, *, delimiter: str = "\\n\\n"):
+    delimiter = delimiter.encode().decode("unicode_escape")
+    chunks = input["text"].split(delimiter)
+    return pd.DataFrame(chunks, columns=["text"])
+
 
 @ops.input_position(input="top")
 @op("Build document graph")
 def build_document_graph(input):
-  return [{'source': i, 'target': i+1} for i in range(len(input)-1)]
+    return [{"source": i, "target": i + 1} for i in range(len(input) - 1)]
+
 
 @ops.input_position(nodes="top", edges="top")
 @op("Predict links")
 def predict_links(nodes, edges):
-  '''A placeholder for a real algorithm. For now just adds 2-hop neighbors.'''
-  edge_map = {} # Source -> [Targets]
-  for edge in edges:
-    edge_map.setdefault(edge['source'], [])
-    edge_map[edge['source']].append(edge['target'])
-  new_edges = []
-  for edge in edges:
-    for t in edge_map.get(edge['target'], []):
-      new_edges.append({'source': edge['source'], 'target': t})
-  return edges + new_edges
+    """A placeholder for a real algorithm. For now just adds 2-hop neighbors."""
+    edge_map = {}  # Source -> [Targets]
+    for edge in edges:
+        edge_map.setdefault(edge["source"], [])
+        edge_map[edge["source"]].append(edge["target"])
+    new_edges = []
+    for edge in edges:
+        for t in edge_map.get(edge["target"], []):
+            new_edges.append({"source": edge["source"], "target": t})
+    return edges + new_edges
+
 
 @ops.input_position(nodes="top", edges="top")
 @op("Add neighbors")
 def add_neighbors(nodes, edges, item):
-  nodes = pd.DataFrame(nodes)
-  edges = pd.DataFrame(edges)
-  matches = item['rag']
-  additional_matches = []
-  for m in matches:
-    node = nodes[nodes['text'] == m].index[0]
-    neighbors = edges[edges['source'] == node]['target'].to_list()
-    additional_matches.extend(nodes.loc[neighbors, 'text'])
-  return {**item, 'rag': matches + additional_matches}
+    nodes = pd.DataFrame(nodes)
+    edges = pd.DataFrame(edges)
+    matches = item["rag"]
+    additional_matches = []
+    for m in matches:
+        node = nodes[nodes["text"] == m].index[0]
+        neighbors = edges[edges["source"] == node]["target"].to_list()
+        additional_matches.extend(nodes.loc[neighbors, "text"])
+    return {**item, "rag": matches + additional_matches}
+
 
 @op("Create prompt")
-def create_prompt(input, *, save_as='prompt', template: ops.LongStr):
-  assert template, 'Please specify the template. Refer to columns using the Jinja2 syntax.'
-  t = jinja.from_string(template)
-  prompt = t.render(**input)
-  return {**input, save_as: prompt}
+def create_prompt(input, *, save_as="prompt", template: ops.LongStr):
+    assert (
+        template
+    ), "Please specify the template. Refer to columns using the Jinja2 syntax."
+    t = jinja.from_string(template)
+    prompt = t.render(**input)
+    return {**input, save_as: prompt}
+
 
 @op("Ask LLM")
 def ask_llm(input, *, model: str, accepted_regex: str = None, max_tokens: int = 100):
-  assert model, 'Please specify the model.'
-  assert 'prompt' in input, 'Please create the prompt first.'
-  options = {}
-  if accepted_regex:
-    options['extra_body'] = {
-      "guided_regex": accepted_regex,
-    }
-  results = chat(
-    model=model,
-    max_tokens=max_tokens,
-    messages=[
-      {"role": "user", "content": input['prompt']},
-    ],
-    **options,
-  )
-  return [{**input, 'response': r} for r in results]
+    assert model, "Please specify the model."
+    assert "prompt" in input, "Please create the prompt first."
+    options = {}
+    if accepted_regex:
+        options["extra_body"] = {
+            "guided_regex": accepted_regex,
+        }
+    results = chat(
+        model=model,
+        max_tokens=max_tokens,
+        messages=[
+            {"role": "user", "content": input["prompt"]},
+        ],
+        **options,
+    )
+    return [{**input, "response": r} for r in results]
+
 
 @op("View", view="table_view")
 def view(input, *, _ctx: one_by_one.Context):
-  v = _ctx.last_result
-  if v:
-    columns = v['dataframes']['df']['columns']
-    v['dataframes']['df']['data'].append([input[c] for c in columns])
-  else:
-    columns = [str(c) for c in input.keys() if not str(c).startswith('_')]
-    v = {
-      'dataframes': { 'df': {
-        'columns': columns,
-        'data': [[input[c] for c in columns]],
-      }}
-    }
-  return v
+    v = _ctx.last_result
+    if v:
+        columns = v["dataframes"]["df"]["columns"]
+        v["dataframes"]["df"]["data"].append([input[c] for c in columns])
+    else:
+        columns = [str(c) for c in input.keys() if not str(c).startswith("_")]
+        v = {
+            "dataframes": {
+                "df": {
+                    "columns": columns,
+                    "data": [[input[c] for c in columns]],
+                }
+            }
+        }
+    return v
+
 
 @ops.input_position(input="right")
 @ops.output_position(output="left")
 @op("Loop")
 def loop(input, *, max_iterations: int = 3, _ctx: one_by_one.Context):
-  '''Data can flow back here max_iterations-1 times.'''
-  key = f'iterations-{_ctx.node.id}'
-  input[key] = input.get(key, 0) + 1
-  if input[key] < max_iterations:
-    return input
+    """Data can flow back here max_iterations-1 times."""
+    key = f"iterations-{_ctx.node.id}"
+    input[key] = input.get(key, 0) + 1
+    if input[key] < max_iterations:
+        return input
+
 
-@op('Branch', outputs=['true', 'false'])
+@op("Branch", outputs=["true", "false"])
 def branch(input, *, expression: str):
-  res = eval(expression, input)
-  return one_by_one.Output(output_handle=str(bool(res)).lower(), value=input)
+    res = eval(expression, input)
+    return one_by_one.Output(output_handle=str(bool(res)).lower(), value=input)
+
 
 class RagEngine(enum.Enum):
-  Chroma = 'Chroma'
-  Custom = 'Custom'
+    Chroma = "Chroma"
+    Custom = "Custom"
+
 
 @ops.input_position(db="top")
-@op('RAG')
+@op("RAG")
 def rag(
-  input, db, *,
-  engine: RagEngine = RagEngine.Chroma,
-  input_field='text', db_field='text', num_matches: int = 10,
-  _ctx: one_by_one.Context):
-  if engine == RagEngine.Chroma:
-    last = _ctx.last_result
-    if last:
-      collection = last['_collection']
-    else:
-      collection_name = _ctx.node.id.replace(' ', '_')
-      for c in chroma_client.list_collections():
-        if c.name == collection_name:
-          chroma_client.delete_collection(name=collection_name)
-      collection = chroma_client.create_collection(name=collection_name)
-      collection.add(
-        documents=[r[db_field] for r in db],
-        ids=[str(i) for i in range(len(db))],
-      )
-    results = collection.query(
-      query_texts=[input[input_field]],
-      n_results=num_matches,
-    )
-    results = [db[int(r)] for r in results['ids'][0]]
-    return {**input, 'rag': results, '_collection': collection}
-  if engine == RagEngine.Custom:
-    model = 'google/gemma-2-2b-it'
-    chat = input[input_field]
-    embeddings = [embedding(input=[r[db_field]], model=model) for r in db]
-    q = embedding(input=[chat], model=model)
-    def cosine_similarity(a, b):
-      return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
-    scores = [(i, cosine_similarity(q, e)) for i, e in enumerate(embeddings)]
-    scores.sort(key=lambda x: -x[1])
-    matches = [db[i][db_field] for i, _ in scores[:num_matches]]
-    return {**input, 'rag': matches}
-
-@op('Run Python')
+    input,
+    db,
+    *,
+    engine: RagEngine = RagEngine.Chroma,
+    input_field="text",
+    db_field="text",
+    num_matches: int = 10,
+    _ctx: one_by_one.Context,
+):
+    if engine == RagEngine.Chroma:
+        last = _ctx.last_result
+        if last:
+            collection = last["_collection"]
+        else:
+            collection_name = _ctx.node.id.replace(" ", "_")
+            for c in chroma_client.list_collections():
+                if c.name == collection_name:
+                    chroma_client.delete_collection(name=collection_name)
+            collection = chroma_client.create_collection(name=collection_name)
+            collection.add(
+                documents=[r[db_field] for r in db],
+                ids=[str(i) for i in range(len(db))],
+            )
+        results = collection.query(
+            query_texts=[input[input_field]],
+            n_results=num_matches,
+        )
+        results = [db[int(r)] for r in results["ids"][0]]
+        return {**input, "rag": results, "_collection": collection}
+    if engine == RagEngine.Custom:
+        model = "google/gemma-2-2b-it"
+        chat = input[input_field]
+        embeddings = [embedding(input=[r[db_field]], model=model) for r in db]
+        q = embedding(input=[chat], model=model)
+
+        def cosine_similarity(a, b):
+            return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
+
+        scores = [(i, cosine_similarity(q, e)) for i, e in enumerate(embeddings)]
+        scores.sort(key=lambda x: -x[1])
+        matches = [db[i][db_field] for i, _ in scores[:num_matches]]
+        return {**input, "rag": matches}
+
+
+@op("Run Python")
 def run_python(input, *, template: str):
-  '''TODO: Implement.'''
-  return input
+    """TODO: Implement."""
+    return input

server/lynxkite_ops.py

@@ -1,4 +1,5 @@
-'''Some operations. To be split into separate files when we have more.'''
+"""Some operations. To be split into separate files when we have more."""
+
 from . import ops
 from collections import deque
 import dataclasses
@@ -9,72 +10,85 @@ import pandas as pd
 import traceback
 import typing
 
-op = ops.op_registration('LynxKite')
+op = ops.op_registration("LynxKite")
+
 
 @dataclasses.dataclass
 class RelationDefinition:
-  '''Defines a set of edges.'''
-  df: str # The DataFrame that contains the edges.
-  source_column: str # The column in the edge DataFrame that contains the source node ID.
-  target_column: str # The column in the edge DataFrame that contains the target node ID.
-  source_table: str # The DataFrame that contains the source nodes.
-  target_table: str # The DataFrame that contains the target nodes.
-  source_key: str # The column in the source table that contains the node ID.
-  target_key: str # The column in the target table that contains the node ID.
+    """Defines a set of edges."""
+
+    df: str  # The DataFrame that contains the edges.
+    source_column: (
+        str  # The column in the edge DataFrame that contains the source node ID.
+    )
+    target_column: (
+        str  # The column in the edge DataFrame that contains the target node ID.
+    )
+    source_table: str  # The DataFrame that contains the source nodes.
+    target_table: str  # The DataFrame that contains the target nodes.
+    source_key: str  # The column in the source table that contains the node ID.
+    target_key: str  # The column in the target table that contains the node ID.
+
 
 @dataclasses.dataclass
 class Bundle:
-  '''A collection of DataFrames and other data.
-
-  Can efficiently represent a knowledge graph (homogeneous or heterogeneous) or tabular data.
-  It can also carry other data, such as a trained model.
-  '''
-  dfs: dict[str, pd.DataFrame] = dataclasses.field(default_factory=dict)
-  relations: list[RelationDefinition] = dataclasses.field(default_factory=list)
-  other: dict[str, typing.Any] = None
-
-  @classmethod
-  def from_nx(cls, graph: nx.Graph):
-    edges = nx.to_pandas_edgelist(graph)
-    d = dict(graph.nodes(data=True))
-    nodes = pd.DataFrame(d.values(), index=d.keys())
-    nodes['id'] = nodes.index
-    return cls(
-      dfs={'edges': edges, 'nodes': nodes},
-      relations=[
-        RelationDefinition(
-          df='edges',
-          source_column='source',
-          target_column='target',
-          source_table='nodes',
-          target_table='nodes',
-          source_key='id',
-          target_key='id',
+    """A collection of DataFrames and other data.
+
+    Can efficiently represent a knowledge graph (homogeneous or heterogeneous) or tabular data.
+    It can also carry other data, such as a trained model.
+    """
+
+    dfs: dict[str, pd.DataFrame] = dataclasses.field(default_factory=dict)
+    relations: list[RelationDefinition] = dataclasses.field(default_factory=list)
+    other: dict[str, typing.Any] = None
+
+    @classmethod
+    def from_nx(cls, graph: nx.Graph):
+        edges = nx.to_pandas_edgelist(graph)
+        d = dict(graph.nodes(data=True))
+        nodes = pd.DataFrame(d.values(), index=d.keys())
+        nodes["id"] = nodes.index
+        return cls(
+            dfs={"edges": edges, "nodes": nodes},
+            relations=[
+                RelationDefinition(
+                    df="edges",
+                    source_column="source",
+                    target_column="target",
+                    source_table="nodes",
+                    target_table="nodes",
+                    source_key="id",
+                    target_key="id",
+                )
+            ],
         )
-      ]
-    )
 
-  def to_nx(self):
-    graph = nx.from_pandas_edgelist(self.dfs['edges'])
-    nx.set_node_attributes(graph, self.dfs['nodes'].set_index('id').to_dict('index'))
-    return graph
+    def to_nx(self):
+        graph = nx.from_pandas_edgelist(self.dfs["edges"])
+        nx.set_node_attributes(
+            graph, self.dfs["nodes"].set_index("id").to_dict("index")
+        )
+        return graph
 
 
 def nx_node_attribute_func(name):
-  '''Decorator for wrapping a function that adds a NetworkX node attribute.'''
-  def decorator(func):
-    @functools.wraps(func)
-    def wrapper(graph: nx.Graph, **kwargs):
-      graph = graph.copy()
-      attr = func(graph, **kwargs)
-      nx.set_node_attributes(graph, attr, name)
-      return graph
-    return wrapper
-  return decorator
+    """Decorator for wrapping a function that adds a NetworkX node attribute."""
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(graph: nx.Graph, **kwargs):
+            graph = graph.copy()
+            attr = func(graph, **kwargs)
+            nx.set_node_attributes(graph, attr, name)
+            return graph
+
+        return wrapper
+
+    return decorator
 
 
 def disambiguate_edges(ws):
-    '''If an input plug is connected to multiple edges, keep only the last edge.'''
+    """If an input plug is connected to multiple edges, keep only the last edge."""
     seen = set()
     for edge in reversed(ws.edges):
         if (edge.target, edge.targetHandle) in seen:
@@ -82,20 +96,15 @@ def disambiguate_edges(ws):
         seen.add((edge.target, edge.targetHandle))
 
 
-@ops.register_executor('LynxKite')
+@ops.register_executor("LynxKite")
 async def execute(ws):
-    catalog = ops.CATALOGS['LynxKite']
+    catalog = ops.CATALOGS["LynxKite"]
     # Nodes are responsible for interpreting/executing their child nodes.
-    nodes = [n for n in ws.nodes if not n.parentId]
     disambiguate_edges(ws)
-    children = {}
-    for n in ws.nodes:
-        if n.parentId:
-            children.setdefault(n.parentId, []).append(n)
     outputs = {}
     failed = 0
-    while len(outputs) + failed < len(nodes):
-        for node in nodes:
+    while len(outputs) + failed < len(ws.nodes):
+        for node in ws.nodes:
             if node.id in outputs:
                 continue
             # TODO: Take the input/output handles into account.
@@ -107,118 +116,138 @@ async def execute(ws):
                 params = {**data.params}
                 # Convert inputs.
                 for i, (x, p) in enumerate(zip(inputs, op.inputs.values())):
-                  if p.type == nx.Graph and isinstance(x, Bundle):
-                    inputs[i] = x.to_nx()
-                  elif p.type == Bundle and isinstance(x, nx.Graph):
-                    inputs[i] = Bundle.from_nx(x)
+                    if p.type == nx.Graph and isinstance(x, Bundle):
+                        inputs[i] = x.to_nx()
+                    elif p.type == Bundle and isinstance(x, nx.Graph):
+                        inputs[i] = Bundle.from_nx(x)
                 try:
-                  output = op(*inputs, **params)
+                    output = op(*inputs, **params)
                 except Exception as e:
-                  traceback.print_exc()
-                  data.error = str(e)
-                  failed += 1
-                  continue
-                if len(op.inputs) == 1 and op.inputs.get('multi') == '*':
+                    traceback.print_exc()
+                    data.error = str(e)
+                    failed += 1
+                    continue
+                if len(op.inputs) == 1 and op.inputs.get("multi") == "*":
                     # It's a flexible input. Create n+1 handles.
-                    data.inputs = {f'input{i}': None for i in range(len(inputs) + 1)}
+                    data.inputs = {f"input{i}": None for i in range(len(inputs) + 1)}
                 data.error = None
                 outputs[node.id] = output
-                if op.type == 'visualization' or op.type == 'table_view' or op.type == 'image':
+                if (
+                    op.type == "visualization"
+                    or op.type == "table_view"
+                    or op.type == "image"
+                ):
                     data.display = output
 
+
 @op("Import Parquet")
 def import_parquet(*, filename: str):
-  '''Imports a parquet file.'''
-  return pd.read_parquet(filename)
+    """Imports a parquet file."""
+    return pd.read_parquet(filename)
+
 
 @op("Create scale-free graph")
 def create_scale_free_graph(*, nodes: int = 10):
-  '''Creates a scale-free graph with the given number of nodes.'''
-  return nx.scale_free_graph(nodes)
+    """Creates a scale-free graph with the given number of nodes."""
+    return nx.scale_free_graph(nodes)
+
 
 @op("Compute PageRank")
-@nx_node_attribute_func('pagerank')
+@nx_node_attribute_func("pagerank")
 def compute_pagerank(graph: nx.Graph, *, damping=0.85, iterations=100):
-  return nx.pagerank(graph, alpha=damping, max_iter=iterations)
+    return nx.pagerank(graph, alpha=damping, max_iter=iterations)
+
 
 @op("Discard loop edges")
 def discard_loop_edges(graph: nx.Graph):
-  graph = graph.copy()
-  graph.remove_edges_from(nx.selfloop_edges(graph))
-  return graph
+    graph = graph.copy()
+    graph.remove_edges_from(nx.selfloop_edges(graph))
+    return graph
+
 
 @op("Sample graph")
 def sample_graph(graph: nx.Graph, *, nodes: int = 100):
-  '''Takes a (preferably connected) subgraph.'''
-  sample = set()
-  to_expand = deque([0])
-  while to_expand and len(sample) < nodes:
-    node = to_expand.pop()
-    for n in graph.neighbors(node):
-      if n not in sample:
-        sample.add(n)
-        to_expand.append(n)
-      if len(sample) == nodes:
-        break
-  return nx.Graph(graph.subgraph(sample))
+    """Takes a (preferably connected) subgraph."""
+    sample = set()
+    to_expand = deque([0])
+    while to_expand and len(sample) < nodes:
+        node = to_expand.pop()
+        for n in graph.neighbors(node):
+            if n not in sample:
+                sample.add(n)
+                to_expand.append(n)
+            if len(sample) == nodes:
+                break
+    return nx.Graph(graph.subgraph(sample))
+
 
 def _map_color(value):
-  cmap = matplotlib.cm.get_cmap('viridis')
-  value = (value - value.min()) / (value.max() - value.min())
-  rgba = cmap(value)
-  return ['#{:02x}{:02x}{:02x}'.format(int(r*255), int(g*255), int(b*255)) for r, g, b in rgba[:, :3]]
+    cmap = matplotlib.cm.get_cmap("viridis")
+    value = (value - value.min()) / (value.max() - value.min())
+    rgba = cmap(value)
+    return [
+        "#{:02x}{:02x}{:02x}".format(int(r * 255), int(g * 255), int(b * 255))
+        for r, g, b in rgba[:, :3]
+    ]
+
 
 @op("Visualize graph", view="visualization")
 def visualize_graph(graph: Bundle, *, color_nodes_by: ops.NodeAttribute = None):
-  nodes = graph.dfs['nodes'].copy()
-  if color_nodes_by:
-    nodes['color'] = _map_color(nodes[color_nodes_by])
-  nodes = nodes.to_records()
-  edges = graph.dfs['edges'].drop_duplicates(['source', 'target'])
-  edges = edges.to_records()
-  pos = nx.spring_layout(graph.to_nx(), iterations=max(1, int(10000/len(nodes))))
-  v = {
-    'animationDuration': 500,
-    'animationEasingUpdate': 'quinticInOut',
-    'series': [
-      {
-        'type': 'graph',
-        'roam': True,
-        'lineStyle': {
-          'color': 'gray',
-          'curveness': 0.3,
-        },
-        'emphasis': {
-          'focus': 'adjacency',
-          'lineStyle': {
-            'width': 10,
-          }
-        },
-        'data': [
-          {
-            'id': str(n.id),
-            'x': float(pos[n.id][0]), 'y': float(pos[n.id][1]),
-            # Adjust node size to cover the same area no matter how many nodes there are.
-            'symbolSize': 50 / len(nodes) ** 0.5,
-            'itemStyle': {'color': n.color} if color_nodes_by else {},
-          }
-          for n in nodes],
-        'links': [
-          {'source': str(r.source), 'target': str(r.target)}
-          for r in edges],
-      },
-    ],
-  }
-  return v
+    nodes = graph.dfs["nodes"].copy()
+    if color_nodes_by:
+        nodes["color"] = _map_color(nodes[color_nodes_by])
+    nodes = nodes.to_records()
+    edges = graph.dfs["edges"].drop_duplicates(["source", "target"])
+    edges = edges.to_records()
+    pos = nx.spring_layout(graph.to_nx(), iterations=max(1, int(10000 / len(nodes))))
+    v = {
+        "animationDuration": 500,
+        "animationEasingUpdate": "quinticInOut",
+        "series": [
+            {
+                "type": "graph",
+                "roam": True,
+                "lineStyle": {
+                    "color": "gray",
+                    "curveness": 0.3,
+                },
+                "emphasis": {
+                    "focus": "adjacency",
+                    "lineStyle": {
+                        "width": 10,
+                    },
+                },
+                "data": [
+                    {
+                        "id": str(n.id),
+                        "x": float(pos[n.id][0]),
+                        "y": float(pos[n.id][1]),
+                        # Adjust node size to cover the same area no matter how many nodes there are.
+                        "symbolSize": 50 / len(nodes) ** 0.5,
+                        "itemStyle": {"color": n.color} if color_nodes_by else {},
+                    }
+                    for n in nodes
+                ],
+                "links": [
+                    {"source": str(r.source), "target": str(r.target)} for r in edges
+                ],
+            },
+        ],
+    }
+    return v
+
 
 @op("View tables", view="table_view")
 def view_tables(bundle: Bundle):
-  v = {
-    'dataframes': { name: {
-      'columns': [str(c) for c in df.columns],
-      'data': df.values.tolist(),
-    } for name, df in bundle.dfs.items() },
-    'relations': bundle.relations,
-    'other': bundle.other,
-  }
-  return v
+    v = {
+        "dataframes": {
+            name: {
+                "columns": [str(c) for c in df.columns],
+                "data": df.values.tolist(),
+            }
+            for name, df in bundle.dfs.items()
+        },
+        "relations": bundle.relations,
+        "other": bundle.other,
+    }
+    return v

server/ops.py

@@ -1,4 +1,5 @@
-'''API for implementing LynxKite operations.'''
+"""API for implementing LynxKite operations."""
+
 from __future__ import annotations
 import enum
 import functools
@@ -10,180 +11,214 @@ from typing_extensions import Annotated
 CATALOGS = {}
 EXECUTORS = {}
 
-typeof = type # We have some arguments called "type".
+typeof = type  # We have some arguments called "type".
+
+
 def type_to_json(t):
-  if isinstance(t, type) and issubclass(t, enum.Enum):
-    return {'enum': list(t.__members__.keys())}
-  if getattr(t, '__metadata__', None):
-    return t.__metadata__[-1]
-  return {'type': str(t)}
-Type = Annotated[
-  typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)
-]
-LongStr = Annotated[
-  str, {'format': 'textarea'}
-]
-PathStr = Annotated[
-  str, {'format': 'path'}
-]
-CollapsedStr = Annotated[
-  str, {'format': 'collapsed'}
-]
-NodeAttribute = Annotated[
-  str, {'format': 'node attribute'}
-]
-EdgeAttribute = Annotated[
-  str, {'format': 'edge attribute'}
-]
+    if isinstance(t, type) and issubclass(t, enum.Enum):
+        return {"enum": list(t.__members__.keys())}
+    if getattr(t, "__metadata__", None):
+        return t.__metadata__[-1]
+    return {"type": str(t)}
+
+
+Type = Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)]
+LongStr = Annotated[str, {"format": "textarea"}]
+PathStr = Annotated[str, {"format": "path"}]
+CollapsedStr = Annotated[str, {"format": "collapsed"}]
+NodeAttribute = Annotated[str, {"format": "node attribute"}]
+EdgeAttribute = Annotated[str, {"format": "edge attribute"}]
+
+
 class BaseConfig(pydantic.BaseModel):
-  model_config = pydantic.ConfigDict(
-    arbitrary_types_allowed=True,
-  )
+    model_config = pydantic.ConfigDict(
+        arbitrary_types_allowed=True,
+    )
 
 
 class Parameter(BaseConfig):
-  '''Defines a parameter for an operation.'''
-  name: str
-  default: typing.Any
-  type: Type = None
-
-  @staticmethod
-  def options(name, options, default=None):
-    e = enum.Enum(f'OptionsFor_{name}', options)
-    return Parameter.basic(name, e[default or options[0]], e)
-
-  @staticmethod
-  def collapsed(name, default, type=None):
-    return Parameter.basic(name, default, CollapsedStr)
-
-  @staticmethod
-  def basic(name, default=None, type=None):
-    if default is inspect._empty:
-      default = None
-    if type is None or type is inspect._empty:
-      type = typeof(default) if default is not None else None
-    return Parameter(name=name, default=default, type=type)
+    """Defines a parameter for an operation."""
+
+    name: str
+    default: typing.Any
+    type: Type = None
+
+    @staticmethod
+    def options(name, options, default=None):
+        e = enum.Enum(f"OptionsFor_{name}", options)
+        return Parameter.basic(name, e[default or options[0]], e)
+
+    @staticmethod
+    def collapsed(name, default, type=None):
+        return Parameter.basic(name, default, CollapsedStr)
+
+    @staticmethod
+    def basic(name, default=None, type=None):
+        if default is inspect._empty:
+            default = None
+        if type is None or type is inspect._empty:
+            type = typeof(default) if default is not None else None
+        return Parameter(name=name, default=default, type=type)
+
 
 class Input(BaseConfig):
-  name: str
-  type: Type
-  position: str = 'left'
+    name: str
+    type: Type
+    position: str = "left"
+
 
 class Output(BaseConfig):
-  name: str
-  type: Type
-  position: str = 'right'
+    name: str
+    type: Type
+    position: str = "right"
+
+
+MULTI_INPUT = Input(name="multi", type="*")
+
 
-MULTI_INPUT = Input(name='multi', type='*')
 def basic_inputs(*names):
-  return {name: Input(name=name, type=None) for name in names}
+    return {name: Input(name=name, type=None) for name in names}
+
+
 def basic_outputs(*names):
-  return {name: Output(name=name, type=None) for name in names}
+    return {name: Output(name=name, type=None) for name in names}
 
 
 class Op(BaseConfig):
-  func: typing.Callable = pydantic.Field(exclude=True)
-  name: str
-  params: dict[str, Parameter]
-  inputs: dict[str, Input]
-  outputs: dict[str, Output]
-  type: str = 'basic' # The UI to use for this operation.
-  sub_nodes: list[Op] = None # If set, these nodes can be placed inside the operation's node.
-
-  def __call__(self, *inputs, **params):
-    # Convert parameters.
-    for p in params:
-      if p in self.params:
-        if self.params[p].type == int:
-          params[p] = int(params[p])
-        elif self.params[p].type == float:
-          params[p] = float(params[p])
-        elif isinstance(self.params[p].type, enum.EnumMeta):
-          params[p] = self.params[p].type[params[p]]
-    res = self.func(*inputs, **params)
-    return res
-
-
-def op(env: str, name: str, *, view='basic', sub_nodes=None, outputs=None):
-  '''Decorator for defining an operation.'''
-  def decorator(func):
-    sig = inspect.signature(func)
-    # Positional arguments are inputs.
-    inputs = {
-      name: Input(name=name, type=param.annotation)
-      for name, param in sig.parameters.items()
-      if param.kind != param.KEYWORD_ONLY}
-    params = {}
-    for n, param in sig.parameters.items():
-      if param.kind == param.KEYWORD_ONLY and not n.startswith('_'):
-        params[n] = Parameter.basic(n, param.default, param.annotation)
-    if outputs:
-      _outputs = {name: Output(name=name, type=None) for name in outputs}
-    else:
-      _outputs = {'output': Output(name='output', type=None)} if view == 'basic' else {}
-    op = Op(func=func, name=name, params=params, inputs=inputs, outputs=_outputs, type=view)
-    if sub_nodes is not None:
-      op.sub_nodes = sub_nodes
-      op.type = 'sub_flow'
-    CATALOGS.setdefault(env, {})
-    CATALOGS[env][name] = op
-    func.__op__ = op
-    return func
-  return decorator
+    func: typing.Callable = pydantic.Field(exclude=True)
+    name: str
+    params: dict[str, Parameter]
+    inputs: dict[str, Input]
+    outputs: dict[str, Output]
+    type: str = "basic"  # The UI to use for this operation.
+
+    def __call__(self, *inputs, **params):
+        # Convert parameters.
+        for p in params:
+            if p in self.params:
+                if self.params[p].type == int:
+                    params[p] = int(params[p])
+                elif self.params[p].type == float:
+                    params[p] = float(params[p])
+                elif isinstance(self.params[p].type, enum.EnumMeta):
+                    params[p] = self.params[p].type[params[p]]
+        res = self.func(*inputs, **params)
+        return res
+
+
+def op(env: str, name: str, *, view="basic", outputs=None):
+    """Decorator for defining an operation."""
+
+    def decorator(func):
+        sig = inspect.signature(func)
+        # Positional arguments are inputs.
+        inputs = {
+            name: Input(name=name, type=param.annotation)
+            for name, param in sig.parameters.items()
+            if param.kind != param.KEYWORD_ONLY
+        }
+        params = {}
+        for n, param in sig.parameters.items():
+            if param.kind == param.KEYWORD_ONLY and not n.startswith("_"):
+                params[n] = Parameter.basic(n, param.default, param.annotation)
+        if outputs:
+            _outputs = {name: Output(name=name, type=None) for name in outputs}
+        else:
+            _outputs = (
+                {"output": Output(name="output", type=None)} if view == "basic" else {}
+            )
+        op = Op(
+            func=func,
+            name=name,
+            params=params,
+            inputs=inputs,
+            outputs=_outputs,
+            type=view,
+        )
+        CATALOGS.setdefault(env, {})
+        CATALOGS[env][name] = op
+        func.__op__ = op
+        return func
+
+    return decorator
+
 
 def input_position(**kwargs):
-  '''Decorator for specifying unusual positions for the inputs.'''
-  def decorator(func):
-    op = func.__op__
-    for k, v in kwargs.items():
-      op.inputs[k].position = v
-    return func
-  return decorator
+    """Decorator for specifying unusual positions for the inputs."""
+
+    def decorator(func):
+        op = func.__op__
+        for k, v in kwargs.items():
+            op.inputs[k].position = v
+        return func
+
+    return decorator
+
 
 def output_position(**kwargs):
-  '''Decorator for specifying unusual positions for the outputs.'''
-  def decorator(func):
-    op = func.__op__
-    for k, v in kwargs.items():
-      op.outputs[k].position = v
-    return func
-  return decorator
+    """Decorator for specifying unusual positions for the outputs."""
+
+    def decorator(func):
+        op = func.__op__
+        for k, v in kwargs.items():
+            op.outputs[k].position = v
+        return func
+
+    return decorator
+
 
 def no_op(*args, **kwargs):
-  if args:
-    return args[0]
-  return None
-
-def register_passive_op(env: str, name: str, inputs=[], outputs=['output'], params=[]):
-  '''A passive operation has no associated code.'''
-  op = Op(
-    func=no_op,
-    name=name,
-    params={p.name: p for p in params},
-    inputs=dict(
-      (i, Input(name=i, type=None)) if isinstance(i, str)
-      else (i.name, i) for i in inputs),
-    outputs=dict(
-      (o, Output(name=o, type=None)) if isinstance(o, str)
-      else (o.name, o) for o in outputs))
-  CATALOGS.setdefault(env, {})
-  CATALOGS[env][name] = op
-  return op
+    if args:
+        return args[0]
+    return None
+
+
+def register_passive_op(env: str, name: str, inputs=[], outputs=["output"], params=[]):
+    """A passive operation has no associated code."""
+    op = Op(
+        func=no_op,
+        name=name,
+        params={p.name: p for p in params},
+        inputs=dict(
+            (i, Input(name=i, type=None)) if isinstance(i, str) else (i.name, i)
+            for i in inputs
+        ),
+        outputs=dict(
+            (o, Output(name=o, type=None)) if isinstance(o, str) else (o.name, o)
+            for o in outputs
+        ),
+    )
+    CATALOGS.setdefault(env, {})
+    CATALOGS[env][name] = op
+    return op
+
 
 def register_executor(env: str):
-  '''Decorator for registering an executor.'''
-  def decorator(func):
-    EXECUTORS[env] = func
-    return func
-  return decorator
+    """Decorator for registering an executor."""
+
+    def decorator(func):
+        EXECUTORS[env] = func
+        return func
+
+    return decorator
+
 
 def op_registration(env: str):
-  return functools.partial(op, env)
+    return functools.partial(op, env)
+
 
 def passive_op_registration(env: str):
-  return functools.partial(register_passive_op, env)
+    return functools.partial(register_passive_op, env)
+
 
 def register_area(env, name, params=[]):
-  '''A node that represents an area. It can contain other nodes, but does not restrict movement in any way.'''
-  op = Op(func=no_op, name=name, params={p.name: p for p in params}, inputs={}, outputs={}, type='area')
-  CATALOGS[env][name] = op
+    """A node that represents an area. It can contain other nodes, but does not restrict movement in any way."""
+    op = Op(
+        func=no_op,
+        name=name,
+        params={p.name: p for p in params},
+        inputs={},
+        outputs={},
+        type="area",
+    )
+    CATALOGS[env][name] = op

server/workspace.py

@@ -1,4 +1,5 @@
-'''For working with LynxKite workspaces.'''
+"""For working with LynxKite workspaces."""
+
 from typing import Optional
 import dataclasses
 import os
@@ -6,15 +7,18 @@ import pydantic
 import tempfile
 from . import ops
 
+
 class BaseConfig(pydantic.BaseModel):
     model_config = pydantic.ConfigDict(
-        extra='allow',
+        extra="allow",
     )
 
+
 class Position(BaseConfig):
     x: float
     y: float
 
+
 class WorkspaceNodeData(BaseConfig):
     title: str
     params: dict
@@ -23,12 +27,13 @@ class WorkspaceNodeData(BaseConfig):
     # Also contains a "meta" field when going out.
     # This is ignored when coming back from the frontend.
 
+
 class WorkspaceNode(BaseConfig):
     id: str
     type: str
     data: WorkspaceNodeData
     position: Position
-    parentId: Optional[str] = None
+
 
 class WorkspaceEdge(BaseConfig):
     id: str
@@ -37,8 +42,9 @@ class WorkspaceEdge(BaseConfig):
     sourceHandle: str
     targetHandle: str
 
+
 class Workspace(BaseConfig):
-    env: str = ''
+    env: str = ""
     nodes: list[WorkspaceNode] = dataclasses.field(default_factory=list)
     edges: list[WorkspaceEdge] = dataclasses.field(default_factory=list)
 
@@ -52,7 +58,9 @@ def save(ws: Workspace, path: str):
     j = ws.model_dump_json(indent=2)
     dirname, basename = os.path.split(path)
     # Create temp file in the same directory to make sure it's on the same filesystem.
-    with tempfile.NamedTemporaryFile('w', prefix=f'.{basename}.', dir=dirname, delete_on_close=False) as f:
+    with tempfile.NamedTemporaryFile(
+        "w", prefix=f".{basename}.", dir=dirname, delete_on_close=False
+    ) as f:
         f.write(j)
         f.close()
         os.replace(f.name, path)
@@ -76,22 +84,13 @@ def _update_metadata(ws):
             if node.id in done:
                 continue
             data = node.data
-            if node.parentId is None:
-                op = catalog.get(data.title)
-            elif node.parentId not in nodes:
-                data.error = f'Parent not found: {node.parentId}'
-                done.add(node.id)
-                continue
-            elif node.parentId in done:
-                op = nodes[node.parentId].data.meta.sub_nodes[data.title]
-            else:
-                continue
+            op = catalog.get(data.title)
             if op:
                 data.meta = op
                 node.type = op.type
-                if data.error == 'Unknown operation.':
+                if data.error == "Unknown operation.":
                     data.error = None
             else:
-                data.error = 'Unknown operation.'
+                data.error = "Unknown operation."
             done.add(node.id)
     return ws

web/.gitignore

@@ -11,3 +11,14 @@ node_modules
 dist
 dist-ssr
 *.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

web/.vscode/extensions.json

@@ -1,3 +0,0 @@
-{
-  "recommendations": ["svelte.svelte-vscode"]
-}

web/README.md

@@ -1,27 +1,7 @@
-# Vite Svelte Flow Template
+To set up:
 
-This template creates a very basic [Svelte Flow](https://svelteflow.dev) app with [Vite](https://vite.dev).
+    npm i
 
-## Get it!
+To start dev server:
 
-```sh
-npx degit xyflow/vite-svelte-flow-template app-name
-```
-
-## Installation
-
-```sh
-npm install
-```
-
-## Development
-
-```sh
-npm run dev
-```
-
-## Build
-
-```sh
-npm run build
-```
+    npm run dev

web/eslint.config.js

@@ -0,0 +1,28 @@
+import js from '@eslint/js'
+import globals from 'globals'
+import reactHooks from 'eslint-plugin-react-hooks'
+import reactRefresh from 'eslint-plugin-react-refresh'
+import tseslint from 'typescript-eslint'
+
+export default tseslint.config(
+  { ignores: ['dist'] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ['**/*.{ts,tsx}'],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      'react-hooks': reactHooks,
+      'react-refresh': reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      'react-refresh/only-export-components': [
+        'warn',
+        { allowConstantExport: true },
+      ],
+    },
+  },
+)

web/index.html

@@ -2,12 +2,12 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/png" href="/favicon.ico" />
+    <link rel="icon" type="image/svg+xml" href="/src/assets/favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>LynxKite 2024</title>
+    <title>LynxKite 2025</title>
   </head>
   <body>
-    <div id="app"></div>
-    <script type="module" src="/src/main.ts"></script>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
   </body>
 </html>

web/package.json

@@ -1,37 +1,48 @@
 {
-  "name": "vite-svelte-flow-template",
+  "name": "lynxkite",
   "private": true,
   "version": "0.0.0",
   "type": "module",
   "scripts": {
     "dev": "vite",
-    "build": "vite build",
-    "preview": "vite preview",
-    "check": "svelte-check --tsconfig ./tsconfig.json"
-  },
-  "devDependencies": {
-    "@sveltejs/vite-plugin-svelte": "3.1.2",
-    "@syncedstore/core": "0.6.0",
-    "@syncedstore/svelte": "0.6.0",
-    "@tsconfig/svelte": "5.0.4",
-    "sass": "1.79.5",
-    "svelte": "4.2.19",
-    "svelte-check": "4.0.5",
-    "svelte-markdown": "^0.4.1",
-    "tslib": "2.7.0",
-    "typescript": "5.6.3",
-    "unplugin-icons": "0.19.3",
-    "vite": "5.4.9",
-    "y-websocket": "2.0.4"
+    "build": "tsc -b && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
   },
   "dependencies": {
-    "@iconify-json/tabler": "1.2.5",
-    "@popperjs/core": "2.11.8",
-    "@sveltestack/svelte-query": "1.6.0",
-    "@xyflow/svelte": "0.1.21",
-    "bootstrap": "5.3.3",
-    "echarts": "5.5.1",
-    "fuse.js": "7.0.0",
-    "svelte-echarts": "^1.0.0-rc3"
+    "@iconify-json/tabler": "^1.2.10",
+    "@svgr/core": "^8.1.0",
+    "@svgr/plugin-jsx": "^8.1.0",
+    "@syncedstore/core": "^0.6.0",
+    "@syncedstore/react": "^0.6.0",
+    "@types/node": "^22.10.1",
+    "@xyflow/react": "^12.3.5",
+    "daisyui": "^4.12.20",
+    "echarts": "^5.5.1",
+    "fuse.js": "^7.0.0",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-markdown": "^9.0.1",
+    "react-router-dom": "^7.0.2",
+    "swr": "^2.2.5",
+    "unplugin-icons": "^0.21.0",
+    "y-websocket": "^2.0.4",
+    "yjs": "^13.6.20"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.15.0",
+    "@types/react": "^18.3.14",
+    "@types/react-dom": "^18.3.2",
+    "@vitejs/plugin-react-swc": "^3.5.0",
+    "autoprefixer": "^10.4.20",
+    "eslint": "^9.15.0",
+    "eslint-plugin-react-hooks": "^5.0.0",
+    "eslint-plugin-react-refresh": "^0.4.14",
+    "globals": "^15.12.0",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.16",
+    "typescript": "~5.6.2",
+    "typescript-eslint": "^8.15.0",
+    "vite": "^6.0.1"
   }
 }

web/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

web/src/App.svelte

@@ -1,22 +0,0 @@
-<script lang="ts">
-  import Directory from './Directory.svelte';
-  import Workspace from './Workspace.svelte';
-  let page = '';
-  let parameters = {};
-  function onHashChange() {
-    const parts = location.hash.split('?');
-    page = parts[0].substring(1);
-    parameters = {};
-    if (parts.length > 1) {
-      parameters = Object.fromEntries(new URLSearchParams(parts[1]));
-    }
-  }
-  onHashChange();
-</script>
-
-<svelte:window on:hashchange={onHashChange} />
-{#if page === 'edit'}
-  <Workspace {...parameters} />
-{:else}
-  <Directory {...parameters} />
-{/if}

web/src/Directory.svelte

@@ -1,177 +0,0 @@
-<script lang="ts">
-  // The directory browser.
-  import logo from './assets/logo.png';
-  import Home from 'virtual:icons/tabler/home'
-  import Folder from 'virtual:icons/tabler/folder'
-  import FolderPlus from 'virtual:icons/tabler/folder-plus'
-  import File from 'virtual:icons/tabler/file'
-  import FilePlus from 'virtual:icons/tabler/file-plus'
-
-  export let path = '';
-  async function fetchList(path) {
-    const encodedPath = encodeURIComponent(path || '');
-    const res = await fetch(`/api/dir/list?path=${encodedPath}`);
-    const j = await res.json();
-    return j;
-  }
-  $: list = fetchList(path);
-  function link(item) {
-    if (item.type === 'directory') {
-      return `#dir?path=${item.name}`;
-    } else {
-      return `#edit?path=${item.name}`;
-    }
-  }
-  function shortName(item) {
-    return item.name.split('/').pop();
-  }
-  function newName(list) {
-    let i = 0;
-    while (true) {
-      const name = `Untitled${i ? ` ${i}` : ''}`;
-      if (!list.find(item => item.name === name)) {
-        return name;
-      }
-      i++;
-    }
-  }
-  function newWorkspaceIn(path, list) {
-    const pathSlash = path ? `${path}/` : '';
-    return `#edit?path=${pathSlash}${newName(list)}`;
-  }
-  async function newFolderIn(path, list) {
-    const pathSlash = path ? `${path}/` : '';
-    const name = newName(list);
-    const res = await fetch(`/api/dir/mkdir`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ path: pathSlash + name }),
-    });
-    list = await res.json();
-  }
-</script>
-
-<div class="directory-page">
-  <div class="logo">
-    <a href="https://lynxkite.com/"><img src="{logo}" class="logo-image"></a>
-    <div class="tagline">The Complete Graph Data Science Platform</div>
-  </div>
-  <div class="entry-list">
-    {#await list}
-      <div class="loading spinner-border" role="status">
-        <span class="visually-hidden">Loading...</span>
-      </div>
-    {:then list}
-      <div class="actions">
-        <a href="{newWorkspaceIn(path, list)}"><FilePlus /> New workspace</a>
-        <a href on:click="{newFolderIn(path, list)}"><FolderPlus /> New folder</a>
-      </div>
-      {#if path} <div class="breadcrumbs"><a href="#dir"><Home /></a> {path} </div> {/if}
-      {#each list as item}
-        <a class="entry" href={link(item)}>
-          {#if item.type === 'directory'}
-            <Folder />
-          {:else}
-            <File />
-          {/if}
-          {shortName(item)}
-        </a>
-      {/each}
-    {:catch error}
-      <p style="color: red">{error.message}</p>
-    {/await}
-  </div>
-</div>
-
-<style>
-  .entry-list {
-    width: 100%;
-    margin: 10px auto;
-    background-color: white;
-    border-radius: 10px;
-    box-shadow: 0px 2px 4px;
-    padding: 0 0 10px 0;
-  }
-  @media (min-width: 768px) {
-    .entry-list {
-      width: 768px;
-    }
-  }
-  @media (min-width: 960px) {
-    .entry-list {
-      width: 80%;
-    }
-  }
-
-  .logo {
-    margin: 0;
-    padding-top: 50px;
-    text-align: center;
-  }
-  .logo-image {
-    max-width: 50%;
-  }
-  .tagline {
-    color: #39bcf3;
-    font-size: 14px;
-    font-weight: 500;
-  }
-  @media (min-width: 1400px) {
-    .tagline {
-      font-size: 18px;
-    }
-  }
-
-  .actions {
-    display: flex;
-    justify-content: space-evenly;
-    padding: 5px;
-  }
-  .actions a {
-    padding: 2px 10px;
-    border-radius: 5px;
-  }
-  .actions a:hover {
-    background: #39bcf3;
-    color: white;
-  }
-
-  .breadcrumbs {
-    padding-left: 10px;
-    font-size: 20px;
-    background: #002a4c20;
-  }
-  .breadcrumbs a:hover {
-    color: #39bcf3;
-  }
-  .entry-list .entry {
-    display: block;
-    border-bottom: 1px solid whitesmoke;
-    padding-left: 10px;
-    color: #004165;
-    cursor: pointer;
-    user-select: none;
-    text-decoration: none;
-  }
-  .entry-list .open .entry,
-  .entry-list .entry:hover,
-  .entry-list .entry:focus {
-    background: #39bcf3;
-    color: white;
-  }
-  .entry-list .entry:last-child {
-    border-bottom: none;
-  }
-  .directory-page {
-    background: #002a4c;
-    height: 100vh;
-  }
-  a {
-    color: black;
-    text-decoration: none;
-  }
-  .loading {
-    color: #39bcf3;
-    margin: 10px;
-  }
-</style>

web/src/Directory.tsx

@@ -0,0 +1,88 @@
+// The directory browser.
+import { useParams } from "react-router";
+import useSWR from 'swr'
+
+import logo from './assets/logo.png';
+// @ts-ignore
+import Home from '~icons/tabler/home'
+// @ts-ignore
+import Folder from '~icons/tabler/folder'
+// @ts-ignore
+import FolderPlus from '~icons/tabler/folder-plus'
+// @ts-ignore
+import File from '~icons/tabler/file'
+// @ts-ignore
+import FilePlus from '~icons/tabler/file-plus'
+
+const fetcher = (url: string) => fetch(url).then((res) => res.json());
+
+export default function () {
+  const { path } = useParams();
+  const encodedPath = encodeURIComponent(path || '');
+  const list = useSWR(`/api/dir/list?path=${encodedPath}`, fetcher)
+  function link(item: any) {
+    if (item.type === 'directory') {
+      return `/dir/${item.name}`;
+    } else {
+      return `/edit/${item.name}`;
+    }
+  }
+  function shortName(item: any) {
+    return item.name.split('/').pop();
+  }
+  function newName(list: any[]) {
+    let i = 0;
+    while (true) {
+      const name = `Untitled${i ? ` ${i}` : ''}`;
+      if (!list.find(item => item.name === name)) {
+        return name;
+      }
+      i++;
+    }
+  }
+  function newWorkspaceIn(path: string, list: any[]) {
+    const pathSlash = path ? `${path}/` : '';
+    return `/edit/${pathSlash}${newName(list)}`;
+  }
+  async function newFolderIn(path: string, list: any[]) {
+    const pathSlash = path ? `${path}/` : '';
+    const name = newName(list);
+    const res = await fetch(`/api/dir/mkdir`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ path: pathSlash + name }),
+    });
+    list = await res.json();
+  }
+
+  return (
+    <div className="directory">
+      <div className="logo">
+        <a href="https://lynxkite.com/"><img src={logo} className="logo-image" alt="LynxKite logo" /></a>
+        <div className="tagline">The Complete Graph Data Science Platform</div>
+      </div>
+      <div className="entry-list">
+        {list.error && <p className="error">{list.error.message}</p>}
+        {list.isLoading &&
+          <div className="loading spinner-border" role="status">
+            <span className="visually-hidden">Loading...</span>
+          </div>}
+        {list.data &&
+          <>
+            <div className="actions">
+              <a href={newWorkspaceIn(path || "", list.data)}><FilePlus /> New workspace</a>
+              <a href="" onClick={() => newFolderIn(path || "", list.data)}><FolderPlus /> New folder</a>
+            </div>
+            {path && <div className="breadcrumbs"><a href="/dir/"><Home /></a> {path} </div>}
+            {list.data.map((item: any) =>
+              <a key={link(item)} className="entry" href={link(item)}>
+                {item.type === 'directory' ? <Folder /> : <File />}
+                {shortName(item)}
+              </a>
+            )}
+          </>
+        }
+      </div>
+    </div>
+  );
+}

web/src/EnvironmentSelector.svelte

@@ -1,14 +0,0 @@
-<script lang="ts">
-  export let options;
-  export let value;
-  export let onChange;
-</script>
-
-<select class="form-select form-select-sm"
-  value={value}
-  on:change={(evt) => onChange(evt.currentTarget.value)}
-  >
-  {#each options as option}
-    <option value={option}>{option}</option>
-  {/each}
-</select>

web/src/LynxKiteFlow.svelte

@@ -1,230 +0,0 @@
-<script lang="ts">
-  import { setContext } from 'svelte';
-  import { writable } from 'svelte/store';
-  import {
-    SvelteFlow,
-    Controls,
-    MiniMap,
-    MarkerType,
-    useSvelteFlow,
-    useUpdateNodeInternals,
-    type XYPosition,
-    type Node,
-    type Edge,
-    type Connection,
-    type NodeTypes,
-  } from '@xyflow/svelte';
-  import ArrowBack from 'virtual:icons/tabler/arrow-back'
-  import Backspace from 'virtual:icons/tabler/backspace'
-  import Atom from 'virtual:icons/tabler/Atom'
-  import { useQuery } from '@sveltestack/svelte-query';
-  import NodeWithParams from './NodeWithParams.svelte';
-  import NodeWithVisualization from './NodeWithVisualization.svelte';
-  import NodeWithImage from './NodeWithImage.svelte';
-  import NodeWithTableView from './NodeWithTableView.svelte';
-  import NodeWithSubFlow from './NodeWithSubFlow.svelte';
-  import NodeWithArea from './NodeWithArea.svelte';
-  import NodeSearch from './NodeSearch.svelte';
-  import EnvironmentSelector from './EnvironmentSelector.svelte';
-  import '@xyflow/svelte/dist/style.css';
-  import { syncedStore, getYjsDoc } from "@syncedstore/core";
-  import { svelteSyncedStore } from "@syncedstore/svelte";
-  import { WebsocketProvider } from "y-websocket";
-  const updateNodeInternals = useUpdateNodeInternals();
-
-  function getCRDTStore(path) {
-    const sstore = syncedStore({ workspace: {} });
-    const doc = getYjsDoc(sstore);
-    const wsProvider = new WebsocketProvider("ws://localhost:8000/ws/crdt", path, doc);
-    return {store: svelteSyncedStore(sstore), sstore, doc};
-  }
-  $: connection = getCRDTStore(path);
-  $: store = connection.store;
-  $: store.subscribe((value) => {
-    if (!value?.workspace?.edges) return;
-    $nodes = [...value.workspace.nodes];
-    $edges = [...value.workspace.edges];
-    updateNodeInternals();
-  });
-  $: setContext('LynxKite store', store);
-
-  export let path = '';
-
-  const { screenToFlowPosition } = useSvelteFlow();
-
-  const nodeTypes: NodeTypes = {
-    basic: NodeWithParams,
-    visualization: NodeWithVisualization,
-    image: NodeWithImage,
-    table_view: NodeWithTableView,
-    sub_flow: NodeWithSubFlow,
-    area: NodeWithArea,
-  };
-
-  const nodes = writable<Node[]>([]);
-  const edges = writable<Edge[]>([]);
-
-  function closeNodeSearch() {
-    nodeSearchSettings = undefined;
-  }
-  function toggleNodeSearch({ detail: { event } }) {
-    if (nodeSearchSettings) {
-      closeNodeSearch();
-      return;
-    }
-    event.preventDefault();
-    nodeSearchSettings = {
-      pos: { x: event.clientX, y: event.clientY },
-      boxes: $catalog.data[$store.workspace.env],
-    };
-  }
-  function addNode(e) {
-    const meta = {...e.detail};
-    const node = {
-      type: meta.type,
-      data: {
-        meta: meta,
-        title: meta.name,
-        params: Object.fromEntries(
-          Object.values(meta.params).map((p) => [p.name, p.default])),
-      },
-    };
-    node.position = screenToFlowPosition({x: nodeSearchSettings.pos.x, y: nodeSearchSettings.pos.y});
-    const title = node.data.title;
-    let i = 1;
-    node.id = `${title} ${i}`;
-    const nodes = $store.workspace.nodes;
-    while (nodes.find((x) => x.id === node.id)) {
-      i += 1;
-      node.id = `${title} ${i}`;
-    }
-    node.parentId = nodeSearchSettings.parentId;
-    if (node.parentId) {
-      node.extent = 'parent';
-      const parent = nodes.find((x) => x.id === node.parentId);
-      node.position = { x: node.position.x - parent.position.x, y: node.position.y - parent.position.y };
-    }
-    nodes.push(node);
-    closeNodeSearch();
-  }
-  const catalog = useQuery(['catalog'], async () => {
-    const res = await fetch('/api/catalog');
-    return res.json();
-  }, {staleTime: 60000, retry: false});
-
-  let nodeSearchSettings: {
-    pos: XYPosition,
-    boxes: any[],
-    parentId: string,
-  };
-
-  function nodeClick(e) {
-    const node = e.detail.node;
-    const meta = node.data.meta;
-    if (!meta) return;
-    const sub_nodes = meta.sub_nodes;
-    if (!sub_nodes) return;
-    const event = e.detail.event;
-    if (event.target.classList.contains('title')) return;
-    nodeSearchSettings = {
-      pos: { x: event.clientX, y: event.clientY },
-      boxes: sub_nodes,
-      parentId: node.id,
-    };
-  }
-  function onConnect(params: Connection) {
-    const edge = {
-      id: `${params.source} ${params.target}`,
-      source: params.source,
-      sourceHandle: params.sourceHandle,
-      target: params.target,
-      targetHandle: params.targetHandle,
-    };
-    $store.workspace.edges.push(edge);
-  }
-  function onDelete(params) {
-    const { nodes, edges } = params;
-    for (const node of nodes) {
-      const index = $store.workspace.nodes.findIndex((x) => x.id === node.id);
-      if (index !== -1) $store.workspace.nodes.splice(index, 1);
-    }
-    for (const edge of edges) {
-      const index = $store.workspace.edges.findIndex((x) => x.id === edge.id);
-      if (index !== -1) $store.workspace.edges.splice(index, 1);
-    }
-  }
-  $: parentDir = path.split('/').slice(0, -1).join('/');
-</script>
-
-<div class="page">
-  {#if $store.workspace !== undefined}
-  <div class="top-bar">
-    <div class="ws-name">
-      <a href><img src="/favicon.ico"></a>
-      {path}
-    </div>
-    <div class="tools">
-      <EnvironmentSelector
-        options={Object.keys($catalog.data || {})}
-        value={$store.workspace.env}
-        onChange={(env) => {
-          $store.workspace.env = env;
-        }}
-        />
-      <a href><Atom /></a>
-      <a href><Backspace /></a>
-      <a href="#dir?path={parentDir}"><ArrowBack /></a>
-    </div>
-  </div>
-  <div style:height="100%">
-    <SvelteFlow {nodes} {edges} {nodeTypes} fitView
-      on:paneclick={toggleNodeSearch}
-      on:nodeclick={nodeClick}
-      onconnect={onConnect}
-      ondelete={onDelete}
-      proOptions={{ hideAttribution: true }}
-      maxZoom={3}
-      minZoom={0.3}
-      defaultEdgeOptions={{ markerEnd: { type: MarkerType.Arrow } }}
-      >
-      <Controls />
-      <MiniMap />
-      {#if nodeSearchSettings}
-      <NodeSearch pos={nodeSearchSettings.pos} boxes={nodeSearchSettings.boxes} on:cancel={closeNodeSearch} on:add={addNode} />
-      {/if}
-    </SvelteFlow>
-  </div>
-  {/if}
-</div>
-
-<style>
-  .top-bar {
-    display: flex;
-    justify-content: space-between;
-    background: oklch(30% 0.13 230);
-    color: white;
-  }
-  .ws-name {
-    font-size: 1.5em;
-  }
-  .ws-name img {
-    height: 1.5em;
-    vertical-align: middle;
-    margin: 4px;
-  }
-  .page {
-    display: flex;
-    flex-direction: column;
-    height: 100vh;
-  }
-
-  .tools {
-    display: flex;
-    align-items: center;
-  }
-  .tools a {
-    color: oklch(75% 0.13 230);
-    font-size: 1.5em;
-    padding: 0 10px;
-  }
-</style>

web/src/LynxKiteNode.svelte

@@ -1,174 +0,0 @@
-<script lang="ts">
-  import { getContext } from 'svelte';
-  import { Handle, useSvelteFlow, useUpdateNodeInternals, type NodeProps, NodeResizeControl } from '@xyflow/svelte';
-  import ChevronDownRight from 'virtual:icons/tabler/chevron-down-right';
-
-  const { updateNodeData } = useSvelteFlow();
-  const updateNodeInternals = useUpdateNodeInternals();
-  type $$Props = NodeProps;
-
-  export let nodeStyle = '';
-  export let containerStyle = '';
-  export let id: $$Props['id']; id;
-  export let data: $$Props['data'];
-  export let deletable: $$Props['deletable'] = undefined; deletable;
-  export let draggable: $$Props['draggable'] = undefined; draggable;
-  export let parentId: $$Props['parentId'] = undefined; parentId;
-  export let selectable: $$Props['selectable'] = undefined; selectable;
-  export let dragHandle: $$Props['dragHandle'] = undefined; dragHandle;
-  export let type: $$Props['type']  = undefined; type;
-  export let selected: $$Props['selected'] = undefined; selected;
-  export let isConnectable: $$Props['isConnectable'] = undefined; isConnectable;
-  export let zIndex: $$Props['zIndex'] = undefined; zIndex;
-  export let width: $$Props['width'] = undefined; width;
-  export let height: $$Props['height'] = undefined; height;
-  export let dragging: $$Props['dragging']; dragging;
-  export let targetPosition: $$Props['targetPosition'] = undefined; targetPosition;
-  export let sourcePosition: $$Props['sourcePosition'] = undefined; sourcePosition;
-  export let positionAbsoluteX: $$Props['positionAbsoluteX'] = undefined; positionAbsoluteX;
-  export let positionAbsoluteY: $$Props['positionAbsoluteY'] = undefined; positionAbsoluteY;
-  export let onToggle = () => {};
-
-  $: store = getContext('LynxKite store');
-  $: expanded = !data.collapsed;
-  function titleClicked() {
-    const i = $store.workspace.nodes.findIndex((n) => n.id === id);
-    $store.workspace.nodes[i].data.collapsed = expanded;
-    onToggle({ expanded });
-    // Trigger update.
-    data = data;
-    updateNodeInternals();
-  }
-  function asPx(n: number | undefined) {
-    return n ? n + 'px' : undefined;
-  }
-  function getHandles(inputs, outputs) {
-    const handles: {
-      position: 'top' | 'bottom' | 'left' | 'right',
-      name: string,
-      index: number,
-      offsetPercentage: number,
-      showLabel: boolean,
-    }[] = [];
-    for (const e of Object.values(inputs)) {
-      handles.push({ ...e, type: 'target' });
-    }
-    for (const e of Object.values(outputs)) {
-      handles.push({ ...e, type: 'source' });
-    }
-    const counts = { top: 0, bottom: 0, left: 0, right: 0 };
-    for (const e of handles) {
-      e.index = counts[e.position];
-      counts[e.position]++;
-    }
-    for (const e of handles) {
-      e.offsetPercentage = 100 * (e.index + 1) / (counts[e.position] + 1);
-      const simpleHorizontal = counts.top === 0 && counts.bottom === 0 && handles.length <= 2;
-      const simpleVertical = counts.left === 0 && counts.right === 0 && handles.length <= 2;
-      e.showLabel = !simpleHorizontal && !simpleVertical;
-    }
-    return handles;
-  }
-  $: handles = getHandles(data.meta?.inputs || {}, data.meta?.outputs || {});
-  const handleOffsetDirection = { top: 'left', bottom: 'left', left: 'top', right: 'top' };
-</script>
-
-<div class="node-container" class:expanded={expanded}
-  style:width={asPx(width)} style:height={asPx(expanded ? height : undefined)} style={containerStyle}>
-  <div class="lynxkite-node" style={nodeStyle}>
-    <div class="title" on:click={titleClicked}>
-      {data.title}
-      {#if data.error}<span class="title-icon">⚠️</span>{/if}
-      {#if !expanded}<span class="title-icon">⋯</span>{/if}
-    </div>
-    {#if expanded}
-      {#if data.error}
-        <div class="error">{data.error}</div>
-      {/if}
-      <slot />
-    {/if}
-    {#each handles as handle}
-      <Handle
-        id={handle.name} type={handle.type} position={handle.position}
-        style="{handleOffsetDirection[handle.position]}: {handle.offsetPercentage}%">
-        {#if handle.showLabel}<span class="handle-name">{handle.name.replace(/_/g, " ")}</span>{/if}
-      </Handle>
-    {/each}
-  </div>
-  {#if expanded}
-    <NodeResizeControl
-      minWidth={100}
-      minHeight={50}
-      style="background: transparent; border: none;"
-      onResizeStart={() => updateNodeData(id, { beingResized: true })}
-      onResizeEnd={() => updateNodeData(id, { beingResized: false })}
-      >
-      <ChevronDownRight class="node-resizer" />
-    </NodeResizeControl>
-  {/if}
-</div>
-
-<style>
-  .error {
-    background: #ffdddd;
-    padding: 8px;
-    font-size: 12px;
-  }
-  .title-icon {
-    margin-left: 5px;
-    float: right;
-  }
-  .node-container {
-    padding: 8px;
-    position: relative;
-  }
-  .lynxkite-node {
-    box-shadow: 0px 5px 50px 0px rgba(0, 0, 0, 0.3);
-    border-radius: 4px;
-    background: white;
-  }
-  .expanded .lynxkite-node {
-    overflow-y: auto;
-    height: 100%;
-  }
-  .title {
-    background: oklch(75% 0.2 55);
-    font-weight: bold;
-    padding: 8px;
-  }
-  .handle-name {
-    font-size: 10px;
-    color: black;
-    letter-spacing: 0.05em;
-    text-align: right;
-    white-space: nowrap;
-    position: absolute;
-    top: -5px;
-    backdrop-filter: blur(10px);
-    padding: 2px 8px;
-    border-radius: 4px;
-    visibility: hidden;
-  }
-  :global(.left) .handle-name {
-    right: 20px;
-  }
-  :global(.right) .handle-name {
-    left: 20px;
-  }
-  :global(.top) .handle-name,
-  :global(.bottom) .handle-name {
-    top: -5px;
-    left: 5px;
-    backdrop-filter: none;
-  }
-  .node-container:hover .handle-name {
-    visibility: visible;
-  }
-  :global(.node-resizer) {
-    position: absolute;
-    bottom: 8px;
-    right: 8px;
-    cursor: nwse-resize;
-    color: var(--bs-border-color);
-  }
-</style>

web/src/NodeParameter.svelte

@@ -1,69 +0,0 @@
-<script lang="ts">
-  export let name: string;
-  export let value;
-  export let meta;
-  export let onChange;
-const BOOLEAN = "<class 'bool'>";
-</script>
-
-<label class="param">
-  {#if meta?.type?.format === 'collapsed'}
-    <span class="param-name">{name.replace(/_/g, ' ')}</span>
-    <button class="collapsed-param form-control form-control-sm">
-      ⋯
-    </button>
-  {:else if meta?.type?.format === 'textarea'}
-    <span class="param-name">{name.replace(/_/g, ' ')}</span>
-    <textarea class="form-control form-control-sm"
-      rows="6"
-      value={value}
-      on:change={(evt) => onChange(evt.currentTarget.value)}
-      />
-  {:else if meta?.type?.enum}
-    <span class="param-name">{name.replace(/_/g, ' ')}</span>
-    <select class="form-select form-select-sm"
-      value={value || meta.type.enum[0]}
-      on:change={(evt) => onChange(evt.currentTarget.value)}
-    >
-      {#each meta.type.enum as option}
-        <option value={option}>{option}</option>
-      {/each}
-    </select>
-  {:else if meta?.type?.type === BOOLEAN}
-    <label class="form-check-label">
-      <input class="form-check-input"
-      type="checkbox"
-      checked={value}
-      on:change={(evt) => onChange(evt.currentTarget.checked)}
-      />
-      {name.replace(/_/g, ' ')}
-    </label>
-  {:else}
-    <span class="param-name">{name.replace(/_/g, ' ')}</span>
-    <input class="form-control form-control-sm"
-      value={value}
-      on:change={(evt) => onChange(evt.currentTarget.value)}
-      />
-  {/if}
-</label>
-
-<style>
-  .param {
-    padding: 4px 8px 4px 8px;
-    display: block;
-  }
-  .param-name {
-    display: block;
-    font-size: 10px;
-    letter-spacing: 0.05em;
-    margin-left: 10px;
-    background: var(--bs-border-color);
-    width: fit-content;
-    padding: 2px 8px;
-    border-radius: 4px 4px 0 0;
-  }
-  .collapsed-param {
-    min-height: 20px;
-    line-height: 10px;
-  }
-</style>

web/src/NodeSearch.svelte

@@ -1,100 +0,0 @@
-<script lang="ts">
-  import { createEventDispatcher, onMount } from 'svelte';
-  import Fuse from 'fuse.js'
-  const dispatch = createEventDispatcher();
-  export let pos;
-  export let boxes;
-  let searchBox: HTMLInputElement;
-  let hits = Object.values(boxes).map(box => ({item: box}));
-  let selectedIndex = 0;
-  onMount(() => searchBox.focus());
-  $: fuse = new Fuse(Object.values(boxes), {
-    keys: ['name']
-  })
-  function onInput() {
-    hits = fuse.search(searchBox.value);
-    selectedIndex = Math.max(0, Math.min(selectedIndex, hits.length - 1));
-  }
-  function onKeyDown(e) {
-    if (e.key === 'ArrowDown') {
-      e.preventDefault();
-      selectedIndex = Math.min(selectedIndex + 1, hits.length - 1);
-    } else if (e.key === 'ArrowUp') {
-      e.preventDefault();
-      selectedIndex = Math.max(selectedIndex - 1, 0);
-    } else if (e.key === 'Enter') {
-      addSelected();
-    } else if (e.key === 'Escape') {
-      dispatch('cancel');
-    }
-  }
-  function addSelected() {
-    const node = {...hits[selectedIndex].item};
-    delete node.sub_nodes;
-    node.position = pos;
-    dispatch('add', node);
-  }
-  async function lostFocus(e) {
-    // If it's a click on a result, let the click handler handle it.
-    if (e.relatedTarget && e.relatedTarget.closest('.node-search')) return;
-    dispatch('cancel');
-  }
-
-</script>
-
-<div class="node-search" style="top: {pos.y}px; left: {pos.x}px;">
-  <input
-    bind:this={searchBox}
-    on:input={onInput}
-    on:keydown={onKeyDown}
-    on:focusout={lostFocus}
-    placeholder="Search for box">
-  <div class="matches">
-    {#each hits as box, index}
-      <div
-        tabindex="0"
-        on:focus={() => selectedIndex = index}
-        on:mouseenter={() => selectedIndex = index}
-        on:click={addSelected}
-        class="search-result"
-        class:selected={index == selectedIndex}>
-        {box.item.name}
-      </div>
-    {/each}
-  </div>
-</div>
-
-<style>
-  input {
-    width: calc(100% - 26px);
-    font-size: 20px;
-    padding: 8px;
-    border-radius: 4px;
-    border: 1px solid #eee;
-    margin: 4px;
-  }
-  .search-result {
-    padding: 4px;
-    cursor: pointer;
-  }
-  .search-result.selected {
-    background-color: oklch(75% 0.2 55);
-    border-radius: 4px;
-  }
-  .node-search {
-    position: fixed;
-    width: 300px;
-    z-index: 5;
-    padding: 4px;
-    border-radius: 4px;
-    border: 1px solid #888;
-    background-color: white;
-    max-height: -webkit-fill-available;
-    max-height: -moz-available;
-    display: flex;
-    flex-direction: column;
-  }
-  .matches {
-    overflow-y: auto;
-  }
-</style>

web/src/NodeWithArea.svelte

@@ -1,60 +0,0 @@
-<script lang="ts">
-  import { type NodeProps, useSvelteFlow } from '@xyflow/svelte';
-  import NodeParameter from './NodeParameter.svelte';
-
-  type $$Props = NodeProps;
-
-  export let containerStyle = '';
-  export let id: $$Props['id']; id;
-  export let data: $$Props['data'];
-  export let dragHandle: $$Props['dragHandle'] = undefined; dragHandle;
-  export let type: $$Props['type']  = undefined; type;
-  export let selected: $$Props['selected'] = undefined; selected;
-  export let isConnectable: $$Props['isConnectable'] = undefined; isConnectable;
-  export let zIndex: $$Props['zIndex'] = undefined; zIndex;
-  export let width: $$Props['width'] = undefined; width;
-  export let height: $$Props['height'] = undefined; height;
-  export let dragging: $$Props['dragging']; dragging;
-  export let targetPosition: $$Props['targetPosition'] = undefined; targetPosition;
-  export let sourcePosition: $$Props['sourcePosition'] = undefined; sourcePosition;
-  export let positionAbsoluteX: $$Props['positionAbsoluteX'] = undefined; positionAbsoluteX;
-  export let positionAbsoluteY: $$Props['positionAbsoluteY'] = undefined; positionAbsoluteY;
-
-  function asPx(n: number | undefined) {
-    return n ? n + 'px' : undefined;
-  }
-  const { updateNodeData } = useSvelteFlow();
-  $: metaParams = data.meta?.params;
-</script>
-
-<div class="area" style:width={asPx(width)} style:height={asPx(height)} style={containerStyle}>
-  <div class="title">
-    {data.title}
-  </div>
-  {#each Object.entries(data.params) as [name, value]}
-    <NodeParameter
-      {name}
-      {value}
-      meta={metaParams?.[name]}
-      onChange={(newValue) => updateNodeData(id, { params: { ...data.params, [name]: newValue } })}
-      />
-  {/each}
-</div>
-
-<style>
-  .area {
-    border-radius: 10px;
-    border: 3px dashed oklch(75% 0.2 55);
-    z-index: 0 !important;
-  }
-  .title {
-    color: oklch(75% 0.2 55);
-    width: 100%;
-    text-align: center;
-    top: -1.5em;
-    position: absolute;
-    -webkit-text-stroke: 5px white;
-    paint-order: stroke fill;
-    font-weight: bold;
-  }
-</style>

web/src/NodeWithImage.svelte

@@ -1,14 +0,0 @@
-<script lang="ts">
-  import { type NodeProps } from '@xyflow/svelte';
-  import NodeWithParams from './NodeWithParams.svelte';
-  type $$Props = NodeProps;
-  export let data: $$Props['data'];
-</script>
-
-<NodeWithParams {...$$props}>
-  {#if data.display}
-    <img src={data.display}/>
-  {/if}
-</NodeWithParams>
-<style>
-</style>

web/src/NodeWithParams.svelte

@@ -1,30 +0,0 @@
-<script lang="ts">
-  import { getContext } from 'svelte';
-  import { type NodeProps, useNodes } from '@xyflow/svelte';
-  import LynxKiteNode from './LynxKiteNode.svelte';
-  import NodeParameter from './NodeParameter.svelte';
-  type $$Props = NodeProps;
-  export let id: $$Props['id'];
-  export let data: $$Props['data'];
-  $: metaParams = data.meta?.params;
-  $: store = getContext('LynxKite store');
-  function setParam(name, newValue) {
-    const i = $store.workspace.nodes.findIndex((n) => n.id === id);
-    $store.workspace.nodes[i].data.params[name] = newValue;
-  }
-  $: params = $nodes && data?.params ? Object.entries(data.params) : [];
-  const nodes = useNodes(); // We don't properly get updates to "data". This is a hack.
-  $: props = $nodes && $$props;
-</script>
-
-<LynxKiteNode {...props}>
-  {#each params as [name, value]}
-    <NodeParameter
-      {name}
-      {value}
-      meta={metaParams?.[name]}
-      onChange={(value) => setParam(name, value)}
-      />
-  {/each}
-  <slot />
-</LynxKiteNode>

web/src/NodeWithSubFlow.svelte

@@ -1,37 +0,0 @@
-<script lang="ts">
-  import { type NodeProps, useNodes } from '@xyflow/svelte';
-  import LynxKiteNode from './LynxKiteNode.svelte';
-  type $$Props = NodeProps;
-  const nodes = useNodes();
-  export let id: $$Props['id'];
-  export let data: $$Props['data'];
-  let isExpanded = true;
-  function onToggle({ expanded }) {
-    isExpanded = expanded;
-    nodes.update((n) =>
-      n.map((node) =>
-        node.parentId === id
-        ? { ...node, hidden: !expanded }
-        : node));
-  }
-  function computeSize(nodes) {
-    let width = 200;
-    let height = 200;
-    for (const node of nodes) {
-      if (node.parentId === id) {
-        width = Math.max(width, node.position.x + 300);
-        height = Math.max(height, node.position.y + 200);
-      }
-    }
-    return { width, height };
-  }
-  $: ({ width, height } = computeSize($nodes));
-</script>
-
-<LynxKiteNode
-  {...$$props}
-  width={isExpanded && width} height={isExpanded && height}
-  nodeStyle="background: transparent;" containerStyle="max-width: none; max-height: none;" {onToggle}>
-</LynxKiteNode>
-<style>
-</style>

web/src/NodeWithTableView.svelte

@@ -1,58 +0,0 @@
-<script lang="ts">
-  import { useNodes, type NodeProps } from '@xyflow/svelte';
-  import LynxKiteNode from './LynxKiteNode.svelte';
-  import Table from './Table.svelte';
-  import SvelteMarkdown from 'svelte-markdown'
-  type $$Props = NodeProps;
-  export let data: $$Props['data'];
-  const nodes = useNodes(); // We don't properly get updates to "data". This is a hack.
-  $: D = $nodes && data;
-  const open = {};
-  $: single = D.display?.value?.dataframes && Object.keys(D.display.value.dataframes).length === 1;
-  function toMD(v) {
-    if (typeof v === 'string') {
-      return v;
-    }
-    if (Array.isArray(v)) {
-      return v.map(toMD).join('\n\n');
-    }
-    return JSON.stringify(v);
-  }
-</script>
-
-<LynxKiteNode {...$$props}>
-  {#if D?.display?.value}
-    {#each Object.entries(D.display.value.dataframes || {}) as [name, df]}
-      {#if !single}<div class="df-head" on:click={() => open[name] = !open[name]}>{name}</div>{/if}
-      {#if single || open[name]}
-        {#if df.data.length > 1}
-          <Table columns={df.columns} data={df.data} />
-        {:else}
-          <dl>
-          {#each df.columns as c, i}
-            <dt>{c}</dt>
-            <dd><SvelteMarkdown source={toMD(df.data[0][i])} /></dd>
-          {/each}
-          </dl>
-        {/if}
-      {/if}
-    {/each}
-    {#each Object.entries(D.display.value.others || {}) as [name, o]}
-      <div class="df-head" on:click={() => open[name] = !open[name]}>{name}</div>
-      {#if open[name]}
-      <pre>{o}</pre>
-      {/if}
-    {/each}
-  {/if}
-</LynxKiteNode>
-<style>
-  .df-head {
-    font-weight: bold;
-    padding: 8px;
-    background: #f0f0f0;
-    cursor: pointer;
-  }
-  dl {
-    margin: 10px;
-  }
-</style>

web/src/NodeWithVisualization.svelte

@@ -1,17 +0,0 @@
-<script lang="ts">
-  import { useNodes, type NodeProps } from '@xyflow/svelte';
-  import NodeWithParams from './NodeWithParams.svelte';
-  import { Chart } from 'svelte-echarts';
-  import { init } from 'echarts';
-  type $$Props = NodeProps;
-  export let data: $$Props['data'];
-
-  const nodes = useNodes(); // We don't properly get updates to "data". This is a hack.
-  $: D = $nodes && data;
-</script>
-
-<NodeWithParams {...$$props}>
-  <Chart {init} options={D?.display?.value || {}} initOptions={{renderer: 'canvas', width: 250, height: 250}}/>
-</NodeWithParams>
-<style>
-</style>

web/src/Table.svelte

@@ -1,22 +0,0 @@
-<script lang="ts">
-  export let data, columns;
-</script>
-
-<table>
-  <thead>
-    <tr>
-      {#each columns as column}
-        <th>{column}</th>
-      {/each}
-    </tr>
-  </thead>
-  <tbody>
-    {#each data as row}
-      <tr>
-        {#each columns as column}
-          <td>{row[column]}</td>
-        {/each}
-      </tr>
-    {/each}
-  </tbody>
-</table>

web/src/Workspace.svelte

@@ -1,14 +0,0 @@
-<script lang="ts">
-  // This is the whole LynxKite workspace editor page.
-  import { QueryClient, QueryClientProvider } from '@sveltestack/svelte-query'
-  import { SvelteFlowProvider } from '@xyflow/svelte';
-  import LynxKiteFlow from './LynxKiteFlow.svelte';
-  export let path = '';
-  const queryClient = new QueryClient()
-</script>
-
-<QueryClientProvider client={queryClient}>
-  <SvelteFlowProvider>
-    <LynxKiteFlow path={path} />
-  </SvelteFlowProvider>
-</QueryClientProvider>

web/src/apiTypes.ts

@@ -0,0 +1,45 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+/* This file was automatically generated from pydantic models by running pydantic2ts.
+/* Do not modify it by hand - just update the pydantic models and then re-run the script
+*/
+
+export interface BaseConfig {
+  [k: string]: unknown;
+}
+export interface Position {
+  x: number;
+  y: number;
+  [k: string]: unknown;
+}
+export interface Workspace {
+  env?: string;
+  nodes?: WorkspaceNode[];
+  edges?: WorkspaceEdge[];
+  [k: string]: unknown;
+}
+export interface WorkspaceNode {
+  id: string;
+  type: string;
+  data: WorkspaceNodeData;
+  position: Position;
+  [k: string]: unknown;
+}
+export interface WorkspaceNodeData {
+  title: string;
+  params: {
+    [k: string]: unknown;
+  };
+  display?: unknown;
+  error?: string | null;
+  [k: string]: unknown;
+}
+export interface WorkspaceEdge {
+  id: string;
+  source: string;
+  target: string;
+  sourceHandle: string;
+  targetHandle: string;
+  [k: string]: unknown;
+}

web/src/app.scss

@@ -1,38 +0,0 @@
-// Import all of Bootstrap's CSS
-$form-select-indicator-color: oklch(90% 0.01 55);
-@import "bootstrap/scss/bootstrap";
-:root {
-  --bs-border-color: oklch(90% 0.01 55);
-}
-
-path.svelte-flow__edge-path {
-  stroke-width: 2;
-  stroke: black;
-}
-.svelte-flow__edge.selected path.svelte-flow__edge-path {
-  outline: var(--xy-selection-border, var(--xy-selection-border-default));
-  outline-offset: 10px;
-  border-radius: 1px;
-}
-.svelte-flow__handle {
-  border-color: black;
-  background: white;
-  width: 10px;
-  height: 10px;
-}
-.svelte-flow__arrowhead * {
-  stroke: none;
-  fill: black;
-}
-// We want the area node to be above the sub-flow node if its inside the sub-flow.
-// This will need some more thinking for a general solution.
-.svelte-flow__node-sub_flow {
-  z-index: -20 !important;
-}
-.svelte-flow__node-area {
-  z-index: -10 !important;
-}
-.selected .lynxkite-node {
-  outline: var(--xy-selection-border, var(--xy-selection-border-default));
-  outline-offset: 7.5px;
-}

web/src/directory.css

@@ -1,10 +0,0 @@
-@media (min-width: 640px) {
-  .directory {
-    width: 100%;
-  }
-}
-
-.directory {
-  width: 800px;
-  background: white;
-}

web/src/index.css

@@ -0,0 +1,350 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+
+  background: #002a4c;
+}
+
+img,
+svg {
+  display: inline-block;
+}
+
+body {
+  color: var(--foreground);
+  background: var(--background);
+  font-family: Arial, Helvetica, sans-serif;
+}
+
+.workspace {
+  background: white;
+  display: flex;
+  flex-direction: column;
+  height: 100vh;
+
+  .top-bar {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    background: #002a4c;
+
+    .ws-name {
+      font-size: 1.5em;
+      flex: 1;
+      color: white;
+    }
+
+    .logo img {
+      height: 2em;
+      vertical-align: middle;
+      margin: 4px;
+    }
+
+    .tools {
+      display: flex;
+      align-items: center;
+
+      a {
+        color: oklch(75% 0.13 230);
+        font-size: 1.5em;
+        padding: 0 10px;
+      }
+    }
+  }
+
+  .error {
+    background: #ffdddd;
+    padding: 8px;
+    font-size: 12px;
+  }
+
+  .title-icon {
+    margin-left: 5px;
+    float: right;
+  }
+
+  .node-container {
+    padding: 8px;
+    position: relative;
+  }
+
+  .lynxkite-node {
+    box-shadow: 0px 5px 50px 0px rgba(0, 0, 0, 0.3);
+    border-radius: 4px;
+    background: white;
+  }
+
+  .expanded .lynxkite-node {
+    overflow-y: auto;
+    height: 100%;
+  }
+
+  .lynxkite-node .title {
+    /* background: oklch(75% 0.2 55); */
+    font-weight: bold;
+    padding: 8px;
+  }
+
+  .handle-name {
+    font-size: 10px;
+    color: black;
+    letter-spacing: 0.05em;
+    text-align: right;
+    white-space: nowrap;
+    position: absolute;
+    top: -5px;
+    backdrop-filter: blur(10px);
+    padding: 2px 8px;
+    border-radius: 4px;
+    visibility: hidden;
+  }
+
+  .react-flow__handle-left .handle-name {
+    right: 20px;
+  }
+
+  .react-flow__handle-right .handle-name {
+    left: 20px;
+  }
+
+  .react-flow__handle-top .handle-name {
+    top: -20px;
+    left: 5px;
+    backdrop-filter: none;
+  }
+
+  .react-flow__handle-bottom .handle-name {
+    top: 0px;
+    left: 5px;
+    backdrop-filter: none;
+  }
+
+  .node-container:hover .handle-name {
+    visibility: visible;
+  }
+
+  .node-resizer {
+    position: absolute;
+    bottom: 8px;
+    right: 8px;
+    cursor: nwse-resize;
+  }
+
+  .lynxkite-node {
+    .param {
+      padding: 4px 8px 4px 8px;
+      display: block;
+    }
+
+    .param-name {
+      display: block;
+      font-size: 10px;
+      letter-spacing: 0.05em;
+      margin-left: 10px;
+      width: fit-content;
+      padding: 2px 8px;
+      border-radius: 4px 4px 0 0;
+      ;
+    }
+
+    .collapsed-param {
+      min-height: 20px;
+      line-height: 10px;
+    }
+  }
+
+  .node-search {
+    position: fixed;
+    width: 300px;
+    z-index: 5;
+    padding: 4px;
+    border-radius: 4px;
+    border: 1px solid #888;
+    background-color: white;
+    max-height: -webkit-fill-available;
+    max-height: -moz-available;
+    display: flex;
+    flex-direction: column;
+
+    input {
+      width: calc(100% - 26px);
+      font-size: 20px;
+      padding: 8px;
+      border-radius: 4px;
+      border: 1px solid #eee;
+      margin: 4px;
+    }
+
+    .search-result {
+      padding: 4px;
+      cursor: pointer;
+    }
+
+    .search-result.selected {
+      background-color: oklch(75% 0.2 55);
+      border-radius: 4px;
+    }
+
+    .matches {
+      overflow-y: auto;
+    }
+  }
+
+  .react-flow__node-table_view {
+    .df-head {
+      font-weight: bold;
+      padding: 8px;
+      background: #f0f0f0;
+      cursor: pointer;
+    }
+
+    dl {
+      margin: 10px;
+    }
+  }
+}
+
+.directory {
+  .entry-list {
+    width: 100%;
+    margin: 10px auto;
+    background-color: white;
+    border-radius: 10px;
+    box-shadow: 0px 2px 4px;
+    padding: 0 0 10px 0;
+  }
+
+  @media (min-width: 768px) {
+    .entry-list {
+      width: 768px;
+    }
+  }
+
+  @media (min-width: 960px) {
+    .entry-list {
+      width: 80%;
+    }
+  }
+
+  .logo {
+    margin: 0;
+    padding-top: 50px;
+    text-align: center;
+  }
+
+  .logo-image {
+    max-width: 50%;
+  }
+
+  .tagline {
+    color: #39bcf3;
+    font-size: 14px;
+    font-weight: 500;
+  }
+
+  @media (min-width: 1400px) {
+    .tagline {
+      font-size: 18px;
+    }
+  }
+
+  .actions {
+    display: flex;
+    justify-content: space-evenly;
+    padding: 5px;
+  }
+
+  .actions a {
+    padding: 2px 10px;
+    border-radius: 5px;
+  }
+
+  .actions a:hover {
+    background: #39bcf3;
+    color: white;
+  }
+
+  .breadcrumbs {
+    padding-left: 10px;
+    font-size: 20px;
+    background: #002a4c20;
+  }
+
+  .breadcrumbs a:hover {
+    color: #39bcf3;
+  }
+
+  .entry-list .entry {
+    display: block;
+    border-bottom: 1px solid whitesmoke;
+    padding-left: 10px;
+    color: #004165;
+    cursor: pointer;
+    user-select: none;
+    text-decoration: none;
+  }
+
+  .entry-list .open .entry,
+  .entry-list .entry:hover,
+  .entry-list .entry:focus {
+    background: #39bcf3;
+    color: white;
+  }
+
+  .entry-list .entry:last-child {
+    border-bottom: none;
+  }
+
+  a {
+    color: black;
+    text-decoration: none;
+  }
+
+  .loading {
+    color: #39bcf3;
+    margin: 10px;
+  }
+}
+
+
+path.react-flow__edge-path {
+  stroke-width: 2;
+  stroke: black;
+}
+
+.react-flow__edge.selected path.react-flow__edge-path {
+  outline: var(--xy-selection-border, var(--xy-selection-border-default));
+  outline-offset: 10px;
+  border-radius: 1px;
+}
+
+.react-flow__handle {
+  border-color: black;
+  background: white;
+  width: 10px;
+  height: 10px;
+}
+
+.react-flow__arrowhead * {
+  stroke: none;
+  fill: black;
+}
+
+.react-flow__node-area {
+  z-index: -10 !important;
+}
+
+.selected .lynxkite-node {
+  outline: var(--xy-selection-border, var(--xy-selection-border-default));
+  outline-offset: 7.5px;
+}

web/src/main.ts

@@ -1,10 +0,0 @@
-import App from './App.svelte';
-
-import './app.scss';
-import * as bootstrap from 'bootstrap';
-
-const app = new App({
-  target: document.getElementById('app')!,
-});
-
-export default app;

web/src/main.tsx

@@ -0,0 +1,19 @@
+import { StrictMode } from 'react'
+import { createRoot } from 'react-dom/client'
+import './index.css'
+import Directory from './Directory.tsx'
+import Workspace from './workspace/Workspace.tsx'
+import { BrowserRouter, Routes, Route } from "react-router";
+
+createRoot(document.getElementById('root')!).render(
+  <StrictMode>
+    <BrowserRouter>
+      <Routes>
+        <Route path="/" element={<Directory />} />
+        <Route path="/dir" element={<Directory />} />
+        <Route path="/dir/:path" element={<Directory />} />
+        <Route path="/edit/:path" element={<Workspace />} />
+      </Routes>
+    </BrowserRouter>
+  </StrictMode>,
+)

web/src/vite-env.d.ts

@@ -1,2 +1 @@
-/// <reference types="svelte" />
 /// <reference types="vite/client" />

web/src/workspace/EnvironmentSelector.tsx

@@ -0,0 +1,15 @@
+export default function EnvironmentSelector(props: { options: string[], value: string, onChange: (val: string) => void }) {
+  return (
+    <>
+      <select className="select w-full max-w-xs"
+        name="workspace-env"
+        value={props.value}
+        onChange={(evt) => props.onChange(evt.currentTarget.value)}
+      >
+        {props.options.map(option =>
+          <option key={option} value={option}>{option}</option>
+        )}
+      </select>
+    </>
+  );
+}

web/src/workspace/LynxKiteState.ts

@@ -0,0 +1,4 @@
+import { createContext } from "react";
+import { Workspace } from "../apiTypes.ts";
+
+export const LynxKiteState = createContext({ workspace: {} as Workspace });

web/src/workspace/NodeSearch.tsx

@@ -0,0 +1,75 @@
+import Fuse from 'fuse.js'
+import { useEffect, useMemo, useRef, useState } from 'react';
+
+export type OpsOp = {
+  name: string
+  type: string
+  position: { x: number, y: number }
+  params: { name: string, default: any }[]
+}
+export type Catalog = { [op: string]: OpsOp };
+export type Catalogs = { [env: string]: Catalog };
+
+export default function (props: { boxes: Catalog, onCancel: any, onAdd: any, pos: { x: number, y: number } }) {
+  const searchBox = useRef(null as unknown as HTMLInputElement);
+  const [searchText, setSearchText] = useState('');
+  const fuse = useMemo(() => new Fuse(Object.values(props.boxes), {
+    keys: ['name']
+  }), [props.boxes]);
+  const hits: { item: OpsOp }[] = searchText ? fuse.search<OpsOp>(searchText) : Object.values(props.boxes).map(box => ({ item: box }));
+  const [selectedIndex, setSelectedIndex] = useState(0);
+  useEffect(() => searchBox.current.focus());
+  function typed(text: string) {
+    setSearchText(text);
+    setSelectedIndex(Math.max(0, Math.min(selectedIndex, hits.length - 1)));
+  }
+  function onKeyDown(e: React.KeyboardEvent<HTMLInputElement>) {
+    if (e.key === 'ArrowDown') {
+      e.preventDefault();
+      setSelectedIndex(Math.min(selectedIndex + 1, hits.length - 1));
+    } else if (e.key === 'ArrowUp') {
+      e.preventDefault();
+      setSelectedIndex(Math.max(selectedIndex - 1, 0));
+    } else if (e.key === 'Enter') {
+      addSelected();
+    } else if (e.key === 'Escape') {
+      props.onCancel();
+    }
+  }
+  function addSelected() {
+    const node = { ...hits[selectedIndex].item };
+    node.position = props.pos;
+    props.onAdd(node);
+  }
+  async function lostFocus(e: any) {
+    // If it's a click on a result, let the click handler handle it.
+    if (e.relatedTarget && e.relatedTarget.closest('.node-search')) return;
+    props.onCancel();
+  }
+
+
+  return (
+    <div className="node-search" style={{ top: props.pos.y, left: props.pos.x }}>
+      <input
+        ref={searchBox}
+        value={searchText}
+        onChange={event => typed(event.target.value)}
+        onKeyDown={onKeyDown}
+        onBlur={lostFocus}
+        placeholder="Search for box" />
+      <div className="matches">
+        {hits.map((box, index) =>
+          <div
+            key={box.item.name}
+            tabIndex={0}
+            onFocus={() => setSelectedIndex(index)}
+            onMouseEnter={() => setSelectedIndex(index)}
+            onClick={addSelected}
+            className={`search-result ${index === selectedIndex ? 'selected' : ''}`}>
+            {box.item.name}
+          </div>
+        )}
+      </div>
+    </div >
+  );
+}

web/src/workspace/Workspace.tsx

@@ -0,0 +1,246 @@
+// The LynxKite workspace editor.
+import { useParams } from "react-router";
+import useSWR, { Fetcher } from 'swr';
+import { useEffect, useMemo, useCallback, useState, MouseEvent } from "react";
+import favicon from '../assets/favicon.ico';
+import {
+  ReactFlow,
+  Controls,
+  MarkerType,
+  ReactFlowProvider,
+  applyEdgeChanges,
+  applyNodeChanges,
+  useUpdateNodeInternals,
+  type XYPosition,
+  type Node,
+  type Edge,
+  type Connection,
+  useReactFlow,
+  MiniMap,
+} from '@xyflow/react';
+// @ts-ignore
+import ArrowBack from '~icons/tabler/arrow-back.jsx';
+// @ts-ignore
+import Backspace from '~icons/tabler/backspace.jsx';
+// @ts-ignore
+import Atom from '~icons/tabler/atom.jsx';
+import { syncedStore, getYjsDoc } from "@syncedstore/core";
+import { WebsocketProvider } from "y-websocket";
+import NodeWithParams from './nodes/NodeWithParams';
+// import NodeWithTableView from './NodeWithTableView';
+import EnvironmentSelector from './EnvironmentSelector';
+import { LynxKiteState } from './LynxKiteState';
+import '@xyflow/react/dist/style.css';
+import { Workspace, WorkspaceNode } from "../apiTypes.ts";
+import NodeSearch, { OpsOp, Catalog, Catalogs } from "./NodeSearch.tsx";
+import NodeWithVisualization from "./nodes/NodeWithVisualization.tsx";
+import NodeWithImage from "./nodes/NodeWithImage.tsx";
+import NodeWithTableView from "./nodes/NodeWithTableView.tsx";
+
+export default function (props: any) {
+  return (
+    <ReactFlowProvider>
+      <LynxKiteFlow {...props} />
+    </ReactFlowProvider>
+  );
+}
+
+
+function LynxKiteFlow() {
+  const updateNodeInternals = useUpdateNodeInternals()
+  const reactFlow = useReactFlow();
+  const [nodes, setNodes] = useState([] as Node[]);
+  const [edges, setEdges] = useState([] as Edge[]);
+  const { path } = useParams();
+  const [state, setState] = useState({ workspace: {} as Workspace });
+  useEffect(() => {
+    const state = syncedStore({ workspace: {} as Workspace });
+    setState(state);
+    const doc = getYjsDoc(state);
+    const wsProvider = new WebsocketProvider("ws://localhost:8000/ws/crdt", path!, doc);
+    const onChange = (_update: any, origin: any, _doc: any, _tr: any) => {
+      if (origin === wsProvider) {
+        // An update from the CRDT. Apply it to the local state.
+        // This is only necessary because ReactFlow keeps secret internal copies of our stuff.
+        if (!state.workspace) return;
+        if (!state.workspace.nodes) return;
+        if (!state.workspace.edges) return;
+        console.log('update', JSON.parse(JSON.stringify(state.workspace)));
+        setNodes([...state.workspace.nodes] as Node[]);
+        setEdges([...state.workspace.edges] as Edge[]);
+        for (const node of state.workspace.nodes) {
+          // Make sure the internal copies are updated.
+          updateNodeInternals(node.id);
+        }
+      }
+    };
+    doc.on('update', onChange);
+    return () => {
+      doc.destroy();
+      wsProvider.destroy();
+    }
+  }, [path]);
+
+  const onNodesChange = useCallback((changes: any[]) => {
+    // An update from the UI. Apply it to the local state...
+    setNodes((nds) => applyNodeChanges(changes, nds));
+    // ...and to the CRDT state. (Which could be the same, except for ReactFlow's internal copies.)
+    const wnodes = state.workspace?.nodes;
+    if (!wnodes) return;
+    for (const ch of changes) {
+      const nodeIndex = wnodes.findIndex((n) => n.id === ch.id);
+      if (nodeIndex === -1) continue;
+      const node = wnodes[nodeIndex];
+      if (!node) continue;
+      // Position events sometimes come with NaN values. Ignore them.
+      if (ch.type === 'position' && !isNaN(ch.position.x) && !isNaN(ch.position.y)) {
+        getYjsDoc(state).transact(() => {
+          Object.assign(node.position, ch.position);
+        });
+      } else if (ch.type === 'select') {
+      } else if (ch.type === 'dimensions') {
+        getYjsDoc(state).transact(() => Object.assign(node, ch.dimensions));
+      } else if (ch.type === 'remove') {
+        wnodes.splice(nodeIndex, 1);
+      } else if (ch.type === 'replace') {
+        // Ideally we would only update the parameter that changed. But ReactFlow does not give us that detail.
+        const u = {
+          collapsed: ch.item.data.collapsed,
+          // The "..." expansion on a Y.map returns an empty object. Copying with fromEntries/entries instead.
+          params: { ...Object.fromEntries(Object.entries(ch.item.data.params)) },
+          __execution_delay: ch.item.data.__execution_delay,
+        };
+        getYjsDoc(state).transact(() => Object.assign(node.data, u));
+      } else {
+        console.log('Unknown node change', ch);
+      }
+    }
+  }, [state]);
+  const onEdgesChange = useCallback((changes: any[]) => {
+    setEdges((eds) => applyEdgeChanges(changes, eds));
+    const wedges = state.workspace?.edges;
+    if (!wedges) return;
+    for (const ch of changes) {
+      console.log('edge change', ch);
+      const edgeIndex = wedges.findIndex((e) => e.id === ch.id);
+      if (ch.type === 'remove') {
+        wedges.splice(edgeIndex, 1);
+      } else {
+        console.log('Unknown edge change', ch);
+      }
+    }
+  }, [state]);
+
+  const fetcher: Fetcher<Catalogs> = (resource: string, init?: RequestInit) => fetch(resource, init).then(res => res.json());
+  const catalog = useSWR('/api/catalog', fetcher);
+  const [suppressSearchUntil, setSuppressSearchUntil] = useState(0);
+  const [nodeSearchSettings, setNodeSearchSettings] = useState(undefined as {
+    pos: XYPosition,
+    boxes: Catalog,
+  } | undefined);
+  const nodeTypes = useMemo(() => ({
+    basic: NodeWithParams,
+    visualization: NodeWithVisualization,
+    image: NodeWithImage,
+    table_view: NodeWithTableView,
+  }), []);
+  const closeNodeSearch = useCallback(() => {
+    setNodeSearchSettings(undefined);
+    setSuppressSearchUntil(Date.now() + 200);
+  }, [setNodeSearchSettings, setSuppressSearchUntil]);
+  const toggleNodeSearch = useCallback((event: MouseEvent) => {
+    if (suppressSearchUntil > Date.now()) return;
+    if (nodeSearchSettings) {
+      closeNodeSearch();
+      return;
+    }
+    event.preventDefault();
+    setNodeSearchSettings({
+      pos: { x: event.clientX, y: event.clientY },
+      boxes: catalog.data![state.workspace.env!],
+    });
+  }, [catalog, state, setNodeSearchSettings, suppressSearchUntil]);
+  const addNode = useCallback((meta: OpsOp) => {
+    const node: Partial<WorkspaceNode> = {
+      type: meta.type,
+      data: {
+        meta: meta,
+        title: meta.name,
+        params: Object.fromEntries(
+          Object.values(meta.params).map((p) => [p.name, p.default])),
+      },
+    };
+    const nss = nodeSearchSettings!;
+    node.position = reactFlow.screenToFlowPosition({ x: nss.pos.x, y: nss.pos.y });
+    const title = meta.name;
+    let i = 1;
+    node.id = `${title} ${i}`;
+    const wnodes = state.workspace.nodes!;
+    while (wnodes.find((x) => x.id === node.id)) {
+      i += 1;
+      node.id = `${title} ${i}`;
+    }
+    wnodes.push(node as WorkspaceNode);
+    setNodes([...nodes, node as WorkspaceNode]);
+    closeNodeSearch();
+  }, [nodeSearchSettings, state, reactFlow, setNodes]);
+
+  const onConnect = useCallback((connection: Connection) => {
+    setSuppressSearchUntil(Date.now() + 200);
+    const edge = {
+      id: `${connection.source} ${connection.target}`,
+      source: connection.source,
+      sourceHandle: connection.sourceHandle!,
+      target: connection.target,
+      targetHandle: connection.targetHandle!,
+    };
+    console.log(JSON.stringify(edge));
+    state.workspace.edges!.push(edge);
+    setEdges((oldEdges) => [...oldEdges, edge]);
+  }, [state, setEdges]);
+  const parentDir = path!.split('/').slice(0, -1).join('/');
+  return (
+    <div className="workspace">
+      <div className="top-bar bg-neutral">
+        <a className="logo" href=""><img src={favicon} /></a>
+        <div className="ws-name">
+          {path}
+        </div>
+        <EnvironmentSelector
+          options={Object.keys(catalog.data || {})}
+          value={state.workspace.env!}
+          onChange={(env) => { state.workspace.env = env; }}
+        />
+        <div className="tools text-secondary">
+          <a href=""><Atom /></a>
+          <a href=""><Backspace /></a>
+          <a href={'/dir/' + parentDir}><ArrowBack /></a>
+        </div>
+      </div>
+      <div style={{ height: "100%", width: '100vw' }}>
+        <LynxKiteState.Provider value={state}>
+          <ReactFlow
+            nodes={nodes}
+            edges={edges}
+            nodeTypes={nodeTypes} fitView
+            onNodesChange={onNodesChange}
+            onEdgesChange={onEdgesChange}
+            onPaneClick={toggleNodeSearch}
+            onConnect={onConnect}
+            proOptions={{ hideAttribution: true }}
+            maxZoom={3}
+            minZoom={0.3}
+            defaultEdgeOptions={{ markerEnd: { type: MarkerType.Arrow } }}
+          >
+            <Controls />
+            <MiniMap />
+            {nodeSearchSettings &&
+              <NodeSearch pos={nodeSearchSettings.pos} boxes={nodeSearchSettings.boxes} onCancel={closeNodeSearch} onAdd={addNode} />
+            }
+          </ReactFlow>
+        </LynxKiteState.Provider>
+      </div>
+    </div>
+
+  );
+}

web/src/workspace/nodes/LynxKiteNode.tsx

@@ -0,0 +1,89 @@
+import { useReactFlow, Handle, NodeResizeControl, Position } from '@xyflow/react';
+// @ts-ignore
+import ChevronDownRight from '~icons/tabler/chevron-down-right.jsx';
+
+interface LynxKiteNodeProps {
+  id: string;
+  width: number;
+  height: number;
+  nodeStyle: any;
+  data: any;
+  children: any;
+}
+
+function getHandles(inputs: object, outputs: object) {
+  const handles: {
+    position: 'top' | 'bottom' | 'left' | 'right',
+    name: string,
+    index: number,
+    offsetPercentage: number,
+    showLabel: boolean,
+    type: 'source' | 'target',
+  }[] = [];
+  for (const e of Object.values(inputs)) {
+    handles.push({ ...e, type: 'target' });
+  }
+  for (const e of Object.values(outputs)) {
+    handles.push({ ...e, type: 'source' });
+  }
+  const counts = { top: 0, bottom: 0, left: 0, right: 0 };
+  for (const e of handles) {
+    e.index = counts[e.position];
+    counts[e.position]++;
+  }
+  for (const e of handles) {
+    e.offsetPercentage = 100 * (e.index + 1) / (counts[e.position] + 1);
+    const simpleHorizontal = counts.top === 0 && counts.bottom === 0 && handles.length <= 2;
+    const simpleVertical = counts.left === 0 && counts.right === 0 && handles.length <= 2;
+    e.showLabel = !simpleHorizontal && !simpleVertical;
+  }
+  return handles;
+}
+
+export default function LynxKiteNode(props: LynxKiteNodeProps) {
+  const reactFlow = useReactFlow();
+  const data = props.data;
+  const expanded = !data.collapsed;
+  const handles = getHandles(data.meta?.inputs || {}, data.meta?.outputs || {});
+  function asPx(n: number | undefined) {
+    return (n ? n + 'px' : undefined) || '200px';
+  }
+  function titleClicked() {
+    reactFlow.updateNodeData(props.id, { collapsed: expanded });
+  }
+  const handleOffsetDirection = { top: 'left', bottom: 'left', left: 'top', right: 'top' };
+
+  return (
+    <div className={'node-container ' + (expanded ? 'expanded' : 'collapsed')}
+      style={{ width: asPx(props.width), height: asPx(expanded ? props.height : undefined) }}>
+      <div className="lynxkite-node" style={props.nodeStyle}>
+        <div className="title bg-primary" onClick={titleClicked}>
+          {data.title}
+          {data.error && <span className="title-icon">⚠️</span>}
+          {expanded || <span className="title-icon">⋯</span>}
+        </div>
+        {expanded && <>
+          {data.error &&
+            <div className="error">{data.error}</div>
+          }
+          {props.children}
+          {handles.map(handle => (
+            <Handle
+              key={handle.name}
+              id={handle.name} type={handle.type} position={handle.position as Position}
+              style={{ [handleOffsetDirection[handle.position]]: handle.offsetPercentage + '%' }}>
+              {handle.showLabel && <span className="handle-name">{handle.name.replace(/_/g, " ")}</span>}
+            </Handle >
+          ))}
+          <NodeResizeControl
+            minWidth={100}
+            minHeight={50}
+            style={{ 'background': 'transparent', 'border': 'none' }}
+          >
+            <ChevronDownRight className="node-resizer" />
+          </NodeResizeControl>
+        </>}
+      </div>
+    </div>
+  );
+}

web/src/workspace/nodes/NodeParameter.tsx

@@ -0,0 +1,52 @@
+const BOOLEAN = "<class 'bool'>";
+
+function ParamName({ name }) {
+  return <span className="param-name bg-base-200">{name.replace(/_/g, ' ')}</span>;
+}
+
+export default function NodeParameter({ name, value, meta, onChange }) {
+  return (
+    <label className="param">
+      {meta?.type?.format === 'collapsed' ? <>
+        <ParamName name={name} />
+        <button className="collapsed-param">
+          ⋯
+        </button>
+      </> : meta?.type?.format === 'textarea' ? <>
+        <ParamName name={name} />
+        <textarea className="textarea textarea-bordered w-full max-w-xs"
+          rows={6}
+          value={value}
+          onChange={(evt) => onChange(evt.currentTarget.value)}
+        />
+      </> : meta?.type?.enum ? <>
+        <ParamName name={name} />
+        <select className="select select-bordered w-full max-w-xs"
+          value={value || meta.type.enum[0]}
+          onChange={(evt) => onChange(evt.currentTarget.value)}
+        >
+          {meta.type.enum.map(option =>
+            <option key={option} value={option}>{option}</option>
+          )}
+        </select>
+      </> : meta?.type?.type === BOOLEAN ? <div className="form-control">
+        <label className="label cursor-pointer">
+          <input className="checkbox"
+            type="checkbox"
+            checked={value}
+            onChange={(evt) => onChange(evt.currentTarget.checked)}
+          />
+          {name.replace(/_/g, ' ')}
+        </label>
+      </div> : <>
+        <ParamName name={name} />
+        <input className="input input-bordered w-full max-w-xs"
+          value={value || ""}
+          onChange={(evt) => onChange(evt.currentTarget.value, { delay: 2 })}
+          onBlur={(evt) => onChange(evt.currentTarget.value, { delay: 0 })}
+        />
+      </>
+      }
+    </label >
+  );
+}

web/src/workspace/nodes/NodeWithImage.tsx

@@ -0,0 +1,11 @@
+import NodeWithParams from './NodeWithParams';
+
+const NodeWithImage = (props: any) => {
+  return (
+    <NodeWithParams {...props}>
+      {props.data.display && <img src={props.data.display} alt="Node Display" />}
+    </NodeWithParams>
+  );
+};
+
+export default NodeWithImage;

web/src/workspace/nodes/NodeWithParams.tsx

@@ -0,0 +1,31 @@
+import LynxKiteNode from './LynxKiteNode';
+import { useReactFlow } from '@xyflow/react';
+import NodeParameter from './NodeParameter';
+
+export type UpdateOptions = { delay?: number };
+
+function NodeWithParams(props: any) {
+  const reactFlow = useReactFlow();
+  const metaParams = props.data.meta?.params;
+  function setParam(name: string, newValue: any, opts: UpdateOptions) {
+    reactFlow.updateNodeData(props.id, { params: { ...props.data.params, [name]: newValue }, __execution_delay: opts.delay || 0 });
+  }
+  const params = props.data?.params ? Object.entries(props.data.params) : [];
+
+  return (
+    <LynxKiteNode {...props}>
+      {params.map(([name, value]) =>
+        <NodeParameter
+          name={name}
+          key={name}
+          value={value}
+          meta={metaParams?.[name]}
+          onChange={(value: any, opts?: UpdateOptions) => setParam(name, value, opts || {})}
+        />
+      )}
+      {props.children}
+    </LynxKiteNode >
+  );
+}
+
+export default NodeWithParams;

web/src/workspace/nodes/NodeWithTableView.tsx

@@ -0,0 +1,46 @@
+import { useState } from 'react';
+import Markdown from 'react-markdown'
+import LynxKiteNode from './LynxKiteNode';
+import Table from './Table';
+import React from 'react';
+
+function toMD(v: any): string {
+  if (typeof v === 'string') {
+    return v;
+  }
+  if (Array.isArray(v)) {
+    return v.map(toMD).join('\n\n');
+  }
+  return JSON.stringify(v);
+}
+
+export default function NodeWithTableView(props: any) {
+  const [open, setOpen] = useState({} as { [name: string]: boolean });
+  const display = props.data.display?.value;
+  const single = display?.dataframes && Object.keys(display?.dataframes).length === 1;
+  return (
+    <LynxKiteNode {...props}>
+      {display && [
+        Object.entries(display.dataframes || {}).map(([name, df]: [string, any]) => <React.Fragment key={name}>
+          {!single && <div key={name} className="df-head" onClick={() => setOpen({ ...open, [name]: !open[name] })}>{name}</div>}
+          {(single || open[name]) &&
+            (df.data.length > 1 ?
+              <Table key={name} columns={df.columns} data={df.data} />
+              :
+              <dl key={name}>
+                {df.columns.map((c: string, i: number) =>
+                  <React.Fragment key={name + '-' + c}>
+                    <dt>{c}</dt>
+                    <dd><Markdown>{toMD(df.data[0][i])}</Markdown></dd>
+                  </React.Fragment>)
+                }
+              </dl>)}
+        </React.Fragment>),
+        Object.entries(display.others || {}).map(([name, o]) => <>
+          <div className="df-head" onClick={() => setOpen({ ...open, [name]: !open[name] })}>{name}</div>
+          {open[name] && <pre>{(o as any).toString()}</pre>}
+        </>
+        )]}
+    </LynxKiteNode >
+  );
+}

web/src/workspace/nodes/NodeWithVisualization.tsx

@@ -0,0 +1,28 @@
+import React, { useEffect } from 'react';
+import NodeWithParams from './NodeWithParams';
+import * as echarts from 'echarts';
+
+const NodeWithVisualization = (props: any) => {
+  const chartsRef = React.useRef<HTMLDivElement>(null);
+  const chartsInstanceRef = React.useRef<echarts.ECharts>();
+  useEffect(() => {
+    const opts = props.data?.display?.value;
+    if (!opts || !chartsRef.current) return;
+    console.log(chartsRef.current);
+    chartsInstanceRef.current = echarts.init(chartsRef.current, null, { renderer: 'canvas', width: 250, height: 250 });
+    chartsInstanceRef.current.setOption(opts);
+    const onResize = () => chartsInstanceRef.current?.resize();
+    window.addEventListener('resize', onResize);
+    return () => {
+      window.removeEventListener('resize', onResize);
+      chartsInstanceRef.current?.dispose();
+    };
+  }, [props.data?.display?.value]);
+  return (
+    <NodeWithParams {...props}>
+      <div className="box" draggable={false} ref={chartsRef} />;
+    </NodeWithParams>
+  );
+};
+
+export default NodeWithVisualization;