Start switching to CRDT. The env setting works!!!

requirements.txt

@@ -5,6 +5,8 @@ numpy
 pandas
 scipy
 uvicorn[standard]
+pycrdt
+pycrdt-websocket
 # For llm_ops
 chromadb
 Jinja2

server/crdt.py

@@ -0,0 +1,62 @@
+'''CRDT is used to synchronize workspace state for backend and frontend(s).'''
+import contextlib
+import fastapi
+import os.path
+import pycrdt
+import pycrdt_websocket
+
+import pycrdt_websocket.ystore
+
+router = fastapi.APIRouter()
+
+def ws_exception_handler(exception, log):
+    log.exception(exception)
+    return True
+
+class WebsocketServer(pycrdt_websocket.WebsocketServer):
+    async def init_room(self, name):
+        ystore = pycrdt_websocket.ystore.FileYStore(f'crdt_data/{name}.crdt')
+        ydoc = pycrdt.Doc()
+        ydoc['workspace'] = ws = pycrdt.Map()
+        ws['nodes'] = []
+        ws['edges'] = []
+        ws['env'] = 'unset'
+        # Replay updates from the store.
+        try:
+            for update, timestamp in [(item[0], item[-1]) async for item in ystore.read()]:
+                ydoc.apply_update(update)
+        except pycrdt_websocket.ystore.YDocNotFound:
+            pass
+        print('init_room', name, ws)
+        _subscription_id = ws.observe_deep(handle_deep_changes)
+        return pycrdt_websocket.YRoom(ystore=ystore, ydoc=ydoc)
+
+    async def get_room(self, name: str) -> pycrdt_websocket.YRoom:
+        print('get_room', name, self.rooms)
+        if name not in self.rooms:
+            self.rooms[name] = await self.init_room(name)
+        print('get_room2', name, self.rooms)
+        room = self.rooms[name]
+        await self.start_room(room)
+        return room
+
+print('new WebsocketServer')
+websocket_server = WebsocketServer(exception_handler=ws_exception_handler, auto_clean_rooms=False)
+asgi_server = pycrdt_websocket.ASGIServer(websocket_server)
+
+def handle_deep_changes(events):
+    print('events', events)
+
+@contextlib.asynccontextmanager
+async def lifespan(app):
+    async with websocket_server:
+        yield
+
+def sanitize_path(path):
+    return os.path.relpath(os.path.normpath(os.path.join("/", path)), "/")
+
+@router.websocket("/ws/crdt/{room_name}")
+async def crdt_websocket(websocket: fastapi.WebSocket, room_name: str):
+    room_name = sanitize_path(room_name)
+    print('room_name', room_name)
+    await asgi_server({'path': room_name}, websocket._receive, websocket._send)

server/main.py

@@ -2,6 +2,7 @@ import dataclasses
 import fastapi
 import pathlib
 import pkgutil
+from . import crdt
 from . import ops
 from . import workspace
 
@@ -11,8 +12,8 @@ for _, name, _ in pkgutil.iter_modules([str(here)]):
         print(f'Importing {name}')
         __import__(f'server.{name}')
 
-app = fastapi.FastAPI()
-
+app = fastapi.FastAPI(lifespan=crdt.lifespan)
+app.include_router(crdt.router)
 
 @app.get("/api/catalog")
 def get_catalog():

web/package-lock.json

@@ -21,6 +21,8 @@
       },
       "devDependencies": {
         "@sveltejs/vite-plugin-svelte": "^3.0.2",
+        "@syncedstore/core": "^0.6.0",
+        "@syncedstore/svelte": "^0.6.0",
         "@tsconfig/svelte": "^5.0.4",
         "sass": "^1.77.2",
         "svelte": "^4.2.12",
@@ -28,7 +30,8 @@
         "tslib": "^2.6.2",
         "typescript": "^5.4.4",
         "unplugin-icons": "^0.18.5",
-        "vite": "^5.2.8"
+        "vite": "^5.2.8",
+        "y-websocket": "^2.0.4"
       }
     },
     "node_modules/@ampproject/remapping": {
@@ -575,6 +578,12 @@
         "url": "https://opencollective.com/popperjs"
       }
     },
+    "node_modules/@reactivedata/reactive": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/@reactivedata/reactive/-/reactive-0.2.2.tgz",
+      "integrity": "sha512-KnINM/Sng25QAv6sHkJO9q/XyslLegCF5jTsTSVu+AouY3uZDVf4Am99xNCqsfqFZFvnTBBDvCsHNdvTVGvPEA==",
+      "dev": true
+    },
     "node_modules/@rollup/rollup-android-arm-eabi": {
       "version": "4.18.0",
       "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz",
@@ -843,6 +852,39 @@
         }
       }
     },
+    "node_modules/@syncedstore/core": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/@syncedstore/core/-/core-0.6.0.tgz",
+      "integrity": "sha512-6TtjEoYJsceYi8u1oRecXwbbLmjHaU0S7HvVfOaEdDfphZLGm/faVuA2fpazqc28F0yIFGvYzvPEBUJn9vqRNw==",
+      "dev": true,
+      "dependencies": {
+        "@reactivedata/reactive": "^0.2.0",
+        "@syncedstore/yjs-reactive-bindings": "^0.6.0"
+      },
+      "peerDependencies": {
+        "yjs": "^13.5.13"
+      }
+    },
+    "node_modules/@syncedstore/svelte": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/@syncedstore/svelte/-/svelte-0.6.0.tgz",
+      "integrity": "sha512-MZcA1v4Qcwkix4P3/aax8azXFV1bqwyNnUZOT3XFCgpGgbRTq0NqNfKVDbDFRerpkZdrYqyWYxpRus2KtqPSjQ==",
+      "dev": true,
+      "peerDependencies": {
+        "@reactivedata/reactive": "*",
+        "@syncedstore/core": "*",
+        "svelte": "*"
+      }
+    },
+    "node_modules/@syncedstore/yjs-reactive-bindings": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/@syncedstore/yjs-reactive-bindings/-/yjs-reactive-bindings-0.6.0.tgz",
+      "integrity": "sha512-VF78h0J4iOt79YU9d6j5E6bFKu7WXYuiI2ue9ZnA+T4SNVn8viRvg0AHm3NqHzudZZUgYT3dpnbv1/ZmU7yPZQ==",
+      "dev": true,
+      "peerDependencies": {
+        "yjs": "^13.5.13"
+      }
+    },
     "node_modules/@tsconfig/svelte": {
       "version": "5.0.4",
       "resolved": "https://registry.npmjs.org/@tsconfig/svelte/-/svelte-5.0.4.tgz",
@@ -1081,6 +1123,17 @@
       "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz",
       "integrity": "sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg=="
     },
+    "node_modules/@types/node": {
+      "version": "22.5.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz",
+      "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==",
+      "dev": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
     "node_modules/@types/pug": {
       "version": "2.0.10",
       "resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.10.tgz",
@@ -1114,6 +1167,23 @@
         "d3-zoom": "^3.0.0"
       }
     },
+    "node_modules/abstract-leveldown": {
+      "version": "6.2.3",
+      "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz",
+      "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "buffer": "^5.5.0",
+        "immediate": "^3.2.3",
+        "level-concat-iterator": "~2.0.0",
+        "level-supports": "~1.0.0",
+        "xtend": "~4.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/acorn": {
       "version": "8.11.3",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@@ -1146,6 +1216,13 @@
         "dequal": "^2.0.3"
       }
     },
+    "node_modules/async-limiter": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz",
+      "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==",
+      "dev": true,
+      "optional": true
+    },
     "node_modules/axobject-query": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.0.0.tgz",
@@ -1159,6 +1236,27 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
+    "node_modules/base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "optional": true
+    },
     "node_modules/binary-extensions": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",
@@ -1210,6 +1308,31 @@
         "node": ">=8"
       }
     },
+    "node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "optional": true,
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "node_modules/buffer-crc32": {
       "version": "0.2.13",
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -1438,6 +1561,20 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/deferred-leveldown": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz",
+      "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "abstract-leveldown": "~6.2.1",
+        "inherits": "^2.0.3"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/dequal": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
@@ -1468,6 +1605,35 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz",
       "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg=="
     },
+    "node_modules/encoding-down": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz",
+      "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "abstract-leveldown": "^6.2.1",
+        "inherits": "^2.0.3",
+        "level-codec": "^9.0.0",
+        "level-errors": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/errno": {
+      "version": "0.1.8",
+      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz",
+      "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "prr": "~1.0.1"
+      },
+      "bin": {
+        "errno": "cli.js"
+      }
+    },
     "node_modules/es6-promise": {
       "version": "3.3.1",
       "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz",
@@ -1680,6 +1846,34 @@
         "node": ">=10.17.0"
       }
     },
+    "node_modules/ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "optional": true
+    },
+    "node_modules/immediate": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz",
+      "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==",
+      "dev": true,
+      "optional": true
+    },
     "node_modules/immutable": {
       "version": "4.3.6",
       "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz",
@@ -1785,6 +1979,16 @@
       "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
       "dev": true
     },
+    "node_modules/isomorphic.js": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz",
+      "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==",
+      "dev": true,
+      "funding": {
+        "type": "GitHub Sponsors ❤",
+        "url": "https://github.com/sponsors/dmonad"
+      }
+    },
     "node_modules/kleur": {
       "version": "4.1.5",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz",
@@ -1800,6 +2004,170 @@
       "integrity": "sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==",
       "dev": true
     },
+    "node_modules/level": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz",
+      "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "level-js": "^5.0.0",
+        "level-packager": "^5.1.0",
+        "leveldown": "^5.4.0"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/level"
+      }
+    },
+    "node_modules/level-codec": {
+      "version": "9.0.2",
+      "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz",
+      "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "buffer": "^5.6.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/level-concat-iterator": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz",
+      "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==",
+      "dev": true,
+      "optional": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/level-errors": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz",
+      "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "errno": "~0.1.1"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/level-iterator-stream": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz",
+      "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0",
+        "xtend": "^4.0.2"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/level-js": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz",
+      "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "abstract-leveldown": "~6.2.3",
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.3",
+        "ltgt": "^2.1.2"
+      }
+    },
+    "node_modules/level-packager": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz",
+      "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "encoding-down": "^6.3.0",
+        "levelup": "^4.3.2"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/level-supports": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz",
+      "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "xtend": "^4.0.2"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/leveldown": {
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz",
+      "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "dependencies": {
+        "abstract-leveldown": "~6.2.1",
+        "napi-macros": "~2.0.0",
+        "node-gyp-build": "~4.1.0"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      }
+    },
+    "node_modules/levelup": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz",
+      "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "deferred-leveldown": "~5.3.0",
+        "level-errors": "~2.0.0",
+        "level-iterator-stream": "~4.0.0",
+        "level-supports": "~1.0.0",
+        "xtend": "~4.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/lib0": {
+      "version": "0.2.98",
+      "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.98.tgz",
+      "integrity": "sha512-XteTiNO0qEXqqweWx+b21p/fBnNHUA1NwAtJNJek1oPrewEZs2uiT4gWivHKr9GqCjDPAhchz0UQO8NwU3bBNA==",
+      "dev": true,
+      "dependencies": {
+        "isomorphic.js": "^0.2.4"
+      },
+      "bin": {
+        "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js",
+        "0gentesthtml": "bin/gentesthtml.js",
+        "0serve": "bin/0serve.js"
+      },
+      "engines": {
+        "node": ">=16"
+      },
+      "funding": {
+        "type": "GitHub Sponsors ❤",
+        "url": "https://github.com/sponsors/dmonad"
+      }
+    },
     "node_modules/local-pkg": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.0.tgz",
@@ -1836,6 +2204,19 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/lodash.debounce": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
+      "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
+      "dev": true
+    },
+    "node_modules/ltgt": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz",
+      "integrity": "sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==",
+      "dev": true,
+      "optional": true
+    },
     "node_modules/magic-string": {
       "version": "0.30.10",
       "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.10.tgz",
@@ -1969,6 +2350,25 @@
         "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
       }
     },
+    "node_modules/napi-macros": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz",
+      "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==",
+      "dev": true,
+      "optional": true
+    },
+    "node_modules/node-gyp-build": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz",
+      "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==",
+      "dev": true,
+      "optional": true,
+      "bin": {
+        "node-gyp-build": "bin.js",
+        "node-gyp-build-optional": "optional.js",
+        "node-gyp-build-test": "build-test.js"
+      }
+    },
     "node_modules/normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -2154,6 +2554,13 @@
         "node": "^10 || ^12 || >=14"
       }
     },
+    "node_modules/prr": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
+      "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==",
+      "dev": true,
+      "optional": true
+    },
     "node_modules/queue-microtask": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@@ -2174,6 +2581,21 @@
         }
       ]
     },
+    "node_modules/readable-stream": {
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "inherits": "^2.0.3",
+        "string_decoder": "^1.1.1",
+        "util-deprecate": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/readdirp": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -2287,6 +2709,27 @@
         "node": ">=6"
       }
     },
+    "node_modules/safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "optional": true
+    },
     "node_modules/sander": {
       "version": "0.5.1",
       "resolved": "https://registry.npmjs.org/sander/-/sander-0.5.1.tgz",
@@ -2364,6 +2807,16 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/string_decoder": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+      "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "safe-buffer": "~5.2.0"
+      }
+    },
     "node_modules/string-argv": {
       "version": "0.3.2",
       "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz",
@@ -2572,6 +3025,14 @@
       "integrity": "sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw==",
       "dev": true
     },
+    "node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "dev": true,
+      "optional": true,
+      "peer": true
+    },
     "node_modules/unplugin": {
       "version": "1.10.1",
       "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-1.10.1.tgz",
@@ -2629,6 +3090,13 @@
         }
       }
     },
+    "node_modules/util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
+      "dev": true,
+      "optional": true
+    },
     "node_modules/vite": {
       "version": "5.2.11",
       "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.11.tgz",
@@ -2733,6 +3201,112 @@
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
+    "node_modules/ws": {
+      "version": "6.2.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.3.tgz",
+      "integrity": "sha512-jmTjYU0j60B+vHey6TfR3Z7RD61z/hmxBS3VMSGIrroOWXQEneK1zNuotOUrGyBHQj0yrpsLHPWtigEFd13ndA==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "async-limiter": "~1.0.0"
+      }
+    },
+    "node_modules/xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+      "dev": true,
+      "optional": true,
+      "engines": {
+        "node": ">=0.4"
+      }
+    },
+    "node_modules/y-leveldb": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.2.tgz",
+      "integrity": "sha512-6ulEn5AXfXJYi89rXPEg2mMHAyyw8+ZfeMMdOtBbV8FJpQ1NOrcgi6DTAcXof0dap84NjHPT2+9d0rb6cFsjEg==",
+      "dev": true,
+      "optional": true,
+      "dependencies": {
+        "level": "^6.0.1",
+        "lib0": "^0.2.31"
+      },
+      "funding": {
+        "type": "GitHub Sponsors ❤",
+        "url": "https://github.com/sponsors/dmonad"
+      },
+      "peerDependencies": {
+        "yjs": "^13.0.0"
+      }
+    },
+    "node_modules/y-protocols": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz",
+      "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==",
+      "dev": true,
+      "dependencies": {
+        "lib0": "^0.2.85"
+      },
+      "engines": {
+        "node": ">=16.0.0",
+        "npm": ">=8.0.0"
+      },
+      "funding": {
+        "type": "GitHub Sponsors ❤",
+        "url": "https://github.com/sponsors/dmonad"
+      },
+      "peerDependencies": {
+        "yjs": "^13.0.0"
+      }
+    },
+    "node_modules/y-websocket": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/y-websocket/-/y-websocket-2.0.4.tgz",
+      "integrity": "sha512-UbrkOU4GPNFFTDlJYAxAmzZhia8EPxHkngZ6qjrxgIYCN3gI2l+zzLzA9p4LQJ0IswzpioeIgmzekWe7HoBBjg==",
+      "dev": true,
+      "dependencies": {
+        "lib0": "^0.2.52",
+        "lodash.debounce": "^4.0.8",
+        "y-protocols": "^1.0.5"
+      },
+      "bin": {
+        "y-websocket": "bin/server.cjs",
+        "y-websocket-server": "bin/server.cjs"
+      },
+      "engines": {
+        "node": ">=16.0.0",
+        "npm": ">=8.0.0"
+      },
+      "funding": {
+        "type": "GitHub Sponsors ❤",
+        "url": "https://github.com/sponsors/dmonad"
+      },
+      "optionalDependencies": {
+        "ws": "^6.2.1",
+        "y-leveldb": "^0.1.0"
+      },
+      "peerDependencies": {
+        "yjs": "^13.5.6"
+      }
+    },
+    "node_modules/yjs": {
+      "version": "13.6.19",
+      "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.19.tgz",
+      "integrity": "sha512-GNKw4mEUn5yWU2QPHRx8jppxmCm9KzbBhB4qJLUJFiiYD0g/tDVgXQ7aPkyh01YO28kbs2J/BEbWBagjuWyejw==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "lib0": "^0.2.86"
+      },
+      "engines": {
+        "node": ">=16.0.0",
+        "npm": ">=8.0.0"
+      },
+      "funding": {
+        "type": "GitHub Sponsors ❤",
+        "url": "https://github.com/sponsors/dmonad"
+      }
+    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

web/package.json

@@ -11,6 +11,8 @@
   },
   "devDependencies": {
     "@sveltejs/vite-plugin-svelte": "^3.0.2",
+    "@syncedstore/core": "^0.6.0",
+    "@syncedstore/svelte": "^0.6.0",
     "@tsconfig/svelte": "^5.0.4",
     "sass": "^1.77.2",
     "svelte": "^4.2.12",
@@ -18,7 +20,8 @@
     "tslib": "^2.6.2",
     "typescript": "^5.4.4",
     "unplugin-icons": "^0.18.5",
-    "vite": "^5.2.8"
+    "vite": "^5.2.8",
+    "y-websocket": "^2.0.4"
   },
   "dependencies": {
     "@iconify-json/tabler": "^1.1.110",

web/src/LynxKiteFlow.svelte

@@ -16,7 +16,7 @@
   import ArrowBack from 'virtual:icons/tabler/arrow-back'
   import Backspace from 'virtual:icons/tabler/backspace'
   import Atom from 'virtual:icons/tabler/Atom'
-  import { useQuery, useMutation, useQueryClient } from '@sveltestack/svelte-query';
+  import { useQuery } from '@sveltestack/svelte-query';
   import NodeWithParams from './NodeWithParams.svelte';
   import NodeWithVisualization from './NodeWithVisualization.svelte';
   import NodeWithImage from './NodeWithImage.svelte';
@@ -26,27 +26,28 @@
   import NodeSearch from './NodeSearch.svelte';
   import EnvironmentSelector from './EnvironmentSelector.svelte';
   import '@xyflow/svelte/dist/style.css';
+  import { syncedStore, getYjsDoc } from "@syncedstore/core";
+  import { svelteSyncedStore } from "@syncedstore/svelte";
+  import { WebsocketProvider } from "y-websocket";
+
+  function getCRDTStore(path) {
+    const sstore = syncedStore({ workspace: {} });
+    console.log('ss', sstore.workspace);
+    const doc = getYjsDoc(sstore);
+    console.log('doc', doc.toJSON());
+    const wsProvider = new WebsocketProvider("ws://localhost:8000/ws/crdt", path, doc);
+    wsProvider.on('sync', function(isSynced: boolean) {
+      console.log('synced', isSynced, 'ydoc', doc.toJSON());
+    });
+    return {store: svelteSyncedStore(sstore), doc};
+  }
+  $: connection = getCRDTStore(path);
+  $: store = connection.store;
+  $: ws = connection.doc?.getMap('workspace');
 
   export let path = '';
 
   const { screenToFlowPosition } = useSvelteFlow();
-  const queryClient = useQueryClient();
-  const backendWorkspace = useQuery(['workspace', path], async () => {
-    const res = await fetch(`/api/load?path=${path}`);
-    return res.json();
-  }, {staleTime: 10000, retry: false});
-  const mutation = useMutation(async(update) => {
-    const res = await fetch('/api/save', {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify(update),
-    });
-    return await res.json();
-  }, {
-    onSuccess: data => queryClient.setQueryData(['workspace', path], data),
-  });
 
   const nodeTypes: NodeTypes = {
     basic: NodeWithParams,
@@ -57,16 +58,6 @@
     area: NodeWithArea,
   };
 
-  const nodes = writable<Node[]>([]);
-  const edges = writable<Edge[]>([]);
-  let doNotSave = true;
-  $: if ($backendWorkspace.isSuccess) {
-    doNotSave = true; // Change is coming from the backend.
-    nodes.set(JSON.parse(JSON.stringify($backendWorkspace.data?.nodes || [])));
-    edges.set(JSON.parse(JSON.stringify($backendWorkspace.data?.edges || [])));
-    doNotSave = false;
-  }
-
   function closeNodeSearch() {
     nodeSearchSettings = undefined;
   }
@@ -78,12 +69,12 @@
     event.preventDefault();
     nodeSearchSettings = {
       pos: { x: event.clientX, y: event.clientY },
-      boxes: $catalog.data[$backendWorkspace.data?.env],
+      boxes: $catalog.data[ws.env],
     };
   }
   function addNode(e) {
     const meta = {...e.detail};
-    nodes.update((n) => {
+    store.update((ws) => {
       const node = {
         type: meta.type,
         data: {
@@ -97,7 +88,7 @@
       const title = node.data.title;
       let i = 1;
       node.id = `${title} ${i}`;
-      while (n.find((x) => x.id === node.id)) {
+      while (ws.nodes.find((x) => x.id === node.id)) {
         i += 1;
         node.id = `${title} ${i}`;
       }
@@ -107,7 +98,7 @@
         const parent = n.find((x) => x.id === node.parentId);
         node.position = { x: node.position.x - parent.position.x, y: node.position.y - parent.position.y };
       }
-      return [...n, node]
+      return {...ws, nodes: [...n, node]};
     });
     closeNodeSearch();
   }
@@ -122,46 +113,6 @@
     parentId: string,
   };
 
-  const graph = derived([nodes, edges], ([nodes, edges]) => ({ nodes, edges }));
-  // Like JSON.stringify, but with keys sorted.
-  function orderedJSON(obj: any) {
-    const allKeys = new Set();
-    JSON.stringify(obj, (key, value) => (allKeys.add(key), value));
-    return JSON.stringify(obj, Array.from(allKeys).sort());
-  }
-  graph.subscribe(async (g) => {
-    if (doNotSave) return;
-    const dragging = g.nodes.find((n) => n.dragging);
-    if (dragging) return;
-    const resizing = g.nodes.find((n) => n.data?.beingResized);
-    if (resizing) return;
-    scheduleSave(g);
-  });
-  let saveTimeout;
-  function scheduleSave(g) {
-    // A slight delay, so we don't send a million requests when a node is resized, for example.
-    clearTimeout(saveTimeout);
-    saveTimeout = setTimeout(() => save(g), 500);
-  }
-  function save(g) {
-    g = JSON.parse(JSON.stringify(g));
-    for (const node of g.nodes) {
-      delete node.measured;
-      delete node.selected;
-      delete node.dragging;
-      delete node.beingResized;
-    }
-    for (const node of g.edges) {
-      delete node.markerEnd;
-      delete node.selected;
-    }
-    g.env = $backendWorkspace.data?.env;
-    const ws = orderedJSON(g);
-    const bd = orderedJSON($backendWorkspace.data);
-    if (ws === bd) return;
-    console.log('changed', JSON.stringify(diff(g, $backendWorkspace.data), null, 2));
-    $mutation.mutate({ path, ws: g });
-  }
   function nodeClick(e) {
     const node = e.detail.node;
     const meta = node.data.meta;
@@ -181,6 +132,11 @@
 </script>
 
 <div class="page">
+  <br>doc: {JSON.stringify(connection.doc)}
+  <br>w2j: {JSON.parse(JSON.stringify(connection.doc)).workspace}
+  <br>ws: {connection.doc?.getMap("workspace")}
+  {#if ws !== undefined}
+  {{ws}}
   <div class="top-bar">
     <div class="ws-name">
       <a href><img src="/favicon.ico"></a>
@@ -189,8 +145,11 @@
     <div class="tools">
       <EnvironmentSelector
         options={Object.keys($catalog.data || {})}
-        value={$backendWorkspace.data?.env}
-        onChange={(env) => $mutation.mutate({ path, ws: { ...$backendWorkspace.data, env } })}
+        value={$store.workspace.env}
+        onChange={(env) => {
+          console.log('env change', env);
+          $store.workspace.env = env;
+        }}
         />
       <a href><Atom /></a>
       <a href><Backspace /></a>
@@ -198,7 +157,7 @@
     </div>
   </div>
   <div style:height="100%">
-    <SvelteFlow {nodes} {edges} {nodeTypes} fitView
+    <SvelteFlow nodes={ws.nodes} edges={ws.edges} {nodeTypes} fitView
       on:paneclick={toggleNodeSearch}
       on:nodeclick={nodeClick}
       proOptions={{ hideAttribution: true }}
@@ -213,6 +172,7 @@
       {/if}
     </SvelteFlow>
   </div>
+  {/if}
 </div>
 
 <style>

web/src/Workspace.svelte

@@ -3,6 +3,29 @@
   import { QueryClient, QueryClientProvider } from '@sveltestack/svelte-query'
   import { SvelteFlowProvider } from '@xyflow/svelte';
   import LynxKiteFlow from './LynxKiteFlow.svelte';
+  // import { syncedStore, getYjsDoc } from "@syncedstore/core";
+  // import { svelteSyncedStore } from "@syncedstore/svelte";
+  // import { WebsocketProvider } from "y-websocket";
+  // const todoStore = syncedStore({ todos: [] });
+  // const doc = getYjsDoc(todoStore);
+  // const wsProvider = new WebsocketProvider("ws://localhost:8000/ws/crdt", "my-roomname3", doc);
+  // wsProvider.on('sync', function(isSynced: boolean) {
+  //   console.log('synced test', isSynced, 'ydoc', doc.toJSON());
+  // });
+
+  // console.log('ydoc', doc.toJSON());
+  // console.log('todoStore', todoStore);
+  // todoStore.todos.push('asdfx');
+  // console.log('ydoc', doc.toJSON());
+  // // console.log(doc.toJSON());
+  // // console.log(doc.getMap().toJSON());
+  // // console.log(doc.getMap().get('todos'));
+  // // doc.getMap().get('todos').observe(() => {
+  // //   console.log('todos changed', todoStore.todos);
+  // // });
+  // // doc.getMap().get('todos').add('hello');
+
+  // export const store = svelteSyncedStore(todoStore);
   export let path = '';
   const queryClient = new QueryClient()
 </script>

web/vite.config.ts

@@ -9,6 +9,13 @@ export default defineConfig({
     Icons({ compiler: 'svelte', defaultStyle: 'vertical-align: sub;' }),
   ],
   server: {
-    proxy: { '/api': 'http://127.0.0.1:8000' },
+    proxy: {
+      '/api': 'http://127.0.0.1:8000',
+      '/ws': {
+        target: 'ws://127.0.0.1:8000',
+        ws: true,
+        changeOrigin: true,
+      },
+    },
   },
 })