Merge pull request #69 from biggraph/darabos-plugin-by-naming

.github/workflows/test.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      
+
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
@@ -37,8 +37,8 @@ jobs:
         run: |
           cd lynxkite-app
           pytest
-    
-      - name: Run graph analytics tests  
+
+      - name: Run graph analytics tests
         run: |
           cd lynxkite-graph-analytics
           pytest
@@ -51,17 +51,17 @@ jobs:
         run: |
           cd lynxkite-app/web
           npm i
-          npx playwright install --with-deps 
+          npx playwright install --with-deps
 
       - name: Run Playwright tests
         run: |
           cd lynxkite-app/web
           npm run test
-      
+
       - uses: actions/upload-artifact@v4
         name: Upload playwright report
         if: ${{ !cancelled() }}
         with:
           name: playwright-report
           path: lynxkite-app/web/playwright-report/
-          retention-days: 30
+          retention-days: 30

README.md

@@ -24,14 +24,13 @@ Install everything like this:
 uv venv
 source .venv/bin/activate
 # The [dev] tag is only needed if you intend on running tests
-uv pip install -e lynxkite-core/[dev] lynxkite-app/[dev] lynxkite-graph-analytics/[dev] lynxkite-lynxscribe/ lynxkite-pillow-example/
+uv pip install -e lynxkite-core/[dev] -e lynxkite-app/[dev] -e lynxkite-graph-analytics/[dev] -e lynxkite-lynxscribe/ -e lynxkite-pillow-example/
 ```
 
 This also builds the frontend, hopefully very quickly. To run it:
 
 ```bash
-cd lynxkite-app # just because the "data" directory with the examples is here
-LYNXKITE_RELOAD=1 lynxkite
+LYNXKITE_DATA=examples LYNXKITE_RELOAD=1 lynxkite
 ```
 
 If you also want to make changes to the frontend with hot reloading:

{lynxkite-app/data → examples}/Graph RAG

@@ -7,7 +7,7 @@
       "data": {
         "title": "Input document",
         "params": {
-          "filename": "data/example-pizza.md"
+          "filename": "examples/example-pizza.md"
         },
         "display": null,
         "error": null,

lynxkite-app/README.md

@@ -13,7 +13,7 @@ To run the backend:
 
 ```bash
 uv pip install -e .
-LYNXKITE_RELOAD=1 lynxkite
+LYNXKITE_DATA=../examples LYNXKITE_RELOAD=1 lynxkite
 ```
 
 To run the frontend:

lynxkite-app/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "pytest",
+    "pytest>=8.3.4",
 ]
 
 [tool.uv.sources]
@@ -30,8 +30,8 @@ namespaces = true
 where = ["src"]
 
 [tool.setuptools.package-data]
-"lynxkite.app.web_assets" = ["*"]
-"lynxkite.app.web_assets.assets" = ["*"]
+"lynxkite_app.web_assets" = ["*"]
+"lynxkite_app.web_assets.assets" = ["*"]
 
 [tool.setuptools]
 py-modules = ["build_frontend"]
@@ -41,4 +41,4 @@ include-package-data = true
 build_py = "build_frontend.build_py"
 
 [project.scripts]
-lynxkite = "lynxkite.app.__main__:main"
+lynxkite = "lynxkite_app.__main__:main"

lynxkite-app/src/build_frontend.py

@@ -11,7 +11,7 @@ class build_py(_build_py):
         print("\n\nBuilding frontend...", __file__)
         here = Path(__file__).parent.parent
         frontend_dir = here / "web"
-        package_dir = here / "src" / "lynxkite" / "app" / "web_assets"
+        package_dir = here / "src" / "lynxkite_app" / "web_assets"
         subprocess.check_call(["npm", "install"], cwd=frontend_dir)
         subprocess.check_call(["npm", "run", "build"], cwd=frontend_dir)
         print("files in", frontend_dir / "dist")

lynxkite-app/src/{lynxkite/app → lynxkite_app}/__main__.py

@@ -6,7 +6,7 @@ import os
 def main():
     port = int(os.environ.get("PORT", "8000"))
     reload = bool(os.environ.get("LYNXKITE_RELOAD", ""))
-    uvicorn.run("lynxkite.app.main:app", host="0.0.0.0", port=port, reload=reload)
+    uvicorn.run("lynxkite_app.main:app", host="0.0.0.0", port=port, reload=reload)
 
 
 if __name__ == "__main__":

lynxkite-app/src/lynxkite_app/config.py

@@ -0,0 +1,8 @@
+"""Some common configuration."""
+
+import os
+import pathlib
+
+
+DATA_PATH = pathlib.Path(os.environ.get("LYNXKITE_DATA", "lynxkite_data"))
+CRDT_PATH = pathlib.Path(os.environ.get("LYNXKITE_CRDT_DATA", "lynxkite_crdt_data"))

lynxkite-app/src/{lynxkite/app → lynxkite_app}/crdt.py

@@ -12,10 +12,9 @@ import pycrdt_websocket.ystore
 import uvicorn
 import builtins
 from lynxkite.core import workspace, ops
+from . import config
 
 router = fastapi.APIRouter()
-DATA_PATH = pathlib.Path.cwd() / "data"
-CRDT_PATH = pathlib.Path.cwd() / "crdt_data"
 
 
 def ws_exception_handler(exception, log):
@@ -34,8 +33,8 @@ class WebsocketServer(pycrdt_websocket.WebsocketServer):
 
         The workspace is loaded from "crdt_data" if it exists there, or from "data", or a new workspace is created.
         """
-        path = CRDT_PATH / f"{name}.crdt"
-        assert path.is_relative_to(CRDT_PATH)
+        path = config.CRDT_PATH / f"{name}.crdt"
+        assert path.is_relative_to(config.CRDT_PATH)
         ystore = pycrdt_websocket.ystore.FileYStore(path)
         ydoc = pycrdt.Doc()
         ydoc["workspace"] = ws = pycrdt.Map()
@@ -52,7 +51,7 @@ class WebsocketServer(pycrdt_websocket.WebsocketServer):
         if "edges" not in ws:
             ws["edges"] = pycrdt.Array()
         if "env" not in ws:
-            ws["env"] = next(iter(ops.CATALOGS), 'unset') 
+            ws["env"] = next(iter(ops.CATALOGS), "unset")
             # We have two possible sources of truth for the workspaces, the YStore and the JSON files.
             # In case we didn't find the workspace in the YStore, we try to load it from the JSON files.
             try_to_load_workspace(ws, name)
@@ -162,7 +161,7 @@ def try_to_load_workspace(ws: pycrdt.Map, name: str):
         ws: CRDT object to udpate with the workspace contents.
         name: Name of the workspace to load.
     """
-    json_path = f"data/{name}"
+    json_path = f"{config.DATA_PATH}/{name}"
     if os.path.exists(json_path):
         ws_pyd = workspace.load(json_path)
         # We treat the display field as a black box, since it is a large
@@ -222,8 +221,8 @@ async def execute(
             await asyncio.sleep(delay)
         except asyncio.CancelledError:
             return
-    path = DATA_PATH / name
-    assert path.is_relative_to(DATA_PATH),"Provided workspace path is invalid"
+    path = config.DATA_PATH / name
+    assert path.is_relative_to(config.DATA_PATH), "Provided workspace path is invalid"
     # Save user changes before executing, in case the execution fails.
     workspace.save(ws_pyd, path)
     await workspace.execute(ws_pyd)

lynxkite-app/src/{lynxkite/app → lynxkite_app}/main.py

@@ -1,5 +1,8 @@
+"""The FastAPI server for serving the LynxKite application."""
+
 import os
 import shutil
+
 if os.environ.get("NX_CUGRAPH_AUTOCONFIG", "").strip().lower() == "true":
     import cudf.pandas
 
@@ -13,23 +16,17 @@ from fastapi.staticfiles import StaticFiles
 import starlette
 from lynxkite.core import ops
 from lynxkite.core import workspace
-from . import crdt
+from . import crdt, config
 
 
 def detect_plugins():
-    try:
-        import lynxkite_plugins
-    except ImportError:
-        print("No modules found in lynxkite_plugins. Be sure to install some plugins.")
-        return {}
-
     plugins = {}
-    for _, name, _ in pkgutil.iter_modules(lynxkite_plugins.__path__):
-        name = f"lynxkite_plugins.{name}"
-        print(f"Importing {name}")
-        plugins[name] = importlib.import_module(name)
+    for _, name, _ in pkgutil.iter_modules():
+        if name.startswith("lynxkite_"):
+            print(f"Importing {name}")
+            plugins[name] = importlib.import_module(name)
     if not plugins:
-        print("No modules found in lynxkite_plugins. Be sure to install some plugins.")
+        print("No LynxKite plugins found. Be sure to install some!")
     return plugins
 
 
@@ -53,8 +50,8 @@ class SaveRequest(workspace.BaseConfig):
 
 
 def save(req: SaveRequest):
-    path = DATA_PATH / req.path
-    assert path.is_relative_to(DATA_PATH)
+    path = config.DATA_PATH / req.path
+    assert path.is_relative_to(config.DATA_PATH)
     workspace.save(req.ws, path)
 
 
@@ -68,26 +65,23 @@ async def save_and_execute(req: SaveRequest):
 
 @app.post("/api/delete")
 async def delete_workspace(req: dict):
-    json_path: pathlib.Path = DATA_PATH / req["path"]
-    crdt_path: pathlib.Path = CRDT_PATH / f"{req["path"]}.crdt"
-    assert json_path.is_relative_to(DATA_PATH)
-    assert crdt_path.is_relative_to(CRDT_PATH)
+    json_path: pathlib.Path = config.DATA_PATH / req["path"]
+    crdt_path: pathlib.Path = config.CRDT_PATH / f"{req['path']}.crdt"
+    assert json_path.is_relative_to(config.DATA_PATH)
+    assert crdt_path.is_relative_to(config.CRDT_PATH)
     json_path.unlink()
     crdt_path.unlink()
 
 
 @app.get("/api/load")
 def load(path: str):
-    path = DATA_PATH / path
-    assert path.is_relative_to(DATA_PATH)
+    path = config.DATA_PATH / path
+    assert path.is_relative_to(config.DATA_PATH)
     if not path.exists():
         return workspace.Workspace()
     return workspace.load(path)
 
 
-DATA_PATH = pathlib.Path.cwd() / "data"
-CRDT_PATH = pathlib.Path.cwd() / "crdt_data"
-
 @dataclasses.dataclass(order=True)
 class DirectoryEntry:
     name: str
@@ -96,12 +90,13 @@ class DirectoryEntry:
 
 @app.get("/api/dir/list")
 def list_dir(path: str):
-    path = DATA_PATH / path
-    assert path.is_relative_to(DATA_PATH)
+    path = config.DATA_PATH / path
+    assert path.is_relative_to(config.DATA_PATH)
     return sorted(
         [
             DirectoryEntry(
-                p.relative_to(DATA_PATH), "directory" if p.is_dir() else "workspace"
+                p.relative_to(config.DATA_PATH),
+                "directory" if p.is_dir() else "workspace",
             )
             for p in path.iterdir()
         ]
@@ -110,23 +105,17 @@ def list_dir(path: str):
 
 @app.post("/api/dir/mkdir")
 def make_dir(req: dict):
-    path = DATA_PATH / req["path"]
-    assert path.is_relative_to(DATA_PATH)
-    assert not path.exists()
+    path = config.DATA_PATH / req["path"]
+    assert path.is_relative_to(config.DATA_PATH)
+    assert not path.exists(), f"{path} already exists"
     path.mkdir()
-    return list_dir(path.parent)
 
 
 @app.post("/api/dir/delete")
 def delete_dir(req: dict):
-    path: pathlib.Path = DATA_PATH / req["path"]
-    assert all([
-        path.is_relative_to(DATA_PATH),
-        path.exists(),
-        path.is_dir()
-    ])
+    path: pathlib.Path = config.DATA_PATH / req["path"]
+    assert all([path.is_relative_to(config.DATA_PATH), path.exists(), path.is_dir()])
     shutil.rmtree(path)
-    return list_dir(path.parent)
 
 
 @app.get("/api/service/{module_path:path}")
@@ -159,5 +148,5 @@ class SPAStaticFiles(StaticFiles):
                 raise ex
 
 
-static_dir = SPAStaticFiles(packages=[("lynxkite.app", "web_assets")], html=True)
+static_dir = SPAStaticFiles(packages=[("lynxkite_app", "web_assets")], html=True)
 app.mount("/", static_dir, name="web_assets")

lynxkite-app/tests/test_crdt.py

@@ -1,7 +1,7 @@
 from enum import Enum
 import pycrdt
 import pytest
-from lynxkite.app.crdt import crdt_update
+from lynxkite_app.crdt import crdt_update
 
 
 @pytest.fixture

lynxkite-app/tests/test_main.py

@@ -1,6 +1,7 @@
 import uuid
 from fastapi.testclient import TestClient
-from lynxkite.app.main import app, detect_plugins, DATA_PATH
+from lynxkite_app.main import app, detect_plugins
+from lynxkite_app.config import DATA_PATH
 import os
 
 
@@ -13,9 +14,9 @@ def test_detect_plugins_with_plugins():
     assert all(
         plugin in plugins.keys()
         for plugin in [
-            "lynxkite_plugins.graph_analytics",
-            "lynxkite_plugins.lynxscribe",
-            "lynxkite_plugins.pillow_example",
+            "lynxkite_graph_analytics",
+            "lynxkite_lynxscribe",
+            "lynxkite_pillow_example",
         ]
     )
 
@@ -57,7 +58,7 @@ def test_save_and_load():
 def test_list_dir():
     test_dir = str(uuid.uuid4())
     test_dir_full_path = DATA_PATH / test_dir
-    test_dir_full_path.mkdir(exist_ok=True)
+    test_dir_full_path.mkdir(parents=True, exist_ok=True)
     test_file = test_dir_full_path / "test_file.txt"
     test_file.touch()
     response = client.get(f"/api/dir/list?path={str(test_dir)}")

lynxkite-app/web/playwright.config.ts

@@ -24,7 +24,7 @@ export default defineConfig({
     },
   ],
   webServer: {
-    command: 'cd .. && lynxkite',
+    command: 'cd ../.. && LYNXKITE_DATA=examples lynxkite',
     url: 'http://127.0.0.1:8000',
     reuseExistingServer: false,
   },

lynxkite-app/web/src/Directory.tsx

@@ -29,8 +29,8 @@ export default function () {
   const navigate = useNavigate();
   const [isCreatingDir, setIsCreatingDir] = useState(false);
   const [isCreatingWorkspace, setIsCreatingWorkspace] = useState(false);
-  
-  
+
+
   function link(item: any) {
     if (item.type === 'directory') {
       return `/dir/${item.name}`;
@@ -38,7 +38,7 @@ export default function () {
       return `/edit/${item.name}`;
     }
   }
-  
+
   function shortName(item: any) {
     return item.name.split('/').pop();
   }
@@ -53,18 +53,18 @@ export default function () {
       i++;
     }
   }
-  
+
   function newWorkspaceIn(path: string, list: any[], workspaceName?: string) {
     const pathSlash = path ? `${path}/` : "";
     const name = workspaceName || newName(list);
-    navigate(`/edit/${pathSlash}${name}`, {replace: true});
+    navigate(`/edit/${pathSlash}${name}`, { replace: true });
   }
-  
+
 
   async function newFolderIn(path: string, list: any[], folderName?: string) {
     const name = folderName || newName(list, "New Folder");
     const pathSlash = path ? `${path}/` : "";
-  
+
     const res = await fetch(`/api/dir/mkdir`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
@@ -76,12 +76,12 @@ export default function () {
       alert("Failed to create folder.");
     }
   }
-  
+
   async function deleteItem(item: any) {
     if (!window.confirm(`Are you sure you want to delete "${item.name}"?`)) return;
     const pathSlash = path ? `${path}/` : "";
 
-    const apiPath = item.type === "directory" ? `/api/dir/delete`: `/api/delete`;
+    const apiPath = item.type === "directory" ? `/api/dir/delete` : `/api/delete`;
     await fetch(apiPath, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
@@ -109,39 +109,39 @@ export default function () {
         {list.data && (
           <>
             <div className="actions">
-                <div className="new-workspace">
-                  {isCreatingWorkspace && 
-                    // @ts-ignore
-                    <form onSubmit={(e) => {e.preventDefault(); newWorkspaceIn(path || "", list.data, e.target.workspaceName.value.trim())}}>
-                      <input
-                        type="text"
-                        name="workspaceName"
-                        defaultValue={newName(list.data)}
-                        placeholder={newName(list.data)}
-                      />
-                    </form>
-                  }
-                  <button type="button" onClick={() => setIsCreatingWorkspace(true)}>
-                    <FolderPlus /> New workspace
-                  </button>
-                </div>
-
-                <div className="new-folder">
-                  {isCreatingDir && 
-                    // @ts-ignore
-                    <form onSubmit={(e) =>{e.preventDefault(); newFolderIn(path || "", list.data, e.target.folderName.value.trim())}}>
-                      <input
-                        type="text"
-                        name="folderName"
-                        defaultValue={newName(list.data)}
-                        placeholder={newName(list.data)}
-                      />
-                    </form>
-                  }
-                  <button type="button" onClick={() => setIsCreatingDir(true)}>
-                    <FolderPlus /> New folder
-                  </button>
-                </div>
+              <div className="new-workspace">
+                {isCreatingWorkspace &&
+                  // @ts-ignore
+                  <form onSubmit={(e) => { e.preventDefault(); newWorkspaceIn(path || "", list.data, e.target.workspaceName.value.trim()) }}>
+                    <input
+                      type="text"
+                      name="workspaceName"
+                      defaultValue={newName(list.data)}
+                      placeholder={newName(list.data)}
+                    />
+                  </form>
+                }
+                <button type="button" onClick={() => setIsCreatingWorkspace(true)}>
+                  <FolderPlus /> New workspace
+                </button>
+              </div>
+
+              <div className="new-folder">
+                {isCreatingDir &&
+                  // @ts-ignore
+                  <form onSubmit={(e) => { e.preventDefault(); newFolderIn(path || "", list.data, e.target.folderName.value.trim()) }}>
+                    <input
+                      type="text"
+                      name="folderName"
+                      defaultValue={newName(list.data)}
+                      placeholder={newName(list.data)}
+                    />
+                  </form>
+                }
+                <button type="button" onClick={() => setIsCreatingDir(true)}>
+                  <FolderPlus /> New folder
+                </button>
+              </div>
             </div>
 
             {path && (
@@ -155,7 +155,7 @@ export default function () {
 
             {list.data.map((item: any) => (
               <div key={item.name} className="entry">
-                <a key={link(item)} className="entry" href={link(item)}>
+                <a key={link(item)} href={link(item)}>
                   {item.type === 'directory' ? <Folder /> : <File />}
                   {shortName(item)}
                 </a>

lynxkite-app/web/src/index.css

@@ -308,7 +308,6 @@ body {
   }
 
   a {
-    color: black;
     text-decoration: none;
   }
 

lynxkite-core/src/lynxkite/core/ops.py

@@ -206,8 +206,10 @@ def register_executor(env: str):
 
 
 def op_registration(env: str):
+    """Returns a decorator that can be used for registering functions as operations."""
     return functools.partial(op, env)
 
 
 def passive_op_registration(env: str):
+    """Returns a function that can be used to register operations without associated code."""
     return functools.partial(register_passive_op, env)

lynxkite-core/src/lynxkite/core/workspace.py

@@ -67,6 +67,7 @@ def save(ws: Workspace, path: str):
     """Persist a workspace to a local file in JSON format."""
     j = ws.model_dump_json(indent=2)
     dirname, basename = os.path.split(path)
+    os.makedirs(dirname, exist_ok=True)
     # Create temp file in the same directory to make sure it's on the same filesystem.
     with tempfile.NamedTemporaryFile(
         "w", prefix=f".{basename}.", dir=dirname, delete=False

lynxkite-graph-analytics/pyproject.toml

@@ -17,8 +17,8 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "pytest",
-    "pytest-asyncio",
+    "pytest>=8.3.4",
+    "pytest-asyncio>=0.25.3",
 ]
 gpu = [
     "nx-cugraph-cu12>=24.12.0",
@@ -28,4 +28,4 @@ gpu = [
 lynxkite-core = { path = "../lynxkite-core" }
 
 [tool.pytest.ini_options]
-asyncio_mode = "auto"
+asyncio_mode = "auto"

lynxkite-graph-analytics/src/lynxkite_graph_analytics/__init__.py

@@ -0,0 +1,3 @@
+from . import lynxkite_ops  # noqa (imported to trigger registration)
+from . import networkx_ops  # noqa (imported to trigger registration)
+from . import pytorch_model_ops  # noqa (imported to trigger registration)

lynxkite-graph-analytics/src/lynxkite_plugins/graph_analytics/__init__.py

@@ -1,3 +0,0 @@
-from . import lynxkite_ops
-from . import networkx_ops
-from . import pytorch_model_ops

lynxkite-graph-analytics/tests/test_lynxkite_ops.py

@@ -3,7 +3,7 @@ import pytest
 import networkx as nx
 
 from lynxkite.core import workspace
-from lynxkite_plugins.graph_analytics.lynxkite_ops import Bundle, execute, op
+from lynxkite_graph_analytics.lynxkite_ops import Bundle, execute, op
 
 
 async def test_execute_operation_not_in_catalog():

lynxkite-lynxscribe/README.md

@@ -5,7 +5,7 @@ LynxKite UI for building LynxScribe chat applications. Also runs the chat applic
 To run a chat UI for LynxScribe workspaces:
 
 ```bash
-WEBUI_AUTH=false OPENAI_API_BASE_URL=http://localhost:8000/api/service/server.lynxscribe_ops uvx open-webui serve
+WEBUI_AUTH=false OPENAI_API_BASE_URL=http://localhost:8000/api/service/lynxscribe/lynxscribe_ops uvx open-webui serve
 ```
 
 Or use [Lynx WebUI](https://github.com/biggraph/lynx-webui/) instead of Open WebUI.

lynxkite-lynxscribe/src/{lynxkite_plugins/lynxscribe → lynxkite_lynxscribe}/__init__.py

@@ -1,5 +1,5 @@
-from . import lynxscribe_ops
-from . import llm_ops
+from . import lynxscribe_ops  # noqa (imported to trigger registration)
+from . import llm_ops  # noqa (imported to trigger registration)
 from .lynxscribe_ops import api_service_post, api_service_get
 
 __all__ = ["api_service_post", "api_service_get"]

lynxkite-pillow-example/pyproject.toml

@@ -8,8 +8,8 @@ dependencies = [
     "fsspec>=2025.2.0",
     "lynxkite-core",
     "pillow>=11.1.0",
-    "requests",
-    "aiohttp",
+    "requests>=2.32.3",
+    "aiohttp>=3.11.11",
 ]
 
 [tool.uv.sources]