JMLizano JMLizano commited on
Commit
45b3519
·
unverified ·
1 Parent(s): ce575c3

47 smooth graph creation flow

Browse files

* Add Graph creation box

* Wrap Operation result into a Result class

* Complex Yjs objects for parameters dont work well, use JSON string representation instead

---------

Co-authored-by: JMLizano <[email protected]>

.gitignore CHANGED
@@ -15,4 +15,4 @@ build
15
  joblib-cache
16
  *.egg-info
17
 
18
- lynxkite-app/crdt_data
 
15
  joblib-cache
16
  *.egg-info
17
 
18
+ lynxkite_crdt_data
lynxkite-app/.gitignore CHANGED
@@ -1,3 +1,4 @@
1
- /src/lynxkite/app/web_assets
2
- !/src/lynxkite/app/web_assets/__init__.py
3
- !/src/lynxkite/app/web_assets/assets/__init__.py
 
 
1
+ /src/lynxkite_app/web_assets
2
+ !/src/lynxkite_app/web_assets/__init__.py
3
+ !/src/lynxkite_app/web_assets/assets/__init__.py
4
+ data/
lynxkite-app/web/src/index.css CHANGED
@@ -347,3 +347,85 @@ path.react-flow__edge-path {
347
  outline: var(--xy-selection-border, var(--xy-selection-border-default));
348
  outline-offset: 7.5px;
349
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347
  outline: var(--xy-selection-border, var(--xy-selection-border-default));
348
  outline-offset: 7.5px;
349
  }
350
+
351
+ .graph-creation-view {
352
+ display: flex;
353
+ width: 100%;
354
+ margin-top: 10px;
355
+ }
356
+
357
+ .graph-tables, .graph-relations {
358
+ flex: 1;
359
+ padding-left: 10px;
360
+ padding-right: 10px;
361
+ }
362
+
363
+ .graph-table-header{
364
+ display: flex;
365
+ justify-content: space-between;
366
+ font-weight: bold;
367
+ text-align: left;
368
+ background-color: #333;
369
+ color: white;
370
+ padding: 10px;
371
+ border-bottom: 2px solid #222;
372
+ font-size: 16px;
373
+ }
374
+
375
+ .graph-creation-view .df-head {
376
+ font-weight: bold;
377
+ display: flex;
378
+ justify-content: space-between;
379
+ padding: 8px 12px;
380
+ border-bottom: 1px solid #ccc; /* Adds a separator between rows */
381
+ }
382
+
383
+ /* Alternating background colors for table-like effect */
384
+ .graph-creation-view .df-head:nth-child(odd) {
385
+ background-color: #f9f9f9;
386
+ }
387
+ .graph-creation-view .df-head:nth-child(even) {
388
+ background-color: #e0e0e0;
389
+ }
390
+
391
+ .graph-relation-attributes {
392
+ display: flex;
393
+ flex-direction: column;
394
+ gap: 10px; /* Adds space between each label-input pair */
395
+ width: 100%;
396
+ }
397
+
398
+ .graph-relation-attributes label {
399
+ font-size: 12px;
400
+ font-weight: bold;
401
+ display: block;
402
+ margin-bottom: 2px;
403
+ color: #666; /* Lighter text for labels */
404
+ }
405
+
406
+ .graph-relation-attributes input {
407
+ width: 100%;
408
+ padding: 8px;
409
+ font-size: 14px;
410
+ border: 1px solid #ccc;
411
+ border-radius: 4px;
412
+ outline: none;
413
+ }
414
+
415
+ .graph-relation-attributes input:focus {
416
+ border-color: #007bff; /* Highlight input on focus */
417
+ }
418
+
419
+ .add-relationship-button {
420
+ background-color: #28a745;
421
+ color: white;
422
+ border: none;
423
+ font-size: 16px;
424
+ cursor: pointer;
425
+ padding: 4px 10px;
426
+ border-radius: 4px;
427
+ }
428
+
429
+ .add-relationship-button:hover {
430
+ background-color: #218838;
431
+ }
lynxkite-app/web/src/workspace/Workspace.tsx CHANGED
@@ -45,6 +45,7 @@ import NodeWithImage from "./nodes/NodeWithImage.tsx";
45
  import NodeWithParams from "./nodes/NodeWithParams";
46
  import NodeWithTableView from "./nodes/NodeWithTableView.tsx";
47
  import NodeWithVisualization from "./nodes/NodeWithVisualization.tsx";
 
48
 
49
  export default function (props: any) {
50
  return (
@@ -172,6 +173,7 @@ function LynxKiteFlow() {
172
  visualization: NodeWithVisualization,
173
  image: NodeWithImage,
174
  table_view: NodeWithTableView,
 
175
  }),
176
  [],
177
  );
 
45
  import NodeWithParams from "./nodes/NodeWithParams";
46
  import NodeWithTableView from "./nodes/NodeWithTableView.tsx";
47
  import NodeWithVisualization from "./nodes/NodeWithVisualization.tsx";
48
+ import NodeWithGraphCreationView from "./nodes/GraphCreationNode.tsx";
49
 
50
  export default function (props: any) {
51
  return (
 
173
  visualization: NodeWithVisualization,
174
  image: NodeWithImage,
175
  table_view: NodeWithTableView,
176
+ graph_creation_view: NodeWithGraphCreationView,
177
  }),
178
  [],
179
  );
lynxkite-app/web/src/workspace/nodes/GraphCreationNode.tsx ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useReactFlow } from "@xyflow/react";
2
+ import { useState } from "react";
3
+ import React from "react";
4
+ import Markdown from "react-markdown";
5
+ // @ts-ignore
6
+ import Trash from "~icons/tabler/trash";
7
+ import LynxKiteNode from "./LynxKiteNode";
8
+ import Table from "./Table";
9
+
10
+ function toMD(v: any): string {
11
+ if (typeof v === "string") {
12
+ return v;
13
+ }
14
+ if (Array.isArray(v)) {
15
+ return v.map(toMD).join("\n\n");
16
+ }
17
+ return JSON.stringify(v);
18
+ }
19
+
20
+ function displayTable(name: string, df: any) {
21
+ if (df.data.length > 1) {
22
+ return (
23
+ <Table
24
+ key={`${name}-table`}
25
+ name={`${name}-table`}
26
+ columns={df.columns}
27
+ data={df.data}
28
+ />
29
+ );
30
+ }
31
+ if (df.data.length) {
32
+ return (
33
+ <dl key={`${name}-dl`}>
34
+ {df.columns.map((c: string, i: number) => (
35
+ <React.Fragment key={`${name}-${c}`}>
36
+ <dt>{c}</dt>
37
+ <dd>
38
+ <Markdown>{toMD(df.data[0][i])}</Markdown>
39
+ </dd>
40
+ </React.Fragment>
41
+ ))}
42
+ </dl>
43
+ );
44
+ }
45
+ return JSON.stringify(df.data);
46
+ }
47
+
48
+ function relationsToDict(relations: any[]) {
49
+ if (!relations) {
50
+ return {};
51
+ }
52
+ return Object.assign({}, ...relations.map((r: any) => ({ [r.name]: r })));
53
+ }
54
+
55
+ export type UpdateOptions = { delay?: number };
56
+
57
+ export default function NodeWithGraphCreationView(props: any) {
58
+ const reactFlow = useReactFlow();
59
+ const [open, setOpen] = useState({} as { [name: string]: boolean });
60
+ const display = props.data.display?.value;
61
+ const tables = display?.dataframes || {};
62
+ const singleTable = tables && Object.keys(tables).length === 1;
63
+ const [relations, setRelations] = useState(
64
+ relationsToDict(display?.relations) || {},
65
+ );
66
+ const singleRelation = relations && Object.keys(relations).length === 1;
67
+ function setParam(name: string, newValue: any, opts: UpdateOptions) {
68
+ reactFlow.updateNodeData(props.id, {
69
+ params: { ...props.data.params, [name]: newValue },
70
+ __execution_delay: opts.delay || 0,
71
+ });
72
+ }
73
+
74
+ function updateRelation(event: any, relation: any) {
75
+ event.preventDefault();
76
+
77
+ const updatedRelation = {
78
+ ...relation,
79
+ ...Object.fromEntries(new FormData(event.target).entries()),
80
+ };
81
+
82
+ // Avoid mutating React state directly
83
+ const newRelations = { ...relations };
84
+ if (relation.name !== updatedRelation.name) {
85
+ delete newRelations[relation.name];
86
+ }
87
+ newRelations[updatedRelation.name] = updatedRelation;
88
+ setRelations(newRelations);
89
+ // There is some issue with how Yjs handles complex objects (maps, arrays)
90
+ // so we need to serialize the relations object to a string
91
+ setParam("relations", JSON.stringify(newRelations), {});
92
+ }
93
+
94
+ const addRelation = () => {
95
+ const new_relation = {
96
+ name: "new_relation",
97
+ df: "",
98
+ source_column: "",
99
+ target_column: "",
100
+ source_table: "",
101
+ target_table: "",
102
+ source_key: "",
103
+ target_key: "",
104
+ };
105
+ setRelations({
106
+ ...relations,
107
+ [new_relation.name]: new_relation,
108
+ });
109
+ setOpen({ ...open, [new_relation.name]: true });
110
+ };
111
+
112
+ const deleteRelation = (relation: any) => {
113
+ const newOpen = { ...open };
114
+ delete newOpen[relation.name];
115
+ setOpen(newOpen);
116
+ const newRelations = { ...relations };
117
+ delete newRelations[relation.name];
118
+ setRelations(newRelations);
119
+ // There is some issue with how Yjs handles complex objects (maps, arrays)
120
+ // so we need to serialize the relations object to a string
121
+ setParam("relations", JSON.stringify(newRelations), {});
122
+ };
123
+
124
+ function displayRelation(relation: any) {
125
+ // TODO: Dynamic autocomplete
126
+ return (
127
+ <form
128
+ className="graph-relation-attributes"
129
+ onSubmit={(e) => {
130
+ updateRelation(e, relation);
131
+ }}
132
+ >
133
+ <label htmlFor="name">Name:</label>
134
+ <input type="text" id="name" name="name" defaultValue={relation.name} />
135
+
136
+ <label htmlFor="df">DataFrame:</label>
137
+ <input
138
+ type="text"
139
+ id="df"
140
+ name="df"
141
+ defaultValue={relation.df}
142
+ list="df-options"
143
+ required
144
+ />
145
+
146
+ <label htmlFor="source_column">Source Column:</label>
147
+ <input
148
+ type="text"
149
+ id="source_column"
150
+ name="source_column"
151
+ defaultValue={relation.source_column}
152
+ list="edges-column-options"
153
+ required
154
+ />
155
+
156
+ <label htmlFor="target_column">Target Column:</label>
157
+ <input
158
+ type="text"
159
+ id="target_column"
160
+ name="target_column"
161
+ defaultValue={relation.target_column}
162
+ list="edges-column-options"
163
+ required
164
+ />
165
+
166
+ <label htmlFor="source_table">Source Table:</label>
167
+ <input
168
+ type="text"
169
+ id="source_table"
170
+ name="source_table"
171
+ defaultValue={relation.source_table}
172
+ list="df-options"
173
+ required
174
+ />
175
+
176
+ <label htmlFor="target_table">Target Table:</label>
177
+ <input
178
+ type="text"
179
+ id="target_table"
180
+ name="target_table"
181
+ defaultValue={relation.target_table}
182
+ list="df-options"
183
+ required
184
+ />
185
+
186
+ <label htmlFor="source_key">Source Key:</label>
187
+ <input
188
+ type="text"
189
+ id="source_key"
190
+ name="source_key"
191
+ defaultValue={relation.source_key}
192
+ list="source-node-column-options"
193
+ required
194
+ />
195
+
196
+ <label htmlFor="target_key">Target Key:</label>
197
+ <input
198
+ type="text"
199
+ id="target_key"
200
+ name="target_key"
201
+ defaultValue={relation.target_key}
202
+ list="target-node-column-options"
203
+ required
204
+ />
205
+
206
+ <datalist id="df-options">
207
+ {Object.keys(tables).map((name) => (
208
+ <option key={name} value={name} />
209
+ ))}
210
+ </datalist>
211
+
212
+ <datalist id="edges-column-options">
213
+ {tables[relation.source_table] &&
214
+ tables[relation.df].columns.map((name: string) => (
215
+ <option key={name} value={name} />
216
+ ))}
217
+ </datalist>
218
+
219
+ <datalist id="source-node-column-options">
220
+ {tables[relation.source_table] &&
221
+ tables[relation.source_table].columns.map((name: string) => (
222
+ <option key={name} value={name} />
223
+ ))}
224
+ </datalist>
225
+
226
+ <datalist id="target-node-column-options">
227
+ {tables[relation.source_table] &&
228
+ tables[relation.target_table].columns.map((name: string) => (
229
+ <option key={name} value={name} />
230
+ ))}
231
+ </datalist>
232
+
233
+ <button className="submit-relationship-button" type="submit">
234
+ Create
235
+ </button>
236
+ </form>
237
+ );
238
+ }
239
+
240
+ return (
241
+ <LynxKiteNode {...props}>
242
+ <div className="graph-creation-view">
243
+ <div className="graph-tables">
244
+ <div className="graph-table-header">Node Tables</div>
245
+ {display && [
246
+ Object.entries(tables).map(([name, df]: [string, any]) => (
247
+ <React.Fragment key={name}>
248
+ {!singleTable && (
249
+ <div
250
+ key={`${name}-header`}
251
+ className="df-head"
252
+ onClick={() => setOpen({ ...open, [name]: !open[name] })}
253
+ >
254
+ {name}
255
+ </div>
256
+ )}
257
+ {(singleTable || open[name]) && displayTable(name, df)}
258
+ </React.Fragment>
259
+ )),
260
+ Object.entries(display.others || {}).map(([name, o]) => (
261
+ <>
262
+ <div
263
+ key={name}
264
+ className="df-head"
265
+ onClick={() => setOpen({ ...open, [name]: !open[name] })}
266
+ >
267
+ {name}
268
+ </div>
269
+ {open[name] && <pre>{(o as any).toString()}</pre>}
270
+ </>
271
+ )),
272
+ ]}
273
+ </div>
274
+ <div className="graph-relations">
275
+ <div className="graph-table-header">
276
+ Relationships
277
+ <button
278
+ className="add-relationship-button"
279
+ onClick={(_) => addRelation()}
280
+ >
281
+ +
282
+ </button>
283
+ </div>
284
+ {relations &&
285
+ Object.entries(relations).map(([name, relation]: [string, any]) => (
286
+ <React.Fragment key={name}>
287
+ <div
288
+ key={`${name}-header`}
289
+ className="df-head"
290
+ onClick={() => setOpen({ ...open, [name]: !open[name] })}
291
+ >
292
+ {name}
293
+ <button
294
+ onClick={() => {
295
+ deleteRelation(relation);
296
+ }}
297
+ >
298
+ <Trash />
299
+ </button>
300
+ </div>
301
+ {(singleRelation || open[name]) && displayRelation(relation)}
302
+ </React.Fragment>
303
+ ))}
304
+ </div>
305
+ </div>
306
+ </LynxKiteNode>
307
+ );
308
+ }
lynxkite-app/web/src/workspace/nodes/Table.tsx CHANGED
@@ -1,6 +1,6 @@
1
  export default function Table(props: any) {
2
  return (
3
- <table>
4
  <thead>
5
  <tr>
6
  {props.columns.map((column: string) => (
 
1
  export default function Table(props: any) {
2
  return (
3
+ <table id={props.name || "table"}>
4
  <thead>
5
  <tr>
6
  {props.columns.map((column: string) => (
lynxkite-app/web/tests/directory.spec.ts CHANGED
@@ -1,4 +1,4 @@
1
- // Tests some basic operations like box creation, deletion, and dragging.
2
  import { expect, test } from "@playwright/test";
3
  import { Splash, Workspace } from "./lynxkite";
4
 
 
1
+ // Tests the basic directory operations, such as creating and deleting folders and workspaces.
2
  import { expect, test } from "@playwright/test";
3
  import { Splash, Workspace } from "./lynxkite";
4
 
lynxkite-app/web/tests/graph_creation.spec.ts ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Test the graph creation box in LynxKite
2
+ import { expect, test } from "@playwright/test";
3
+ import { Splash, Workspace } from "./lynxkite";
4
+
5
+ let workspace: Workspace;
6
+
7
+ test.beforeEach(async ({ browser }) => {
8
+ workspace = await Workspace.empty(
9
+ await browser.newPage(),
10
+ "graph_creation_spec_test",
11
+ );
12
+ await workspace.addBox("Create scale-free graph");
13
+ await workspace.addBox("Create graph");
14
+ await workspace.connectBoxes("Create scale-free graph 1", "Create graph 1");
15
+ });
16
+
17
+ test.afterEach(async () => {
18
+ await workspace.close();
19
+ const splash = await new Splash(workspace.page);
20
+ splash.page.on("dialog", async (dialog) => {
21
+ await dialog.accept();
22
+ });
23
+ await splash.deleteEntry("graph_creation_spec_test");
24
+ });
25
+
26
+ test("Tables are displayed in the Graph creation box", async () => {
27
+ const graphBox = await workspace.getBox("Create graph 1");
28
+ const nodesTableHeader = await graphBox.locator(".graph-tables .df-head", {
29
+ hasText: "nodes",
30
+ });
31
+ const edgesTableHeader = await graphBox.locator(".graph-tables .df-head", {
32
+ hasText: "edges",
33
+ });
34
+ await expect(nodesTableHeader).toBeVisible();
35
+ await expect(edgesTableHeader).toBeVisible();
36
+ nodesTableHeader.click();
37
+ await expect(graphBox.locator("#nodes-table")).toBeVisible();
38
+ edgesTableHeader.click();
39
+ await expect(graphBox.locator("#edges-table")).toBeVisible();
40
+ });
41
+
42
+ test("Adding and removing relationships", async () => {
43
+ const graphBox = await workspace.getBox("Create graph 1");
44
+ const addRelationshipButton = await graphBox.locator(
45
+ ".add-relationship-button",
46
+ );
47
+ await addRelationshipButton.click();
48
+ const formData: Record<string, string> = {
49
+ name: "relation_1",
50
+ df: "edges",
51
+ source_column: "source_id",
52
+ target_column: "target_id",
53
+ source_table: "nodes",
54
+ target_table: "nodes",
55
+ source_key: "node_id",
56
+ target_key: "node_id",
57
+ };
58
+ for (const [fieldName, fieldValue] of Object.entries(formData)) {
59
+ const inputField = await graphBox.locator(
60
+ `.graph-relation-attributes input[name="${fieldName}"]`,
61
+ );
62
+ await inputField.fill(fieldValue);
63
+ }
64
+ await graphBox.locator(".submit-relationship-button").click();
65
+ // check that the relationship has been saved in the backend
66
+ await workspace.page.reload();
67
+ const graphBoxAfterReload = await workspace.getBox("Create graph 1");
68
+ const relationHeader = await graphBoxAfterReload.locator(
69
+ ".graph-relations .df-head",
70
+ { hasText: "relation_1" },
71
+ );
72
+ await expect(relationHeader).toBeVisible();
73
+ await relationHeader.locator("button").click(); // Delete the relationship
74
+ await expect(relationHeader).not.toBeVisible();
75
+ });
76
+
77
+ test("Output of the box is a bundle", async () => {
78
+ await workspace.addBox("View tables");
79
+ const tableView = await workspace.getBox("View tables 1");
80
+ await workspace.connectBoxes("Create graph 1", "View tables 1");
81
+ const nodesTableHeader = await tableView.locator(".df-head", {
82
+ hasText: "nodes",
83
+ });
84
+ const edgesTableHeader = await tableView.locator(".df-head", {
85
+ hasText: "edges",
86
+ });
87
+ await expect(nodesTableHeader).toBeVisible();
88
+ await expect(edgesTableHeader).toBeVisible();
89
+ });
lynxkite-app/web/tests/lynxkite.ts CHANGED
@@ -57,8 +57,9 @@ export class Workspace {
57
  const allBoxes = await this.getBoxes();
58
  if (allBoxes) {
59
  // Avoid overlapping with existing nodes
60
- const numNodes = allBoxes.length;
61
- await this.page.mouse.wheel(0, numNodes * 500);
 
62
  }
63
 
64
  // Some x,y offset, otherwise the box handle may fall outside the viewport.
@@ -97,7 +98,12 @@ export class Workspace {
97
  return this.page.locator(".react-flow__node").all();
98
  }
99
 
100
- getBoxHandle(boxId: string) {
 
 
 
 
 
101
  return this.page.getByTestId(boxId);
102
  }
103
 
@@ -130,8 +136,8 @@ export class Workspace {
130
  }
131
 
132
  async connectBoxes(sourceId: string, targetId: string) {
133
- const sourceHandle = this.getBoxHandle(sourceId);
134
- const targetHandle = this.getBoxHandle(targetId);
135
  await sourceHandle.hover();
136
  await this.page.mouse.down();
137
  await targetHandle.hover();
 
57
  const allBoxes = await this.getBoxes();
58
  if (allBoxes) {
59
  // Avoid overlapping with existing nodes
60
+ const numNodes = allBoxes.length || 1;
61
+ await this.page.mouse.wheel(0, numNodes * 400);
62
+ await new Promise((resolve) => setTimeout(resolve, 200));
63
  }
64
 
65
  // Some x,y offset, otherwise the box handle may fall outside the viewport.
 
98
  return this.page.locator(".react-flow__node").all();
99
  }
100
 
101
+ getBoxHandle(boxId: string, pos?: string) {
102
+ if (pos) {
103
+ return this.page.locator(
104
+ `[data-id="${boxId}"] [data-handlepos="${pos}"]`,
105
+ );
106
+ }
107
  return this.page.getByTestId(boxId);
108
  }
109
 
 
136
  }
137
 
138
  async connectBoxes(sourceId: string, targetId: string) {
139
+ const sourceHandle = this.getBoxHandle(sourceId, "right");
140
+ const targetHandle = this.getBoxHandle(targetId, "left");
141
  await sourceHandle.hover();
142
  await this.page.mouse.down();
143
  await targetHandle.hover();
lynxkite-core/src/lynxkite/core/executors/one_by_one.py CHANGED
@@ -141,28 +141,27 @@ async def execute(ws: workspace.Workspace, catalog, cache=None):
141
  if cache is not None:
142
  key = make_cache_key((inputs, params))
143
  if key not in cache:
144
- cache[key] = await await_if_needed(op(*inputs, **params))
145
- result = cache[key]
 
 
146
  else:
147
- result = await await_if_needed(op(*inputs, **params))
 
148
  except Exception as e:
149
  traceback.print_exc()
150
  data.error = str(e)
151
  break
152
- contexts[node.id].last_result = result
153
  # Returned lists and DataFrames are considered multiple tasks.
154
- if isinstance(result, pd.DataFrame):
155
- result = df_to_list(result)
156
- elif not isinstance(result, list):
157
- result = [result]
158
- results.extend(result)
159
  else: # Finished all tasks without errors.
160
- if (
161
- op.type == "visualization"
162
- or op.type == "table_view"
163
- or op.type == "image"
164
- ):
165
- data.display = results[0]
166
  for edge in edges[node.id]:
167
  t = nodes[edge.target]
168
  op = catalog[t.data.title]
 
141
  if cache is not None:
142
  key = make_cache_key((inputs, params))
143
  if key not in cache:
144
+ result: ops.Result = op(*inputs, **params)
145
+ output = await await_if_needed(result.output)
146
+ cache[key] = output
147
+ output = cache[key]
148
  else:
149
+ result = op(*inputs, **params)
150
+ output = await await_if_needed(result.output)
151
  except Exception as e:
152
  traceback.print_exc()
153
  data.error = str(e)
154
  break
155
+ contexts[node.id].last_result = output
156
  # Returned lists and DataFrames are considered multiple tasks.
157
+ if isinstance(output, pd.DataFrame):
158
+ output = df_to_list(output)
159
+ elif not isinstance(output, list):
160
+ output = [output]
161
+ results.extend(output)
162
  else: # Finished all tasks without errors.
163
+ if result.display:
164
+ data.display = await await_if_needed(result.display)
 
 
 
 
165
  for edge in edges[node.id]:
166
  t = nodes[edge.target]
167
  op = catalog[t.data.title]
lynxkite-core/src/lynxkite/core/ops.py CHANGED
@@ -6,6 +6,7 @@ import functools
6
  import inspect
7
  import pydantic
8
  import typing
 
9
  from typing_extensions import Annotated
10
 
11
  CATALOGS = {}
@@ -28,6 +29,16 @@ PathStr = Annotated[str, {"format": "path"}]
28
  CollapsedStr = Annotated[str, {"format": "collapsed"}]
29
  NodeAttribute = Annotated[str, {"format": "node attribute"}]
30
  EdgeAttribute = Annotated[str, {"format": "edge attribute"}]
 
 
 
 
 
 
 
 
 
 
31
 
32
 
33
  class BaseConfig(pydantic.BaseModel):
@@ -74,6 +85,19 @@ class Output(BaseConfig):
74
  position: str = "right"
75
 
76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  MULTI_INPUT = Input(name="multi", type="*")
78
 
79
 
@@ -105,6 +129,18 @@ class Op(BaseConfig):
105
  elif isinstance(self.params[p].type, enum.EnumMeta):
106
  params[p] = self.params[p].type[params[p]]
107
  res = self.func(*inputs, **params)
 
 
 
 
 
 
 
 
 
 
 
 
108
  return res
109
 
110
 
 
6
  import inspect
7
  import pydantic
8
  import typing
9
+ from dataclasses import dataclass
10
  from typing_extensions import Annotated
11
 
12
  CATALOGS = {}
 
29
  CollapsedStr = Annotated[str, {"format": "collapsed"}]
30
  NodeAttribute = Annotated[str, {"format": "node attribute"}]
31
  EdgeAttribute = Annotated[str, {"format": "edge attribute"}]
32
+ # https://github.com/python/typing/issues/182#issuecomment-1320974824
33
+ ReadOnlyJSON: typing.TypeAlias = (
34
+ typing.Mapping[str, "ReadOnlyJSON"]
35
+ | typing.Sequence["ReadOnlyJSON"]
36
+ | str
37
+ | int
38
+ | float
39
+ | bool
40
+ | None
41
+ )
42
 
43
 
44
  class BaseConfig(pydantic.BaseModel):
 
85
  position: str = "right"
86
 
87
 
88
+ @dataclass
89
+ class Result:
90
+ """Represents the result of an operation.
91
+
92
+ The `output` attribute is what will be used as input for other operations.
93
+ The `display` attribute is used to send data to display on the UI. The value has to be
94
+ JSON-serializable.
95
+ """
96
+
97
+ output: typing.Any
98
+ display: ReadOnlyJSON | None = None
99
+
100
+
101
  MULTI_INPUT = Input(name="multi", type="*")
102
 
103
 
 
129
  elif isinstance(self.params[p].type, enum.EnumMeta):
130
  params[p] = self.params[p].type[params[p]]
131
  res = self.func(*inputs, **params)
132
+ if not isinstance(res, Result):
133
+ # Automatically wrap the result in a Result object, if it isn't already.
134
+ res = Result(output=res)
135
+ if self.type in [
136
+ "visualization",
137
+ "table_view",
138
+ "graph_creation_view",
139
+ "image",
140
+ ]:
141
+ # If the operation is some kind of visualization, we use the output as the
142
+ # value to display by default.
143
+ res.display = res.output
144
  return res
145
 
146
 
lynxkite-core/tests/test_ops.py CHANGED
@@ -76,10 +76,44 @@ def test_op_decorator_with_complex_types():
76
  assert complex_op.__op__.inputs == {
77
  "color": ops.Input(name="color", type=Color, position="left"),
78
  "color_list": ops.Input(name="color_list", type=list[Color], position="left"),
79
- "color_dict": ops.Input(name="color_dict", type=dict[str, Color], position="left"),
 
 
80
  }
81
  assert complex_op.__op__.type == "basic"
82
  assert complex_op.__op__.outputs == {
83
  "result": ops.Output(name="result", type=None, position="right")
84
  }
85
  assert ops.CATALOGS["test"]["color_op"] == complex_op.__op__
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
  assert complex_op.__op__.inputs == {
77
  "color": ops.Input(name="color", type=Color, position="left"),
78
  "color_list": ops.Input(name="color_list", type=list[Color], position="left"),
79
+ "color_dict": ops.Input(
80
+ name="color_dict", type=dict[str, Color], position="left"
81
+ ),
82
  }
83
  assert complex_op.__op__.type == "basic"
84
  assert complex_op.__op__.outputs == {
85
  "result": ops.Output(name="result", type=None, position="right")
86
  }
87
  assert ops.CATALOGS["test"]["color_op"] == complex_op.__op__
88
+
89
+
90
+ def test_operation_can_return_non_result_instance():
91
+ @ops.op(env="test", name="subtract", view="basic", outputs=["result"])
92
+ def subtract(a, b):
93
+ return a - b
94
+
95
+ result = ops.CATALOGS["test"]["subtract"](5, 3)
96
+ assert isinstance(result, ops.Result)
97
+ assert result.output == 2
98
+ assert result.display is None
99
+
100
+
101
+ def test_operation_can_return_result_instance():
102
+ @ops.op(env="test", name="subtract", view="basic", outputs=["result"])
103
+ def subtract(a, b):
104
+ return ops.Result(output=a - b, display=None)
105
+
106
+ result = ops.CATALOGS["test"]["subtract"](5, 3)
107
+ assert isinstance(result, ops.Result)
108
+ assert result.output == 2
109
+ assert result.display is None
110
+
111
+
112
+ def test_visualization_operations_display_is_populated_automatically():
113
+ @ops.op(env="test", name="display_op", view="visualization", outputs=["result"])
114
+ def display_op():
115
+ return {"display_value": 1}
116
+
117
+ result = ops.CATALOGS["test"]["display_op"]()
118
+ assert isinstance(result, ops.Result)
119
+ assert result.output == result.display == {"display_value": 1}
lynxkite-graph-analytics/src/lynxkite_graph_analytics/lynxkite_ops.py CHANGED
@@ -14,6 +14,8 @@ import pandas as pd
14
  import polars as pl
15
  import traceback
16
  import typing
 
 
17
 
18
  mem = joblib.Memory("../joblib-cache")
19
  ENV = "LynxKite Graph Analytics"
@@ -35,6 +37,7 @@ class RelationDefinition:
35
  target_table: str # The DataFrame that contains the target nodes.
36
  source_key: str # The column in the source table that contains the node ID.
37
  target_key: str # The column in the target table that contains the node ID.
 
38
 
39
 
40
  @dataclasses.dataclass
@@ -96,6 +99,19 @@ class Bundle:
96
  other=dict(self.other) if self.other else None,
97
  )
98
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
 
100
  def nx_node_attribute_func(name):
101
  """Decorator for wrapping a function that adds a NetworkX node attribute."""
@@ -153,7 +169,7 @@ async def execute(ws):
153
  inputs[i] = Bundle.from_nx(x)
154
  elif p.type == Bundle and isinstance(x, pd.DataFrame):
155
  inputs[i] = Bundle.from_df(x)
156
- output = op(*inputs, **params)
157
  except Exception as e:
158
  traceback.print_exc()
159
  data.error = str(e)
@@ -163,13 +179,9 @@ async def execute(ws):
163
  # It's a flexible input. Create n+1 handles.
164
  data.inputs = {f"input{i}": None for i in range(len(inputs) + 1)}
165
  data.error = None
166
- outputs[node.id] = output
167
- if (
168
- op.type == "visualization"
169
- or op.type == "table_view"
170
- or op.type == "image"
171
- ):
172
- data.display = output
173
 
174
 
175
  @op("Import Parquet")
@@ -404,15 +416,33 @@ def collect(df: pd.DataFrame):
404
 
405
  @op("View tables", view="table_view")
406
  def view_tables(bundle: Bundle, *, limit: int = 100):
407
- v = {
408
- "dataframes": {
409
- name: {
410
- "columns": [str(c) for c in df.columns],
411
- "data": collect(df)[:limit],
412
- }
413
- for name, df in bundle.dfs.items()
414
- },
415
- "relations": bundle.relations,
416
- "other": bundle.other,
417
- }
418
- return v
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  import polars as pl
15
  import traceback
16
  import typing
17
+ import json
18
+
19
 
20
  mem = joblib.Memory("../joblib-cache")
21
  ENV = "LynxKite Graph Analytics"
 
37
  target_table: str # The DataFrame that contains the target nodes.
38
  source_key: str # The column in the source table that contains the node ID.
39
  target_key: str # The column in the target table that contains the node ID.
40
+ name: str | None = None # Descriptive name for the relation.
41
 
42
 
43
  @dataclasses.dataclass
 
99
  other=dict(self.other) if self.other else None,
100
  )
101
 
102
+ def to_dict(self, limit: int = 100):
103
+ return {
104
+ "dataframes": {
105
+ name: {
106
+ "columns": [str(c) for c in df.columns],
107
+ "data": collect(df)[:limit],
108
+ }
109
+ for name, df in self.dfs.items()
110
+ },
111
+ "relations": [dataclasses.asdict(relation) for relation in self.relations],
112
+ "other": self.other,
113
+ }
114
+
115
 
116
  def nx_node_attribute_func(name):
117
  """Decorator for wrapping a function that adds a NetworkX node attribute."""
 
169
  inputs[i] = Bundle.from_nx(x)
170
  elif p.type == Bundle and isinstance(x, pd.DataFrame):
171
  inputs[i] = Bundle.from_df(x)
172
+ result = op(*inputs, **params)
173
  except Exception as e:
174
  traceback.print_exc()
175
  data.error = str(e)
 
179
  # It's a flexible input. Create n+1 handles.
180
  data.inputs = {f"input{i}": None for i in range(len(inputs) + 1)}
181
  data.error = None
182
+ outputs[node.id] = result.output
183
+ if result.display:
184
+ data.display = result.display
 
 
 
 
185
 
186
 
187
  @op("Import Parquet")
 
416
 
417
  @op("View tables", view="table_view")
418
  def view_tables(bundle: Bundle, *, limit: int = 100):
419
+ return bundle.to_dict(limit=limit)
420
+
421
+
422
+ @op(
423
+ "Create graph",
424
+ view="graph_creation_view",
425
+ outputs=["output"],
426
+ )
427
+ def create_graph(bundle: Bundle, *, relations: str = None) -> Bundle:
428
+ """Replace relations of the given bundle
429
+
430
+ relations is a stringified JSON, instead of a dict, because complex Yjs types (arrays, maps)
431
+ are not currently supported in the UI.
432
+
433
+ Args:
434
+ bundle: Bundle to modify
435
+ relations (str, optional): Set of relations to set for the bundle. The parameter
436
+ should be a JSON object where the keys are relation names and the values are
437
+ a dictionary representation of a `RelationDefinition`.
438
+ Defaults to None.
439
+
440
+ Returns:
441
+ Bundle: The input bundle with the new relations set.
442
+ """
443
+ bundle = bundle.copy()
444
+ if not (relations is None or relations.strip() == ""):
445
+ bundle.relations = [
446
+ RelationDefinition(**r) for r in json.loads(relations).values()
447
+ ]
448
+ return ops.Result(output=bundle, display=bundle.to_dict(limit=100))