lynxkite-graph-analytics/src/lynxkite_graph_analytics/core.py

"""Graph analytics executor and data types."""

import os
from lynxkite.core import ops, workspace
import dataclasses
import functools
import networkx as nx
import pandas as pd
import polars as pl
import traceback
import typing


ENV = "LynxKite Graph Analytics"


@dataclasses.dataclass
class RelationDefinition:
    """Defines a set of edges."""

    df: str  # The DataFrame that contains the edges.
    source_column: (
        str  # The column in the edge DataFrame that contains the source node ID.
    )
    target_column: (
        str  # The column in the edge DataFrame that contains the target node ID.
    )
    source_table: str  # The DataFrame that contains the source nodes.
    target_table: str  # The DataFrame that contains the target nodes.
    source_key: str  # The column in the source table that contains the node ID.
    target_key: str  # The column in the target table that contains the node ID.
    name: str | None = None  # Descriptive name for the relation.


@dataclasses.dataclass
class Bundle:
    """A collection of DataFrames and other data.

    Can efficiently represent a knowledge graph (homogeneous or heterogeneous) or tabular data.
    It can also carry other data, such as a trained model.
    """

    dfs: dict[str, pd.DataFrame] = dataclasses.field(default_factory=dict)
    relations: list[RelationDefinition] = dataclasses.field(default_factory=list)
    other: dict[str, typing.Any] = None

    @classmethod
    def from_nx(cls, graph: nx.Graph):
        edges = nx.to_pandas_edgelist(graph)
        d = dict(graph.nodes(data=True))
        nodes = pd.DataFrame(d.values(), index=d.keys())
        nodes["id"] = nodes.index
        if "index" in nodes.columns:
            nodes.drop(columns=["index"], inplace=True)
        return cls(
            dfs={"edges": edges, "nodes": nodes},
            relations=[
                RelationDefinition(
                    df="edges",
                    source_column="source",
                    target_column="target",
                    source_table="nodes",
                    target_table="nodes",
                    source_key="id",
                    target_key="id",
                )
            ],
        )

    @classmethod
    def from_df(cls, df: pd.DataFrame):
        return cls(dfs={"df": df})

    def to_nx(self):
        # TODO: Use relations.
        graph = nx.DiGraph()
        if "nodes" in self.dfs:
            df = self.dfs["nodes"]
            if df.index.name != "id":
                df = df.set_index("id")
            graph.add_nodes_from(df.to_dict("index").items())
        if "edges" in self.dfs:
            edges = self.dfs["edges"]
            graph.add_edges_from(
                [
                    (
                        e["source"],
                        e["target"],
                        {
                            k: e[k]
                            for k in edges.columns
                            if k not in ["source", "target"]
                        },
                    )
                    for e in edges.to_records()
                ]
            )
        return graph

    def copy(self):
        """Returns a medium depth copy of the bundle. The Bundle is completely new, but the DataFrames and RelationDefinitions are shared."""
        return Bundle(
            dfs=dict(self.dfs),
            relations=list(self.relations),
            other=dict(self.other) if self.other else None,
        )

    def to_dict(self, limit: int = 100):
        return {
            "dataframes": {
                name: {
                    "columns": [str(c) for c in df.columns],
                    "data": df_for_frontend(df, limit).values.tolist(),
                }
                for name, df in self.dfs.items()
            },
            "relations": [dataclasses.asdict(relation) for relation in self.relations],
            "other": self.other,
        }


def nx_node_attribute_func(name):
    """Decorator for wrapping a function that adds a NetworkX node attribute."""

    def decorator(func):
        @functools.wraps(func)
        def wrapper(graph: nx.Graph, **kwargs):
            graph = graph.copy()
            attr = func(graph, **kwargs)
            nx.set_node_attributes(graph, attr, name)
            return graph

        return wrapper

    return decorator


def disambiguate_edges(ws: workspace.Workspace):
    """If an input plug is connected to multiple edges, keep only the last edge."""
    catalog = ops.CATALOGS[ws.env]
    nodes = {node.id: node for node in ws.nodes}
    seen = set()
    for edge in reversed(ws.edges):
        dst_node = nodes[edge.target]
        op = catalog.get(dst_node.data.title)
        if op.inputs[edge.targetHandle].type == list[Bundle]:
            # Takes multiple bundles as an input. No need to disambiguate.
            continue
        if (edge.target, edge.targetHandle) in seen:
            i = ws.edges.index(edge)
            del ws.edges[i]
            if hasattr(ws, "_crdt"):
                del ws._crdt["edges"][i]
        seen.add((edge.target, edge.targetHandle))


@ops.register_executor(ENV)
async def execute(ws: workspace.Workspace):
    catalog: dict[str, ops.Op] = ops.CATALOGS[ws.env]
    disambiguate_edges(ws)
    outputs = {}
    nodes = {node.id: node for node in ws.nodes}
    todo = set(nodes.keys())
    progress = True
    while progress:
        progress = False
        for id in list(todo):
            node = nodes[id]
            input_nodes = [edge.source for edge in ws.edges if edge.target == id]
            if all(input in outputs for input in input_nodes):
                # All inputs for this node are ready, we can compute the output.
                todo.remove(id)
                progress = True
                _execute_node(node, ws, catalog, outputs)


def _execute_node(node, ws, catalog, outputs):
    params = {**node.data.params}
    op = catalog.get(node.data.title)
    if not op:
        node.publish_error("Operation not found in catalog")
        return
    node.publish_started()
    # TODO: Handle multi-inputs.
    input_map = {
        edge.targetHandle: outputs[edge.source]
        for edge in ws.edges
        if edge.target == node.id
    }
    try:
        # Convert inputs types to match operation signature.
        inputs = []
        for p in op.inputs.values():
            if p.name not in input_map:
                node.publish_error(f"Missing input: {p.name}")
                return
            x = input_map[p.name]
            if p.type == nx.Graph and isinstance(x, Bundle):
                x = x.to_nx()
            elif p.type == Bundle and isinstance(x, nx.Graph):
                x = Bundle.from_nx(x)
            elif p.type == Bundle and isinstance(x, pd.DataFrame):
                x = Bundle.from_df(x)
            inputs.append(x)
        result = op(*inputs, **params)
    except Exception as e:
        if os.environ.get("LYNXKITE_LOG_OP_ERRORS"):
            traceback.print_exc()
        node.publish_error(e)
        return
    outputs[node.id] = result.output
    node.publish_result(result)


def df_for_frontend(df: pd.DataFrame, limit: int) -> pd.DataFrame:
    """Returns a DataFrame with values that are safe to send to the frontend."""
    df = df[:limit]
    if isinstance(df, pl.LazyFrame):
        df = df.collect()
    if isinstance(df, pl.DataFrame):
        df = df.to_pandas()
    # Convert non-numeric columns to strings.
    for c in df.columns:
        if not pd.api.types.is_numeric_dtype(df[c]):
            df[c] = df[c].astype(str)
    return df