feat: fetch flow (#1068)

### What problem does this PR solve?
feat: fetch flow #918
feat: save graph

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "author": "zhaofengchao <[email protected]>",
+  "author": "bill",
   "scripts": {
     "build": "umi build",
     "dev": "cross-env UMI_DEV_SERVER_COMPRESS=none umi dev",

web/src/components/similarity-slider/index.tsx

@@ -3,14 +3,18 @@ import { Form, Slider } from 'antd';
 
 type FieldType = {
   similarity_threshold?: number;
-  vector_similarity_weight?: number;
+  // vector_similarity_weight?: number;
 };
 
 interface IProps {
   isTooltipShown?: boolean;
+  vectorSimilarityWeightName?: string;
 }
 
-const SimilaritySlider = ({ isTooltipShown = false }: IProps) => {
+const SimilaritySlider = ({
+  isTooltipShown = false,
+  vectorSimilarityWeightName = 'vector_similarity_weight',
+}: IProps) => {
   const { t } = useTranslate('knowledgeDetails');
 
   return (
@@ -23,9 +27,9 @@ const SimilaritySlider = ({ isTooltipShown = false }: IProps) => {
       >
         <Slider max={1} step={0.01} />
       </Form.Item>
-      <Form.Item<FieldType>
+      <Form.Item
         label={t('vectorSimilarityWeight')}
-        name={'vector_similarity_weight'}
+        name={vectorSimilarityWeightName}
         initialValue={1 - 0.3}
         tooltip={isTooltipShown && t('vectorSimilarityWeightTip')}
       >

web/src/constants/chat.ts

@@ -2,3 +2,11 @@ export enum MessageType {
   Assistant = 'assistant',
   User = 'user',
 }
+
+export const variableEnabledFieldMap = {
+  temperatureEnabled: 'temperature',
+  topPEnabled: 'top_p',
+  presencePenaltyEnabled: 'presence_penalty',
+  frequencyPenaltyEnabled: 'frequency_penalty',
+  maxTokensEnabled: 'max_tokens',
+};

web/src/hooks/flow-hooks.ts

@@ -1,5 +1,9 @@
+import { DSL, IFlow } from '@/interfaces/database/flow';
+import i18n from '@/locales/config';
 import flowService from '@/services/flow-service';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { message } from 'antd';
+import { useParams } from 'umi';
 
 export const useFetchFlowTemplates = () => {
   const { data } = useQuery({
@@ -15,7 +19,7 @@ export const useFetchFlowTemplates = () => {
   return data;
 };
 
-export const useFetchFlowList = () => {
+export const useFetchFlowList = (): { data: IFlow[]; loading: boolean } => {
   const { data, isFetching: loading } = useQuery({
     queryKey: ['fetchFlowList'],
     initialData: [],
@@ -29,6 +33,21 @@ export const useFetchFlowList = () => {
   return { data, loading };
 };
 
+export const useFetchFlow = (): { data: IFlow; loading: boolean } => {
+  const { id } = useParams();
+  const { data, isFetching: loading } = useQuery({
+    queryKey: ['flowDetail'],
+    initialData: {} as IFlow,
+    queryFn: async () => {
+      const { data } = await flowService.getCanvas({}, id);
+
+      return data?.data ?? {};
+    },
+  });
+
+  return { data, loading };
+};
+
 export const useSetFlow = () => {
   const queryClient = useQueryClient();
   const {
@@ -37,9 +56,12 @@ export const useSetFlow = () => {
     mutateAsync,
   } = useMutation({
     mutationKey: ['setFlow'],
-    mutationFn: async (params: any) => {
+    mutationFn: async (params: { id?: string; title?: string; dsl?: DSL }) => {
       const { data } = await flowService.setCanvas(params);
       if (data.retcode === 0) {
+        message.success(
+          i18n.t(`message.${params?.id ? 'modified' : 'created'}`),
+        );
         queryClient.invalidateQueries({ queryKey: ['fetchFlowList'] });
       }
       return data?.retcode;

web/src/interfaces/database/flow.ts

@@ -1,10 +1,13 @@
+import { Edge, Node } from 'reactflow';
+
 export type DSLComponents = Record<string, IOperator>;
 
 export interface DSL {
   components: DSLComponents;
-  history: any[];
-  path: string[];
-  answer: any[];
+  history?: any[];
+  path?: string[];
+  answer?: any[];
+  graph?: IGraph;
 }
 
 export interface IOperator {
@@ -17,3 +20,28 @@ export interface IOperatorNode {
   component_name: string;
   params: Record<string, unknown>;
 }
+
+export interface IGraph {
+  nodes: Node[];
+  edges: Edge[];
+}
+
+export interface IFlow {
+  avatar: null;
+  canvas_type: null;
+  create_date: string;
+  create_time: number;
+  description: null;
+  dsl: {
+    answer: any[];
+    components: DSLComponents;
+    graph: IGraph;
+    history: any[];
+    path: string[];
+  };
+  id: string;
+  title: string;
+  update_date: string;
+  update_time: number;
+  user_id: string;
+}

web/src/pages/chat/chat-configuration-modal/index.tsx

@@ -8,11 +8,8 @@ import { IDialog } from '@/interfaces/database/chat';
 import { Divider, Flex, Form, Modal, Segmented, UploadFile } from 'antd';
 import { SegmentedValue } from 'antd/es/segmented';
 import camelCase from 'lodash/camelCase';
-import omit from 'lodash/omit';
 import { useEffect, useRef, useState } from 'react';
-import { variableEnabledFieldMap } from '../constants';
 import { IPromptConfigParameters } from '../interface';
-import { excludeUnEnabledVariables } from '../utils';
 import AssistantSetting from './assistant-setting';
 import { useFetchLlmModelOnVisible, useFetchModelId } from './hooks';
 import ModelSetting from './model-setting';
@@ -20,6 +17,7 @@ import PromptEngine from './prompt-engine';
 
 import { useTranslate } from '@/hooks/commonHooks';
 import { getBase64FromUploadFileList } from '@/utils/fileUtil';
+import { removeUselessFieldsFromValues } from '@/utils/form';
 import styles from './index.less';
 
 const layout = {
@@ -76,11 +74,10 @@ const ChatConfigurationModal = ({
 
   const handleOk = async () => {
     const values = await form.validateFields();
-    const nextValues: any = omit(values, [
-      ...Object.keys(variableEnabledFieldMap),
-      'parameters',
-      ...excludeUnEnabledVariables(values),
-    ]);
+    const nextValues: any = removeUselessFieldsFromValues(
+      values,
+      'llm_setting.',
+    );
     const emptyResponse = nextValues.prompt_config?.empty_response ?? '';
 
     const icon = await getBase64FromUploadFileList(values.icon);

web/src/pages/chat/chat-configuration-modal/model-setting.tsx

@@ -7,8 +7,8 @@ import { useEffect } from 'react';
 import { ISegmentedContentProps } from '../interface';
 
 import LlmSettingItems from '@/components/llm-setting-items';
+import { variableEnabledFieldMap } from '@/constants/chat';
 import { Variable } from '@/interfaces/database/chat';
-import { variableEnabledFieldMap } from '../constants';
 import styles from './index.less';
 
 const ModelSetting = ({

web/src/pages/chat/constants.ts

@@ -1,11 +1,3 @@
-export const variableEnabledFieldMap = {
-  temperatureEnabled: 'temperature',
-  topPEnabled: 'top_p',
-  presencePenaltyEnabled: 'presence_penalty',
-  frequencyPenaltyEnabled: 'frequency_penalty',
-  maxTokensEnabled: 'max_tokens',
-};
-
 export enum ChatSearchParams {
   DialogId = 'dialogId',
   ConversationId = 'conversationId',

web/src/pages/chat/utils.ts

@@ -1,19 +1,8 @@
 import { MessageType } from '@/constants/chat';
 import { IConversation, IReference } from '@/interfaces/database/chat';
-import { EmptyConversationId, variableEnabledFieldMap } from './constants';
+import { EmptyConversationId } from './constants';
 import { IClientConversation, IMessage } from './interface';
 
-export const excludeUnEnabledVariables = (values: any) => {
-  const unEnabledFields: Array<keyof typeof variableEnabledFieldMap> =
-    Object.keys(variableEnabledFieldMap).filter((key) => !values[key]) as Array<
-      keyof typeof variableEnabledFieldMap
-    >;
-
-  return unEnabledFields.map(
-    (key) => `llm_setting.${variableEnabledFieldMap[key]}`,
-  );
-};
-
 export const isConversationIdExist = (conversationId: string) => {
   return conversationId !== EmptyConversationId && conversationId !== '';
 };

web/src/pages/flow/begin-form/index.tsx

@@ -15,9 +15,8 @@ const onFinishFailed: FormProps<FieldType>['onFinishFailed'] = (errorInfo) => {
   console.log('Failed:', errorInfo);
 };
 
-const BeginForm = ({ onValuesChange }: IOperatorForm) => {
+const BeginForm = ({ onValuesChange, form }: IOperatorForm) => {
   const { t } = useTranslate('chat');
-  const [form] = Form.useForm();
 
   return (
     <Form

web/src/pages/flow/constant.ts

@@ -1,6 +1,32 @@
+import { ModelVariableType } from '@/constants/knowledge';
+
 export enum Operator {
   Begin = 'Begin',
   Retrieval = 'Retrieval',
   Generate = 'Generate',
   Answer = 'Answer',
 }
+
+export const initialRetrievalValues = {
+  similarity_threshold: 0.2,
+  keywords_similarity_weight: 0.3,
+  top_n: 8,
+};
+
+export const initialBeginValues = {
+  prologue: `Hi! I'm your assistant, what can I do for you?`,
+};
+
+export const initialGenerateValues = {
+  parameters: ModelVariableType.Precise,
+  temperatureEnabled: false,
+  temperature: 0.1,
+  top_p: 0.3,
+  frequency_penalty: 0.7,
+  presence_penalty: 0.4,
+  max_tokens: 512,
+  prompt: `Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:
+  {cluster_content}
+The above is the content you need to summarize.`,
+  cite: true,
+};

web/src/pages/flow/flow-drawer/index.tsx

@@ -1,5 +1,6 @@
 import { IModalProps } from '@/interfaces/common';
-import { Drawer } from 'antd';
+import { Drawer, Form } from 'antd';
+import { useEffect } from 'react';
 import { Node } from 'reactflow';
 import AnswerForm from '../answer-form';
 import BeginForm from '../begin-form';
@@ -26,8 +27,16 @@ const FlowDrawer = ({
 }: IModalProps<any> & IProps) => {
   const operatorName: Operator = node?.data.label;
   const OperatorForm = FormMap[operatorName];
+  const [form] = Form.useForm();
+
   const { handleValuesChange } = useHandleFormValuesChange(node?.id);
 
+  useEffect(() => {
+    if (visible) {
+      form.setFieldsValue(node?.data?.form);
+    }
+  }, [visible, form, node?.data?.form]);
+
   return (
     <Drawer
       title={node?.data.label}
@@ -39,7 +48,10 @@ const FlowDrawer = ({
       width={470}
     >
       {visible && (
-        <OperatorForm onValuesChange={handleValuesChange}></OperatorForm>
+        <OperatorForm
+          onValuesChange={handleValuesChange}
+          form={form}
+        ></OperatorForm>
       )}
     </Drawer>
   );

web/src/pages/flow/generate-form/index.tsx

@@ -1,24 +1,23 @@
 import LlmSettingItems from '@/components/llm-setting-items';
+import { variableEnabledFieldMap } from '@/constants/chat';
 import {
   ModelVariableType,
   settledModelVariableMap,
 } from '@/constants/knowledge';
 import { useTranslate } from '@/hooks/commonHooks';
 import { Variable } from '@/interfaces/database/chat';
-import { variableEnabledFieldMap } from '@/pages/chat/constants';
 import { Form, Input, Switch } from 'antd';
 import { useCallback, useEffect } from 'react';
 import { IOperatorForm } from '../interface';
 
-const GenerateForm = ({ onValuesChange }: IOperatorForm) => {
+const GenerateForm = ({ onValuesChange, form }: IOperatorForm) => {
   const { t } = useTranslate('flow');
-  const [form] = Form.useForm();
   const initialLlmSetting = undefined;
 
   const handleParametersChange = useCallback(
     (value: ModelVariableType) => {
       const variable = settledModelVariableMap[value];
-      form.setFieldsValue(variable);
+      form?.setFieldsValue(variable);
     },
     [form],
   );
@@ -38,7 +37,7 @@ const GenerateForm = ({ onValuesChange }: IOperatorForm) => {
       return pre;
     }, {});
     const otherValues = settledModelVariableMap[ModelVariableType.Precise];
-    form.setFieldsValue({ ...switchBoxValues, ...otherValues });
+    form?.setFieldsValue({ ...switchBoxValues, ...otherValues });
   }, [form, initialLlmSetting]);
 
   return (

web/src/pages/flow/hooks.ts

@@ -1,9 +1,22 @@
 import { useSetModalState } from '@/hooks/commonHooks';
-import { useFetchFlowTemplates } from '@/hooks/flow-hooks';
+import {
+  useFetchFlow,
+  useFetchFlowTemplates,
+  useSetFlow,
+} from '@/hooks/flow-hooks';
 import { useFetchLlmList } from '@/hooks/llmHooks';
-import React, { KeyboardEventHandler, useCallback, useState } from 'react';
+import { IGraph } from '@/interfaces/database/flow';
+import { useIsFetching } from '@tanstack/react-query';
+import React, {
+  KeyboardEventHandler,
+  useCallback,
+  useEffect,
+  useState,
+} from 'react';
 import { Node, Position, ReactFlowInstance } from 'reactflow';
 import { v4 as uuidv4 } from 'uuid';
+// import { shallow } from 'zustand/shallow';
+import { useParams } from 'umi';
 import useStore, { RFState } from './store';
 import { buildDslComponentsByGraph } from './utils';
 
@@ -18,7 +31,8 @@ const selector = (state: RFState) => ({
 });
 
 export const useSelectCanvasData = () => {
-  // return useStore(useShallow(selector)); throw error
+  // return useStore(useShallow(selector)); // throw error
+  // return useStore(selector, shallow);
   return useStore(selector);
 };
 
@@ -121,11 +135,19 @@ export const useHandleKeyUp = () => {
 };
 
 export const useSaveGraph = () => {
+  const { data } = useFetchFlow();
+  const { setFlow } = useSetFlow();
+  const { id } = useParams();
   const { nodes, edges } = useStore((state) => state);
   const saveGraph = useCallback(() => {
-    const x = buildDslComponentsByGraph(nodes, edges);
-    console.info('components:', x);
-  }, [nodes, edges]);
+    const dslComponents = buildDslComponentsByGraph(nodes, edges);
+    console.info('components:', dslComponents);
+    setFlow({
+      id,
+      title: data.title,
+      dsl: { ...data.dsl, graph: { nodes, edges }, components: dslComponents },
+    });
+  }, [nodes, edges, setFlow, id, data]);
 
   return { saveGraph };
 };
@@ -145,7 +167,34 @@ export const useHandleFormValuesChange = (id?: string) => {
   return { handleValuesChange };
 };
 
+const useSetGraphInfo = () => {
+  const { setEdges, setNodes } = useStore((state) => state);
+  const setGraphInfo = useCallback(
+    ({ nodes = [], edges = [] }: IGraph) => {
+      if (nodes.length && edges.length) {
+        setNodes(nodes);
+        setEdges(edges);
+      }
+    },
+    [setEdges, setNodes],
+  );
+  return setGraphInfo;
+};
+
 export const useFetchDataOnMount = () => {
+  const { loading, data } = useFetchFlow();
+  const setGraphInfo = useSetGraphInfo();
+
+  useEffect(() => {
+    setGraphInfo(data?.dsl?.graph ?? {});
+  }, [setGraphInfo, data?.dsl?.graph]);
+
   useFetchFlowTemplates();
   useFetchLlmList();
+
+  return { loading, flowDetail: data };
+};
+
+export const useFlowIsFetching = () => {
+  return useIsFetching({ queryKey: ['flowDetail'] }) > 0;
 };

web/src/pages/flow/interface.ts

@@ -1,4 +1,4 @@
-import { Edge, Node } from 'reactflow';
+import { FormInstance } from 'antd';
 
 export interface DSLComponentList {
   id: string;
@@ -7,6 +7,7 @@ export interface DSLComponentList {
 
 export interface IOperatorForm {
   onValuesChange?(changedValues: any, values: any): void;
+  form?: FormInstance;
 }
 
 export interface IBeginForm {
@@ -40,23 +41,3 @@ export type NodeData = {
   color: string;
   form: IBeginForm | IRetrievalForm | IGenerateForm;
 };
-
-export interface IFlow {
-  avatar: null;
-  canvas_type: null;
-  create_date: string;
-  create_time: number;
-  description: null;
-  dsl: {
-    answer: any[];
-    components: DSLComponentList;
-    graph: { nodes: Node[]; edges: Edge[] };
-    history: any[];
-    path: string[];
-  };
-  id: string;
-  title: string;
-  update_date: string;
-  update_time: number;
-  user_id: string;
-}

web/src/pages/flow/list/flow-card/index.tsx

@@ -11,7 +11,7 @@ import { useTranslation } from 'react-i18next';
 import { useNavigate } from 'umi';
 
 import { useDeleteFlow } from '@/hooks/flow-hooks';
-import { IFlow } from '../../interface';
+import { IFlow } from '@/interfaces/database/flow';
 import styles from './index.less';
 
 interface IProps {

web/src/pages/flow/list/index.tsx

@@ -31,8 +31,8 @@ const FlowList = () => {
       <Spin spinning={loading}>
         <Flex gap={'large'} wrap="wrap" className={styles.flowCardContainer}>
           {list.length > 0 ? (
-            list.map((item: any) => {
-              return <FlowCard item={item} key={item.name}></FlowCard>;
+            list.map((item) => {
+              return <FlowCard item={item} key={item.id}></FlowCard>;
             })
           ) : (
             <Empty className={styles.knowledgeEmpty}></Empty>

web/src/pages/flow/retrieval-form/index.tsx

@@ -18,9 +18,7 @@ const onFinishFailed: FormProps<FieldType>['onFinishFailed'] = (errorInfo) => {
   console.log('Failed:', errorInfo);
 };
 
-const RetrievalForm = ({ onValuesChange }: IOperatorForm) => {
-  const [form] = Form.useForm();
-
+const RetrievalForm = ({ onValuesChange, form }: IOperatorForm) => {
   return (
     <Form
       name="basic"
@@ -32,7 +30,10 @@ const RetrievalForm = ({ onValuesChange }: IOperatorForm) => {
       onValuesChange={onValuesChange}
       form={form}
     >
-      <SimilaritySlider isTooltipShown></SimilaritySlider>
+      <SimilaritySlider
+        isTooltipShown
+        vectorSimilarityWeightName="keywords_similarity_weight"
+      ></SimilaritySlider>
       <TopNItem></TopNItem>
       <Rerank></Rerank>
       <KnowledgeBaseItem></KnowledgeBaseItem>

web/src/pages/flow/store.ts

@@ -17,9 +17,6 @@ import {
 import { create } from 'zustand';
 import { devtools } from 'zustand/middleware';
 import { NodeData } from './interface';
-import { dsl } from './mock';
-
-const { nodes: initialNodes, edges: initialEdges } = dsl.graph;
 
 export type RFState = {
   nodes: Node<NodeData>[];
@@ -41,8 +38,8 @@ export type RFState = {
 // this is our useStore hook that we can use in our components to get parts of the store and call actions
 const useStore = create<RFState>()(
   devtools((set, get) => ({
-    nodes: initialNodes as Node[],
-    edges: initialEdges as Edge[],
+    nodes: [] as Node[],
+    edges: [] as Edge[],
     selectedNodeIds: [],
     selectedEdgeIds: [],
     onNodesChange: (changes: NodeChange[]) => {

web/src/pages/flow/utils.ts

@@ -1,4 +1,5 @@
 import { DSLComponents } from '@/interfaces/database/flow';
+import { removeUselessFieldsFromValues } from '@/utils/form';
 import dagre from 'dagre';
 import { Edge, MarkerType, Node, Position } from 'reactflow';
 import { v4 as uuidv4 } from 'uuid';
@@ -108,6 +109,16 @@ const buildComponentDownstreamOrUpstream = (
     .map((y) => y[isBuildDownstream ? 'target' : 'source']);
 };
 
+const removeUselessDataInTheOperator = (
+  operatorName: string,
+  params: Record<string, unknown>,
+) => {
+  if (operatorName === 'Generate') {
+    return removeUselessFieldsFromValues(params, '');
+  }
+  return params;
+};
+
 // construct a dsl based on the node information of the graph
 export const buildDslComponentsByGraph = (
   nodes: Node<NodeData>[],
@@ -117,10 +128,15 @@ export const buildDslComponentsByGraph = (
 
   nodes.forEach((x) => {
     const id = x.id;
+    const operatorName = x.data.label;
     components[id] = {
       obj: {
-        component_name: x.data.label,
-        params: x.data.form as Record<string, unknown>,
+        component_name: operatorName,
+        params:
+          removeUselessDataInTheOperator(
+            operatorName,
+            x.data.form as Record<string, unknown>,
+          ) ?? {},
       },
       downstream: buildComponentDownstreamOrUpstream(edges, id, true),
       upstream: buildComponentDownstreamOrUpstream(edges, id, false),

web/src/utils/form.ts

@@ -0,0 +1,28 @@
+import { variableEnabledFieldMap } from '@/constants/chat';
+import omit from 'lodash/omit';
+
+// chat model setting and generate operator
+export const excludeUnEnabledVariables = (
+  values: any,
+  prefix = 'llm_setting.',
+) => {
+  const unEnabledFields: Array<keyof typeof variableEnabledFieldMap> =
+    Object.keys(variableEnabledFieldMap).filter((key) => !values[key]) as Array<
+      keyof typeof variableEnabledFieldMap
+    >;
+
+  return unEnabledFields.map(
+    (key) => `${prefix}${variableEnabledFieldMap[key]}`,
+  );
+};
+
+// chat model setting and generate operator
+export const removeUselessFieldsFromValues = (values: any, prefix?: string) => {
+  const nextValues: any = omit(values, [
+    ...Object.keys(variableEnabledFieldMap),
+    'parameters',
+    ...excludeUnEnabledVariables(values, prefix),
+  ]);
+
+  return nextValues;
+};

web/src/utils/registerServer.ts

@@ -1,7 +1,10 @@
 import omit from 'lodash/omit';
 import { RequestMethod } from 'umi-request';
 
-type Service<T extends string> = Record<T, (params?: any) => any>;
+type Service<T extends string> = Record<
+  T,
+  (params?: any, urlAppendix?: string) => any
+>;
 
 const registerServer = <T extends string>(
   opt: Record<T, { url: string; method: string }>,