balibabu
commited on
Commit
·
7b030d6
1
Parent(s):
1f75d02
feat: After selecting the parsing method as knowledge graph, the delimiter and chunk token number are displayed. #1594 (#1929)
Browse files### What problem does this PR solve?
feat: After selecting the parsing method as knowledge graph, the
delimiter and chunk token number are displayed. #1594
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- web/src/components/chunk-method-modal/hooks.ts +5 -1
- web/src/components/chunk-method-modal/index.tsx +8 -4
- web/src/components/edit-tag/index.less +5 -0
- web/src/components/max-token-number.tsx +10 -4
- web/src/hooks/logic-hooks.ts +23 -0
- web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx +12 -2
- web/src/pages/add-knowledge/components/knowledge-setting/hooks.ts +4 -0
web/src/components/chunk-method-modal/hooks.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
|
|
| 1 |
import { useSelectParserList } from '@/hooks/user-setting-hooks';
|
|
|
|
| 2 |
import { useEffect, useMemo, useState } from 'react';
|
| 3 |
|
| 4 |
const ParserListMap = new Map([
|
|
@@ -84,9 +86,11 @@ export const useFetchParserListOnMount = (
|
|
| 84 |
documentId: string,
|
| 85 |
parserId: string,
|
| 86 |
documentExtension: string,
|
|
|
|
| 87 |
) => {
|
| 88 |
const [selectedTag, setSelectedTag] = useState('');
|
| 89 |
const parserList = useSelectParserList();
|
|
|
|
| 90 |
|
| 91 |
const nextParserList = useMemo(() => {
|
| 92 |
const key = [...ParserListMap.keys()].find((x) =>
|
|
@@ -108,7 +112,7 @@ export const useFetchParserListOnMount = (
|
|
| 108 |
}, [parserId, documentId]);
|
| 109 |
|
| 110 |
const handleChange = (tag: string) => {
|
| 111 |
-
|
| 112 |
setSelectedTag(tag);
|
| 113 |
};
|
| 114 |
|
|
|
|
| 1 |
+
import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks';
|
| 2 |
import { useSelectParserList } from '@/hooks/user-setting-hooks';
|
| 3 |
+
import { FormInstance } from 'antd';
|
| 4 |
import { useEffect, useMemo, useState } from 'react';
|
| 5 |
|
| 6 |
const ParserListMap = new Map([
|
|
|
|
| 86 |
documentId: string,
|
| 87 |
parserId: string,
|
| 88 |
documentExtension: string,
|
| 89 |
+
form: FormInstance,
|
| 90 |
) => {
|
| 91 |
const [selectedTag, setSelectedTag] = useState('');
|
| 92 |
const parserList = useSelectParserList();
|
| 93 |
+
const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
|
| 94 |
|
| 95 |
const nextParserList = useMemo(() => {
|
| 96 |
const key = [...ParserListMap.keys()].find((x) =>
|
|
|
|
| 112 |
}, [parserId, documentId]);
|
| 113 |
|
| 114 |
const handleChange = (tag: string) => {
|
| 115 |
+
handleChunkMethodSelectChange(tag);
|
| 116 |
setSelectedTag(tag);
|
| 117 |
};
|
| 118 |
|
web/src/components/chunk-method-modal/index.tsx
CHANGED
|
@@ -62,12 +62,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
| 62 |
parserConfig,
|
| 63 |
loading,
|
| 64 |
}) => {
|
|
|
|
| 65 |
const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
|
| 66 |
documentId,
|
| 67 |
parserId,
|
| 68 |
documentExtension,
|
|
|
|
| 69 |
);
|
| 70 |
-
const [form] = Form.useForm();
|
| 71 |
const { t } = useTranslate('knowledgeDetails');
|
| 72 |
|
| 73 |
const handleOk = async () => {
|
|
@@ -89,12 +90,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
| 89 |
return (
|
| 90 |
isPdf &&
|
| 91 |
hidePagesChunkMethods
|
| 92 |
-
.filter((x) => x !== 'one'
|
| 93 |
.every((x) => x !== selectedTag)
|
| 94 |
);
|
| 95 |
}, [selectedTag, isPdf]);
|
| 96 |
|
| 97 |
-
const showMaxTokenNumber =
|
|
|
|
| 98 |
|
| 99 |
const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
|
| 100 |
(x) => x === false,
|
|
@@ -271,7 +273,9 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
| 271 |
)}
|
| 272 |
{showMaxTokenNumber && (
|
| 273 |
<>
|
| 274 |
-
<MaxTokenNumber
|
|
|
|
|
|
|
| 275 |
<Delimiter></Delimiter>
|
| 276 |
</>
|
| 277 |
)}
|
|
|
|
| 62 |
parserConfig,
|
| 63 |
loading,
|
| 64 |
}) => {
|
| 65 |
+
const [form] = Form.useForm();
|
| 66 |
const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
|
| 67 |
documentId,
|
| 68 |
parserId,
|
| 69 |
documentExtension,
|
| 70 |
+
form,
|
| 71 |
);
|
|
|
|
| 72 |
const { t } = useTranslate('knowledgeDetails');
|
| 73 |
|
| 74 |
const handleOk = async () => {
|
|
|
|
| 90 |
return (
|
| 91 |
isPdf &&
|
| 92 |
hidePagesChunkMethods
|
| 93 |
+
.filter((x) => x !== 'one')
|
| 94 |
.every((x) => x !== selectedTag)
|
| 95 |
);
|
| 96 |
}, [selectedTag, isPdf]);
|
| 97 |
|
| 98 |
+
const showMaxTokenNumber =
|
| 99 |
+
selectedTag === 'naive' || selectedTag === 'knowledge_graph';
|
| 100 |
|
| 101 |
const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
|
| 102 |
(x) => x === false,
|
|
|
|
| 273 |
)}
|
| 274 |
{showMaxTokenNumber && (
|
| 275 |
<>
|
| 276 |
+
<MaxTokenNumber
|
| 277 |
+
max={selectedTag === 'knowledge_graph' ? 8192 * 2 : 2048}
|
| 278 |
+
></MaxTokenNumber>
|
| 279 |
<Delimiter></Delimiter>
|
| 280 |
</>
|
| 281 |
)}
|
web/src/components/edit-tag/index.less
CHANGED
|
@@ -1,3 +1,8 @@
|
|
| 1 |
.tweenGroup {
|
| 2 |
display: inline-block;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
}
|
|
|
|
| 1 |
.tweenGroup {
|
| 2 |
display: inline-block;
|
| 3 |
+
:global(.ant-tag) {
|
| 4 |
+
margin-bottom: 8px;
|
| 5 |
+
font-size: 14px;
|
| 6 |
+
padding: 2px 8px;
|
| 7 |
+
}
|
| 8 |
}
|
web/src/components/max-token-number.tsx
CHANGED
|
@@ -1,7 +1,12 @@
|
|
| 1 |
import { useTranslate } from '@/hooks/common-hooks';
|
| 2 |
import { Flex, Form, InputNumber, Slider } from 'antd';
|
| 3 |
|
| 4 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
const { t } = useTranslate('knowledgeConfiguration');
|
| 6 |
|
| 7 |
return (
|
|
@@ -11,18 +16,19 @@ const MaxTokenNumber = () => {
|
|
| 11 |
<Form.Item
|
| 12 |
name={['parser_config', 'chunk_token_num']}
|
| 13 |
noStyle
|
| 14 |
-
initialValue={
|
| 15 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
| 16 |
>
|
| 17 |
-
<Slider max={
|
| 18 |
</Form.Item>
|
| 19 |
</Flex>
|
| 20 |
<Form.Item
|
| 21 |
name={['parser_config', 'chunk_token_num']}
|
| 22 |
noStyle
|
|
|
|
| 23 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
| 24 |
>
|
| 25 |
-
<InputNumber max={
|
| 26 |
</Form.Item>
|
| 27 |
</Flex>
|
| 28 |
</Form.Item>
|
|
|
|
| 1 |
import { useTranslate } from '@/hooks/common-hooks';
|
| 2 |
import { Flex, Form, InputNumber, Slider } from 'antd';
|
| 3 |
|
| 4 |
+
interface IProps {
|
| 5 |
+
initialValue?: number;
|
| 6 |
+
max?: number;
|
| 7 |
+
}
|
| 8 |
+
|
| 9 |
+
const MaxTokenNumber = ({ initialValue = 128, max = 2048 }: IProps) => {
|
| 10 |
const { t } = useTranslate('knowledgeConfiguration');
|
| 11 |
|
| 12 |
return (
|
|
|
|
| 16 |
<Form.Item
|
| 17 |
name={['parser_config', 'chunk_token_num']}
|
| 18 |
noStyle
|
| 19 |
+
initialValue={initialValue}
|
| 20 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
| 21 |
>
|
| 22 |
+
<Slider max={max} style={{ width: '100%' }} />
|
| 23 |
</Form.Item>
|
| 24 |
</Flex>
|
| 25 |
<Form.Item
|
| 26 |
name={['parser_config', 'chunk_token_num']}
|
| 27 |
noStyle
|
| 28 |
+
initialValue={initialValue}
|
| 29 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
| 30 |
>
|
| 31 |
+
<InputNumber max={max} min={0} />
|
| 32 |
</Form.Item>
|
| 33 |
</Flex>
|
| 34 |
</Form.Item>
|
web/src/hooks/logic-hooks.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
|
|
| 8 |
import api from '@/utils/api';
|
| 9 |
import { getAuthorization } from '@/utils/authorization-util';
|
| 10 |
import { PaginationProps } from 'antd';
|
|
|
|
| 11 |
import axios from 'axios';
|
| 12 |
import { EventSourceParserStream } from 'eventsource-parser/stream';
|
| 13 |
import {
|
|
@@ -337,3 +338,25 @@ export const useFetchModelId = () => {
|
|
| 337 |
|
| 338 |
return tenantInfo?.llm_id ?? '';
|
| 339 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
import api from '@/utils/api';
|
| 9 |
import { getAuthorization } from '@/utils/authorization-util';
|
| 10 |
import { PaginationProps } from 'antd';
|
| 11 |
+
import { FormInstance } from 'antd/lib';
|
| 12 |
import axios from 'axios';
|
| 13 |
import { EventSourceParserStream } from 'eventsource-parser/stream';
|
| 14 |
import {
|
|
|
|
| 338 |
|
| 339 |
return tenantInfo?.llm_id ?? '';
|
| 340 |
};
|
| 341 |
+
|
| 342 |
+
const ChunkTokenNumMap = {
|
| 343 |
+
naive: 128,
|
| 344 |
+
knowledge_graph: 8192,
|
| 345 |
+
};
|
| 346 |
+
|
| 347 |
+
export const useHandleChunkMethodSelectChange = (form: FormInstance) => {
|
| 348 |
+
// const form = Form.useFormInstance();
|
| 349 |
+
const handleChange = useCallback(
|
| 350 |
+
(value: string) => {
|
| 351 |
+
if (value in ChunkTokenNumMap) {
|
| 352 |
+
form.setFieldValue(
|
| 353 |
+
['parser_config', 'chunk_token_num'],
|
| 354 |
+
ChunkTokenNumMap[value as keyof typeof ChunkTokenNumMap],
|
| 355 |
+
);
|
| 356 |
+
}
|
| 357 |
+
},
|
| 358 |
+
[form],
|
| 359 |
+
);
|
| 360 |
+
|
| 361 |
+
return handleChange;
|
| 362 |
+
};
|
web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx
CHANGED
|
@@ -6,6 +6,7 @@ import ParseConfiguration, {
|
|
| 6 |
showRaptorParseConfiguration,
|
| 7 |
} from '@/components/parse-configuration';
|
| 8 |
import { useTranslate } from '@/hooks/common-hooks';
|
|
|
|
| 9 |
import { normFile } from '@/utils/file-util';
|
| 10 |
import { PlusOutlined } from '@ant-design/icons';
|
| 11 |
import { Button, Form, Input, Radio, Select, Space, Upload } from 'antd';
|
|
@@ -24,6 +25,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
|
|
| 24 |
const { parserList, embeddingModelOptions, disabled } =
|
| 25 |
useFetchKnowledgeConfigurationOnMount(form);
|
| 26 |
const { t } = useTranslate('knowledgeConfiguration');
|
|
|
|
| 27 |
|
| 28 |
return (
|
| 29 |
<Form form={form} name="validateOnly" layout="vertical" autoComplete="off">
|
|
@@ -91,7 +93,11 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
|
|
| 91 |
tooltip={t('chunkMethodTip')}
|
| 92 |
rules={[{ required: true }]}
|
| 93 |
>
|
| 94 |
-
<Select
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
{parserList.map((x) => (
|
| 96 |
<Option value={x.value} key={x.value}>
|
| 97 |
{x.label}
|
|
@@ -107,7 +113,11 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
|
|
| 107 |
return (
|
| 108 |
<>
|
| 109 |
{parserId === 'knowledge_graph' && (
|
| 110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
)}
|
| 112 |
{parserId === 'naive' && (
|
| 113 |
<>
|
|
|
|
| 6 |
showRaptorParseConfiguration,
|
| 7 |
} from '@/components/parse-configuration';
|
| 8 |
import { useTranslate } from '@/hooks/common-hooks';
|
| 9 |
+
import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks';
|
| 10 |
import { normFile } from '@/utils/file-util';
|
| 11 |
import { PlusOutlined } from '@ant-design/icons';
|
| 12 |
import { Button, Form, Input, Radio, Select, Space, Upload } from 'antd';
|
|
|
|
| 25 |
const { parserList, embeddingModelOptions, disabled } =
|
| 26 |
useFetchKnowledgeConfigurationOnMount(form);
|
| 27 |
const { t } = useTranslate('knowledgeConfiguration');
|
| 28 |
+
const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
|
| 29 |
|
| 30 |
return (
|
| 31 |
<Form form={form} name="validateOnly" layout="vertical" autoComplete="off">
|
|
|
|
| 93 |
tooltip={t('chunkMethodTip')}
|
| 94 |
rules={[{ required: true }]}
|
| 95 |
>
|
| 96 |
+
<Select
|
| 97 |
+
placeholder={t('chunkMethodPlaceholder')}
|
| 98 |
+
disabled={disabled}
|
| 99 |
+
onChange={handleChunkMethodSelectChange}
|
| 100 |
+
>
|
| 101 |
{parserList.map((x) => (
|
| 102 |
<Option value={x.value} key={x.value}>
|
| 103 |
{x.label}
|
|
|
|
| 113 |
return (
|
| 114 |
<>
|
| 115 |
{parserId === 'knowledge_graph' && (
|
| 116 |
+
<>
|
| 117 |
+
<EntityTypesItem></EntityTypesItem>
|
| 118 |
+
<MaxTokenNumber max={8192 * 2}></MaxTokenNumber>
|
| 119 |
+
<Delimiter></Delimiter>
|
| 120 |
+
</>
|
| 121 |
)}
|
| 122 |
{parserId === 'naive' && (
|
| 123 |
<>
|
web/src/pages/add-knowledge/components/knowledge-setting/hooks.ts
CHANGED
|
@@ -80,5 +80,9 @@ export const useHandleChunkMethodChange = () => {
|
|
| 80 |
const [form] = Form.useForm();
|
| 81 |
const chunkMethod = Form.useWatch('parser_id', form);
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
return { form, chunkMethod };
|
| 84 |
};
|
|
|
|
| 80 |
const [form] = Form.useForm();
|
| 81 |
const chunkMethod = Form.useWatch('parser_id', form);
|
| 82 |
|
| 83 |
+
useEffect(() => {
|
| 84 |
+
console.log('🚀 ~ useHandleChunkMethodChange ~ chunkMethod:', chunkMethod);
|
| 85 |
+
}, [chunkMethod]);
|
| 86 |
+
|
| 87 |
return { form, chunkMethod };
|
| 88 |
};
|