You've already forked DataMate
knowledge base pages (#43)
* feat: Update site name to DataMate and refine text for AI data processing * feat: Refactor settings page and implement model access functionality - Created a new ModelAccess component for managing model configurations. - Removed the old Settings component and replaced it with a new SettingsPage component that integrates ModelAccess, SystemConfig, and WebhookConfig. - Added SystemConfig component for managing system settings. - Implemented WebhookConfig component for managing webhook configurations. - Updated API functions for model management in settings.apis.ts. - Adjusted routing to point to the new SettingsPage component. * feat: Implement Data Collection Page with Task Management and Execution Log - Created DataCollectionPage component to manage data collection tasks. - Added TaskManagement and ExecutionLog components for task handling and logging. - Integrated task operations including start, stop, edit, and delete functionalities. - Implemented filtering and searching capabilities in task management. - Introduced SimpleCronScheduler for scheduling tasks with cron expressions. - Updated CreateTask component to utilize new scheduling and template features. - Enhanced BasicInformation component to conditionally render fields based on visibility settings. - Refactored ImportConfiguration component to remove NAS import section. * feat: Update task creation API endpoint and enhance task creation form with new fields and validation * Refactor file upload and operator management components - Removed unnecessary console logs from file download and export functions. - Added size property to TaskItem interface for better task management. - Simplified TaskUpload component by utilizing useFileSliceUpload hook for file upload logic. - Enhanced OperatorPluginCreate component to handle file uploads and parsing more efficiently. - Updated ConfigureStep component to use Ant Design Form for better data handling and validation. - Improved PreviewStep component to navigate back to the operator market. - Added support for additional file types in UploadStep component. - Implemented delete operator functionality in OperatorMarketPage with confirmation prompts. - Cleaned up unused API functions in operator.api.ts to streamline the codebase. - Fixed number formatting utility to handle zero values correctly. * Refactor Knowledge Generation to Knowledge Base - Created new API service for Knowledge Base operations including querying, creating, updating, and deleting knowledge bases and files. - Added constants for Knowledge Base status and type mappings. - Defined models for Knowledge Base and related files. - Removed obsolete Knowledge Base creation and home components, replacing them with new implementations under the Knowledge Base structure. - Updated routing to reflect the new Knowledge Base paths. - Adjusted menu items to align with the new Knowledge Base terminology. - Modified ModelAccess interface to include modelName and type properties. * feat: Implement Knowledge Base Page with CRUD operations and data management - Added KnowledgeBasePage component for displaying and managing knowledge bases. - Integrated search and filter functionalities with SearchControls component. - Implemented CreateKnowledgeBase component for creating and editing knowledge bases. - Enhanced AddDataDialog for file uploads and dataset selections. - Introduced TableTransfer component for managing data transfers between tables. - Updated API functions for knowledge base operations, including file management. - Refactored knowledge base model to include file status and metadata. - Adjusted routing to point to the new KnowledgeBasePage.
This commit is contained in:
chenghh-9609
GitHub
@@ -1,65 +1,320 @@
|
||||
export default function AddDataDialog() {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const [selectedFiles, setSelectedFiles] = useState<File[]>([]);
|
||||
const { message } = App.useApp();
|
||||
import { useEffect, useState } from "react";
|
||||
import {
|
||||
Button,
|
||||
App,
|
||||
Input,
|
||||
Select,
|
||||
Form,
|
||||
Modal,
|
||||
UploadFile,
|
||||
Radio,
|
||||
Tree,
|
||||
} from "antd";
|
||||
import { InboxOutlined, PlusOutlined } from "@ant-design/icons";
|
||||
import { KnowledgeBaseItem } from "../knowledge-base.model";
|
||||
import Dragger from "antd/es/upload/Dragger";
|
||||
import {
|
||||
queryDatasetFilesUsingGet,
|
||||
queryDatasetsUsingGet,
|
||||
} from "@/pages/DataManagement/dataset.api";
|
||||
import { datasetTypeMap } from "@/pages/DataManagement/dataset.const";
|
||||
import { addKnowledgeBaseFilesUsingPost } from "../knowledge-base.api";
|
||||
import { DatasetType } from "@/pages/DataManagement/dataset.model";
|
||||
|
||||
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
if (e.target.files) {
|
||||
setSelectedFiles(Array.from(e.target.files));
|
||||
}
|
||||
const dataSourceOptions = [
|
||||
{ label: "本地上传", value: "local" },
|
||||
{ label: "数据集", value: "dataset" },
|
||||
];
|
||||
|
||||
const sliceOptions = [
|
||||
{ label: "章节分块", value: "CHAPTER_CHUNK" },
|
||||
{ label: "段落分块", value: "PARAGRAPH_CHUNK" },
|
||||
{ label: "长度分块", value: "LENGTH_CHUNK" },
|
||||
{ label: "自定义分割符分块", value: "CUSTOM_SEPARATOR_CHUNK" },
|
||||
{ label: "默认分块", value: "DEFAULT_CHUNK" },
|
||||
];
|
||||
|
||||
const columns = [
|
||||
{
|
||||
dataIndex: "name",
|
||||
title: "名称",
|
||||
ellipsis: true,
|
||||
},
|
||||
{
|
||||
dataIndex: "datasetType",
|
||||
title: "类型",
|
||||
ellipsis: true,
|
||||
render: (type) => datasetTypeMap[type].label,
|
||||
},
|
||||
{
|
||||
dataIndex: "size",
|
||||
title: "大小",
|
||||
ellipsis: true,
|
||||
},
|
||||
{
|
||||
dataIndex: "fileCount",
|
||||
title: "文件数",
|
||||
ellipsis: true,
|
||||
},
|
||||
];
|
||||
|
||||
export default function AddDataDialog({ knowledgeBase }) {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const { message } = App.useApp();
|
||||
const [form] = Form.useForm();
|
||||
const [fileList, setFileList] = useState<UploadFile[]>([]);
|
||||
|
||||
// Form initial values
|
||||
const [newKB, setNewKB] = useState<Partial<KnowledgeBaseItem>>({
|
||||
dataSource: "dataset",
|
||||
processType: "DEFAULT_CHUNK",
|
||||
chunkSize: 500,
|
||||
overlap: 50,
|
||||
datasetIds: [],
|
||||
});
|
||||
|
||||
const [filesTree, setFilesTree] = useState<any[]>([]);
|
||||
|
||||
const fetchDatasets = async () => {
|
||||
const { data } = await queryDatasetsUsingGet({
|
||||
page: 0,
|
||||
size: 1000,
|
||||
type: DatasetType.TEXT,
|
||||
});
|
||||
const datasets =
|
||||
data.content.map((item) => ({
|
||||
...item,
|
||||
key: item.id,
|
||||
title: item.name,
|
||||
isLeaf: item.fileCount === 0,
|
||||
disabled: item.fileCount === 0,
|
||||
})) || [];
|
||||
setFilesTree(datasets);
|
||||
};
|
||||
|
||||
const handleUpload = async () => {
|
||||
if (selectedFiles.length === 0) {
|
||||
message.error("请先选择文件");
|
||||
return;
|
||||
}
|
||||
useEffect(() => {
|
||||
if (isOpen) fetchDatasets();
|
||||
}, [isOpen]);
|
||||
|
||||
try {
|
||||
const formData = new FormData();
|
||||
selectedFiles.forEach((file) => {
|
||||
formData.append("files", file);
|
||||
const updateTreeData = (list, key: React.Key, children) =>
|
||||
list.map((node) => {
|
||||
if (node.key === key) {
|
||||
return {
|
||||
...node,
|
||||
children,
|
||||
};
|
||||
}
|
||||
if (node.children) {
|
||||
return {
|
||||
...node,
|
||||
children: updateTreeData(node.children, key, children),
|
||||
};
|
||||
}
|
||||
return node;
|
||||
});
|
||||
|
||||
const onLoadFiles = async ({ key, children }) =>
|
||||
new Promise<void>((resolve) => {
|
||||
if (children) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
queryDatasetFilesUsingGet(key, {
|
||||
page: 0,
|
||||
size: 1000,
|
||||
}).then(({ data }) => {
|
||||
const children = data.content.map((file) => ({
|
||||
title: file.fileName,
|
||||
key: file.id,
|
||||
isLeaf: true,
|
||||
}));
|
||||
setFilesTree((origin) => updateTreeData(origin, key, children));
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
await uploadDataFilesUsingPost(formData);
|
||||
message.success("文件上传成功");
|
||||
setIsOpen(false);
|
||||
setSelectedFiles([]);
|
||||
} catch (error) {
|
||||
message.error("文件上传失败");
|
||||
}
|
||||
const handleBeforeUpload = (_, files: UploadFile[]) => {
|
||||
setFileList([...fileList, ...files]);
|
||||
return false;
|
||||
};
|
||||
|
||||
const handleRemoveFile = (file: UploadFile) => {
|
||||
setFileList((prev) => prev.filter((f) => f.uid !== file.uid));
|
||||
};
|
||||
|
||||
const handleAddData = async () => {
|
||||
await addKnowledgeBaseFilesUsingPost(knowledgeBase.id, {
|
||||
knowledgeBaseId: knowledgeBase.id,
|
||||
files: newKB.dataSource === "local" ? fileList : newKB.files,
|
||||
processType: newKB.processType,
|
||||
chunkSize: newKB.chunkSize,
|
||||
overlap: newKB.overlap,
|
||||
delimiter: newKB.delimiter,
|
||||
});
|
||||
message.success("数据添加成功");
|
||||
form.resetFields();
|
||||
setIsOpen(false);
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Button type="primary" onClick={() => setIsOpen(true)}>
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<PlusOutlined />}
|
||||
onClick={() => setIsOpen(true)}
|
||||
>
|
||||
添加数据
|
||||
</Button>
|
||||
<Modal
|
||||
title="添加数据文件"
|
||||
title="添加数据"
|
||||
open={isOpen}
|
||||
onCancel={() => setIsOpen(false)}
|
||||
onOk={handleUpload}
|
||||
okText="上传"
|
||||
onOk={handleAddData}
|
||||
okText="确定"
|
||||
cancelText="取消"
|
||||
width={1000}
|
||||
>
|
||||
<input
|
||||
type="file"
|
||||
multiple
|
||||
onChange={handleFileChange}
|
||||
accept=".txt,.pdf,.docx,.csv,.json"
|
||||
/>
|
||||
{selectedFiles.length > 0 && (
|
||||
<div className="mt-4">
|
||||
<h4>已选择的文件：</h4>
|
||||
<ul>
|
||||
{selectedFiles.map((file, index) => (
|
||||
<li key={index}>
|
||||
{file.name} - {(file.size / 1024).toFixed(2)} KB
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
<div className="overflow-auto p-6">
|
||||
<Form
|
||||
form={form}
|
||||
layout="vertical"
|
||||
initialValues={newKB}
|
||||
onValuesChange={(_, allValues) => setNewKB(allValues)}
|
||||
>
|
||||
<Form.Item
|
||||
label="分块方式"
|
||||
name="processType"
|
||||
required
|
||||
rules={[{ required: true }]}
|
||||
>
|
||||
<Select options={sliceOptions}></Select>
|
||||
</Form.Item>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<Form.Item
|
||||
label="分块大小"
|
||||
name="chunkSize"
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: "请输入分块大小",
|
||||
},
|
||||
]}
|
||||
>
|
||||
<Input type="number" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="重叠长度"
|
||||
name="overlap"
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: "请输入重叠长度",
|
||||
},
|
||||
]}
|
||||
>
|
||||
<Input type="number" />
|
||||
</Form.Item>
|
||||
</div>
|
||||
{newKB.processType === "CUSTOM_SEPARATOR_CHUNK" && (
|
||||
<Form.Item
|
||||
label="分隔符"
|
||||
name="delimiter"
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: "请输入分隔符",
|
||||
},
|
||||
]}
|
||||
>
|
||||
<Input placeholder="输入分隔符，如 \\n\\n" />
|
||||
</Form.Item>
|
||||
)}
|
||||
<Form.Item
|
||||
label="数据来源"
|
||||
name="dataSource"
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: "请选择数据来源",
|
||||
},
|
||||
]}
|
||||
>
|
||||
<Radio.Group options={dataSourceOptions} />
|
||||
</Form.Item>
|
||||
{newKB.dataSource === "local" && (
|
||||
<Form.Item
|
||||
label="上传文件"
|
||||
name="files"
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: "请上传文件",
|
||||
},
|
||||
]}
|
||||
>
|
||||
<Dragger
|
||||
className="w-full"
|
||||
onRemove={handleRemoveFile}
|
||||
beforeUpload={handleBeforeUpload}
|
||||
multiple
|
||||
>
|
||||
<p className="ant-upload-drag-icon">
|
||||
<InboxOutlined />
|
||||
</p>
|
||||
<p className="ant-upload-text">本地文件上传</p>
|
||||
<p className="ant-upload-hint">
|
||||
拖拽文件到此处或点击选择文件
|
||||
</p>
|
||||
</Dragger>
|
||||
</Form.Item>
|
||||
)}
|
||||
{newKB.dataSource === "dataset" && (
|
||||
<Form.Item
|
||||
label="选择数据集文件"
|
||||
name="datasetId"
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: "请选择数据集",
|
||||
},
|
||||
]}
|
||||
>
|
||||
<div className="border-card p-4 overflow-auto h-[300px]">
|
||||
<Tree
|
||||
blockNode
|
||||
multiple
|
||||
loadData={onLoadFiles}
|
||||
treeData={filesTree}
|
||||
onSelect={(_, { selectedNodes }) => {
|
||||
console.log({
|
||||
...newKB,
|
||||
files: selectedNodes
|
||||
.filter((node) => node.isLeaf)
|
||||
.map((node) => ({
|
||||
...node,
|
||||
id: node.key,
|
||||
name: node.title,
|
||||
})),
|
||||
});
|
||||
|
||||
setNewKB({
|
||||
...newKB,
|
||||
files: selectedNodes
|
||||
.filter((node) => node.isLeaf)
|
||||
.map((node) => ({
|
||||
...node,
|
||||
id: node.key,
|
||||
name: node.title,
|
||||
})),
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</Form.Item>
|
||||
)}
|
||||
</Form>
|
||||
</div>
|
||||
</Modal>
|
||||
</>
|
||||
);
|
||||
|
||||
@@ -12,11 +12,15 @@ import { KnowledgeBaseItem } from "../knowledge-base.model";
|
||||
export default function CreateKnowledgeBase({
|
||||
isEdit,
|
||||
data,
|
||||
showBtn = true,
|
||||
onUpdate,
|
||||
onClose,
|
||||
}: {
|
||||
isEdit?: boolean;
|
||||
showBtn?: boolean;
|
||||
data?: Partial<KnowledgeBaseItem> | null;
|
||||
onUpdate: () => void;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const [form] = Form.useForm();
|
||||
@@ -74,24 +78,32 @@ export default function CreateKnowledgeBase({
|
||||
}
|
||||
};
|
||||
|
||||
const handleCloseModal = () => {
|
||||
setOpen(false);
|
||||
onClose?.();
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<PlusOutlined />}
|
||||
onClick={() => {
|
||||
form.resetFields();
|
||||
setOpen(true);
|
||||
}}
|
||||
>
|
||||
创建知识库
|
||||
</Button>
|
||||
{showBtn && (
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<PlusOutlined />}
|
||||
onClick={() => {
|
||||
form.resetFields();
|
||||
setOpen(true);
|
||||
}}
|
||||
>
|
||||
创建知识库
|
||||
</Button>
|
||||
)}
|
||||
<Modal
|
||||
title={isEdit ? "编辑知识库" : "创建知识库"}
|
||||
open={open}
|
||||
okText="确定"
|
||||
cancelText="取消"
|
||||
onCancel={() => setOpen(false)}
|
||||
maskClosable={false}
|
||||
onCancel={handleCloseModal}
|
||||
onOk={handleCreateKnowledgeBase}
|
||||
>
|
||||
<Form form={form} layout="vertical">
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import React from "react";
|
||||
import { Table, Transfer } from "antd";
|
||||
import type {
|
||||
GetProp,
|
||||
TableColumnsType,
|
||||
TableProps,
|
||||
TransferProps,
|
||||
} from "antd";
|
||||
|
||||
type TransferItem = GetProp<TransferProps, "dataSource">[number];
|
||||
type TableRowSelection<T extends object> = TableProps<T>["rowSelection"];
|
||||
|
||||
interface DataType {
|
||||
key: string;
|
||||
title: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface TableTransferProps extends TransferProps<TransferItem> {
|
||||
dataSource: DataType[];
|
||||
leftColumns: TableColumnsType<DataType>;
|
||||
rightColumns: TableColumnsType<DataType>;
|
||||
}
|
||||
|
||||
// Customize Table Transfer
|
||||
const TableTransfer: React.FC<TableTransferProps> = (props) => {
|
||||
const { leftColumns, rightColumns, ...restProps } = props;
|
||||
return (
|
||||
<Transfer style={{ width: "100%" }} {...restProps}>
|
||||
{({
|
||||
direction,
|
||||
filteredItems,
|
||||
onItemSelect,
|
||||
onItemSelectAll,
|
||||
selectedKeys: listSelectedKeys,
|
||||
disabled: listDisabled,
|
||||
}) => {
|
||||
const columns = direction === "left" ? leftColumns : rightColumns;
|
||||
const rowSelection: TableRowSelection<TransferItem> = {
|
||||
getCheckboxProps: () => ({ disabled: listDisabled }),
|
||||
onChange(selectedRowKeys) {
|
||||
onItemSelectAll(selectedRowKeys, "replace");
|
||||
},
|
||||
selectedRowKeys: listSelectedKeys,
|
||||
selections: [
|
||||
Table.SELECTION_ALL,
|
||||
Table.SELECTION_INVERT,
|
||||
Table.SELECTION_NONE,
|
||||
],
|
||||
};
|
||||
|
||||
return (
|
||||
<Table
|
||||
rowSelection={rowSelection}
|
||||
columns={columns}
|
||||
dataSource={filteredItems}
|
||||
size="small"
|
||||
scroll={{ y: 300 }}
|
||||
style={{ pointerEvents: listDisabled ? "none" : undefined }}
|
||||
onRow={({ key, disabled: itemDisabled }) => ({
|
||||
onClick: () => {
|
||||
if (itemDisabled || listDisabled) {
|
||||
return;
|
||||
}
|
||||
onItemSelect(key, !listSelectedKeys.includes(key));
|
||||
},
|
||||
})}
|
||||
/>
|
||||
);
|
||||
}}
|
||||
</Transfer>
|
||||
);
|
||||
};
|
||||
|
||||
export default TableTransfer;
|
||||
Reference in New Issue
Block a user