You've already forked DataMate
fix: 修复评估时模型输出json格式不对导致读取错误的问题 (#133)
* feature: add cot data evaluation function * fix: added verification to evaluation results * fix: fix the prompt for evaluating * fix: 修复当评估结果为空导致读取失败的问题
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
// TypeScript
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { Button, Form, Input, Select, message, Modal, Row, Col, Table, Space } from 'antd';
|
||||
import { EyeOutlined } from '@ant-design/icons';
|
||||
@@ -36,6 +37,7 @@ interface CreateTaskModalProps {
|
||||
|
||||
const TASK_TYPES = [
|
||||
{ label: 'QA评估', value: 'QA' },
|
||||
{ label: 'COT评估', value: 'COT' },
|
||||
];
|
||||
|
||||
const EVAL_METHODS = [
|
||||
@@ -55,7 +57,7 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
|
||||
dimension: '',
|
||||
description: ''
|
||||
});
|
||||
const [taskType, setTaskType] = useState<string>("QA");
|
||||
const [taskType, setTaskType] = useState<string>(DEFAULT_TASK_TYPE);
|
||||
const [promptTemplates, setPromptTemplates] = useState<PromptTemplate[]>([]);
|
||||
const [previewVisible, setPreviewVisible] = useState(false);
|
||||
const [evaluationPrompt, setEvaluationPrompt] = useState('');
|
||||
@@ -82,9 +84,24 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
|
||||
fetchDatasets().then();
|
||||
fetchModels().then();
|
||||
fetchPromptTemplates().then();
|
||||
// sync form with local taskType default
|
||||
form.setFieldsValue({ taskType: DEFAULT_TASK_TYPE });
|
||||
}
|
||||
}, [visible]);
|
||||
|
||||
// when promptTemplates or taskType change, switch dimensions to template defaults (COT/QA)
|
||||
useEffect(() => {
|
||||
if (!promptTemplates || promptTemplates.length === 0) return;
|
||||
const template = promptTemplates.find(t => t.evalType === taskType);
|
||||
if (template && template.defaultDimensions) {
|
||||
setDimensions(template.defaultDimensions.map((dim: any, index: number) => ({
|
||||
key: `dim-${index}`,
|
||||
dimension: dim.dimension,
|
||||
description: dim.description
|
||||
})));
|
||||
}
|
||||
}, [taskType, promptTemplates]);
|
||||
|
||||
const fetchDatasets = async () => {
|
||||
try {
|
||||
const { data } = await queryDatasetsUsingGet({ page: 1, size: 1000 });
|
||||
@@ -106,31 +123,46 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
|
||||
};
|
||||
|
||||
const formatDimensionsForPrompt = (dimensions: Dimension[]) => {
|
||||
let result = "\n";
|
||||
let result = "";
|
||||
dimensions.forEach((dim, index) => {
|
||||
result += `### ${index + 1}. ${dim.dimension}\n**评估标准:**\n${dim.description}\n\n`;
|
||||
if (index > 0) {
|
||||
result += "\n";
|
||||
}
|
||||
result += `### ${index + 1}. ${dim.dimension}\n**评估标准:**\n${dim.description}`;
|
||||
if (index < dimensions.length - 1) {
|
||||
result += "\n";
|
||||
}
|
||||
});
|
||||
return result;
|
||||
};
|
||||
|
||||
const formatResultExample = (dimensions: Dimension[]) => {
|
||||
return dimensions.map(dim => `\n "${dim.dimension}": "Y",`).join('');
|
||||
let result = "";
|
||||
dimensions.forEach((dim, index) => {
|
||||
if (index > 0) {
|
||||
result += "\n ";
|
||||
}
|
||||
result += `"${dim.dimension}": "Y"`;
|
||||
if (index < dimensions.length - 1) {
|
||||
result += ",";
|
||||
}
|
||||
});
|
||||
return result;
|
||||
};
|
||||
|
||||
const fetchPromptTemplates = async () => {
|
||||
try {
|
||||
const response = await queryPromptTemplatesUsingGet();
|
||||
const templates: PromptTemplate[] = response.data?.templates
|
||||
setPromptTemplates(templates)
|
||||
if (taskType) {
|
||||
const template = templates.find(t => t.evalType === taskType);
|
||||
if (template) {
|
||||
setDimensions(template.defaultDimensions.map((dim: any, index: number) => ({
|
||||
key: `dim-${index}`,
|
||||
dimension: dim.dimension,
|
||||
description: dim.description
|
||||
})));
|
||||
}
|
||||
const templates: PromptTemplate[] = response.data?.templates || [];
|
||||
setPromptTemplates(templates);
|
||||
// if a template exists for current taskType, initialize dimensions (handled also by useEffect)
|
||||
const template = templates.find(t => t.evalType === taskType);
|
||||
if (template) {
|
||||
setDimensions(template.defaultDimensions.map((dim: any, index: number) => ({
|
||||
key: `dim-${index}`,
|
||||
dimension: dim.dimension,
|
||||
description: dim.description
|
||||
})));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error fetching prompt templates:', error);
|
||||
@@ -144,8 +176,11 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
|
||||
return;
|
||||
}
|
||||
const template = promptTemplates.find(t => t.evalType === taskType);
|
||||
setEvaluationPrompt(template?.prompt.replace("{dimensions}", formatDimensionsForPrompt(dimensions))
|
||||
.replace('{result_example}', formatResultExample(dimensions)));
|
||||
const basePrompt = template?.prompt || '';
|
||||
const filled = basePrompt
|
||||
.replace('{dimensions}', formatDimensionsForPrompt(dimensions))
|
||||
.replace('{result_example}', formatResultExample(dimensions));
|
||||
setEvaluationPrompt(filled);
|
||||
setPreviewVisible(true);
|
||||
};
|
||||
|
||||
@@ -243,6 +278,13 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
|
||||
evalMethod: DEFAULT_EVAL_METHOD,
|
||||
taskType: DEFAULT_TASK_TYPE,
|
||||
}}
|
||||
onValuesChange={(changed) => {
|
||||
if (changed.taskType) {
|
||||
setTaskType(changed.taskType);
|
||||
setEvaluationPrompt('');
|
||||
setPreviewVisible(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Row gutter={16}>
|
||||
<Col span={12}>
|
||||
|
||||
Reference in New Issue
Block a user