fix: 修复评估时模型输出json格式不对导致读取错误的问题 (#133)

* feature: add cot data evaluation function

* fix: added verification to evaluation results

* fix: fix the prompt for evaluating

* fix: 修复当评估结果为空导致读取失败的问题
This commit is contained in:
hefanli
2025-12-04 18:49:50 +08:00
committed by GitHub
parent 31c4966608
commit 744d15ba24
14 changed files with 373 additions and 219 deletions

View File

@@ -1,3 +1,4 @@
// TypeScript
import React, { useState, useEffect } from 'react';
import { Button, Form, Input, Select, message, Modal, Row, Col, Table, Space } from 'antd';
import { EyeOutlined } from '@ant-design/icons';
@@ -36,6 +37,7 @@ interface CreateTaskModalProps {
const TASK_TYPES = [
{ label: 'QA评估', value: 'QA' },
{ label: 'COT评估', value: 'COT' },
];
const EVAL_METHODS = [
@@ -55,7 +57,7 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
dimension: '',
description: ''
});
const [taskType, setTaskType] = useState<string>("QA");
const [taskType, setTaskType] = useState<string>(DEFAULT_TASK_TYPE);
const [promptTemplates, setPromptTemplates] = useState<PromptTemplate[]>([]);
const [previewVisible, setPreviewVisible] = useState(false);
const [evaluationPrompt, setEvaluationPrompt] = useState('');
@@ -82,9 +84,24 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
fetchDatasets().then();
fetchModels().then();
fetchPromptTemplates().then();
// sync form with local taskType default
form.setFieldsValue({ taskType: DEFAULT_TASK_TYPE });
}
}, [visible]);
// when promptTemplates or taskType change, switch dimensions to template defaults (COT/QA)
useEffect(() => {
if (!promptTemplates || promptTemplates.length === 0) return;
const template = promptTemplates.find(t => t.evalType === taskType);
if (template && template.defaultDimensions) {
setDimensions(template.defaultDimensions.map((dim: any, index: number) => ({
key: `dim-${index}`,
dimension: dim.dimension,
description: dim.description
})));
}
}, [taskType, promptTemplates]);
const fetchDatasets = async () => {
try {
const { data } = await queryDatasetsUsingGet({ page: 1, size: 1000 });
@@ -106,31 +123,46 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
};
const formatDimensionsForPrompt = (dimensions: Dimension[]) => {
let result = "\n";
let result = "";
dimensions.forEach((dim, index) => {
result += `### ${index + 1}. ${dim.dimension}\n**评估标准:**\n${dim.description}\n\n`;
if (index > 0) {
result += "\n";
}
result += `### ${index + 1}. ${dim.dimension}\n**评估标准:**\n${dim.description}`;
if (index < dimensions.length - 1) {
result += "\n";
}
});
return result;
};
const formatResultExample = (dimensions: Dimension[]) => {
return dimensions.map(dim => `\n "${dim.dimension}": "Y",`).join('');
let result = "";
dimensions.forEach((dim, index) => {
if (index > 0) {
result += "\n ";
}
result += `"${dim.dimension}": "Y"`;
if (index < dimensions.length - 1) {
result += ",";
}
});
return result;
};
const fetchPromptTemplates = async () => {
try {
const response = await queryPromptTemplatesUsingGet();
const templates: PromptTemplate[] = response.data?.templates
setPromptTemplates(templates)
if (taskType) {
const template = templates.find(t => t.evalType === taskType);
if (template) {
setDimensions(template.defaultDimensions.map((dim: any, index: number) => ({
key: `dim-${index}`,
dimension: dim.dimension,
description: dim.description
})));
}
const templates: PromptTemplate[] = response.data?.templates || [];
setPromptTemplates(templates);
// if a template exists for current taskType, initialize dimensions (handled also by useEffect)
const template = templates.find(t => t.evalType === taskType);
if (template) {
setDimensions(template.defaultDimensions.map((dim: any, index: number) => ({
key: `dim-${index}`,
dimension: dim.dimension,
description: dim.description
})));
}
} catch (error) {
console.error('Error fetching prompt templates:', error);
@@ -144,8 +176,11 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
return;
}
const template = promptTemplates.find(t => t.evalType === taskType);
setEvaluationPrompt(template?.prompt.replace("{dimensions}", formatDimensionsForPrompt(dimensions))
.replace('{result_example}', formatResultExample(dimensions)));
const basePrompt = template?.prompt || '';
const filled = basePrompt
.replace('{dimensions}', formatDimensionsForPrompt(dimensions))
.replace('{result_example}', formatResultExample(dimensions));
setEvaluationPrompt(filled);
setPreviewVisible(true);
};
@@ -243,6 +278,13 @@ const CreateTaskModal: React.FC<CreateTaskModalProps> = ({ visible, onCancel, on
evalMethod: DEFAULT_EVAL_METHOD,
taskType: DEFAULT_TASK_TYPE,
}}
onValuesChange={(changed) => {
if (changed.taskType) {
setTaskType(changed.taskType);
setEvaluationPrompt('');
setPreviewVisible(false);
}
}}
>
<Row gutter={16}>
<Col span={12}>