q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

配置文件更改 (#186)

Browse Source 
* feat(generation_service): add image URL extraction and random QA generation logic

* fix(generation_service): increase batch size from 20 to 100 for improved chunk processing

* fix(generation_service): increase batch size from 20 to 100 for improved chunk processing
This commit is contained in:
Dallas98
2025-12-22 09:29:00 +08:00
committed by GitHub
parent 85eb5a99ba
commit 8fc4455b57
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
runtime/datamate-python/app/module/generation/service/generation_service.py
View File

@@ -49,7 +49,7 @@ class GenerationService:
def __init__(self, db: AsyncSession): def __init__(self, db: AsyncSession):
self.db = db self.db = db
# 全局并发信号量：保证任意时刻最多 10 次模型调用 # 全局并发信号量：保证任意时刻最多 10 次模型调用
self.question_semaphore = asyncio.Semaphore(10) self.question_semaphore = asyncio.Semaphore(20)
self.answer_semaphore = asyncio.Semaphore(100) self.answer_semaphore = asyncio.Semaphore(100)
async def process_task(self, task_id: str): async def process_task(self, task_id: str):
@@ -175,7 +175,7 @@ class GenerationService:
answer_chat = get_chat_client(answer_model) answer_chat = get_chat_client(answer_model)
# 分批次从 DB 读取并处理 chunk # 分批次从 DB 读取并处理 chunk
batch_size = 20 batch_size = 100
current_index = 1 current_index = 1
while current_index <= total_chunks: while current_index <= total_chunks: