feat: 支持运行data-juicer算子 (#215)

* feature: 增加data-juicer算子

* feat: 支持运行data-juicer算子

* feat: 支持data-juicer任务下发

* feat: 支持data-juicer结果数据集归档

* feat: 支持data-juicer结果数据集归档
This commit is contained in:
hhhhsc701
2025-12-31 09:20:41 +08:00
committed by GitHub
parent 63f4e3e447
commit 6a1eb85e8e
26 changed files with 709 additions and 120 deletions

View File

@@ -1,6 +1,7 @@
from . import data_juicer_wrapper, datamate_wrapper
WRAPPERS = {
"data_juicer": data_juicer_wrapper,
"ray": data_juicer_wrapper,
"default": data_juicer_wrapper,
"datamate": datamate_wrapper
}