q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add export type settings and enhance metadata structure (#181)

Browse Source 
* fix(session): enhance database connection settings with pool pre-ping and recycle options

* feat(metadata): add export type settings and enhance metadata structure

* fix(base_op): improve sample handling by introducing target_type key and consolidating text/data retrieval logic

* feat(metadata): add export type settings and enhance metadata structure

* feat(metadata): add export type settings and enhance metadata structure
This commit is contained in:
hhhhsc701
2025-12-19 11:54:08 +08:00
committed by GitHub
parent d70a3eda0d
commit ab4523b556
6 changed files with 57 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
runtime/python-executor/datamate/core/base_op.py
View File

@@ -72,6 +72,7 @@ class BaseOp:
self.filesize_key = kwargs.get('fileSize_key', "fileSize")
self.export_path_key = kwargs.get('export_path_key', "export_path")
self.ext_params_key = kwargs.get('ext_params_key', "ext_params")
self.target_type_key = kwargs.get('target_type_key', "target_type")
@property
def name(self):
@@ -437,7 +438,7 @@ class FileExporter(BaseOp):
elif file_type in self.medical_support_ext:
sample, save_path = self.get_medicalfile_handler(sample)
else:
raise TypeError(f"{file_type} is unsupported! please check support_ext in FileExporter Ops")
return False
if sample[self.text_key] == '' and sample[self.data_key] == b'':
sample[self.filesize_key] = "0"
@@ -483,11 +484,11 @@ class FileExporter(BaseOp):
# target_type存在则保存为扫描件, docx格式
if target_type:
sample = self._get_from_data(sample)
sample = self._get_from_text_or_data(sample)
save_path = self.get_save_path(sample, target_type)
# 不存在则保存为txt文件，正常文本清洗
else:
sample = self._get_from_text(sample)
sample = self._get_from_text_or_data(sample)
save_path = self.get_save_path(sample, 'txt')
return sample, save_path
@@ -496,11 +497,11 @@ class FileExporter(BaseOp):
# target_type存在, 图转文保存为target_type，markdown格式
if target_type:
sample = self._get_from_text(sample)
sample = self._get_from_text_or_data(sample)
save_path = self.get_save_path(sample, target_type)
# 不存在则保存为原本图片文件格式，正常图片清洗
else:
sample = self._get_from_data(sample)
sample = self._get_from_text_or_data(sample)
save_path = self.get_save_path(sample, sample[self.filetype_key])
return sample, save_path
@@ -533,6 +534,13 @@ class FileExporter(BaseOp):
sample[self.text_key] = str(sample[self.text_key])
return sample
def _get_from_text_or_data(self, sample: Dict[str, Any]) -> Dict[str, Any]:
if sample[self.data_key] is not None and sample[self.data_key] != b'':
return self._get_from_data(sample)
else:
return self._get_from_text(sample)
@staticmethod
def _get_uuid():
return str(uuid.uuid4())