q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: prevent deletion of predefined operators and improve error handling (#192)

Browse Source 
* fix: prevent deletion of predefined operators and improve error handling

* fix: prevent deletion of predefined operators and improve error handling
This commit is contained in:
hhhhsc701
2025-12-22 19:30:41 +08:00
committed by GitHub
parent c1516c87b6
commit d82bff441a
15 changed files with 98 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
runtime/ops/filter/remove_duplicate_file/process.py
View File

@@ -118,7 +118,7 @@ class DuplicateFilesFilter(Filter):
query_sql = self.sql_dict.get("query_sql")
for i in range(0, total_count, self.page_size):
rows = connection.execute(
text(query_sql), {"task_uuid": self.task_uuid, "ge": self.page_size, "le": i}).fetchall()
text(query_sql), {"task_uuid": self.task_uuid, "file_name": file_name, "ge": self.page_size, "le": i}).fetchall()
# 对应任务uuid，最后一页没有数据，跳出循环
if not rows:
break

2
runtime/ops/filter/remove_duplicate_file/sql/sql_config.json
View File

@@ -1,5 +1,5 @@
{
"query_sql": "SELECT * FROM operators_similar_text_features WHERE task_uuid = :task_uuid ORDER BY timestamp LIMIT :ge OFFSET :le",
"query_sql": "SELECT * FROM operators_similar_text_features WHERE task_uuid = :task_uuid AND file_name != :file_name ORDER BY timestamp LIMIT :ge OFFSET :le",
"create_tables_sql": "CREATE TABLE IF NOT EXISTS operators_similar_text_features (id INT AUTO_INCREMENT PRIMARY KEY, task_uuid VARCHAR(255),file_feature TEXT,file_name TEXT,timestamp DATETIME);",
"insert_sql": "INSERT INTO operators_similar_text_features (task_uuid, file_feature, file_name, timestamp) VALUES (:task_uuid, :file_feature, :file_name, :timestamp)",
"query_task_uuid_sql": "SELECT * FROM operators_similar_text_features WHERE task_uuid = :task_uuid"