You've already forked DataMate
init datamate
This commit is contained in:
Dallas98
@@ -0,0 +1,23 @@
|
||||
datamate:
|
||||
data-collection:
|
||||
# DataX配置
|
||||
datax:
|
||||
home-path: ${DATAX_HOME:D:/datax}
|
||||
python-path: ${DATAX_PYTHON_PATH:python3}
|
||||
job-config-path: ${DATAX_JOB_PATH:./data/temp/datax/jobs}
|
||||
log-path: ${DATAX_LOG_PATH:./logs/datax}
|
||||
max-memory: ${DATAX_MAX_MEMORY:2048}
|
||||
channel-count: ${DATAX_CHANNEL_COUNT:5}
|
||||
|
||||
# 执行配置
|
||||
execution:
|
||||
max-concurrent-tasks: ${DATA_COLLECTION_MAX_CONCURRENT_TASKS:10}
|
||||
task-timeout-minutes: ${DATA_COLLECTION_TASK_TIMEOUT:120}
|
||||
retry-count: ${DATA_COLLECTION_RETRY_COUNT:3}
|
||||
retry-interval-seconds: ${DATA_COLLECTION_RETRY_INTERVAL:30}
|
||||
|
||||
# 监控配置
|
||||
monitoring:
|
||||
status-check-interval-seconds: ${DATA_COLLECTION_STATUS_CHECK_INTERVAL:30}
|
||||
log-retention-days: ${DATA_COLLECTION_LOG_RETENTION:30}
|
||||
enable-metrics: ${DATA_COLLECTION_ENABLE_METRICS:true}
|
||||
@@ -0,0 +1,188 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
|
||||
<mapper namespace="com.datamate.collection.infrastructure.persistence.mapper.CollectionTaskMapper">
|
||||
|
||||
<!-- Result Map -->
|
||||
<resultMap id="CollectionTaskResultMap" type="com.datamate.collection.domain.model.CollectionTask">
|
||||
<id property="id" column="id"/>
|
||||
<result property="name" column="name"/>
|
||||
<result property="description" column="description"/>
|
||||
<result property="config" column="config"/>
|
||||
<result property="status" column="status" typeHandler="org.apache.ibatis.type.EnumTypeHandler"/>
|
||||
<result property="syncMode" column="sync_mode"/>
|
||||
<result property="scheduleExpression" column="schedule_expression"/>
|
||||
<result property="retryCount" column="retry_count"/>
|
||||
<result property="timeoutSeconds" column="timeout_seconds"/>
|
||||
<result property="maxRecords" column="max_records"/>
|
||||
<result property="sortField" column="sort_field"/>
|
||||
<result property="lastExecutionId" column="last_execution_id"/>
|
||||
<result property="createdAt" column="created_at"/>
|
||||
<result property="updatedAt" column="updated_at"/>
|
||||
<result property="createdBy" column="created_by"/>
|
||||
<result property="updatedBy" column="updated_by"/>
|
||||
</resultMap>
|
||||
|
||||
<!-- 结果映射 (模板) -->
|
||||
<resultMap id="DataxTemplateResultMap" type="com.datamate.collection.domain.model.DataxTemplate">
|
||||
<id column="id" property="id" jdbcType="VARCHAR"/>
|
||||
<result column="name" property="name" jdbcType="VARCHAR"/>
|
||||
<result column="source_type" property="sourceType" jdbcType="VARCHAR"/>
|
||||
<result column="target_type" property="targetType" jdbcType="VARCHAR"/>
|
||||
<result column="template_content" property="templateContent" jdbcType="VARCHAR"/>
|
||||
<result column="description" property="description" jdbcType="VARCHAR"/>
|
||||
<result column="version" property="version" jdbcType="VARCHAR"/>
|
||||
<result column="is_system" property="isSystem" jdbcType="BOOLEAN"/>
|
||||
<result column="created_at" property="createdAt" jdbcType="TIMESTAMP"/>
|
||||
<result column="updated_at" property="updatedAt" jdbcType="TIMESTAMP"/>
|
||||
<result column="created_by" property="createdBy" jdbcType="VARCHAR"/>
|
||||
</resultMap>
|
||||
|
||||
<!-- Base Column List (tasks) -->
|
||||
<sql id="Base_Column_List">
|
||||
id,
|
||||
name, description, config, status, sync_mode,
|
||||
schedule_expression, retry_count, timeout_seconds, max_records, sort_field,
|
||||
last_execution_id, created_at, updated_at, created_by, updated_by
|
||||
</sql>
|
||||
|
||||
<!-- Template Column List -->
|
||||
<sql id="Template_Column_List">
|
||||
id, name, source_type, target_type, template_content, description, version, is_system, created_at, updated_at, created_by
|
||||
</sql>
|
||||
|
||||
<!-- Insert -->
|
||||
<insert id="insert" parameterType="com.datamate.collection.domain.model.CollectionTask">
|
||||
INSERT INTO t_dc_collection_tasks (id, name, description, config, status, sync_mode,
|
||||
schedule_expression, retry_count, timeout_seconds, max_records, sort_field,
|
||||
last_execution_id, created_at, updated_at, created_by, updated_by)
|
||||
VALUES (#{id}, #{name}, #{description}, #{config}, #{status}, #{syncMode},
|
||||
#{scheduleExpression}, #{retryCount}, #{timeoutSeconds}, #{maxRecords}, #{sortField},
|
||||
#{lastExecutionId}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy})
|
||||
</insert>
|
||||
|
||||
<!-- Update -->
|
||||
<update id="update" parameterType="com.datamate.collection.domain.model.CollectionTask">
|
||||
UPDATE t_dc_collection_tasks
|
||||
SET name = #{name},
|
||||
description = #{description},
|
||||
config = #{config},
|
||||
status = #{status},
|
||||
sync_mode = #{syncMode},
|
||||
schedule_expression = #{scheduleExpression},
|
||||
retry_count = #{retryCount},
|
||||
timeout_seconds = #{timeoutSeconds},
|
||||
max_records = #{maxRecords},
|
||||
sort_field = #{sortField},
|
||||
last_execution_id = #{lastExecutionId},
|
||||
updated_at = #{updatedAt},
|
||||
updated_by = #{updatedBy}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Delete by ID -->
|
||||
<delete id="deleteById" parameterType="java.lang.String">
|
||||
DELETE FROM t_dc_collection_tasks WHERE id = #{id}
|
||||
</delete>
|
||||
|
||||
<!-- Select by ID -->
|
||||
<select id="selectById" parameterType="java.lang.String" resultMap="CollectionTaskResultMap">
|
||||
SELECT <include refid="Base_Column_List"/> FROM t_dc_collection_tasks WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<!-- Select by Name -->
|
||||
<select id="selectByName" parameterType="java.lang.String" resultMap="CollectionTaskResultMap">
|
||||
SELECT <include refid="Base_Column_List"/> FROM t_dc_collection_tasks WHERE name = #{name}
|
||||
</select>
|
||||
|
||||
<!-- Select by Status -->
|
||||
<select id="selectByStatus" parameterType="java.lang.String" resultMap="CollectionTaskResultMap">
|
||||
SELECT <include refid="Base_Column_List"/> FROM t_dc_collection_tasks WHERE status = #{status} ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<!-- Select All with Pagination -->
|
||||
<select id="selectAll" resultMap="CollectionTaskResultMap">
|
||||
SELECT <include refid="Base_Column_List"/> FROM t_dc_collection_tasks
|
||||
<where>
|
||||
<if test="status != null and status != ''">
|
||||
AND status = #{status}
|
||||
</if>
|
||||
<if test="name != null and name != ''">
|
||||
AND name LIKE CONCAT('%', #{name}, '%')
|
||||
</if>
|
||||
</where>
|
||||
ORDER BY created_at DESC
|
||||
<if test="offset != null and limit != null">
|
||||
LIMIT #{offset}, #{limit}
|
||||
</if>
|
||||
</select>
|
||||
|
||||
<!-- Count Total -->
|
||||
<select id="count" resultType="java.lang.Long">
|
||||
SELECT COUNT(*) FROM t_dc_collection_tasks
|
||||
<where>
|
||||
<if test="status != null and status != ''">
|
||||
AND status = #{status}
|
||||
</if>
|
||||
<if test="name != null and name != ''">
|
||||
AND name LIKE CONCAT('%', #{name}, '%')
|
||||
</if>
|
||||
<if test="sourceDataSourceId != null and sourceDataSourceId != ''">
|
||||
AND source_datasource_id = #{sourceDataSourceId}
|
||||
</if>
|
||||
<if test="targetDataSourceId != null and targetDataSourceId != ''">
|
||||
AND target_datasource_id = #{targetDataSourceId}
|
||||
</if>
|
||||
</where>
|
||||
</select>
|
||||
|
||||
<!-- Update Status -->
|
||||
<update id="updateStatus">
|
||||
UPDATE t_dc_collection_tasks SET status = #{status}, updated_at = NOW() WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Update Last Execution -->
|
||||
<update id="updateLastExecution">
|
||||
UPDATE t_dc_collection_tasks SET last_execution_id = #{lastExecutionId}, updated_at = NOW() WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Select Active Tasks for Scheduling -->
|
||||
<select id="selectActiveTasks" resultMap="CollectionTaskResultMap">
|
||||
SELECT <include refid="Base_Column_List"/> FROM t_dc_collection_tasks
|
||||
WHERE status IN ('READY', 'RUNNING')
|
||||
AND schedule_expression IS NOT NULL
|
||||
ORDER BY created_at DESC
|
||||
</select>
|
||||
|
||||
<!-- 查询模板列表 -->
|
||||
<select id="selectList" resultMap="DataxTemplateResultMap">
|
||||
SELECT <include refid="Template_Column_List"/> FROM t_dc_datax_templates
|
||||
<where>
|
||||
<if test="sourceType != null and sourceType != ''">
|
||||
AND source_type = #{sourceType}
|
||||
</if>
|
||||
<if test="targetType != null and targetType != ''">
|
||||
AND target_type = #{targetType}
|
||||
</if>
|
||||
</where>
|
||||
ORDER BY is_system DESC, created_at DESC
|
||||
<if test="limit > 0">
|
||||
LIMIT #{offset}, #{limit}
|
||||
</if>
|
||||
</select>
|
||||
|
||||
<!-- 统计模板数量 -->
|
||||
<select id="countTemplates" resultType="java.lang.Integer">
|
||||
SELECT COUNT(1) FROM t_dc_datax_templates
|
||||
<where>
|
||||
<if test="sourceType != null and sourceType != ''">
|
||||
AND source_type = #{sourceType}
|
||||
</if>
|
||||
<if test="targetType != null and targetType != ''">
|
||||
AND target_type = #{targetType}
|
||||
</if>
|
||||
</where>
|
||||
</select>
|
||||
|
||||
</mapper>
|
||||
@@ -0,0 +1,191 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
|
||||
<mapper namespace="com.datamate.collection.infrastructure.persistence.mapper.TaskExecutionMapper">
|
||||
|
||||
<!-- Result Map -->
|
||||
<resultMap id="TaskExecutionResultMap" type="com.datamate.collection.domain.model.TaskExecution">
|
||||
<id property="id" column="id"/>
|
||||
<result property="taskId" column="task_id"/>
|
||||
<result property="taskName" column="task_name"/>
|
||||
<result property="status" column="status" typeHandler="org.apache.ibatis.type.EnumTypeHandler"/>
|
||||
<result property="progress" column="progress"/>
|
||||
<result property="recordsTotal" column="records_total"/>
|
||||
<result property="recordsProcessed" column="records_processed"/>
|
||||
<result property="recordsSuccess" column="records_success"/>
|
||||
<result property="recordsFailed" column="records_failed"/>
|
||||
<result property="throughput" column="throughput"/>
|
||||
<result property="dataSizeBytes" column="data_size_bytes"/>
|
||||
<result property="startedAt" column="started_at"/>
|
||||
<result property="completedAt" column="completed_at"/>
|
||||
<result property="durationSeconds" column="duration_seconds"/>
|
||||
<result property="errorMessage" column="error_message"/>
|
||||
<result property="dataxJobId" column="datax_job_id"/>
|
||||
<result property="config" column="config"/>
|
||||
<result property="result" column="result"/>
|
||||
<result property="createdAt" column="created_at"/>
|
||||
</resultMap>
|
||||
|
||||
<!-- Base Column List -->
|
||||
<sql id="Base_Column_List">
|
||||
id, task_id, task_name, status, progress, records_total, records_processed,
|
||||
records_success, records_failed, throughput, data_size_bytes, started_at,
|
||||
completed_at, duration_seconds, error_message, datax_job_id, config, result, created_at
|
||||
</sql>
|
||||
|
||||
<!-- Insert -->
|
||||
<insert id="insert" parameterType="com.datamate.collection.domain.model.TaskExecution">
|
||||
INSERT INTO t_dc_task_executions (
|
||||
id, task_id, task_name, status, progress, records_total, records_processed,
|
||||
records_success, records_failed, throughput, data_size_bytes, started_at,
|
||||
completed_at, duration_seconds, error_message, datax_job_id, config, result, created_at
|
||||
) VALUES (
|
||||
#{id}, #{taskId}, #{taskName}, #{status}, #{progress}, #{recordsTotal}, #{recordsProcessed},
|
||||
#{recordsSuccess}, #{recordsFailed}, #{throughput}, #{dataSizeBytes}, #{startedAt},
|
||||
#{completedAt}, #{durationSeconds}, #{errorMessage}, #{dataxJobId}, #{config}, #{result}, #{createdAt}
|
||||
)
|
||||
</insert>
|
||||
|
||||
<!-- Update -->
|
||||
<update id="update" parameterType="com.datamate.collection.domain.model.TaskExecution">
|
||||
UPDATE t_dc_task_executions
|
||||
SET status = #{status},
|
||||
progress = #{progress},
|
||||
records_total = #{recordsTotal},
|
||||
records_processed = #{recordsProcessed},
|
||||
records_success = #{recordsSuccess},
|
||||
records_failed = #{recordsFailed},
|
||||
throughput = #{throughput},
|
||||
data_size_bytes = #{dataSizeBytes},
|
||||
completed_at = #{completedAt},
|
||||
duration_seconds = #{durationSeconds},
|
||||
error_message = #{errorMessage},
|
||||
result = #{result}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Delete by ID -->
|
||||
<delete id="deleteById" parameterType="java.lang.String">
|
||||
DELETE FROM t_dc_task_executions WHERE id = #{id}
|
||||
</delete>
|
||||
|
||||
<!-- Select by ID -->
|
||||
<select id="selectById" parameterType="java.lang.String" resultMap="TaskExecutionResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dc_task_executions
|
||||
WHERE id = #{id}
|
||||
</select>
|
||||
|
||||
<!-- Select by Task ID -->
|
||||
<select id="selectByTaskId" resultMap="TaskExecutionResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dc_task_executions
|
||||
WHERE task_id = #{taskId}
|
||||
ORDER BY started_at DESC
|
||||
<if test="limit != null">
|
||||
LIMIT #{limit}
|
||||
</if>
|
||||
</select>
|
||||
|
||||
<!-- Select by Status -->
|
||||
<select id="selectByStatus" parameterType="java.lang.String" resultMap="TaskExecutionResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dc_task_executions
|
||||
WHERE status = #{status}
|
||||
ORDER BY started_at DESC
|
||||
</select>
|
||||
|
||||
<!-- Select All with Pagination -->
|
||||
<select id="selectAll" resultMap="TaskExecutionResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dc_task_executions
|
||||
<where>
|
||||
<if test="taskId != null and taskId != ''">
|
||||
AND task_id = #{taskId}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND status = #{status}
|
||||
</if>
|
||||
<if test="startDate != null">
|
||||
AND started_at >= #{startDate}
|
||||
</if>
|
||||
<if test="endDate != null">
|
||||
AND started_at <= #{endDate}
|
||||
</if>
|
||||
</where>
|
||||
ORDER BY started_at DESC
|
||||
<if test="offset != null and limit != null">
|
||||
LIMIT #{offset}, #{limit}
|
||||
</if>
|
||||
</select>
|
||||
|
||||
<!-- Count Total -->
|
||||
<select id="count" resultType="java.lang.Long">
|
||||
SELECT COUNT(*)
|
||||
FROM t_dc_task_executions
|
||||
<where>
|
||||
<if test="taskId != null and taskId != ''">
|
||||
AND task_id = #{taskId}
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
AND status = #{status}
|
||||
</if>
|
||||
<if test="startDate != null">
|
||||
AND started_at >= #{startDate}
|
||||
</if>
|
||||
<if test="endDate != null">
|
||||
AND started_at <= #{endDate}
|
||||
</if>
|
||||
</where>
|
||||
</select>
|
||||
|
||||
<!-- Update Status and Progress -->
|
||||
<update id="updateProgress">
|
||||
UPDATE t_dc_task_executions
|
||||
SET status = #{status},
|
||||
progress = #{progress},
|
||||
records_processed = #{recordsProcessed},
|
||||
throughput = #{throughput}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Complete Execution -->
|
||||
<update id="completeExecution">
|
||||
UPDATE t_dc_task_executions
|
||||
SET status = #{status},
|
||||
progress = 100.00,
|
||||
completed_at = #{completedAt},
|
||||
duration_seconds = #{durationSeconds},
|
||||
records_success = #{recordsSuccess},
|
||||
records_failed = #{recordsFailed},
|
||||
data_size_bytes = #{dataSizeBytes},
|
||||
error_message = #{errorMessage},
|
||||
result = #{result}
|
||||
WHERE id = #{id}
|
||||
</update>
|
||||
|
||||
<!-- Select Running Executions -->
|
||||
<select id="selectRunningExecutions" resultMap="TaskExecutionResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dc_task_executions
|
||||
WHERE status = 'RUNNING'
|
||||
ORDER BY started_at ASC
|
||||
</select>
|
||||
|
||||
<!-- Select Latest Execution by Task -->
|
||||
<select id="selectLatestByTaskId" parameterType="java.lang.String" resultMap="TaskExecutionResultMap">
|
||||
SELECT <include refid="Base_Column_List"/>
|
||||
FROM t_dc_task_executions
|
||||
WHERE task_id = #{taskId}
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1
|
||||
</select>
|
||||
|
||||
<!-- Delete Old Executions -->
|
||||
<delete id="deleteOldExecutions">
|
||||
DELETE FROM t_dc_task_executions
|
||||
WHERE started_at < #{beforeDate}
|
||||
</delete>
|
||||
|
||||
</mapper>
|
||||
Reference in New Issue
Block a user