refactor(clickhouse): 优化统计查询SQL性能和代码结构

- 提取进入景区trace_id子查询逻辑到独立方法appendEnterScenicTraceIdSubQuery
- 将count函数替换为uniqExact以提高去重统计性能
- 优化视频预览统计查询，使用WITH子句提取JSON字段减少重复计算
- 简化经纪人ID列表查询，移除不必要的子查询包装
- 修复每日扫码统计查询的时间范围过滤条件
- 优化按小时和按日期的扫码会员图表查询，使用ClickHouse内置时间函数
- 在子查询中添加时间范围过滤以减少数据扫描量

src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java

@@ -61,27 +61,43 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
         return date != null ? "'" + DATE_FORMAT.format(date) + "'" : null;
     }
 
+    /**
+     * 拼接“进入景区”的 trace_id 子查询。
+     * <p>
+     * ClickHouse 上 t_stats_record 往往按时间分区/排序；给子查询补充时间范围可显著减少扫描量。
+     */
+    private void appendEnterScenicTraceIdSubQuery(StringBuilder sql, Long scenicId, Date startTime, Date endTime) {
+        sql.append("SELECT DISTINCT trace_id FROM t_stats_record ");
+        sql.append("WHERE action = 'ENTER_SCENIC' ");
+        if (scenicId != null) {
+            sql.append("AND identifier = '").append(scenicId).append("' ");
+        }
+        if (startTime != null) {
+            sql.append("AND create_time >= ").append(formatDateTime(startTime)).append(" ");
+        }
+        if (endTime != null) {
+            sql.append("AND create_time <= ").append(formatDateTime(endTime)).append(" ");
+        }
+    }
+
     @Override
     public Integer countPreviewVideoOfMember(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
+        sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
-        sql.append("  SELECT 1 FROM t_stats_record r ");
+        sql.append("FROM t_stats_record r ");
-        sql.append("  INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+        sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("  WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("    SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("    WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
+        sql.append(") ");
-        sql.append("  ) ");
+        sql.append("AND r.action = 'LOAD' ");
-        sql.append("  AND r.action = 'LOAD' ");
+        sql.append("AND r.identifier = 'pages/videoSynthesis/buy' ");
-        sql.append("  AND r.identifier = 'pages/videoSynthesis/buy' ");
+        sql.append("AND JSONExtractString(r.params, 'share') = '' ");
-        sql.append("  AND JSONExtractString(r.params, 'share') = '' ");
         if (query.getStartTime() != null) {
-            sql.append("  AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+            sql.append("AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
         }
         if (query.getEndTime() != null) {
-            sql.append("  AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+            sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
         }
-        sql.append("  GROUP BY s.member_id ");
-        sql.append(")");
 
         return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
     }
@@ -89,23 +105,20 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public Integer countScanCodeOfMember(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
+        sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
-        sql.append("  SELECT 1 FROM t_stats_record r ");
+        sql.append("FROM t_stats_record r ");
-        sql.append("  INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+        sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("  WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("    SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("    WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
+        sql.append(") ");
-        sql.append("  ) ");
+        sql.append("AND r.action = 'LAUNCH' ");
-        sql.append("  AND r.action = 'LAUNCH' ");
+        sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
-        sql.append("  AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
         if (query.getStartTime() != null) {
-            sql.append("  AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+            sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
         }
         if (query.getEndTime() != null) {
-            sql.append("  AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+            sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
         }
-        sql.append("  GROUP BY s.member_id ");
-        sql.append(")");
 
         return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
     }
@@ -113,23 +126,20 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public Integer countPushOfMember(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
+        sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
-        sql.append("  SELECT 1 FROM t_stats_record r ");
+        sql.append("FROM t_stats_record r ");
-        sql.append("  INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+        sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("  WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("    SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("    WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
+        sql.append(") ");
-        sql.append("  ) ");
+        sql.append("AND r.action = 'PERM_REQ' ");
-        sql.append("  AND r.action = 'PERM_REQ' ");
+        sql.append("AND r.identifier = 'NOTIFY' ");
-        sql.append("  AND r.identifier = 'NOTIFY' ");
         if (query.getStartTime() != null) {
-            sql.append("  AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+            sql.append("AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
         }
         if (query.getEndTime() != null) {
-            sql.append("  AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+            sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
         }
-        sql.append("  GROUP BY s.member_id ");
-        sql.append(")");
 
         return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
     }
@@ -137,22 +147,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public Integer countUploadFaceOfMember(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
+        sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
-        sql.append("  SELECT 1 FROM t_stats_record r ");
+        sql.append("FROM t_stats_record r ");
-        sql.append("  INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+        sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("  WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("    SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("    WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
+        sql.append(") ");
-        sql.append("  ) ");
+        sql.append("AND r.action = 'FACE_UPLOAD' ");
-        sql.append("  AND r.action = 'FACE_UPLOAD' ");
         if (query.getStartTime() != null) {
-            sql.append("  AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+            sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
         }
         if (query.getEndTime() != null) {
-            sql.append("  AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+            sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
         }
-        sql.append("  GROUP BY s.member_id ");
-        sql.append(")");
 
         return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
     }
@@ -161,9 +168,8 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
         StringBuilder sql = new StringBuilder();
         sql.append("SELECT DISTINCT r.identifier FROM t_stats_record r ");
         sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("  SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("  WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
         sql.append(") ");
         sql.append("AND r.action = 'FACE_UPLOAD' ");
         if (query.getStartTime() != null) {
@@ -197,22 +203,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public Integer countTotalVisitorOfMember(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
+        sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
-        sql.append("  SELECT 1 FROM t_stats_record r ");
+        sql.append("FROM t_stats_record r ");
-        sql.append("  INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+        sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("  WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("    SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("    WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
+        sql.append(") ");
-        sql.append("  ) ");
+        sql.append("AND r.action = 'LAUNCH' ");
-        sql.append("  AND r.action = 'LAUNCH' ");
         if (query.getStartTime() != null) {
-            sql.append("  AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+            sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
         }
         if (query.getEndTime() != null) {
-            sql.append("  AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+            sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
         }
-        sql.append("  GROUP BY s.member_id ");
-        sql.append(")");
 
         return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
     }
@@ -220,25 +223,24 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public Integer countPreviewOfVideo(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
+        sql.append("WITH JSONExtractString(params, 'id') AS videoId, ");
-        sql.append("  SELECT 1 FROM t_stats_record r ");
+        sql.append("     JSONExtractString(params, 'share') AS share ");
-        sql.append("  INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+        sql.append("SELECT toInt32(uniqExact(videoId)) AS count ");
-        sql.append("  WHERE r.trace_id IN ( ");
+        sql.append("FROM t_stats_record r ");
-        sql.append("    SELECT trace_id FROM t_stats_record ");
+        sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("    WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("  ) ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("  AND r.action = 'LOAD' ");
+        sql.append(") ");
-        sql.append("  AND r.identifier = 'pages/videoSynthesis/buy' ");
+        sql.append("AND r.action = 'LOAD' ");
-        sql.append("  AND JSONExtractString(r.params, 'id') != '' ");
+        sql.append("AND r.identifier = 'pages/videoSynthesis/buy' ");
-        sql.append("  AND JSONExtractString(r.params, 'share') = '' ");
+        sql.append("AND videoId != '' ");
+        sql.append("AND share = '' ");
         if (query.getStartTime() != null) {
-            sql.append("  AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+            sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
         }
         if (query.getEndTime() != null) {
-            sql.append("  AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+            sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
         }
-        sql.append("  GROUP BY JSONExtractString(r.params, 'id') ");
-        sql.append(")");
 
         return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
     }
@@ -246,20 +248,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT toInt64(sub.identifier) AS identifier FROM ( ");
+        sql.append("SELECT toInt64(r.identifier) AS identifier ");
-        sql.append("  SELECT identifier, max(r.create_time) AS createTime ");
+        sql.append("FROM t_stats_record r ");
-        sql.append("  FROM t_stats_record r ");
+        sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("  INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+        sql.append("WHERE r.action = 'CODE_SCAN' ");
-        sql.append("  WHERE r.action = 'CODE_SCAN' ");
+        sql.append("  AND s.member_id = ").append(memberId).append(" ");
-        sql.append("    AND s.member_id = ").append(memberId).append(" ");
         if (startTime != null) {
-            sql.append("    AND r.create_time >= ").append(formatDateTime(startTime)).append(" ");
+            sql.append("  AND r.create_time >= ").append(formatDateTime(startTime)).append(" ");
         }
         if (endTime != null) {
-            sql.append("    AND r.create_time <= ").append(formatDateTime(endTime)).append(" ");
+            sql.append("  AND r.create_time <= ").append(formatDateTime(endTime)).append(" ");
         }
-        sql.append("  GROUP BY identifier ");
+        sql.append("GROUP BY r.identifier ");
-        sql.append(") sub ORDER BY sub.createTime DESC");
+        sql.append("ORDER BY max(r.create_time) DESC");
 
         return getJdbcTemplate().queryForList(sql.toString(), Long.class);
     }
@@ -311,13 +312,17 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) {
         String startDateStr = DATE_FORMAT.format(startTime);
         String endDateStr = DATE_FORMAT.format(endTime);
+        String startDateTimeStr = "'" + startDateStr + " 00:00:00'";
+        String endDateTimeStr = "'" + endDateStr + " 23:59:59'";
 
         String sql = "SELECT toDate(create_time) AS date, count(DISTINCT id) AS scanCount " +
                      "FROM t_stats_record " +
                      "WHERE action = 'CODE_SCAN' " +
                      "  AND identifier = '" + brokerId + "' " +
-                     "  AND toDate(create_time) BETWEEN '" + startDateStr + "' AND '" + endDateStr + "' " +
+                     "  AND create_time >= " + startDateTimeStr + " " +
-                     "GROUP BY toDate(create_time)";
+                     "  AND create_time <= " + endDateTimeStr + " " +
+                     "GROUP BY toDate(create_time) " +
+                     "ORDER BY toDate(create_time)";
 
         return getJdbcTemplate().query(sql, (rs, rowNum) -> {
             HashMap<String, Object> map = new HashMap<>();
@@ -330,22 +335,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT formatDateTime(s.create_time, '%m-%d %H') AS t, ");
+        sql.append("SELECT formatDateTime(toStartOfHour(s.create_time), '%m-%d %H') AS t, ");
-        sql.append("       count(DISTINCT s.member_id) AS count ");
+        sql.append("       uniqExact(s.member_id) AS count ");
         sql.append("FROM t_stats_record r ");
         sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("  SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("  WHERE action = 'ENTER_SCENIC' ");
-        if (query.getScenicId() != null) {
-            sql.append("  AND identifier = '").append(query.getScenicId()).append("' ");
-        }
         sql.append(") ");
         sql.append("AND r.action = 'LAUNCH' ");
         sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
         sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime()));
         sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" ");
-        sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d %H')");
+        sql.append("GROUP BY toStartOfHour(s.create_time) ");
+        sql.append("ORDER BY toStartOfHour(s.create_time)");
 
         return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
             HashMap<String, String> map = new HashMap<>();
@@ -358,22 +360,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
     @Override
     public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) {
         StringBuilder sql = new StringBuilder();
-        sql.append("SELECT formatDateTime(s.create_time, '%m-%d') AS t, ");
+        sql.append("SELECT formatDateTime(toStartOfDay(s.create_time), '%m-%d') AS t, ");
-        sql.append("       count(DISTINCT s.member_id) AS count ");
+        sql.append("       uniqExact(s.member_id) AS count ");
         sql.append("FROM t_stats_record r ");
         sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
-        sql.append("WHERE r.trace_id IN ( ");
+        sql.append("WHERE r.trace_id IN (");
-        sql.append("  SELECT trace_id FROM t_stats_record ");
+        appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
-        sql.append("  WHERE action = 'ENTER_SCENIC' ");
-        if (query.getScenicId() != null) {
-            sql.append("  AND identifier = '").append(query.getScenicId()).append("' ");
-        }
         sql.append(") ");
         sql.append("AND r.action = 'LAUNCH' ");
         sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
         sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime()));
         sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" ");
-        sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d')");
+        sql.append("GROUP BY toStartOfDay(s.create_time) ");
+        sql.append("ORDER BY toStartOfDay(s.create_time)");
 
         return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
             HashMap<String, String> map = new HashMap<>();