You've already forked FrameTour-BE
feat(clickhouse): 实现统计数据查询的时间序列填充功能
- 将日期时间处理从旧的 Date 和 SimpleDateFormat 迁移到新的 Java 8 时间 API - 添加小时级别数据序列填充功能,确保每个小时都有数据记录 - 添加日期级别数据序列填充功能,确保每天都有数据记录 - 实现缺失时间段的数据自动补零机制 - 重构查询方法以支持连续时间序列数据返回 - 提高统计图表数据完整性和可视化效果
This commit is contained in:
@@ -12,10 +12,13 @@ import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.Date;
|
import java.time.LocalDate;
|
||||||
import java.util.HashMap;
|
import java.time.LocalDateTime;
|
||||||
import java.util.List;
|
import java.time.ZoneId;
|
||||||
import java.util.TimeZone;
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.time.temporal.ChronoUnit;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ClickHouse 统计数据查询服务实现
|
* ClickHouse 统计数据查询服务实现
|
||||||
@@ -366,12 +369,14 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
|
|||||||
sql.append("GROUP BY toStartOfHour(s.create_time) ");
|
sql.append("GROUP BY toStartOfHour(s.create_time) ");
|
||||||
sql.append("ORDER BY toStartOfHour(s.create_time)");
|
sql.append("ORDER BY toStartOfHour(s.create_time)");
|
||||||
|
|
||||||
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
List<HashMap<String, String>> rawData = getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
||||||
HashMap<String, String> map = new HashMap<>();
|
HashMap<String, String> map = new HashMap<>();
|
||||||
map.put("t", rs.getString("t"));
|
map.put("t", rs.getString("t"));
|
||||||
map.put("count", rs.getString("count"));
|
map.put("count", rs.getString("count"));
|
||||||
return map;
|
return map;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return fillHourSeries(rawData, query.getStartTime(), query.getEndTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -389,12 +394,14 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
|
|||||||
sql.append("GROUP BY toStartOfDay(s.create_time) ");
|
sql.append("GROUP BY toStartOfDay(s.create_time) ");
|
||||||
sql.append("ORDER BY toStartOfDay(s.create_time)");
|
sql.append("ORDER BY toStartOfDay(s.create_time)");
|
||||||
|
|
||||||
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
List<HashMap<String, String>> rawData = getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
||||||
HashMap<String, String> map = new HashMap<>();
|
HashMap<String, String> map = new HashMap<>();
|
||||||
map.put("t", rs.getString("t"));
|
map.put("t", rs.getString("t"));
|
||||||
map.put("count", rs.getString("count"));
|
map.put("count", rs.getString("count"));
|
||||||
return map;
|
return map;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return fillDateSeries(rawData, query.getStartTime(), query.getEndTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -418,12 +425,14 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
|
|||||||
sql.append("GROUP BY toStartOfHour(s.create_time) ");
|
sql.append("GROUP BY toStartOfHour(s.create_time) ");
|
||||||
sql.append("ORDER BY toStartOfHour(s.create_time)");
|
sql.append("ORDER BY toStartOfHour(s.create_time)");
|
||||||
|
|
||||||
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
List<HashMap<String, String>> rawData = getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
||||||
HashMap<String, String> map = new HashMap<>();
|
HashMap<String, String> map = new HashMap<>();
|
||||||
map.put("t", rs.getString("t"));
|
map.put("t", rs.getString("t"));
|
||||||
map.put("count", rs.getString("count"));
|
map.put("count", rs.getString("count"));
|
||||||
return map;
|
return map;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return fillHourSeries(rawData, query.getStartTime(), query.getEndTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -447,11 +456,81 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
|
|||||||
sql.append("GROUP BY toStartOfDay(s.create_time) ");
|
sql.append("GROUP BY toStartOfDay(s.create_time) ");
|
||||||
sql.append("ORDER BY toStartOfDay(s.create_time)");
|
sql.append("ORDER BY toStartOfDay(s.create_time)");
|
||||||
|
|
||||||
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
List<HashMap<String, String>> rawData = getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
|
||||||
HashMap<String, String> map = new HashMap<>();
|
HashMap<String, String> map = new HashMap<>();
|
||||||
map.put("t", rs.getString("t"));
|
map.put("t", rs.getString("t"));
|
||||||
map.put("count", rs.getString("count"));
|
map.put("count", rs.getString("count"));
|
||||||
return map;
|
return map;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return fillDateSeries(rawData, query.getStartTime(), query.getEndTime());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 填充小时序列,确保每个小时都有数据(缺失的填充为0)
|
||||||
|
*/
|
||||||
|
private List<HashMap<String, String>> fillHourSeries(List<HashMap<String, String>> rawData, Date startTime, Date endTime) {
|
||||||
|
if (startTime == null || endTime == null) {
|
||||||
|
return rawData;
|
||||||
|
}
|
||||||
|
|
||||||
|
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd HH");
|
||||||
|
LocalDateTime start = startTime.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().truncatedTo(ChronoUnit.HOURS);
|
||||||
|
LocalDateTime end = endTime.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().truncatedTo(ChronoUnit.HOURS);
|
||||||
|
|
||||||
|
// 将原始数据转为 Map 以便快速查找
|
||||||
|
Map<String, String> dataMap = rawData.stream()
|
||||||
|
.collect(Collectors.toMap(
|
||||||
|
m -> m.get("t"),
|
||||||
|
m -> m.get("count"),
|
||||||
|
(existing, replacement) -> existing
|
||||||
|
));
|
||||||
|
|
||||||
|
List<HashMap<String, String>> result = new ArrayList<>();
|
||||||
|
LocalDateTime current = start;
|
||||||
|
while (!current.isAfter(end)) {
|
||||||
|
String timeKey = current.format(formatter);
|
||||||
|
HashMap<String, String> item = new HashMap<>();
|
||||||
|
item.put("t", timeKey);
|
||||||
|
item.put("count", dataMap.getOrDefault(timeKey, "0"));
|
||||||
|
result.add(item);
|
||||||
|
current = current.plusHours(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 填充日期序列,确保每天都有数据(缺失的填充为0)
|
||||||
|
*/
|
||||||
|
private List<HashMap<String, String>> fillDateSeries(List<HashMap<String, String>> rawData, Date startTime, Date endTime) {
|
||||||
|
if (startTime == null || endTime == null) {
|
||||||
|
return rawData;
|
||||||
|
}
|
||||||
|
|
||||||
|
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd");
|
||||||
|
LocalDate start = startTime.toInstant().atZone(ZoneId.systemDefault()).toLocalDate();
|
||||||
|
LocalDate end = endTime.toInstant().atZone(ZoneId.systemDefault()).toLocalDate();
|
||||||
|
|
||||||
|
// 将原始数据转为 Map 以便快速查找
|
||||||
|
Map<String, String> dataMap = rawData.stream()
|
||||||
|
.collect(Collectors.toMap(
|
||||||
|
m -> m.get("t"),
|
||||||
|
m -> m.get("count"),
|
||||||
|
(existing, replacement) -> existing
|
||||||
|
));
|
||||||
|
|
||||||
|
List<HashMap<String, String>> result = new ArrayList<>();
|
||||||
|
LocalDate current = start;
|
||||||
|
while (!current.isAfter(end)) {
|
||||||
|
String timeKey = current.format(formatter);
|
||||||
|
HashMap<String, String> item = new HashMap<>();
|
||||||
|
item.put("t", timeKey);
|
||||||
|
item.put("count", dataMap.getOrDefault(timeKey, "0"));
|
||||||
|
result.add(item);
|
||||||
|
current = current.plusDays(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user