diff --git a/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java b/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java index 7c5a2b48..df12d4da 100644 --- a/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java +++ b/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java @@ -61,27 +61,43 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { return date != null ? "'" + DATE_FORMAT.format(date) + "'" : null; } + /** + * 拼接“进入景区”的 trace_id 子查询。 + *
+ * ClickHouse 上 t_stats_record 往往按时间分区/排序;给子查询补充时间范围可显著减少扫描量。
+ */
+ private void appendEnterScenicTraceIdSubQuery(StringBuilder sql, Long scenicId, Date startTime, Date endTime) {
+ sql.append("SELECT DISTINCT trace_id FROM t_stats_record ");
+ sql.append("WHERE action = 'ENTER_SCENIC' ");
+ if (scenicId != null) {
+ sql.append("AND identifier = '").append(scenicId).append("' ");
+ }
+ if (startTime != null) {
+ sql.append("AND create_time >= ").append(formatDateTime(startTime)).append(" ");
+ }
+ if (endTime != null) {
+ sql.append("AND create_time <= ").append(formatDateTime(endTime)).append(" ");
+ }
+ }
+
@Override
public Integer countPreviewVideoOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
- sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
- sql.append(" SELECT 1 FROM t_stats_record r ");
- sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
- sql.append(" WHERE r.trace_id IN ( ");
- sql.append(" SELECT trace_id FROM t_stats_record ");
- sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
- sql.append(" ) ");
- sql.append(" AND r.action = 'LOAD' ");
- sql.append(" AND r.identifier = 'pages/videoSynthesis/buy' ");
- sql.append(" AND JSONExtractString(r.params, 'share') = '' ");
+ sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
+ sql.append("FROM t_stats_record r ");
+ sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+ sql.append("WHERE r.trace_id IN (");
+ appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
+ sql.append(") ");
+ sql.append("AND r.action = 'LOAD' ");
+ sql.append("AND r.identifier = 'pages/videoSynthesis/buy' ");
+ sql.append("AND JSONExtractString(r.params, 'share') = '' ");
if (query.getStartTime() != null) {
- sql.append(" AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+ sql.append("AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
- sql.append(" AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+ sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
- sql.append(" GROUP BY s.member_id ");
- sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -89,23 +105,20 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countScanCodeOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
- sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
- sql.append(" SELECT 1 FROM t_stats_record r ");
- sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
- sql.append(" WHERE r.trace_id IN ( ");
- sql.append(" SELECT trace_id FROM t_stats_record ");
- sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
- sql.append(" ) ");
- sql.append(" AND r.action = 'LAUNCH' ");
- sql.append(" AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
+ sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
+ sql.append("FROM t_stats_record r ");
+ sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+ sql.append("WHERE r.trace_id IN (");
+ appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
+ sql.append(") ");
+ sql.append("AND r.action = 'LAUNCH' ");
+ sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
if (query.getStartTime() != null) {
- sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+ sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
- sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+ sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
- sql.append(" GROUP BY s.member_id ");
- sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -113,23 +126,20 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countPushOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
- sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
- sql.append(" SELECT 1 FROM t_stats_record r ");
- sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
- sql.append(" WHERE r.trace_id IN ( ");
- sql.append(" SELECT trace_id FROM t_stats_record ");
- sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
- sql.append(" ) ");
- sql.append(" AND r.action = 'PERM_REQ' ");
- sql.append(" AND r.identifier = 'NOTIFY' ");
+ sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
+ sql.append("FROM t_stats_record r ");
+ sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+ sql.append("WHERE r.trace_id IN (");
+ appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
+ sql.append(") ");
+ sql.append("AND r.action = 'PERM_REQ' ");
+ sql.append("AND r.identifier = 'NOTIFY' ");
if (query.getStartTime() != null) {
- sql.append(" AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+ sql.append("AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
- sql.append(" AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+ sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
- sql.append(" GROUP BY s.member_id ");
- sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -137,22 +147,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countUploadFaceOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
- sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
- sql.append(" SELECT 1 FROM t_stats_record r ");
- sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
- sql.append(" WHERE r.trace_id IN ( ");
- sql.append(" SELECT trace_id FROM t_stats_record ");
- sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
- sql.append(" ) ");
- sql.append(" AND r.action = 'FACE_UPLOAD' ");
+ sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
+ sql.append("FROM t_stats_record r ");
+ sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+ sql.append("WHERE r.trace_id IN (");
+ appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
+ sql.append(") ");
+ sql.append("AND r.action = 'FACE_UPLOAD' ");
if (query.getStartTime() != null) {
- sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+ sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
- sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+ sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
- sql.append(" GROUP BY s.member_id ");
- sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -161,9 +168,8 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
StringBuilder sql = new StringBuilder();
sql.append("SELECT DISTINCT r.identifier FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
- sql.append("WHERE r.trace_id IN ( ");
- sql.append(" SELECT trace_id FROM t_stats_record ");
- sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
+ sql.append("WHERE r.trace_id IN (");
+ appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'FACE_UPLOAD' ");
if (query.getStartTime() != null) {
@@ -197,22 +203,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countTotalVisitorOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
- sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
- sql.append(" SELECT 1 FROM t_stats_record r ");
- sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
- sql.append(" WHERE r.trace_id IN ( ");
- sql.append(" SELECT trace_id FROM t_stats_record ");
- sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
- sql.append(" ) ");
- sql.append(" AND r.action = 'LAUNCH' ");
+ sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
+ sql.append("FROM t_stats_record r ");
+ sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+ sql.append("WHERE r.trace_id IN (");
+ appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
+ sql.append(") ");
+ sql.append("AND r.action = 'LAUNCH' ");
if (query.getStartTime() != null) {
- sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+ sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
- sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+ sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
- sql.append(" GROUP BY s.member_id ");
- sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -220,25 +223,24 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countPreviewOfVideo(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
- sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
- sql.append(" SELECT 1 FROM t_stats_record r ");
- sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
- sql.append(" WHERE r.trace_id IN ( ");
- sql.append(" SELECT trace_id FROM t_stats_record ");
- sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
- sql.append(" ) ");
- sql.append(" AND r.action = 'LOAD' ");
- sql.append(" AND r.identifier = 'pages/videoSynthesis/buy' ");
- sql.append(" AND JSONExtractString(r.params, 'id') != '' ");
- sql.append(" AND JSONExtractString(r.params, 'share') = '' ");
+ sql.append("WITH JSONExtractString(params, 'id') AS videoId, ");
+ sql.append(" JSONExtractString(params, 'share') AS share ");
+ sql.append("SELECT toInt32(uniqExact(videoId)) AS count ");
+ sql.append("FROM t_stats_record r ");
+ sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
+ sql.append("WHERE r.trace_id IN (");
+ appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
+ sql.append(") ");
+ sql.append("AND r.action = 'LOAD' ");
+ sql.append("AND r.identifier = 'pages/videoSynthesis/buy' ");
+ sql.append("AND videoId != '' ");
+ sql.append("AND share = '' ");
if (query.getStartTime() != null) {
- sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
+ sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
- sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
+ sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
- sql.append(" GROUP BY JSONExtractString(r.params, 'id') ");
- sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -246,20 +248,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public List