q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复入库可能重复;筛选逻辑优化 (#226)

Browse Source 
* 修改数据清洗筛选逻辑-筛选修改为多选

* 修改数据清洗筛选逻辑-筛选修改为多选

* antd 组件库样式定制修改

* fix: 修复入库可能重复

* fix: 算子市场筛选逻辑优化

* fix: 清洗任务创建筛选逻辑优化

* fix: 清洗任务创建筛选逻辑优化

---------

Co-authored-by: chase <byzhangxin11@126.com>
This commit is contained in:
hhhhsc701
2026-01-06 17:57:25 +08:00
committed by GitHub
parent 49cc98941f
commit 7d4dcb756b
19 changed files with 247 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
backend/services/operator-market-service/src/main/java/com/datamate/operator/application/CategoryService.java
View File

@@ -1,10 +1,8 @@
package com.datamate.operator.application;
import com.datamate.operator.domain.contants.OperatorConstant;
import com.datamate.operator.domain.repository.CategoryRelationRepository;
import com.datamate.operator.domain.repository.CategoryRepository;
import com.datamate.operator.domain.repository.OperatorRepository;
import com.datamate.operator.interfaces.dto.CategoryDto;
import com.datamate.operator.interfaces.dto.CategoryRelationDto;
import com.datamate.operator.interfaces.dto.CategoryTreeResponse;
@@ -21,7 +19,7 @@ import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class CategoryService {
private final OperatorRepository operatorRepo;
private final CategoryRepository categoryRepo;
@@ -42,7 +40,7 @@ public class CategoryService {
.filter(relation -> !StringUtils.equals(relation.getParentId(), "0"))
.collect(Collectors.groupingBy(CategoryDto::getParentId));
List<CategoryTreeResponse> categoryTreeResponses = groupedByParentId.entrySet().stream()
return groupedByParentId.entrySet().stream()
.sorted(categoryComparator(nameMap))
.map(entry -> {
String parentId = entry.getKey();
@@ -58,10 +56,6 @@ public class CategoryService {
response.setCount(totalCount.get());
return response;
}).collect(Collectors.toCollection(ArrayList::new));
int stars = operatorRepo.countOperatorByStar(true);
categoryTreeResponses.add(buildStarCategoryTree(stars));
return categoryTreeResponses;
}
private Comparator<Map.Entry<String, List<CategoryDto>>> categoryComparator(Map<String, CategoryDto> categoryMap) {
@@ -71,21 +65,4 @@ public class CategoryService {
return index1.compareTo(index2);
};
}
private CategoryTreeResponse buildStarCategoryTree(int stars) {
CategoryTreeResponse starResponse = new CategoryTreeResponse();
starResponse.setName("收藏状态");
starResponse.setCount(stars);
starResponse.setId("257b27e0-bba9-11f0-89d7-00155d0a6153");
CategoryDto star = new CategoryDto();
star.setId(OperatorConstant.CATEGORY_STAR_ID);
star.setName("已收藏");
star.setValue("isStar");
star.setCount(stars);
star.setParentId("257b27e0-bba9-11f0-89d7-00155d0a6153");
star.setCreatedAt(LocalDateTime.now());
star.setType("predefined");
starResponse.setCategories(Collections.singletonList(star));
return starResponse;
}
}

4
backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java
View File

@@ -52,12 +52,12 @@ public class OperatorService {
@Value("${operator.base.path:/operators}")
private String operatorBasePath;
public List<OperatorDto> getOperators(Integer page, Integer size, List<String> categories,
public List<OperatorDto> getOperators(Integer page, Integer size, List<List<String>> categories,
String keyword, Boolean isStar) {
return operatorViewRepo.findOperatorsByCriteria(page, size, keyword, categories, isStar);
}
public int getOperatorsCount(List<String> categories, String keyword, Boolean isStar) {
public int getOperatorsCount(List<List<String>> categories, String keyword, Boolean isStar) {
return operatorViewRepo.countOperatorsByCriteria(keyword, categories, isStar);
}

4
backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorViewRepository.java
View File

@@ -8,9 +8,9 @@ import java.util.List;
public interface OperatorViewRepository extends IRepository<OperatorView> {
List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String keyword,
List<String> categories, Boolean isStar);
List<List<String>> categories, Boolean isStar);
Integer countOperatorsByCriteria(String keyword, List<String> categories, Boolean isStar);
int countOperatorsByCriteria(String keyword, List<List<String>> categories, Boolean isStar);
OperatorView findOperatorById(String id);
}

51
backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java
View File

@@ -16,6 +16,7 @@ import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.stream.Collectors;
@Repository
@RequiredArgsConstructor
@@ -24,18 +25,36 @@ public class OperatorViewRepositoryImpl extends CrudRepository<OperatorViewMappe
@Override
public List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String keyword,
List<String> categories, Boolean isStar) {
List<List<String>> categories, Boolean isStar) {
QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
queryWrapper.in(CollectionUtils.isNotEmpty(categories), "category_id", categories)
.eq(isStar != null, "is_star", isStar);
queryWrapper.eq(isStar != null, "is_star", isStar);
if (StringUtils.isNotEmpty(keyword)) {
queryWrapper.and(w ->
w.like("operator_name", keyword)
.or()
.like("description", keyword));
}
StringBuilder havingSql = new StringBuilder();
if (CollectionUtils.isNotEmpty(categories)) {
queryWrapper.in("category_id", categories.stream().flatMap(List::stream).toList());
int index = 0;
for (List<String> category : categories) {
if (index > 0) {
havingSql.append(" AND ");
}
havingSql.append("SUM(CASE WHEN category_id IN (");
havingSql.append(category.stream()
.map(id -> "'" + id + "'")
.collect(Collectors.joining(",")));
havingSql.append(") THEN 1 ELSE 0 END) > 0");
index++;
}
}
queryWrapper.groupBy("operator_id")
.having(!havingSql.isEmpty(), havingSql.toString())
.orderByDesc("created_at");
Page<OperatorView> queryPage;
if (size != null && page != null) {
queryPage = new Page<>(page + 1, size);
@@ -48,17 +67,35 @@ public class OperatorViewRepositoryImpl extends CrudRepository<OperatorViewMappe
}
@Override
public Integer countOperatorsByCriteria(String keyword, List<String> categories, Boolean isStar) {
public int countOperatorsByCriteria(String keyword, List<List<String>> categories, Boolean isStar) {
QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
queryWrapper.in(CollectionUtils.isNotEmpty(categories),"category_id", categories)
.eq(isStar != null, "is_star", isStar);
queryWrapper.eq(isStar != null, "is_star", isStar);
if (StringUtils.isNotEmpty(keyword)) {
queryWrapper.and(w ->
w.like("operator_name", keyword)
.or()
.like("description", keyword));
}
return mapper.countOperatorsByCriteria(queryWrapper);
StringBuilder havingSql = new StringBuilder();
if (CollectionUtils.isNotEmpty(categories)) {
queryWrapper.in("category_id", categories.stream().flatMap(List::stream).toList());
int index = 0;
for (List<String> category : categories) {
if (index > 0) {
havingSql.append(" AND ");
}
havingSql.append("SUM(CASE WHEN category_id IN (");
havingSql.append(category.stream()
.map(id -> "'" + id + "'")
.collect(Collectors.joining(",")));
havingSql.append(") THEN 1 ELSE 0 END) > 0");
index++;
}
}
queryWrapper.groupBy("operator_id")
.having(!havingSql.isEmpty(), havingSql.toString());
Integer count = mapper.countOperatorsByCriteria(queryWrapper);
return count != null ? count : 0;
}
@Override

2
backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/mapper/OperatorViewMapper.java
View File

@@ -19,7 +19,7 @@ public interface OperatorViewMapper extends BaseMapper<OperatorView> {
IPage<OperatorView> findOperatorsByCriteria(IPage<OperatorView> page,
@Param(Constants.WRAPPER) Wrapper<OperatorView> queryWrapper);
@Select("SELECT COUNT(DISTINCT operator_id) AS count FROM v_operator ${ew.customSqlSegment}")
@Select("SELECT COUNT(1) FROM (SELECT 1 FROM v_operator ${ew.customSqlSegment}) AS t")
Integer countOperatorsByCriteria(@Param(Constants.WRAPPER) Wrapper<OperatorView> queryWrapper);
@Select("SELECT operator_id AS id, operator_name AS name, description, version, inputs, outputs, runtime, " +

24
backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/CategoryTreePagedResponse.java Normal file
View File

@@ -0,0 +1,24 @@
package com.datamate.operator.interfaces.dto;
import com.datamate.common.interfaces.PagedResponse;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
@Getter
@Setter
@AllArgsConstructor
public class CategoryTreePagedResponse extends PagedResponse<CategoryTreeResponse> {
Integer starCount;
public CategoryTreePagedResponse(List<CategoryTreeResponse> content, Integer starCount) {
super(content);
this.starCount = starCount;
}
public static CategoryTreePagedResponse of(List<CategoryTreeResponse> content, Integer starCount) {
return new CategoryTreePagedResponse(content, starCount);
}
}

2
backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/OperatorsListPostRequest.java
View File

@@ -15,7 +15,7 @@ import lombok.Setter;
@Getter
@Setter
public class OperatorsListPostRequest extends PagingQuery {
private List<String> categories = new ArrayList<>();
private List<List<String>> categories = new ArrayList<>();
private String keyword;

6
backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/CategoryController.java
View File

@@ -2,6 +2,8 @@ package com.datamate.operator.interfaces.rest;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.operator.application.CategoryService;
import com.datamate.operator.domain.repository.OperatorRepository;
import com.datamate.operator.interfaces.dto.CategoryTreePagedResponse;
import com.datamate.operator.interfaces.dto.CategoryTreeResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
@@ -17,9 +19,11 @@ import java.util.List;
public class CategoryController {
private final CategoryService categoryService;
private final OperatorRepository operatorRepo;
@GetMapping("/tree")
public PagedResponse<CategoryTreeResponse> categoryTreeGet() {
List<CategoryTreeResponse> allCategories = categoryService.getAllCategories();
return PagedResponse.of(allCategories);
return CategoryTreePagedResponse.of(allCategories, operatorRepo.countOperatorByStar(true));
}
}

14
backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/OperatorController.java
View File

@@ -3,12 +3,10 @@ package com.datamate.operator.interfaces.rest;
import com.datamate.common.infrastructure.common.IgnoreResponseWrap;
import com.datamate.common.interfaces.PagedResponse;
import com.datamate.operator.application.OperatorService;
import com.datamate.operator.domain.contants.OperatorConstant;
import com.datamate.operator.interfaces.dto.OperatorDto;
import com.datamate.operator.interfaces.dto.OperatorsListPostRequest;
import com.datamate.operator.interfaces.dto.UploadOperatorRequest;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
@@ -27,16 +25,10 @@ public class OperatorController {
@PostMapping("/list")
public PagedResponse<OperatorDto> operatorsListPost(@RequestBody OperatorsListPostRequest request) {
Boolean isStar = null;
List<String> categories = request.getCategories();
if (CollectionUtils.isNotEmpty(request.getCategories()) &&
request.getCategories().contains(OperatorConstant.CATEGORY_STAR_ID)) {
isStar = true;
categories.remove(OperatorConstant.CATEGORY_STAR_ID);
}
List<List<String>> categories = request.getCategories();
List<OperatorDto> responses = operatorService.getOperators(request.getPage(), request.getSize(),
categories, request.getKeyword(), isStar);
int count = operatorService.getOperatorsCount(categories, request.getKeyword(), isStar);
categories, request.getKeyword(), request.getIsStar());
int count = operatorService.getOperatorsCount(categories, request.getKeyword(), request.getIsStar());
int totalPages = (count + request.getSize() + 1) / request.getSize();
return PagedResponse.of(responses, request.getPage(), count, totalPages);
}