You've already forked DataMate
Compare commits
119 Commits
1fd70085e8
...
lsf
| Author | SHA1 | Date | |
|---|---|---|---|
| cda22a720c | |||
| 394e2bda18 | |||
| 4220284f5a | |||
| 8415166949 | |||
| 078f303f57 | |||
| 50f2da5503 | |||
| 3af1daf8b6 | |||
| 7c7729434b | |||
| 17a62cd3c2 | |||
| f381d641ab | |||
| c8611d29ff | |||
| 147beb1ec7 | |||
| 699031dae7 | |||
| 88b1383653 | |||
| cc6415c4d9 | |||
| 3d036c4cd6 | |||
| 2445235fd2 | |||
| 893e0a1580 | |||
| 05e6842fc8 | |||
| da5b18e423 | |||
| 31629ab50b | |||
| fb43052ddf | |||
| c44c75be25 | |||
| 05f3efc148 | |||
| 16eb5cacf9 | |||
| e71116d117 | |||
| cac53d7aac | |||
| 43b4a619bc | |||
| 9da187d2c6 | |||
| b36fdd2438 | |||
| daa63bdd13 | |||
| 85433ac071 | |||
| fc2e50b415 | |||
| 26e1ae69d7 | |||
| 7092c3f955 | |||
| b2bdf9e066 | |||
| a5261b33b2 | |||
|
|
52daf30869 | ||
| 07a901043a | |||
| 32e3fc97c6 | |||
| a73571bd73 | |||
| 00fa1b86eb | |||
| 626c0fcd9a | |||
| 2f2e0d6a8d | |||
| 10fad39e02 | |||
| 9014dca1ac | |||
| 0b8fe34586 | |||
| 27e27a09d4 | |||
| d24fea83d8 | |||
| 05088fef1a | |||
| a0239518fb | |||
| 9d185bb10c | |||
| 6c4f05c0b9 | |||
| 438acebb89 | |||
| f06d6e5a7e | |||
| fda283198d | |||
| d535d0ac1b | |||
| 4d2c9e546c | |||
| 02cd16523f | |||
| d4a44f3bf5 | |||
| 340a0ad364 | |||
| 00c41fbbd3 | |||
| 2430db290d | |||
| 40889baacc | |||
| 551248ec76 | |||
| 0bb9abb200 | |||
| d135a7f336 | |||
| 7043a26ab3 | |||
| 906bb39b83 | |||
| dbf8ec53dd | |||
| 5f89968974 | |||
| be313cf425 | |||
| db37de8aee | |||
| aeec19b99f | |||
| a4aefe66cd | |||
| 2f3a8b38d0 | |||
| 150af1a741 | |||
| e28f680abb | |||
| 4f99875670 | |||
| c23a9da8cb | |||
| 310bc356b1 | |||
| c1fb02b0f5 | |||
| 4a3e466210 | |||
| 5d8d25ca8c | |||
| f6788756d3 | |||
| 5a5279869e | |||
| e1c963928a | |||
| 33cf65c9f8 | |||
| 3e0a15ac8e | |||
| 5318ee9641 | |||
| c5c8e6c69e | |||
| 8fdc7d99b8 | |||
| 2bc48fd465 | |||
| a21a632a4b | |||
| 595a758d05 | |||
| 4fa0ac1df4 | |||
| f2403f00ce | |||
| f4fc574687 | |||
| 52a2a73a8e | |||
| b5d7c66240 | |||
| 6c7ea0c25e | |||
| 153066a95f | |||
| 498f23a0c4 | |||
| 85d7141a91 | |||
| 790385bd80 | |||
| 97170a90fe | |||
| fd209c3083 | |||
| 76f70a6847 | |||
| cbad129ce4 | |||
| ca7ff56610 | |||
| a00a6ed3c3 | |||
| 9a205919d7 | |||
| 8b2a19f09a | |||
| 3c3ca130b3 | |||
| a4cdaecf8a | |||
| 6dfed934a5 | |||
| bd37858ccc | |||
| accaa47a83 | |||
| 98d2ef1aa5 |
304
Makefile.offline.mk
Normal file
304
Makefile.offline.mk
Normal file
@@ -0,0 +1,304 @@
|
|||||||
|
# ============================================================================
|
||||||
|
# Makefile 离线构建扩展
|
||||||
|
# 将此文件内容追加到主 Makefile 末尾,或单独包含使用
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# 离线构建配置
|
||||||
|
CACHE_DIR ?= ./build-cache
|
||||||
|
OFFLINE_VERSION ?= latest
|
||||||
|
|
||||||
|
# 创建 buildx 构建器(如果不存在)
|
||||||
|
.PHONY: ensure-buildx
|
||||||
|
ensure-buildx:
|
||||||
|
@if ! docker buildx inspect offline-builder > /dev/null 2>&1; then \
|
||||||
|
echo "创建 buildx 构建器..."; \
|
||||||
|
docker buildx create --name offline-builder --driver docker-container --use 2>/dev/null || docker buildx use offline-builder; \
|
||||||
|
else \
|
||||||
|
docker buildx use offline-builder 2>/dev/null || true; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ========== 离线缓存导出(有网环境) ==========
|
||||||
|
|
||||||
|
.PHONY: offline-export
|
||||||
|
offline-export: ensure-buildx
|
||||||
|
@echo "======================================"
|
||||||
|
@echo "导出离线构建缓存..."
|
||||||
|
@echo "======================================"
|
||||||
|
@mkdir -p $(CACHE_DIR)/buildkit $(CACHE_DIR)/images $(CACHE_DIR)/resources
|
||||||
|
@$(MAKE) _offline-export-base-images
|
||||||
|
@$(MAKE) _offline-export-cache
|
||||||
|
@$(MAKE) _offline-export-resources
|
||||||
|
@$(MAKE) _offline-package
|
||||||
|
|
||||||
|
.PHONY: _offline-export-base-images
|
||||||
|
_offline-export-base-images:
|
||||||
|
@echo ""
|
||||||
|
@echo "1. 导出基础镜像..."
|
||||||
|
@bash -c 'images=( \
|
||||||
|
"maven:3-eclipse-temurin-21" \
|
||||||
|
"maven:3-eclipse-temurin-8" \
|
||||||
|
"eclipse-temurin:21-jdk" \
|
||||||
|
"mysql:8" \
|
||||||
|
"node:20-alpine" \
|
||||||
|
"nginx:1.29" \
|
||||||
|
"ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm" \
|
||||||
|
"ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm" \
|
||||||
|
"ghcr.nju.edu.cn/astral-sh/uv:latest" \
|
||||||
|
"python:3.12-slim" \
|
||||||
|
"python:3.11-slim" \
|
||||||
|
"gcr.nju.edu.cn/distroless/nodejs20-debian12" \
|
||||||
|
); for img in "$${images[@]}"; do echo " Pulling $$img..."; docker pull "$$img" 2>/dev/null || true; done'
|
||||||
|
@echo " Saving base images..."
|
||||||
|
@docker save -o $(CACHE_DIR)/images/base-images.tar \
|
||||||
|
maven:3-eclipse-temurin-21 \
|
||||||
|
maven:3-eclipse-temurin-8 \
|
||||||
|
eclipse-temurin:21-jdk \
|
||||||
|
mysql:8 \
|
||||||
|
node:20-alpine \
|
||||||
|
nginx:1.29 \
|
||||||
|
ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm \
|
||||||
|
ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm \
|
||||||
|
ghcr.nju.edu.cn/astral-sh/uv:latest \
|
||||||
|
python:3.12-slim \
|
||||||
|
python:3.11-slim \
|
||||||
|
gcr.nju.edu.cn/distroless/nodejs20-debian12 2>/dev/null || echo " Warning: Some images may not exist"
|
||||||
|
|
||||||
|
.PHONY: _offline-export-cache
|
||||||
|
_offline-export-cache:
|
||||||
|
@echo ""
|
||||||
|
@echo "2. 导出 BuildKit 缓存..."
|
||||||
|
@echo " backend..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-cache,mode=max -f scripts/images/backend/Dockerfile -t datamate-backend:cache . 2>/dev/null || echo " Warning: backend cache export failed"
|
||||||
|
@echo " backend-python..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-python-cache,mode=max -f scripts/images/backend-python/Dockerfile -t datamate-backend-python:cache . 2>/dev/null || echo " Warning: backend-python cache export failed"
|
||||||
|
@echo " database..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/database-cache,mode=max -f scripts/images/database/Dockerfile -t datamate-database:cache . 2>/dev/null || echo " Warning: database cache export failed"
|
||||||
|
@echo " frontend..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/frontend-cache,mode=max -f scripts/images/frontend/Dockerfile -t datamate-frontend:cache . 2>/dev/null || echo " Warning: frontend cache export failed"
|
||||||
|
@echo " gateway..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/gateway-cache,mode=max -f scripts/images/gateway/Dockerfile -t datamate-gateway:cache . 2>/dev/null || echo " Warning: gateway cache export failed"
|
||||||
|
@echo " runtime..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/runtime-cache,mode=max -f scripts/images/runtime/Dockerfile -t datamate-runtime:cache . 2>/dev/null || echo " Warning: runtime cache export failed"
|
||||||
|
@echo " deer-flow-backend..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-backend-cache,mode=max -f scripts/images/deer-flow-backend/Dockerfile -t deer-flow-backend:cache . 2>/dev/null || echo " Warning: deer-flow-backend cache export failed"
|
||||||
|
@echo " deer-flow-frontend..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-frontend-cache,mode=max -f scripts/images/deer-flow-frontend/Dockerfile -t deer-flow-frontend:cache . 2>/dev/null || echo " Warning: deer-flow-frontend cache export failed"
|
||||||
|
@echo " mineru..."
|
||||||
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/mineru-cache,mode=max -f scripts/images/mineru/Dockerfile -t datamate-mineru:cache . 2>/dev/null || echo " Warning: mineru cache export failed"
|
||||||
|
|
||||||
|
.PHONY: _offline-export-resources
|
||||||
|
_offline-export-resources:
|
||||||
|
@echo ""
|
||||||
|
@echo "3. 预下载外部资源..."
|
||||||
|
@mkdir -p $(CACHE_DIR)/resources/models
|
||||||
|
@echo " PaddleOCR model..."
|
||||||
|
@wget -q -O $(CACHE_DIR)/resources/models/ch_ppocr_mobile_v2.0_cls_infer.tar \
|
||||||
|
https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar 2>/dev/null || echo " Warning: PaddleOCR model download failed"
|
||||||
|
@echo " spaCy model..."
|
||||||
|
@wget -q -O $(CACHE_DIR)/resources/models/zh_core_web_sm-3.8.0-py3-none-any.whl \
|
||||||
|
https://ghproxy.net/https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl 2>/dev/null || echo " Warning: spaCy model download failed"
|
||||||
|
@echo " DataX source..."
|
||||||
|
@if [ ! -d "$(CACHE_DIR)/resources/DataX" ]; then \
|
||||||
|
git clone --depth 1 https://gitee.com/alibaba/DataX.git $(CACHE_DIR)/resources/DataX 2>/dev/null || echo " Warning: DataX clone failed"; \
|
||||||
|
fi
|
||||||
|
@echo " deer-flow source..."
|
||||||
|
@if [ ! -d "$(CACHE_DIR)/resources/deer-flow" ]; then \
|
||||||
|
git clone --depth 1 https://ghproxy.net/https://github.com/ModelEngine-Group/deer-flow.git $(CACHE_DIR)/resources/deer-flow 2>/dev/null || echo " Warning: deer-flow clone failed"; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
.PHONY: _offline-package
|
||||||
|
_offline-package:
|
||||||
|
@echo ""
|
||||||
|
@echo "4. 打包缓存..."
|
||||||
|
@cd $(CACHE_DIR) && tar -czf "build-cache-$$(date +%Y%m%d).tar.gz" buildkit images resources 2>/dev/null && cd - > /dev/null
|
||||||
|
@echo ""
|
||||||
|
@echo "======================================"
|
||||||
|
@echo "✓ 缓存导出完成!"
|
||||||
|
@echo "======================================"
|
||||||
|
@echo "传输文件: $(CACHE_DIR)/build-cache-$$(date +%Y%m%d).tar.gz"
|
||||||
|
|
||||||
|
# ========== 离线构建(无网环境) ==========
|
||||||
|
|
||||||
|
.PHONY: offline-setup
|
||||||
|
offline-setup:
|
||||||
|
@echo "======================================"
|
||||||
|
@echo "设置离线构建环境..."
|
||||||
|
@echo "======================================"
|
||||||
|
@if [ ! -d "$(CACHE_DIR)" ]; then \
|
||||||
|
echo "查找并解压缓存包..."; \
|
||||||
|
cache_file=$$(ls -t build-cache-*.tar.gz 2>/dev/null | head -1); \
|
||||||
|
if [ -z "$$cache_file" ]; then \
|
||||||
|
echo "错误: 未找到缓存压缩包 (build-cache-*.tar.gz)"; \
|
||||||
|
exit 1; \
|
||||||
|
fi; \
|
||||||
|
echo "解压 $$cache_file..."; \
|
||||||
|
tar -xzf "$$cache_file"; \
|
||||||
|
else \
|
||||||
|
echo "缓存目录已存在: $(CACHE_DIR)"; \
|
||||||
|
fi
|
||||||
|
@echo ""
|
||||||
|
@echo "加载基础镜像..."
|
||||||
|
@if [ -f "$(CACHE_DIR)/images/base-images.tar" ]; then \
|
||||||
|
docker load -i $(CACHE_DIR)/images/base-images.tar; \
|
||||||
|
else \
|
||||||
|
echo "警告: 基础镜像文件不存在,假设已手动加载"; \
|
||||||
|
fi
|
||||||
|
@$(MAKE) ensure-buildx
|
||||||
|
@echo ""
|
||||||
|
@echo "✓ 离线环境准备完成"
|
||||||
|
|
||||||
|
.PHONY: offline-build
|
||||||
|
offline-build: offline-setup
|
||||||
|
@echo ""
|
||||||
|
@echo "======================================"
|
||||||
|
@echo "开始离线构建..."
|
||||||
|
@echo "======================================"
|
||||||
|
@$(MAKE) _offline-build-services
|
||||||
|
|
||||||
|
.PHONY: _offline-build-services
|
||||||
|
_offline-build-services: ensure-buildx
|
||||||
|
@echo ""
|
||||||
|
@echo "构建 datamate-database..."
|
||||||
|
@docker buildx build \
|
||||||
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \
|
||||||
|
--pull=false \
|
||||||
|
-f scripts/images/database/Dockerfile \
|
||||||
|
-t datamate-database:$(OFFLINE_VERSION) \
|
||||||
|
--load . || echo " Failed"
|
||||||
|
|
||||||
|
@echo ""
|
||||||
|
@echo "构建 datamate-gateway..."
|
||||||
|
@docker buildx build \
|
||||||
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/gateway-cache \
|
||||||
|
--pull=false \
|
||||||
|
-f scripts/images/gateway/Dockerfile \
|
||||||
|
-t datamate-gateway:$(OFFLINE_VERSION) \
|
||||||
|
--load . || echo " Failed"
|
||||||
|
|
||||||
|
@echo ""
|
||||||
|
@echo "构建 datamate-backend..."
|
||||||
|
@docker buildx build \
|
||||||
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-cache \
|
||||||
|
--pull=false \
|
||||||
|
-f scripts/images/backend/Dockerfile \
|
||||||
|
-t datamate-backend:$(OFFLINE_VERSION) \
|
||||||
|
--load . || echo " Failed"
|
||||||
|
|
||||||
|
@echo ""
|
||||||
|
@echo "构建 datamate-frontend..."
|
||||||
|
@docker buildx build \
|
||||||
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/frontend-cache \
|
||||||
|
--pull=false \
|
||||||
|
-f scripts/images/frontend/Dockerfile \
|
||||||
|
-t datamate-frontend:$(OFFLINE_VERSION) \
|
||||||
|
--load . || echo " Failed"
|
||||||
|
|
||||||
|
@echo ""
|
||||||
|
@echo "构建 datamate-runtime..."
|
||||||
|
@docker buildx build \
|
||||||
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/runtime-cache \
|
||||||
|
--pull=false \
|
||||||
|
--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
|
||||||
|
-f scripts/images/runtime/Dockerfile \
|
||||||
|
-t datamate-runtime:$(OFFLINE_VERSION) \
|
||||||
|
--load . || echo " Failed"
|
||||||
|
|
||||||
|
@echo ""
|
||||||
|
@echo "构建 datamate-backend-python..."
|
||||||
|
@docker buildx build \
|
||||||
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-python-cache \
|
||||||
|
--pull=false \
|
||||||
|
--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
|
||||||
|
-f scripts/images/backend-python/Dockerfile \
|
||||||
|
-t datamate-backend-python:$(OFFLINE_VERSION) \
|
||||||
|
--load . || echo " Failed"
|
||||||
|
|
||||||
|
@echo ""
|
||||||
|
@echo "======================================"
|
||||||
|
@echo "✓ 离线构建完成"
|
||||||
|
@echo "======================================"
|
||||||
|
|
||||||
|
# 单个服务离线构建 (BuildKit)
|
||||||
|
.PHONY: %-offline-build
|
||||||
|
%-offline-build: offline-setup ensure-buildx
|
||||||
|
@echo "离线构建 $*..."
|
||||||
|
@if [ ! -d "$(CACHE_DIR)/buildkit/$*-cache" ]; then \
|
||||||
|
echo "错误: $* 的缓存不存在"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
@$(eval IMAGE_NAME := $(if $(filter deer-flow%,$*),$*,datamate-$*))
|
||||||
|
@docker buildx build \
|
||||||
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/$*-cache \
|
||||||
|
--pull=false \
|
||||||
|
$(if $(filter runtime backend-python deer-flow%,$*),--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources,) \
|
||||||
|
-f scripts/images/$*/Dockerfile \
|
||||||
|
-t $(IMAGE_NAME):$(OFFLINE_VERSION) \
|
||||||
|
--load .
|
||||||
|
|
||||||
|
# 传统 Docker 构建(不使用 BuildKit,更稳定)
|
||||||
|
.PHONY: offline-build-classic
|
||||||
|
offline-build-classic: offline-setup
|
||||||
|
@echo "使用传统 docker build 进行离线构建..."
|
||||||
|
@bash scripts/offline/build-offline-classic.sh $(CACHE_DIR) $(OFFLINE_VERSION)
|
||||||
|
|
||||||
|
# 诊断离线环境
|
||||||
|
.PHONY: offline-diagnose
|
||||||
|
offline-diagnose:
|
||||||
|
@bash scripts/offline/diagnose.sh $(CACHE_DIR)
|
||||||
|
|
||||||
|
# 构建 APT 预装基础镜像(有网环境)
|
||||||
|
.PHONY: offline-build-base-images
|
||||||
|
offline-build-base-images:
|
||||||
|
@echo "构建 APT 预装基础镜像..."
|
||||||
|
@bash scripts/offline/build-base-images.sh $(CACHE_DIR)
|
||||||
|
|
||||||
|
# 使用预装基础镜像进行离线构建(推荐)
|
||||||
|
.PHONY: offline-build-final
|
||||||
|
offline-build-final: offline-setup
|
||||||
|
@echo "使用预装 APT 包的基础镜像进行离线构建..."
|
||||||
|
@bash scripts/offline/build-offline-final.sh $(CACHE_DIR) $(OFFLINE_VERSION)
|
||||||
|
|
||||||
|
# 完整离线导出(包含 APT 预装基础镜像)
|
||||||
|
.PHONY: offline-export-full
|
||||||
|
offline-export-full:
|
||||||
|
@echo "======================================"
|
||||||
|
@echo "完整离线缓存导出(含 APT 预装基础镜像)"
|
||||||
|
@echo "======================================"
|
||||||
|
@$(MAKE) offline-build-base-images
|
||||||
|
@$(MAKE) offline-export
|
||||||
|
@echo ""
|
||||||
|
@echo "导出完成!传输时请包含以下文件:"
|
||||||
|
@echo " - build-cache/images/base-images-with-apt.tar"
|
||||||
|
@echo " - build-cache-YYYYMMDD.tar.gz"
|
||||||
|
|
||||||
|
# ========== 帮助 ==========
|
||||||
|
|
||||||
|
.PHONY: help-offline
|
||||||
|
help-offline:
|
||||||
|
@echo "离线构建命令:"
|
||||||
|
@echo ""
|
||||||
|
@echo "【有网环境】"
|
||||||
|
@echo " make offline-export [CACHE_DIR=./build-cache] - 导出构建缓存"
|
||||||
|
@echo " make offline-export-full - 导出完整缓存(含 APT 预装基础镜像)"
|
||||||
|
@echo " make offline-build-base-images - 构建 APT 预装基础镜像"
|
||||||
|
@echo ""
|
||||||
|
@echo "【无网环境】"
|
||||||
|
@echo " make offline-setup [CACHE_DIR=./build-cache] - 解压并准备离线缓存"
|
||||||
|
@echo " make offline-build-final - 使用预装基础镜像构建(推荐,解决 APT 问题)"
|
||||||
|
@echo " make offline-build-classic - 使用传统 docker build"
|
||||||
|
@echo " make offline-build - 使用 BuildKit 构建"
|
||||||
|
@echo " make offline-diagnose - 诊断离线构建环境"
|
||||||
|
@echo " make <service>-offline-build - 离线构建单个服务"
|
||||||
|
@echo ""
|
||||||
|
@echo "【完整工作流程(推荐)】"
|
||||||
|
@echo " # 1. 有网环境导出完整缓存"
|
||||||
|
@echo " make offline-export-full"
|
||||||
|
@echo ""
|
||||||
|
@echo " # 2. 传输到无网环境(需要传输两个文件)"
|
||||||
|
@echo " scp build-cache/images/base-images-with-apt.tar user@offline-server:/path/"
|
||||||
|
@echo " scp build-cache-*.tar.gz user@offline-server:/path/"
|
||||||
|
@echo ""
|
||||||
|
@echo " # 3. 无网环境构建"
|
||||||
|
@echo " tar -xzf build-cache-*.tar.gz"
|
||||||
|
@echo " docker load -i build-cache/images/base-images-with-apt.tar"
|
||||||
|
@echo " make offline-build-final"
|
||||||
@@ -447,12 +447,12 @@ paths:
|
|||||||
schema:
|
schema:
|
||||||
type: string
|
type: string
|
||||||
|
|
||||||
/data-management/datasets/{datasetId}/files/upload/chunk:
|
/data-management/datasets/{datasetId}/files/upload/chunk:
|
||||||
post:
|
post:
|
||||||
tags: [ DatasetFile ]
|
tags: [ DatasetFile ]
|
||||||
operationId: chunkUpload
|
operationId: chunkUpload
|
||||||
summary: 切片上传
|
summary: 切片上传
|
||||||
description: 使用预上传返回的请求ID进行分片上传
|
description: 使用预上传返回的请求ID进行分片上传
|
||||||
parameters:
|
parameters:
|
||||||
- name: datasetId
|
- name: datasetId
|
||||||
in: path
|
in: path
|
||||||
@@ -466,15 +466,32 @@ paths:
|
|||||||
multipart/form-data:
|
multipart/form-data:
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/components/schemas/UploadFileRequest'
|
$ref: '#/components/schemas/UploadFileRequest'
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
description: 上传成功
|
description: 上传成功
|
||||||
|
|
||||||
/data-management/dataset-types:
|
/data-management/datasets/upload/cancel-upload/{reqId}:
|
||||||
get:
|
put:
|
||||||
operationId: getDatasetTypes
|
tags: [ DatasetFile ]
|
||||||
tags: [DatasetType]
|
operationId: cancelUpload
|
||||||
summary: 获取数据集类型列表
|
summary: 取消上传
|
||||||
|
description: 取消预上传请求并清理临时分片
|
||||||
|
parameters:
|
||||||
|
- name: reqId
|
||||||
|
in: path
|
||||||
|
required: true
|
||||||
|
schema:
|
||||||
|
type: string
|
||||||
|
description: 预上传请求ID
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: 取消成功
|
||||||
|
|
||||||
|
/data-management/dataset-types:
|
||||||
|
get:
|
||||||
|
operationId: getDatasetTypes
|
||||||
|
tags: [DatasetType]
|
||||||
|
summary: 获取数据集类型列表
|
||||||
description: 获取所有支持的数据集类型
|
description: 获取所有支持的数据集类型
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.datamate.datamanagement.application;
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||||
import com.datamate.common.domain.utils.ChunksSaver;
|
import com.datamate.common.domain.utils.ChunksSaver;
|
||||||
@@ -19,8 +20,11 @@ import com.datamate.datamanagement.infrastructure.exception.DataManagementErrorC
|
|||||||
import com.datamate.datamanagement.infrastructure.persistence.mapper.TagMapper;
|
import com.datamate.datamanagement.infrastructure.persistence.mapper.TagMapper;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.dto.DatasetFileCount;
|
||||||
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
|
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
|
||||||
import com.datamate.datamanagement.interfaces.dto.*;
|
import com.datamate.datamanagement.interfaces.dto.*;
|
||||||
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.collections4.CollectionUtils;
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
@@ -53,6 +57,7 @@ public class DatasetApplicationService {
|
|||||||
private static final int SIMILAR_DATASET_MAX_LIMIT = 50;
|
private static final int SIMILAR_DATASET_MAX_LIMIT = 50;
|
||||||
private static final int SIMILAR_DATASET_CANDIDATE_FACTOR = 5;
|
private static final int SIMILAR_DATASET_CANDIDATE_FACTOR = 5;
|
||||||
private static final int SIMILAR_DATASET_CANDIDATE_MAX = 100;
|
private static final int SIMILAR_DATASET_CANDIDATE_MAX = 100;
|
||||||
|
private static final String DERIVED_METADATA_KEY = "derived_from_file_id";
|
||||||
private final DatasetRepository datasetRepository;
|
private final DatasetRepository datasetRepository;
|
||||||
private final TagMapper tagMapper;
|
private final TagMapper tagMapper;
|
||||||
private final DatasetFileRepository datasetFileRepository;
|
private final DatasetFileRepository datasetFileRepository;
|
||||||
@@ -73,7 +78,7 @@ public class DatasetApplicationService {
|
|||||||
Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
|
Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
|
||||||
Dataset parentDataset = resolveParentDataset(createDatasetRequest.getParentDatasetId(), dataset.getId());
|
Dataset parentDataset = resolveParentDataset(createDatasetRequest.getParentDatasetId(), dataset.getId());
|
||||||
dataset.setParentDatasetId(parentDataset == null ? null : parentDataset.getId());
|
dataset.setParentDatasetId(parentDataset == null ? null : parentDataset.getId());
|
||||||
dataset.initCreateParam(datasetBasePath, parentDataset == null ? null : parentDataset.getPath());
|
dataset.initCreateParam(datasetBasePath);
|
||||||
// 处理标签
|
// 处理标签
|
||||||
Set<Tag> processedTags = Optional.ofNullable(createDatasetRequest.getTags())
|
Set<Tag> processedTags = Optional.ofNullable(createDatasetRequest.getTags())
|
||||||
.filter(CollectionUtils::isNotEmpty)
|
.filter(CollectionUtils::isNotEmpty)
|
||||||
@@ -97,6 +102,7 @@ public class DatasetApplicationService {
|
|||||||
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
|
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
|
||||||
Dataset dataset = datasetRepository.getById(datasetId);
|
Dataset dataset = datasetRepository.getById(datasetId);
|
||||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||||
|
|
||||||
if (StringUtils.hasText(updateDatasetRequest.getName())) {
|
if (StringUtils.hasText(updateDatasetRequest.getName())) {
|
||||||
dataset.setName(updateDatasetRequest.getName());
|
dataset.setName(updateDatasetRequest.getName());
|
||||||
}
|
}
|
||||||
@@ -109,13 +115,31 @@ public class DatasetApplicationService {
|
|||||||
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
|
if (Objects.nonNull(updateDatasetRequest.getStatus())) {
|
||||||
dataset.setStatus(updateDatasetRequest.getStatus());
|
dataset.setStatus(updateDatasetRequest.getStatus());
|
||||||
}
|
}
|
||||||
if (updateDatasetRequest.getParentDatasetId() != null) {
|
if (updateDatasetRequest.isParentDatasetIdProvided()) {
|
||||||
|
// 保存原始的 parentDatasetId 值,用于比较是否发生了变化
|
||||||
|
String originalParentDatasetId = dataset.getParentDatasetId();
|
||||||
|
|
||||||
|
// 处理父数据集变更:仅当请求显式包含 parentDatasetId 时处理
|
||||||
|
// handleParentChange 内部通过 normalizeParentId 方法将空字符串和 null 都转换为 null
|
||||||
|
// 这样既支持设置新的父数据集,也支持清除关联
|
||||||
handleParentChange(dataset, updateDatasetRequest.getParentDatasetId());
|
handleParentChange(dataset, updateDatasetRequest.getParentDatasetId());
|
||||||
|
|
||||||
|
// 检查 parentDatasetId 是否发生了变化
|
||||||
|
if (!Objects.equals(originalParentDatasetId, dataset.getParentDatasetId())) {
|
||||||
|
// 使用 LambdaUpdateWrapper 显式地更新 parentDatasetId 字段
|
||||||
|
// 这样即使值为 null 也能被正确更新到数据库
|
||||||
|
datasetRepository.update(null, new LambdaUpdateWrapper<Dataset>()
|
||||||
|
.eq(Dataset::getId, datasetId)
|
||||||
|
.set(Dataset::getParentDatasetId, dataset.getParentDatasetId()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
|
if (StringUtils.hasText(updateDatasetRequest.getDataSource())) {
|
||||||
// 数据源id不为空,使用异步线程进行文件扫盘落库
|
// 数据源id不为空,使用异步线程进行文件扫盘落库
|
||||||
processDataSourceAsync(dataset.getId(), updateDatasetRequest.getDataSource());
|
processDataSourceAsync(dataset.getId(), updateDatasetRequest.getDataSource());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 更新其他字段(不包括 parentDatasetId,因为它已经在上面的代码中更新了)
|
||||||
datasetRepository.updateById(dataset);
|
datasetRepository.updateById(dataset);
|
||||||
return dataset;
|
return dataset;
|
||||||
}
|
}
|
||||||
@@ -142,6 +166,7 @@ public class DatasetApplicationService {
|
|||||||
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
|
||||||
List<DatasetFile> datasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
List<DatasetFile> datasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
||||||
dataset.setFiles(datasetFiles);
|
dataset.setFiles(datasetFiles);
|
||||||
|
applyVisibleFileCounts(Collections.singletonList(dataset));
|
||||||
return dataset;
|
return dataset;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,6 +178,7 @@ public class DatasetApplicationService {
|
|||||||
IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
|
IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
|
||||||
page = datasetRepository.findByCriteria(page, query);
|
page = datasetRepository.findByCriteria(page, query);
|
||||||
String datasetPvcName = getDatasetPvcName();
|
String datasetPvcName = getDatasetPvcName();
|
||||||
|
applyVisibleFileCounts(page.getRecords());
|
||||||
List<DatasetResponse> datasetResponses = DatasetConverter.INSTANCE.convertToResponse(page.getRecords());
|
List<DatasetResponse> datasetResponses = DatasetConverter.INSTANCE.convertToResponse(page.getRecords());
|
||||||
datasetResponses.forEach(dataset -> dataset.setPvcName(datasetPvcName));
|
datasetResponses.forEach(dataset -> dataset.setPvcName(datasetPvcName));
|
||||||
return PagedResponse.of(datasetResponses, page.getCurrent(), page.getTotal(), page.getPages());
|
return PagedResponse.of(datasetResponses, page.getCurrent(), page.getTotal(), page.getPages());
|
||||||
@@ -200,6 +226,7 @@ public class DatasetApplicationService {
|
|||||||
})
|
})
|
||||||
.limit(safeLimit)
|
.limit(safeLimit)
|
||||||
.toList();
|
.toList();
|
||||||
|
applyVisibleFileCounts(sorted);
|
||||||
List<DatasetResponse> responses = DatasetConverter.INSTANCE.convertToResponse(sorted);
|
List<DatasetResponse> responses = DatasetConverter.INSTANCE.convertToResponse(sorted);
|
||||||
responses.forEach(item -> item.setPvcName(datasetPvcName));
|
responses.forEach(item -> item.setPvcName(datasetPvcName));
|
||||||
return responses;
|
return responses;
|
||||||
@@ -291,7 +318,9 @@ public class DatasetApplicationService {
|
|||||||
|
|
||||||
private void handleParentChange(Dataset dataset, String parentDatasetId) {
|
private void handleParentChange(Dataset dataset, String parentDatasetId) {
|
||||||
String normalized = normalizeParentId(parentDatasetId);
|
String normalized = normalizeParentId(parentDatasetId);
|
||||||
if (Objects.equals(dataset.getParentDatasetId(), normalized)) {
|
String expectedPath = buildDatasetPath(datasetBasePath, dataset.getId());
|
||||||
|
if (Objects.equals(dataset.getParentDatasetId(), normalized)
|
||||||
|
&& Objects.equals(dataset.getPath(), expectedPath)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
long childCount = datasetRepository.countByParentId(dataset.getId());
|
long childCount = datasetRepository.countByParentId(dataset.getId());
|
||||||
@@ -299,8 +328,7 @@ public class DatasetApplicationService {
|
|||||||
throw BusinessException.of(DataManagementErrorCode.DATASET_HAS_CHILDREN);
|
throw BusinessException.of(DataManagementErrorCode.DATASET_HAS_CHILDREN);
|
||||||
}
|
}
|
||||||
Dataset parent = normalized == null ? null : resolveParentDataset(normalized, dataset.getId());
|
Dataset parent = normalized == null ? null : resolveParentDataset(normalized, dataset.getId());
|
||||||
String newPath = buildDatasetPath(parent == null ? datasetBasePath : parent.getPath(), dataset.getId());
|
moveDatasetPath(dataset, expectedPath);
|
||||||
moveDatasetPath(dataset, newPath);
|
|
||||||
dataset.setParentDatasetId(parent == null ? null : parent.getId());
|
dataset.setParentDatasetId(parent == null ? null : parent.getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -344,6 +372,61 @@ public class DatasetApplicationService {
|
|||||||
dataset.setPath(newPath);
|
dataset.setPath(newPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void applyVisibleFileCounts(List<Dataset> datasets) {
|
||||||
|
if (CollectionUtils.isEmpty(datasets)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<String> datasetIds = datasets.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.map(Dataset::getId)
|
||||||
|
.filter(StringUtils::hasText)
|
||||||
|
.toList();
|
||||||
|
if (datasetIds.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Map<String, Long> countMap = datasetFileRepository.countNonDerivedByDatasetIds(datasetIds).stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.collect(Collectors.toMap(
|
||||||
|
DatasetFileCount::getDatasetId,
|
||||||
|
count -> Optional.ofNullable(count.getFileCount()).orElse(0L),
|
||||||
|
(left, right) -> left
|
||||||
|
));
|
||||||
|
for (Dataset dataset : datasets) {
|
||||||
|
if (dataset == null || !StringUtils.hasText(dataset.getId())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Long visibleCount = countMap.get(dataset.getId());
|
||||||
|
dataset.setFileCount(visibleCount != null ? visibleCount : 0L);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatasetFile> filterVisibleFiles(List<DatasetFile> files) {
|
||||||
|
if (CollectionUtils.isEmpty(files)) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
return files.stream()
|
||||||
|
.filter(file -> !isDerivedFile(file))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isDerivedFile(DatasetFile datasetFile) {
|
||||||
|
if (datasetFile == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String metadata = datasetFile.getMetadata();
|
||||||
|
if (!StringUtils.hasText(metadata)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
Map<String, Object> metadataMap = mapper.readValue(metadata, new TypeReference<Map<String, Object>>() {});
|
||||||
|
return metadataMap.get(DERIVED_METADATA_KEY) != null;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("Failed to parse dataset file metadata for derived detection: {}", datasetFile.getId(), e);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取数据集统计信息
|
* 获取数据集统计信息
|
||||||
*/
|
*/
|
||||||
@@ -356,27 +439,29 @@ public class DatasetApplicationService {
|
|||||||
|
|
||||||
Map<String, Object> statistics = new HashMap<>();
|
Map<String, Object> statistics = new HashMap<>();
|
||||||
|
|
||||||
// 基础统计
|
List<DatasetFile> allFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
||||||
Long totalFiles = datasetFileRepository.countByDatasetId(datasetId);
|
List<DatasetFile> visibleFiles = filterVisibleFiles(allFiles);
|
||||||
Long completedFiles = datasetFileRepository.countCompletedByDatasetId(datasetId);
|
long totalFiles = visibleFiles.size();
|
||||||
|
long completedFiles = visibleFiles.stream()
|
||||||
|
.filter(file -> "COMPLETED".equalsIgnoreCase(file.getStatus()))
|
||||||
|
.count();
|
||||||
Long totalSize = datasetFileRepository.sumSizeByDatasetId(datasetId);
|
Long totalSize = datasetFileRepository.sumSizeByDatasetId(datasetId);
|
||||||
|
|
||||||
statistics.put("totalFiles", totalFiles != null ? totalFiles.intValue() : 0);
|
statistics.put("totalFiles", (int) totalFiles);
|
||||||
statistics.put("completedFiles", completedFiles != null ? completedFiles.intValue() : 0);
|
statistics.put("completedFiles", (int) completedFiles);
|
||||||
statistics.put("totalSize", totalSize != null ? totalSize : 0L);
|
statistics.put("totalSize", totalSize != null ? totalSize : 0L);
|
||||||
|
|
||||||
// 完成率计算
|
// 完成率计算
|
||||||
float completionRate = 0.0f;
|
float completionRate = 0.0f;
|
||||||
if (totalFiles != null && totalFiles > 0) {
|
if (totalFiles > 0) {
|
||||||
completionRate = (completedFiles != null ? completedFiles.floatValue() : 0.0f) / totalFiles.floatValue() * 100.0f;
|
completionRate = ((float) completedFiles) / (float) totalFiles * 100.0f;
|
||||||
}
|
}
|
||||||
statistics.put("completionRate", completionRate);
|
statistics.put("completionRate", completionRate);
|
||||||
|
|
||||||
// 文件类型分布统计
|
// 文件类型分布统计
|
||||||
Map<String, Integer> fileTypeDistribution = new HashMap<>();
|
Map<String, Integer> fileTypeDistribution = new HashMap<>();
|
||||||
List<DatasetFile> allFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
if (!visibleFiles.isEmpty()) {
|
||||||
if (allFiles != null) {
|
for (DatasetFile file : visibleFiles) {
|
||||||
for (DatasetFile file : allFiles) {
|
|
||||||
String fileType = file.getFileType() != null ? file.getFileType() : "unknown";
|
String fileType = file.getFileType() != null ? file.getFileType() : "unknown";
|
||||||
fileTypeDistribution.put(fileType, fileTypeDistribution.getOrDefault(fileType, 0) + 1);
|
fileTypeDistribution.put(fileType, fileTypeDistribution.getOrDefault(fileType, 0) + 1);
|
||||||
}
|
}
|
||||||
@@ -385,8 +470,8 @@ public class DatasetApplicationService {
|
|||||||
|
|
||||||
// 状态分布统计
|
// 状态分布统计
|
||||||
Map<String, Integer> statusDistribution = new HashMap<>();
|
Map<String, Integer> statusDistribution = new HashMap<>();
|
||||||
if (allFiles != null) {
|
if (!visibleFiles.isEmpty()) {
|
||||||
for (DatasetFile file : allFiles) {
|
for (DatasetFile file : visibleFiles) {
|
||||||
String status = file.getStatus() != null ? file.getStatus() : "unknown";
|
String status = file.getStatus() != null ? file.getStatus() : "unknown";
|
||||||
statusDistribution.put(status, statusDistribution.getOrDefault(status, 0) + 1);
|
statusDistribution.put(status, statusDistribution.getOrDefault(status, 0) + 1);
|
||||||
}
|
}
|
||||||
@@ -413,33 +498,32 @@ public class DatasetApplicationService {
|
|||||||
public void processDataSourceAsync(String datasetId, String dataSourceId) {
|
public void processDataSourceAsync(String datasetId, String dataSourceId) {
|
||||||
try {
|
try {
|
||||||
log.info("Initiating data source file scanning, dataset ID: {}, collection task ID: {}", datasetId, dataSourceId);
|
log.info("Initiating data source file scanning, dataset ID: {}, collection task ID: {}", datasetId, dataSourceId);
|
||||||
List<String> filePaths = getFilePaths(dataSourceId);
|
CollectionTaskDetailResponse taskDetail = collectionTaskClient.getTaskDetail(dataSourceId).getData();
|
||||||
|
if (taskDetail == null) {
|
||||||
|
log.warn("Fail to get collection task detail, task ID: {}", dataSourceId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Path targetPath = Paths.get(taskDetail.getTargetPath());
|
||||||
|
if (!Files.exists(targetPath) || !Files.isDirectory(targetPath)) {
|
||||||
|
log.warn("Target path not exists or is not a directory: {}", taskDetail.getTargetPath());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<String> filePaths = scanFilePaths(targetPath);
|
||||||
if (CollectionUtils.isEmpty(filePaths)) {
|
if (CollectionUtils.isEmpty(filePaths)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
datasetFileApplicationService.copyFilesToDatasetDir(datasetId, new CopyFilesRequest(filePaths));
|
datasetFileApplicationService.copyFilesToDatasetDirWithSourceRoot(datasetId, targetPath, filePaths);
|
||||||
log.info("Success file scan, total files: {}", filePaths.size());
|
log.info("Success file scan, total files: {}", filePaths.size());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("处理数据源文件扫描失败,数据集ID: {}, 数据源ID: {}", datasetId, dataSourceId, e);
|
log.error("处理数据源文件扫描失败,数据集ID: {}, 数据源ID: {}", datasetId, dataSourceId, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<String> getFilePaths(String dataSourceId) {
|
private List<String> scanFilePaths(Path targetPath) {
|
||||||
CollectionTaskDetailResponse taskDetail = collectionTaskClient.getTaskDetail(dataSourceId).getData();
|
try (Stream<Path> paths = Files.walk(targetPath)) {
|
||||||
if (taskDetail == null) {
|
|
||||||
log.warn("Fail to get collection task detail, task ID: {}", dataSourceId);
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
Path targetPath = Paths.get(taskDetail.getTargetPath());
|
|
||||||
if (!Files.exists(targetPath) || !Files.isDirectory(targetPath)) {
|
|
||||||
log.warn("Target path not exists or is not a directory: {}", taskDetail.getTargetPath());
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
|
|
||||||
try (Stream<Path> paths = Files.walk(targetPath, 1)) {
|
|
||||||
return paths
|
return paths
|
||||||
.filter(Files::isRegularFile) // 只保留文件,排除目录
|
.filter(Files::isRegularFile)
|
||||||
.map(Path::toString) // 转换为字符串路径
|
.map(Path::toString)
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error("Fail to scan directory: {}", targetPath, e);
|
log.error("Fail to scan directory: {}", targetPath, e);
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ import com.datamate.datamanagement.interfaces.dto.CopyFilesRequest;
|
|||||||
import com.datamate.datamanagement.interfaces.dto.CreateDirectoryRequest;
|
import com.datamate.datamanagement.interfaces.dto.CreateDirectoryRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
|
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
|
import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
|
||||||
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
@@ -42,6 +43,8 @@ import org.springframework.core.io.UrlResource;
|
|||||||
import org.springframework.http.HttpHeaders;
|
import org.springframework.http.HttpHeaders;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronization;
|
||||||
|
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -70,12 +73,22 @@ public class DatasetFileApplicationService {
|
|||||||
private static final String PDF_FILE_TYPE = "pdf";
|
private static final String PDF_FILE_TYPE = "pdf";
|
||||||
private static final String DOC_FILE_TYPE = "doc";
|
private static final String DOC_FILE_TYPE = "doc";
|
||||||
private static final String DOCX_FILE_TYPE = "docx";
|
private static final String DOCX_FILE_TYPE = "docx";
|
||||||
private static final Set<String> DOCUMENT_TEXT_FILE_TYPES = Set.of(PDF_FILE_TYPE, DOC_FILE_TYPE, DOCX_FILE_TYPE);
|
private static final String XLS_FILE_TYPE = "xls";
|
||||||
|
private static final String XLSX_FILE_TYPE = "xlsx";
|
||||||
|
private static final Set<String> DOCUMENT_TEXT_FILE_TYPES = Set.of(
|
||||||
|
PDF_FILE_TYPE,
|
||||||
|
DOC_FILE_TYPE,
|
||||||
|
DOCX_FILE_TYPE,
|
||||||
|
XLS_FILE_TYPE,
|
||||||
|
XLSX_FILE_TYPE
|
||||||
|
);
|
||||||
|
private static final String DERIVED_METADATA_KEY = "derived_from_file_id";
|
||||||
|
|
||||||
private final DatasetFileRepository datasetFileRepository;
|
private final DatasetFileRepository datasetFileRepository;
|
||||||
private final DatasetRepository datasetRepository;
|
private final DatasetRepository datasetRepository;
|
||||||
private final FileService fileService;
|
private final FileService fileService;
|
||||||
private final PdfTextExtractAsyncService pdfTextExtractAsyncService;
|
private final PdfTextExtractAsyncService pdfTextExtractAsyncService;
|
||||||
|
private final DatasetFilePreviewService datasetFilePreviewService;
|
||||||
|
|
||||||
@Value("${datamate.data-management.base-path:/dataset}")
|
@Value("${datamate.data-management.base-path:/dataset}")
|
||||||
private String datasetBasePath;
|
private String datasetBasePath;
|
||||||
@@ -84,15 +97,17 @@ public class DatasetFileApplicationService {
|
|||||||
private DuplicateMethod duplicateMethod;
|
private DuplicateMethod duplicateMethod;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
public DatasetFileApplicationService(DatasetFileRepository datasetFileRepository,
|
public DatasetFileApplicationService(DatasetFileRepository datasetFileRepository,
|
||||||
DatasetRepository datasetRepository,
|
DatasetRepository datasetRepository,
|
||||||
FileService fileService,
|
FileService fileService,
|
||||||
PdfTextExtractAsyncService pdfTextExtractAsyncService) {
|
PdfTextExtractAsyncService pdfTextExtractAsyncService,
|
||||||
this.datasetFileRepository = datasetFileRepository;
|
DatasetFilePreviewService datasetFilePreviewService) {
|
||||||
this.datasetRepository = datasetRepository;
|
this.datasetFileRepository = datasetFileRepository;
|
||||||
this.fileService = fileService;
|
this.datasetRepository = datasetRepository;
|
||||||
this.pdfTextExtractAsyncService = pdfTextExtractAsyncService;
|
this.fileService = fileService;
|
||||||
}
|
this.pdfTextExtractAsyncService = pdfTextExtractAsyncService;
|
||||||
|
this.datasetFilePreviewService = datasetFilePreviewService;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取数据集文件列表
|
* 获取数据集文件列表
|
||||||
@@ -111,7 +126,7 @@ public class DatasetFileApplicationService {
|
|||||||
* @param status 状态过滤
|
* @param status 状态过滤
|
||||||
* @param name 文件名模糊查询
|
* @param name 文件名模糊查询
|
||||||
* @param hasAnnotation 是否有标注
|
* @param hasAnnotation 是否有标注
|
||||||
* @param excludeSourceDocuments 是否排除已被转换为TXT的源文档(PDF/DOC/DOCX)
|
* @param excludeSourceDocuments 是否排除源文档(PDF/DOC/DOCX/XLS/XLSX)
|
||||||
* @param pagingQuery 分页参数
|
* @param pagingQuery 分页参数
|
||||||
* @return 分页文件列表
|
* @return 分页文件列表
|
||||||
*/
|
*/
|
||||||
@@ -122,19 +137,15 @@ public class DatasetFileApplicationService {
|
|||||||
IPage<DatasetFile> files = datasetFileRepository.findByCriteria(datasetId, fileType, status, name, hasAnnotation, page);
|
IPage<DatasetFile> files = datasetFileRepository.findByCriteria(datasetId, fileType, status, name, hasAnnotation, page);
|
||||||
|
|
||||||
if (excludeSourceDocuments) {
|
if (excludeSourceDocuments) {
|
||||||
// 查询所有作为衍生TXT文件源的文档文件ID
|
// 过滤掉源文档文件(PDF/DOC/DOCX/XLS/XLSX),用于标注场景只展示派生文件
|
||||||
List<String> sourceFileIds = datasetFileRepository.findSourceFileIdsWithDerivedFiles(datasetId);
|
List<DatasetFile> filteredRecords = files.getRecords().stream()
|
||||||
if (!sourceFileIds.isEmpty()) {
|
.filter(file -> !isSourceDocument(file))
|
||||||
// 过滤掉源文件
|
.collect(Collectors.toList());
|
||||||
List<DatasetFile> filteredRecords = files.getRecords().stream()
|
|
||||||
.filter(file -> !sourceFileIds.contains(file.getId()))
|
// 重新构建分页结果
|
||||||
.collect(Collectors.toList());
|
Page<DatasetFile> filteredPage = new Page<>(files.getCurrent(), files.getSize(), files.getTotal());
|
||||||
|
filteredPage.setRecords(filteredRecords);
|
||||||
// 重新构建分页结果
|
return PagedResponse.of(filteredPage);
|
||||||
Page<DatasetFile> filteredPage = new Page<>(files.getCurrent(), files.getSize(), files.getTotal());
|
|
||||||
filteredPage.setRecords(filteredRecords);
|
|
||||||
return PagedResponse.of(filteredPage);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return PagedResponse.of(files);
|
return PagedResponse.of(files);
|
||||||
@@ -144,7 +155,7 @@ public class DatasetFileApplicationService {
|
|||||||
* 获取数据集文件列表
|
* 获取数据集文件列表
|
||||||
*/
|
*/
|
||||||
@Transactional(readOnly = true)
|
@Transactional(readOnly = true)
|
||||||
public PagedResponse<DatasetFile> getDatasetFilesWithDirectory(String datasetId, String prefix, PagingQuery pagingQuery) {
|
public PagedResponse<DatasetFile> getDatasetFilesWithDirectory(String datasetId, String prefix, boolean excludeDerivedFiles, PagingQuery pagingQuery) {
|
||||||
Dataset dataset = datasetRepository.getById(datasetId);
|
Dataset dataset = datasetRepository.getById(datasetId);
|
||||||
int page = Math.max(pagingQuery.getPage(), 1);
|
int page = Math.max(pagingQuery.getPage(), 1);
|
||||||
int size = pagingQuery.getSize() == null || pagingQuery.getSize() < 0 ? 20 : pagingQuery.getSize();
|
int size = pagingQuery.getSize() == null || pagingQuery.getSize() < 0 ? 20 : pagingQuery.getSize();
|
||||||
@@ -153,15 +164,36 @@ public class DatasetFileApplicationService {
|
|||||||
}
|
}
|
||||||
String datasetPath = dataset.getPath();
|
String datasetPath = dataset.getPath();
|
||||||
Path queryPath = Path.of(dataset.getPath() + File.separator + prefix);
|
Path queryPath = Path.of(dataset.getPath() + File.separator + prefix);
|
||||||
Map<String, DatasetFile> datasetFilesMap = datasetFileRepository.findAllByDatasetId(datasetId)
|
Map<String, DatasetFile> datasetFilesMap = datasetFileRepository.findAllByDatasetId(datasetId)
|
||||||
.stream().collect(Collectors.toMap(DatasetFile::getFilePath, Function.identity()));
|
.stream()
|
||||||
|
.filter(file -> file.getFilePath() != null)
|
||||||
|
.collect(Collectors.toMap(
|
||||||
|
file -> normalizeFilePath(file.getFilePath()),
|
||||||
|
Function.identity(),
|
||||||
|
(left, right) -> left
|
||||||
|
));
|
||||||
|
Set<String> derivedFilePaths = excludeDerivedFiles
|
||||||
|
? datasetFilesMap.values().stream()
|
||||||
|
.filter(this::isDerivedFile)
|
||||||
|
.map(DatasetFile::getFilePath)
|
||||||
|
.map(this::normalizeFilePath)
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.collect(Collectors.toSet())
|
||||||
|
: Collections.emptySet();
|
||||||
|
// 如果目录不存在,直接返回空结果(数据集刚创建时目录可能还未生成)
|
||||||
|
if (!Files.exists(queryPath)) {
|
||||||
|
return new PagedResponse<>(page, size, 0, 0, Collections.emptyList());
|
||||||
|
}
|
||||||
try (Stream<Path> pathStream = Files.list(queryPath)) {
|
try (Stream<Path> pathStream = Files.list(queryPath)) {
|
||||||
List<Path> allFiles = pathStream
|
List<Path> allFiles = pathStream
|
||||||
.filter(path -> path.toString().startsWith(datasetPath))
|
.filter(path -> path.toString().startsWith(datasetPath))
|
||||||
.sorted(Comparator
|
.filter(path -> !excludeDerivedFiles
|
||||||
.comparing((Path path) -> !Files.isDirectory(path))
|
|| Files.isDirectory(path)
|
||||||
.thenComparing(path -> path.getFileName().toString()))
|
|| !derivedFilePaths.contains(normalizeFilePath(path.toString())))
|
||||||
.collect(Collectors.toList());
|
.sorted(Comparator
|
||||||
|
.comparing((Path path) -> !Files.isDirectory(path))
|
||||||
|
.thenComparing(path -> path.getFileName().toString()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
// 计算分页
|
// 计算分页
|
||||||
int total = allFiles.size();
|
int total = allFiles.size();
|
||||||
@@ -176,7 +208,9 @@ public class DatasetFileApplicationService {
|
|||||||
if (fromIndex < total) {
|
if (fromIndex < total) {
|
||||||
pageData = allFiles.subList(fromIndex, toIndex);
|
pageData = allFiles.subList(fromIndex, toIndex);
|
||||||
}
|
}
|
||||||
List<DatasetFile> datasetFiles = pageData.stream().map(path -> getDatasetFile(path, datasetFilesMap)).toList();
|
List<DatasetFile> datasetFiles = pageData.stream()
|
||||||
|
.map(path -> getDatasetFile(path, datasetFilesMap, excludeDerivedFiles, derivedFilePaths))
|
||||||
|
.toList();
|
||||||
|
|
||||||
return new PagedResponse<>(page, size, total, totalPages, datasetFiles);
|
return new PagedResponse<>(page, size, total, totalPages, datasetFiles);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
@@ -185,9 +219,12 @@ public class DatasetFileApplicationService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private DatasetFile getDatasetFile(Path path, Map<String, DatasetFile> datasetFilesMap) {
|
private DatasetFile getDatasetFile(Path path,
|
||||||
DatasetFile datasetFile = new DatasetFile();
|
Map<String, DatasetFile> datasetFilesMap,
|
||||||
LocalDateTime localDateTime = LocalDateTime.now();
|
boolean excludeDerivedFiles,
|
||||||
|
Set<String> derivedFilePaths) {
|
||||||
|
DatasetFile datasetFile = new DatasetFile();
|
||||||
|
LocalDateTime localDateTime = LocalDateTime.now();
|
||||||
try {
|
try {
|
||||||
localDateTime = Files.getLastModifiedTime(path).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
|
localDateTime = Files.getLastModifiedTime(path).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
@@ -206,23 +243,32 @@ public class DatasetFileApplicationService {
|
|||||||
long fileCount;
|
long fileCount;
|
||||||
long totalSize;
|
long totalSize;
|
||||||
|
|
||||||
try (Stream<Path> walk = Files.walk(path)) {
|
try (Stream<Path> walk = Files.walk(path)) {
|
||||||
fileCount = walk.filter(Files::isRegularFile).count();
|
Stream<Path> fileStream = walk.filter(Files::isRegularFile);
|
||||||
}
|
if (excludeDerivedFiles && !derivedFilePaths.isEmpty()) {
|
||||||
|
fileStream = fileStream.filter(filePath ->
|
||||||
try (Stream<Path> walk = Files.walk(path)) {
|
!derivedFilePaths.contains(normalizeFilePath(filePath.toString())));
|
||||||
totalSize = walk
|
}
|
||||||
.filter(Files::isRegularFile)
|
fileCount = fileStream.count();
|
||||||
.mapToLong(p -> {
|
}
|
||||||
try {
|
|
||||||
return Files.size(p);
|
try (Stream<Path> walk = Files.walk(path)) {
|
||||||
} catch (IOException e) {
|
Stream<Path> fileStream = walk.filter(Files::isRegularFile);
|
||||||
log.error("get file size error", e);
|
if (excludeDerivedFiles && !derivedFilePaths.isEmpty()) {
|
||||||
return 0L;
|
fileStream = fileStream.filter(filePath ->
|
||||||
}
|
!derivedFilePaths.contains(normalizeFilePath(filePath.toString())));
|
||||||
})
|
}
|
||||||
.sum();
|
totalSize = fileStream
|
||||||
}
|
.mapToLong(p -> {
|
||||||
|
try {
|
||||||
|
return Files.size(p);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("get file size error", e);
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.sum();
|
||||||
|
}
|
||||||
|
|
||||||
datasetFile.setFileCount(fileCount);
|
datasetFile.setFileCount(fileCount);
|
||||||
datasetFile.setFileSize(totalSize);
|
datasetFile.setFileSize(totalSize);
|
||||||
@@ -230,15 +276,55 @@ public class DatasetFileApplicationService {
|
|||||||
log.error("stat directory info error", e);
|
log.error("stat directory info error", e);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
DatasetFile exist = datasetFilesMap.get(path.toString());
|
DatasetFile exist = datasetFilesMap.get(normalizeFilePath(path.toString()));
|
||||||
if (exist == null) {
|
if (exist == null) {
|
||||||
datasetFile.setId("file-" + datasetFile.getFileName());
|
datasetFile.setId("file-" + datasetFile.getFileName());
|
||||||
datasetFile.setFileSize(path.toFile().length());
|
datasetFile.setFileSize(path.toFile().length());
|
||||||
} else {
|
} else {
|
||||||
datasetFile = exist;
|
datasetFile = exist;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return datasetFile;
|
return datasetFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeFilePath(String filePath) {
|
||||||
|
if (filePath == null || filePath.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return Paths.get(filePath).toAbsolutePath().normalize().toString();
|
||||||
|
} catch (Exception e) {
|
||||||
|
return filePath.replace("\\", "/");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isSourceDocument(DatasetFile datasetFile) {
|
||||||
|
if (datasetFile == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String fileType = datasetFile.getFileType();
|
||||||
|
if (fileType == null || fileType.isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return DOCUMENT_TEXT_FILE_TYPES.contains(fileType.toLowerCase(Locale.ROOT));
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isDerivedFile(DatasetFile datasetFile) {
|
||||||
|
if (datasetFile == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String metadata = datasetFile.getMetadata();
|
||||||
|
if (metadata == null || metadata.isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
Map<String, Object> metadataMap = mapper.readValue(metadata, new TypeReference<Map<String, Object>>() {});
|
||||||
|
return metadataMap.get(DERIVED_METADATA_KEY) != null;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("Failed to parse dataset file metadata for derived detection: {}", datasetFile.getId(), e);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -260,18 +346,19 @@ public class DatasetFileApplicationService {
|
|||||||
* 删除文件
|
* 删除文件
|
||||||
*/
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void deleteDatasetFile(String datasetId, String fileId) {
|
public void deleteDatasetFile(String datasetId, String fileId) {
|
||||||
DatasetFile file = getDatasetFile(datasetId, fileId);
|
DatasetFile file = getDatasetFile(datasetId, fileId);
|
||||||
Dataset dataset = datasetRepository.getById(datasetId);
|
Dataset dataset = datasetRepository.getById(datasetId);
|
||||||
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));
|
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));
|
||||||
datasetFileRepository.removeById(fileId);
|
datasetFileRepository.removeById(fileId);
|
||||||
dataset.removeFile(file);
|
dataset.removeFile(file);
|
||||||
datasetRepository.updateById(dataset);
|
datasetRepository.updateById(dataset);
|
||||||
// 删除文件时,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录
|
datasetFilePreviewService.deletePreviewFileQuietly(datasetId, fileId);
|
||||||
if (file.getFilePath().startsWith(dataset.getPath())) {
|
// 删除文件时,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录
|
||||||
try {
|
if (file.getFilePath().startsWith(dataset.getPath())) {
|
||||||
Path filePath = Paths.get(file.getFilePath());
|
try {
|
||||||
Files.deleteIfExists(filePath);
|
Path filePath = Paths.get(file.getFilePath());
|
||||||
|
Files.deleteIfExists(filePath);
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
|
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
|
||||||
}
|
}
|
||||||
@@ -412,11 +499,19 @@ public class DatasetFileApplicationService {
|
|||||||
*
|
*
|
||||||
* @param uploadFileRequest 上传请求
|
* @param uploadFileRequest 上传请求
|
||||||
*/
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void chunkUpload(String datasetId, UploadFileRequest uploadFileRequest) {
|
public void chunkUpload(String datasetId, UploadFileRequest uploadFileRequest) {
|
||||||
FileUploadResult uploadResult = fileService.chunkUpload(DatasetConverter.INSTANCE.toChunkUploadRequest(uploadFileRequest));
|
FileUploadResult uploadResult = fileService.chunkUpload(DatasetConverter.INSTANCE.toChunkUploadRequest(uploadFileRequest));
|
||||||
saveFileInfoToDb(uploadResult, datasetId);
|
saveFileInfoToDb(uploadResult, datasetId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 取消上传
|
||||||
|
*/
|
||||||
|
@Transactional
|
||||||
|
public void cancelUpload(String reqId) {
|
||||||
|
fileService.cancelUpload(reqId);
|
||||||
|
}
|
||||||
|
|
||||||
private void saveFileInfoToDb(FileUploadResult fileUploadResult, String datasetId) {
|
private void saveFileInfoToDb(FileUploadResult fileUploadResult, String datasetId) {
|
||||||
if (Objects.isNull(fileUploadResult.getSavedFile())) {
|
if (Objects.isNull(fileUploadResult.getSavedFile())) {
|
||||||
@@ -637,9 +732,10 @@ public class DatasetFileApplicationService {
|
|||||||
})
|
})
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
for (DatasetFile file : filesToDelete) {
|
for (DatasetFile file : filesToDelete) {
|
||||||
datasetFileRepository.removeById(file.getId());
|
datasetFileRepository.removeById(file.getId());
|
||||||
}
|
datasetFilePreviewService.deletePreviewFileQuietly(datasetId, file.getId());
|
||||||
|
}
|
||||||
|
|
||||||
// 删除文件系统中的目录
|
// 删除文件系统中的目录
|
||||||
try {
|
try {
|
||||||
@@ -739,6 +835,71 @@ public class DatasetFileApplicationService {
|
|||||||
return copiedFiles;
|
return copiedFiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 复制文件到数据集目录(保留相对路径,适用于数据源导入)
|
||||||
|
*
|
||||||
|
* @param datasetId 数据集id
|
||||||
|
* @param sourceRoot 数据源根目录
|
||||||
|
* @param sourcePaths 源文件路径列表
|
||||||
|
* @return 复制的文件列表
|
||||||
|
*/
|
||||||
|
@Transactional
|
||||||
|
public List<DatasetFile> copyFilesToDatasetDirWithSourceRoot(String datasetId, Path sourceRoot, List<String> sourcePaths) {
|
||||||
|
Dataset dataset = datasetRepository.getById(datasetId);
|
||||||
|
BusinessAssert.notNull(dataset, SystemErrorCode.RESOURCE_NOT_FOUND);
|
||||||
|
|
||||||
|
Path normalizedRoot = sourceRoot.toAbsolutePath().normalize();
|
||||||
|
List<DatasetFile> copiedFiles = new ArrayList<>();
|
||||||
|
List<DatasetFile> existDatasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
|
||||||
|
dataset.setFiles(existDatasetFiles);
|
||||||
|
Map<String, DatasetFile> copyTargets = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
for (String sourceFilePath : sourcePaths) {
|
||||||
|
if (sourceFilePath == null || sourceFilePath.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Path sourcePath = Paths.get(sourceFilePath).toAbsolutePath().normalize();
|
||||||
|
if (!sourcePath.startsWith(normalizedRoot)) {
|
||||||
|
log.warn("Source file path is out of root: {}", sourceFilePath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!Files.exists(sourcePath) || !Files.isRegularFile(sourcePath)) {
|
||||||
|
log.warn("Source file does not exist or is not a regular file: {}", sourceFilePath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path relativePath = normalizedRoot.relativize(sourcePath);
|
||||||
|
String fileName = sourcePath.getFileName().toString();
|
||||||
|
File sourceFile = sourcePath.toFile();
|
||||||
|
LocalDateTime currentTime = LocalDateTime.now();
|
||||||
|
Path targetPath = Paths.get(dataset.getPath(), relativePath.toString());
|
||||||
|
|
||||||
|
DatasetFile datasetFile = DatasetFile.builder()
|
||||||
|
.id(UUID.randomUUID().toString())
|
||||||
|
.datasetId(datasetId)
|
||||||
|
.fileName(fileName)
|
||||||
|
.fileType(AnalyzerUtils.getExtension(fileName))
|
||||||
|
.fileSize(sourceFile.length())
|
||||||
|
.filePath(targetPath.toString())
|
||||||
|
.uploadTime(currentTime)
|
||||||
|
.lastAccessTime(currentTime)
|
||||||
|
.build();
|
||||||
|
setDatasetFileId(datasetFile, dataset);
|
||||||
|
dataset.addFile(datasetFile);
|
||||||
|
copiedFiles.add(datasetFile);
|
||||||
|
copyTargets.put(sourceFilePath, datasetFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (copiedFiles.isEmpty()) {
|
||||||
|
return copiedFiles;
|
||||||
|
}
|
||||||
|
datasetFileRepository.saveOrUpdateBatch(copiedFiles, 100);
|
||||||
|
dataset.active();
|
||||||
|
datasetRepository.updateById(dataset);
|
||||||
|
CompletableFuture.runAsync(() -> copyFilesToDatasetDirWithRelativePath(copyTargets, dataset, normalizedRoot));
|
||||||
|
return copiedFiles;
|
||||||
|
}
|
||||||
|
|
||||||
private void copyFilesToDatasetDir(List<String> sourcePaths, Dataset dataset) {
|
private void copyFilesToDatasetDir(List<String> sourcePaths, Dataset dataset) {
|
||||||
for (String sourcePath : sourcePaths) {
|
for (String sourcePath : sourcePaths) {
|
||||||
Path sourceFilePath = Paths.get(sourcePath);
|
Path sourceFilePath = Paths.get(sourcePath);
|
||||||
@@ -757,6 +918,35 @@ public class DatasetFileApplicationService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void copyFilesToDatasetDirWithRelativePath(
|
||||||
|
Map<String, DatasetFile> copyTargets,
|
||||||
|
Dataset dataset,
|
||||||
|
Path sourceRoot
|
||||||
|
) {
|
||||||
|
Path datasetRoot = Paths.get(dataset.getPath()).toAbsolutePath().normalize();
|
||||||
|
Path normalizedRoot = sourceRoot.toAbsolutePath().normalize();
|
||||||
|
for (Map.Entry<String, DatasetFile> entry : copyTargets.entrySet()) {
|
||||||
|
Path sourcePath = Paths.get(entry.getKey()).toAbsolutePath().normalize();
|
||||||
|
if (!sourcePath.startsWith(normalizedRoot)) {
|
||||||
|
log.warn("Source file path is out of root: {}", sourcePath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Path relativePath = normalizedRoot.relativize(sourcePath);
|
||||||
|
Path targetFilePath = datasetRoot.resolve(relativePath).normalize();
|
||||||
|
if (!targetFilePath.startsWith(datasetRoot)) {
|
||||||
|
log.warn("Target file path is out of dataset path: {}", targetFilePath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Files.createDirectories(targetFilePath.getParent());
|
||||||
|
Files.copy(sourcePath, targetFilePath);
|
||||||
|
triggerPdfTextExtraction(dataset, entry.getValue());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Failed to copy file from {} to {}", sourcePath, targetFilePath, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 添加文件到数据集(仅创建数据库记录,不执行文件系统操作)
|
* 添加文件到数据集(仅创建数据库记录,不执行文件系统操作)
|
||||||
*
|
*
|
||||||
@@ -824,6 +1014,20 @@ public class DatasetFileApplicationService {
|
|||||||
if (fileType == null || !DOCUMENT_TEXT_FILE_TYPES.contains(fileType.toLowerCase(Locale.ROOT))) {
|
if (fileType == null || !DOCUMENT_TEXT_FILE_TYPES.contains(fileType.toLowerCase(Locale.ROOT))) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
pdfTextExtractAsyncService.extractPdfText(dataset.getId(), datasetFile.getId());
|
String datasetId = dataset.getId();
|
||||||
|
String fileId = datasetFile.getId();
|
||||||
|
if (datasetId == null || fileId == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||||
|
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||||
|
@Override
|
||||||
|
public void afterCommit() {
|
||||||
|
pdfTextExtractAsyncService.extractPdfText(datasetId, fileId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pdfTextExtractAsyncService.extractPdfText(datasetId, fileId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeItemPreviewStatus;
|
||||||
|
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
||||||
|
import com.datamate.datamanagement.infrastructure.config.DataManagementProperties;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.scheduling.annotation.Async;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据集文件预览转换异步任务
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class DatasetFilePreviewAsyncService {
|
||||||
|
private static final Set<String> OFFICE_EXTENSIONS = Set.of("doc", "docx");
|
||||||
|
private static final String DATASET_PREVIEW_DIR = "dataset-previews";
|
||||||
|
private static final String PREVIEW_FILE_SUFFIX = ".pdf";
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
|
private static final int MAX_ERROR_LENGTH = 500;
|
||||||
|
private static final DateTimeFormatter PREVIEW_TIME_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
|
||||||
|
|
||||||
|
private final DatasetFileRepository datasetFileRepository;
|
||||||
|
private final DataManagementProperties dataManagementProperties;
|
||||||
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
@Async
|
||||||
|
public void convertPreviewAsync(String fileId) {
|
||||||
|
if (StringUtils.isBlank(fileId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
DatasetFile file = datasetFileRepository.getById(fileId);
|
||||||
|
if (file == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String extension = resolveFileExtension(resolveOriginalName(file));
|
||||||
|
if (!OFFICE_EXTENSIONS.contains(extension)) {
|
||||||
|
updatePreviewStatus(file, KnowledgeItemPreviewStatus.FAILED, null, "仅支持 DOC/DOCX 转换");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(file.getFilePath())) {
|
||||||
|
updatePreviewStatus(file, KnowledgeItemPreviewStatus.FAILED, null, "源文件路径为空");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Path sourcePath = Paths.get(file.getFilePath()).toAbsolutePath().normalize();
|
||||||
|
if (!Files.exists(sourcePath) || !Files.isRegularFile(sourcePath)) {
|
||||||
|
updatePreviewStatus(file, KnowledgeItemPreviewStatus.FAILED, null, "源文件不存在");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(file.getMetadata(), objectMapper);
|
||||||
|
String previewRelativePath = StringUtils.defaultIfBlank(
|
||||||
|
previewInfo.pdfPath(),
|
||||||
|
resolvePreviewRelativePath(file.getDatasetId(), file.getId())
|
||||||
|
);
|
||||||
|
Path targetPath = resolvePreviewStoragePath(previewRelativePath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureParentDirectory(targetPath);
|
||||||
|
LibreOfficeConverter.convertToPdf(sourcePath, targetPath);
|
||||||
|
updatePreviewStatus(file, KnowledgeItemPreviewStatus.READY, previewRelativePath, null);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("dataset preview convert failed, fileId: {}", file.getId(), e);
|
||||||
|
updatePreviewStatus(file, KnowledgeItemPreviewStatus.FAILED, previewRelativePath, trimError(e.getMessage()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updatePreviewStatus(
|
||||||
|
DatasetFile file,
|
||||||
|
KnowledgeItemPreviewStatus status,
|
||||||
|
String previewRelativePath,
|
||||||
|
String error
|
||||||
|
) {
|
||||||
|
if (file == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String updatedMetadata = KnowledgeItemPreviewMetadataHelper.applyPreviewInfo(
|
||||||
|
file.getMetadata(),
|
||||||
|
objectMapper,
|
||||||
|
status,
|
||||||
|
previewRelativePath,
|
||||||
|
error,
|
||||||
|
nowText()
|
||||||
|
);
|
||||||
|
file.setMetadata(updatedMetadata);
|
||||||
|
datasetFileRepository.updateById(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveOriginalName(DatasetFile file) {
|
||||||
|
if (file == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(file.getFileName())) {
|
||||||
|
return file.getFileName();
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(file.getFilePath())) {
|
||||||
|
return Paths.get(file.getFilePath()).getFileName().toString();
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveFileExtension(String fileName) {
|
||||||
|
if (StringUtils.isBlank(fileName)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
int dotIndex = fileName.lastIndexOf('.');
|
||||||
|
if (dotIndex <= 0 || dotIndex >= fileName.length() - 1) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return fileName.substring(dotIndex + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePreviewRelativePath(String datasetId, String fileId) {
|
||||||
|
String relativePath = Paths.get(DATASET_PREVIEW_DIR, datasetId, fileId + PREVIEW_FILE_SUFFIX)
|
||||||
|
.toString();
|
||||||
|
return relativePath.replace("\\", PATH_SEPARATOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolvePreviewStoragePath(String relativePath) {
|
||||||
|
String normalizedRelativePath = StringUtils.defaultString(relativePath).replace("/", java.io.File.separator);
|
||||||
|
Path root = resolveUploadRootPath();
|
||||||
|
Path target = root.resolve(normalizedRelativePath).toAbsolutePath().normalize();
|
||||||
|
if (!target.startsWith(root)) {
|
||||||
|
throw new IllegalArgumentException("invalid preview path");
|
||||||
|
}
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolveUploadRootPath() {
|
||||||
|
String uploadDir = dataManagementProperties.getFileStorage().getUploadDir();
|
||||||
|
return Paths.get(uploadDir).toAbsolutePath().normalize();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ensureParentDirectory(Path targetPath) {
|
||||||
|
try {
|
||||||
|
Path parent = targetPath.getParent();
|
||||||
|
if (parent != null) {
|
||||||
|
Files.createDirectories(parent);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IllegalStateException("创建预览目录失败", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String trimError(String error) {
|
||||||
|
if (StringUtils.isBlank(error)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (error.length() <= MAX_ERROR_LENGTH) {
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
return error.substring(0, MAX_ERROR_LENGTH);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nowText() {
|
||||||
|
return LocalDateTime.now().format(PREVIEW_TIME_FORMATTER);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import com.datamate.common.infrastructure.exception.BusinessAssert;
|
||||||
|
import com.datamate.common.infrastructure.exception.CommonErrorCode;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeItemPreviewStatus;
|
||||||
|
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
||||||
|
import com.datamate.datamanagement.infrastructure.config.DataManagementProperties;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.DatasetFilePreviewStatusResponse;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据集文件预览转换服务
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class DatasetFilePreviewService {
|
||||||
|
private static final Set<String> OFFICE_EXTENSIONS = Set.of("doc", "docx");
|
||||||
|
private static final String DATASET_PREVIEW_DIR = "dataset-previews";
|
||||||
|
private static final String PREVIEW_FILE_SUFFIX = ".pdf";
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
|
private static final DateTimeFormatter PREVIEW_TIME_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
|
||||||
|
|
||||||
|
private final DatasetFileRepository datasetFileRepository;
|
||||||
|
private final DataManagementProperties dataManagementProperties;
|
||||||
|
private final DatasetFilePreviewAsyncService datasetFilePreviewAsyncService;
|
||||||
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
public DatasetFilePreviewStatusResponse getPreviewStatus(String datasetId, String fileId) {
|
||||||
|
DatasetFile file = requireDatasetFile(datasetId, fileId);
|
||||||
|
assertOfficeDocument(file);
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(file.getMetadata(), objectMapper);
|
||||||
|
|
||||||
|
if (previewInfo.status() == KnowledgeItemPreviewStatus.READY && !previewPdfExists(file, previewInfo)) {
|
||||||
|
previewInfo = markPreviewFailed(file, previewInfo, "预览文件不存在");
|
||||||
|
}
|
||||||
|
|
||||||
|
return buildResponse(previewInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatasetFilePreviewStatusResponse ensurePreview(String datasetId, String fileId) {
|
||||||
|
DatasetFile file = requireDatasetFile(datasetId, fileId);
|
||||||
|
assertOfficeDocument(file);
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(file.getMetadata(), objectMapper);
|
||||||
|
|
||||||
|
if (previewInfo.status() == KnowledgeItemPreviewStatus.READY && previewPdfExists(file, previewInfo)) {
|
||||||
|
return buildResponse(previewInfo);
|
||||||
|
}
|
||||||
|
if (previewInfo.status() == KnowledgeItemPreviewStatus.PROCESSING) {
|
||||||
|
return buildResponse(previewInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
String previewRelativePath = resolvePreviewRelativePath(file.getDatasetId(), file.getId());
|
||||||
|
String updatedMetadata = KnowledgeItemPreviewMetadataHelper.applyPreviewInfo(
|
||||||
|
file.getMetadata(),
|
||||||
|
objectMapper,
|
||||||
|
KnowledgeItemPreviewStatus.PROCESSING,
|
||||||
|
previewRelativePath,
|
||||||
|
null,
|
||||||
|
nowText()
|
||||||
|
);
|
||||||
|
file.setMetadata(updatedMetadata);
|
||||||
|
datasetFileRepository.updateById(file);
|
||||||
|
datasetFilePreviewAsyncService.convertPreviewAsync(file.getId());
|
||||||
|
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo refreshed = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(updatedMetadata, objectMapper);
|
||||||
|
return buildResponse(refreshed);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isOfficeDocument(String fileName) {
|
||||||
|
String extension = resolveFileExtension(fileName);
|
||||||
|
return StringUtils.isNotBlank(extension) && OFFICE_EXTENSIONS.contains(extension.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
public PreviewFile resolveReadyPreviewFile(String datasetId, DatasetFile file) {
|
||||||
|
if (file == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(file.getMetadata(), objectMapper);
|
||||||
|
if (previewInfo.status() != KnowledgeItemPreviewStatus.READY) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String relativePath = StringUtils.defaultIfBlank(previewInfo.pdfPath(), resolvePreviewRelativePath(datasetId, file.getId()));
|
||||||
|
Path filePath = resolvePreviewStoragePath(relativePath);
|
||||||
|
if (!Files.exists(filePath) || !Files.isRegularFile(filePath)) {
|
||||||
|
markPreviewFailed(file, previewInfo, "预览文件不存在");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String previewName = resolvePreviewPdfName(file);
|
||||||
|
return new PreviewFile(filePath, previewName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deletePreviewFileQuietly(String datasetId, String fileId) {
|
||||||
|
String relativePath = resolvePreviewRelativePath(datasetId, fileId);
|
||||||
|
Path filePath = resolvePreviewStoragePath(relativePath);
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(filePath);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("delete dataset preview pdf error, fileId: {}", fileId, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatasetFilePreviewStatusResponse buildResponse(KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo) {
|
||||||
|
DatasetFilePreviewStatusResponse response = new DatasetFilePreviewStatusResponse();
|
||||||
|
KnowledgeItemPreviewStatus status = previewInfo.status() == null
|
||||||
|
? KnowledgeItemPreviewStatus.PENDING
|
||||||
|
: previewInfo.status();
|
||||||
|
response.setStatus(status);
|
||||||
|
response.setPreviewError(previewInfo.error());
|
||||||
|
response.setUpdatedAt(previewInfo.updatedAt());
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatasetFile requireDatasetFile(String datasetId, String fileId) {
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(datasetId), CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(fileId), CommonErrorCode.PARAM_ERROR);
|
||||||
|
DatasetFile datasetFile = datasetFileRepository.getById(fileId);
|
||||||
|
BusinessAssert.notNull(datasetFile, CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(Objects.equals(datasetFile.getDatasetId(), datasetId), CommonErrorCode.PARAM_ERROR);
|
||||||
|
return datasetFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertOfficeDocument(DatasetFile file) {
|
||||||
|
BusinessAssert.notNull(file, CommonErrorCode.PARAM_ERROR);
|
||||||
|
String extension = resolveFileExtension(resolveOriginalName(file));
|
||||||
|
BusinessAssert.isTrue(OFFICE_EXTENSIONS.contains(extension), CommonErrorCode.PARAM_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveOriginalName(DatasetFile file) {
|
||||||
|
if (file == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(file.getFileName())) {
|
||||||
|
return file.getFileName();
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(file.getFilePath())) {
|
||||||
|
return Paths.get(file.getFilePath()).getFileName().toString();
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveFileExtension(String fileName) {
|
||||||
|
if (StringUtils.isBlank(fileName)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
int dotIndex = fileName.lastIndexOf('.');
|
||||||
|
if (dotIndex <= 0 || dotIndex >= fileName.length() - 1) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return fileName.substring(dotIndex + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePreviewPdfName(DatasetFile file) {
|
||||||
|
String originalName = resolveOriginalName(file);
|
||||||
|
if (StringUtils.isBlank(originalName)) {
|
||||||
|
return "预览.pdf";
|
||||||
|
}
|
||||||
|
int dotIndex = originalName.lastIndexOf('.');
|
||||||
|
if (dotIndex <= 0) {
|
||||||
|
return originalName + PREVIEW_FILE_SUFFIX;
|
||||||
|
}
|
||||||
|
return originalName.substring(0, dotIndex) + PREVIEW_FILE_SUFFIX;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean previewPdfExists(DatasetFile file, KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo) {
|
||||||
|
String relativePath = StringUtils.defaultIfBlank(previewInfo.pdfPath(), resolvePreviewRelativePath(file.getDatasetId(), file.getId()));
|
||||||
|
Path filePath = resolvePreviewStoragePath(relativePath);
|
||||||
|
return Files.exists(filePath) && Files.isRegularFile(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
private KnowledgeItemPreviewMetadataHelper.PreviewInfo markPreviewFailed(
|
||||||
|
DatasetFile file,
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo,
|
||||||
|
String error
|
||||||
|
) {
|
||||||
|
String relativePath = StringUtils.defaultIfBlank(previewInfo.pdfPath(), resolvePreviewRelativePath(file.getDatasetId(), file.getId()));
|
||||||
|
String updatedMetadata = KnowledgeItemPreviewMetadataHelper.applyPreviewInfo(
|
||||||
|
file.getMetadata(),
|
||||||
|
objectMapper,
|
||||||
|
KnowledgeItemPreviewStatus.FAILED,
|
||||||
|
relativePath,
|
||||||
|
error,
|
||||||
|
nowText()
|
||||||
|
);
|
||||||
|
file.setMetadata(updatedMetadata);
|
||||||
|
datasetFileRepository.updateById(file);
|
||||||
|
return KnowledgeItemPreviewMetadataHelper.readPreviewInfo(updatedMetadata, objectMapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePreviewRelativePath(String datasetId, String fileId) {
|
||||||
|
String relativePath = Paths.get(DATASET_PREVIEW_DIR, datasetId, fileId + PREVIEW_FILE_SUFFIX)
|
||||||
|
.toString();
|
||||||
|
return relativePath.replace("\\", PATH_SEPARATOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
Path resolvePreviewStoragePath(String relativePath) {
|
||||||
|
String normalizedRelativePath = StringUtils.defaultString(relativePath).replace("/", java.io.File.separator);
|
||||||
|
Path root = resolveUploadRootPath();
|
||||||
|
Path target = root.resolve(normalizedRelativePath).toAbsolutePath().normalize();
|
||||||
|
BusinessAssert.isTrue(target.startsWith(root), CommonErrorCode.PARAM_ERROR);
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolveUploadRootPath() {
|
||||||
|
String uploadDir = dataManagementProperties.getFileStorage().getUploadDir();
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(uploadDir), CommonErrorCode.PARAM_ERROR);
|
||||||
|
return Paths.get(uploadDir).toAbsolutePath().normalize();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nowText() {
|
||||||
|
return LocalDateTime.now().format(PREVIEW_TIME_FORMATTER);
|
||||||
|
}
|
||||||
|
|
||||||
|
public record PreviewFile(Path filePath, String fileName) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import com.datamate.common.infrastructure.exception.BusinessAssert;
|
||||||
|
import com.datamate.common.infrastructure.exception.CommonErrorCode;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeStatusType;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItemDirectory;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
|
||||||
|
import com.datamate.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemDirectoryRepository;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeSetRepository;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeDirectoryRequest;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeDirectoryQuery;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目目录应用服务
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@Transactional
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class KnowledgeDirectoryApplicationService {
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
|
private static final String INVALID_PATH_SEGMENT = "..";
|
||||||
|
|
||||||
|
private final KnowledgeItemDirectoryRepository knowledgeItemDirectoryRepository;
|
||||||
|
private final KnowledgeItemRepository knowledgeItemRepository;
|
||||||
|
private final KnowledgeSetRepository knowledgeSetRepository;
|
||||||
|
|
||||||
|
@Transactional(readOnly = true)
|
||||||
|
public List<KnowledgeItemDirectory> getKnowledgeDirectories(String setId, KnowledgeDirectoryQuery query) {
|
||||||
|
BusinessAssert.notNull(query, CommonErrorCode.PARAM_ERROR);
|
||||||
|
query.setSetId(setId);
|
||||||
|
return knowledgeItemDirectoryRepository.findByCriteria(query);
|
||||||
|
}
|
||||||
|
|
||||||
|
public KnowledgeItemDirectory createKnowledgeDirectory(String setId, CreateKnowledgeDirectoryRequest request) {
|
||||||
|
BusinessAssert.notNull(request, CommonErrorCode.PARAM_ERROR);
|
||||||
|
KnowledgeSet knowledgeSet = requireKnowledgeSet(setId);
|
||||||
|
BusinessAssert.isTrue(!isReadOnlyStatus(knowledgeSet.getStatus()),
|
||||||
|
DataManagementErrorCode.KNOWLEDGE_SET_STATUS_ERROR);
|
||||||
|
|
||||||
|
String directoryName = normalizeDirectoryName(request.getDirectoryName());
|
||||||
|
validateDirectoryName(directoryName);
|
||||||
|
|
||||||
|
String parentPrefix = normalizeRelativePathPrefix(request.getParentPrefix());
|
||||||
|
String relativePath = normalizeRelativePathValue(parentPrefix + directoryName);
|
||||||
|
validateRelativePath(relativePath);
|
||||||
|
|
||||||
|
BusinessAssert.isTrue(!knowledgeItemRepository.existsBySetIdAndRelativePath(setId, relativePath),
|
||||||
|
CommonErrorCode.PARAM_ERROR);
|
||||||
|
|
||||||
|
KnowledgeItemDirectory existing = knowledgeItemDirectoryRepository.findBySetIdAndPath(setId, relativePath);
|
||||||
|
if (existing != null) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
KnowledgeItemDirectory directory = new KnowledgeItemDirectory();
|
||||||
|
directory.setId(UUID.randomUUID().toString());
|
||||||
|
directory.setSetId(setId);
|
||||||
|
directory.setName(directoryName);
|
||||||
|
directory.setRelativePath(relativePath);
|
||||||
|
knowledgeItemDirectoryRepository.save(directory);
|
||||||
|
return directory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deleteKnowledgeDirectory(String setId, String relativePath) {
|
||||||
|
KnowledgeSet knowledgeSet = requireKnowledgeSet(setId);
|
||||||
|
BusinessAssert.isTrue(!isReadOnlyStatus(knowledgeSet.getStatus()),
|
||||||
|
DataManagementErrorCode.KNOWLEDGE_SET_STATUS_ERROR);
|
||||||
|
|
||||||
|
String normalized = normalizeRelativePathValue(relativePath);
|
||||||
|
validateRelativePath(normalized);
|
||||||
|
|
||||||
|
knowledgeItemRepository.removeByRelativePathPrefix(setId, normalized);
|
||||||
|
knowledgeItemDirectoryRepository.removeByRelativePathPrefix(setId, normalized);
|
||||||
|
}
|
||||||
|
|
||||||
|
private KnowledgeSet requireKnowledgeSet(String setId) {
|
||||||
|
KnowledgeSet knowledgeSet = knowledgeSetRepository.getById(setId);
|
||||||
|
BusinessAssert.notNull(knowledgeSet, DataManagementErrorCode.KNOWLEDGE_SET_NOT_FOUND);
|
||||||
|
return knowledgeSet;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isReadOnlyStatus(KnowledgeStatusType status) {
|
||||||
|
return status == KnowledgeStatusType.ARCHIVED || status == KnowledgeStatusType.DEPRECATED;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeDirectoryName(String name) {
|
||||||
|
return StringUtils.trimToEmpty(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateDirectoryName(String name) {
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(name), CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(!name.contains(PATH_SEPARATOR), CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(!name.contains("\\"), CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(!name.contains(INVALID_PATH_SEGMENT), CommonErrorCode.PARAM_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateRelativePath(String relativePath) {
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(relativePath), CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(!relativePath.contains(INVALID_PATH_SEGMENT), CommonErrorCode.PARAM_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathPrefix(String prefix) {
|
||||||
|
if (StringUtils.isBlank(prefix)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = prefix.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
while (normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(normalized)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
validateRelativePath(normalized);
|
||||||
|
return normalized + PATH_SEPARATOR;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathValue(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
while (normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,15 +16,20 @@ import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
|||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
|
||||||
import com.datamate.datamanagement.infrastructure.config.DataManagementProperties;
|
import com.datamate.datamanagement.infrastructure.config.DataManagementProperties;
|
||||||
import com.datamate.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
import com.datamate.datamanagement.infrastructure.exception.DataManagementErrorCode;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.mapper.TagMapper;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeSetRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeSetRepository;
|
||||||
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
|
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
|
||||||
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeItemRequest;
|
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeItemRequest;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.DeleteKnowledgeItemsRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.ImportKnowledgeItemsRequest;
|
import com.datamate.datamanagement.interfaces.dto.ImportKnowledgeItemsRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemResponse;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemResponse;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchQuery;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeManagementStatisticsResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.ReplaceKnowledgeItemFileRequest;
|
import com.datamate.datamanagement.interfaces.dto.ReplaceKnowledgeItemFileRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeItemRequest;
|
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeItemRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UploadKnowledgeItemsRequest;
|
import com.datamate.datamanagement.interfaces.dto.UploadKnowledgeItemsRequest;
|
||||||
@@ -71,16 +76,20 @@ public class KnowledgeItemApplicationService {
|
|||||||
private static final String EXPORT_FILE_PREFIX = "knowledge_set_";
|
private static final String EXPORT_FILE_PREFIX = "knowledge_set_";
|
||||||
private static final String EXPORT_FILE_SUFFIX = ".zip";
|
private static final String EXPORT_FILE_SUFFIX = ".zip";
|
||||||
private static final String EXPORT_CONTENT_TYPE = "application/zip";
|
private static final String EXPORT_CONTENT_TYPE = "application/zip";
|
||||||
|
private static final String PREVIEW_PDF_CONTENT_TYPE = "application/pdf";
|
||||||
private static final int MAX_FILE_BASE_LENGTH = 120;
|
private static final int MAX_FILE_BASE_LENGTH = 120;
|
||||||
private static final int MAX_TITLE_LENGTH = 200;
|
private static final int MAX_TITLE_LENGTH = 200;
|
||||||
private static final String KNOWLEDGE_ITEM_UPLOAD_DIR = "knowledge-items";
|
private static final String KNOWLEDGE_ITEM_UPLOAD_DIR = "knowledge-items";
|
||||||
private static final String DEFAULT_FILE_EXTENSION = "bin";
|
private static final String DEFAULT_FILE_EXTENSION = "bin";
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
|
|
||||||
private final KnowledgeItemRepository knowledgeItemRepository;
|
private final KnowledgeItemRepository knowledgeItemRepository;
|
||||||
private final KnowledgeSetRepository knowledgeSetRepository;
|
private final KnowledgeSetRepository knowledgeSetRepository;
|
||||||
private final DatasetRepository datasetRepository;
|
private final DatasetRepository datasetRepository;
|
||||||
private final DatasetFileRepository datasetFileRepository;
|
private final DatasetFileRepository datasetFileRepository;
|
||||||
private final DataManagementProperties dataManagementProperties;
|
private final DataManagementProperties dataManagementProperties;
|
||||||
|
private final TagMapper tagMapper;
|
||||||
|
private final KnowledgeItemPreviewService knowledgeItemPreviewService;
|
||||||
|
|
||||||
public KnowledgeItem createKnowledgeItem(String setId, CreateKnowledgeItemRequest request) {
|
public KnowledgeItem createKnowledgeItem(String setId, CreateKnowledgeItemRequest request) {
|
||||||
KnowledgeSet knowledgeSet = requireKnowledgeSet(setId);
|
KnowledgeSet knowledgeSet = requireKnowledgeSet(setId);
|
||||||
@@ -109,6 +118,7 @@ public class KnowledgeItemApplicationService {
|
|||||||
|
|
||||||
List<MultipartFile> files = request.getFiles();
|
List<MultipartFile> files = request.getFiles();
|
||||||
BusinessAssert.isTrue(CollectionUtils.isNotEmpty(files), CommonErrorCode.PARAM_ERROR);
|
BusinessAssert.isTrue(CollectionUtils.isNotEmpty(files), CommonErrorCode.PARAM_ERROR);
|
||||||
|
String parentPrefix = normalizeRelativePathPrefix(request.getParentPrefix());
|
||||||
|
|
||||||
Path uploadRoot = resolveUploadRootPath();
|
Path uploadRoot = resolveUploadRootPath();
|
||||||
Path setDir = uploadRoot.resolve(KNOWLEDGE_ITEM_UPLOAD_DIR).resolve(setId).normalize();
|
Path setDir = uploadRoot.resolve(KNOWLEDGE_ITEM_UPLOAD_DIR).resolve(setId).normalize();
|
||||||
@@ -142,6 +152,7 @@ public class KnowledgeItemApplicationService {
|
|||||||
knowledgeItem.setContentType(KnowledgeContentType.FILE);
|
knowledgeItem.setContentType(KnowledgeContentType.FILE);
|
||||||
knowledgeItem.setSourceType(KnowledgeSourceType.FILE_UPLOAD);
|
knowledgeItem.setSourceType(KnowledgeSourceType.FILE_UPLOAD);
|
||||||
knowledgeItem.setSourceFileId(trimToLength(safeOriginalName, MAX_TITLE_LENGTH));
|
knowledgeItem.setSourceFileId(trimToLength(safeOriginalName, MAX_TITLE_LENGTH));
|
||||||
|
knowledgeItem.setRelativePath(buildRelativePath(parentPrefix, safeOriginalName));
|
||||||
|
|
||||||
items.add(knowledgeItem);
|
items.add(knowledgeItem);
|
||||||
}
|
}
|
||||||
@@ -167,6 +178,9 @@ public class KnowledgeItemApplicationService {
|
|||||||
if (request.getContentType() != null) {
|
if (request.getContentType() != null) {
|
||||||
knowledgeItem.setContentType(request.getContentType());
|
knowledgeItem.setContentType(request.getContentType());
|
||||||
}
|
}
|
||||||
|
if (request.getMetadata() != null) {
|
||||||
|
knowledgeItem.setMetadata(request.getMetadata());
|
||||||
|
}
|
||||||
|
|
||||||
knowledgeItemRepository.updateById(knowledgeItem);
|
knowledgeItemRepository.updateById(knowledgeItem);
|
||||||
return knowledgeItem;
|
return knowledgeItem;
|
||||||
@@ -179,6 +193,22 @@ public class KnowledgeItemApplicationService {
|
|||||||
knowledgeItemRepository.removeById(itemId);
|
knowledgeItemRepository.removeById(itemId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void deleteKnowledgeItems(String setId, DeleteKnowledgeItemsRequest request) {
|
||||||
|
BusinessAssert.notNull(request, CommonErrorCode.PARAM_ERROR);
|
||||||
|
List<String> ids = request.getIds();
|
||||||
|
BusinessAssert.isTrue(CollectionUtils.isNotEmpty(ids), CommonErrorCode.PARAM_ERROR);
|
||||||
|
|
||||||
|
List<KnowledgeItem> items = knowledgeItemRepository.listByIds(ids);
|
||||||
|
BusinessAssert.isTrue(CollectionUtils.isNotEmpty(items), DataManagementErrorCode.KNOWLEDGE_ITEM_NOT_FOUND);
|
||||||
|
BusinessAssert.isTrue(items.size() == ids.size(), DataManagementErrorCode.KNOWLEDGE_ITEM_NOT_FOUND);
|
||||||
|
|
||||||
|
boolean allMatch = items.stream().allMatch(item -> Objects.equals(item.getSetId(), setId));
|
||||||
|
BusinessAssert.isTrue(allMatch, CommonErrorCode.PARAM_ERROR);
|
||||||
|
|
||||||
|
List<String> deleteIds = items.stream().map(KnowledgeItem::getId).toList();
|
||||||
|
knowledgeItemRepository.removeByIds(deleteIds);
|
||||||
|
}
|
||||||
|
|
||||||
@Transactional(readOnly = true)
|
@Transactional(readOnly = true)
|
||||||
public KnowledgeItem getKnowledgeItem(String setId, String itemId) {
|
public KnowledgeItem getKnowledgeItem(String setId, String itemId) {
|
||||||
KnowledgeItem knowledgeItem = knowledgeItemRepository.getById(itemId);
|
KnowledgeItem knowledgeItem = knowledgeItemRepository.getById(itemId);
|
||||||
@@ -196,6 +226,40 @@ public class KnowledgeItemApplicationService {
|
|||||||
return PagedResponse.of(responses, page.getCurrent(), page.getTotal(), page.getPages());
|
return PagedResponse.of(responses, page.getCurrent(), page.getTotal(), page.getPages());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Transactional(readOnly = true)
|
||||||
|
public KnowledgeManagementStatisticsResponse getKnowledgeManagementStatistics() {
|
||||||
|
KnowledgeManagementStatisticsResponse response = new KnowledgeManagementStatisticsResponse();
|
||||||
|
response.setTotalKnowledgeSets(knowledgeSetRepository.count());
|
||||||
|
|
||||||
|
long totalFiles = knowledgeItemRepository.countBySourceTypes(List.of(
|
||||||
|
KnowledgeSourceType.DATASET_FILE,
|
||||||
|
KnowledgeSourceType.FILE_UPLOAD
|
||||||
|
));
|
||||||
|
response.setTotalFiles(totalFiles);
|
||||||
|
|
||||||
|
long datasetFileSize = safeLong(knowledgeItemRepository.sumDatasetFileSize());
|
||||||
|
long uploadFileSize = calculateUploadFileTotalSize();
|
||||||
|
response.setTotalSize(datasetFileSize + uploadFileSize);
|
||||||
|
response.setTotalTags(safeLong(tagMapper.countKnowledgeSetTags()));
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional(readOnly = true)
|
||||||
|
public PagedResponse<KnowledgeItemSearchResponse> searchKnowledgeItems(KnowledgeItemSearchQuery query) {
|
||||||
|
BusinessAssert.notNull(query, CommonErrorCode.PARAM_ERROR);
|
||||||
|
String keyword = StringUtils.trimToEmpty(query.getKeyword());
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(keyword), CommonErrorCode.PARAM_ERROR);
|
||||||
|
|
||||||
|
IPage<KnowledgeItemSearchResponse> page = new Page<>(query.getPage(), query.getSize());
|
||||||
|
IPage<KnowledgeItemSearchResponse> result = knowledgeItemRepository.searchFileItems(page, keyword);
|
||||||
|
List<KnowledgeItemSearchResponse> responses = result.getRecords()
|
||||||
|
.stream()
|
||||||
|
.map(this::normalizeSearchResponse)
|
||||||
|
.toList();
|
||||||
|
return PagedResponse.of(responses, result.getCurrent(), result.getTotal(), result.getPages());
|
||||||
|
}
|
||||||
|
|
||||||
public List<KnowledgeItem> importKnowledgeItems(String setId, ImportKnowledgeItemsRequest request) {
|
public List<KnowledgeItem> importKnowledgeItems(String setId, ImportKnowledgeItemsRequest request) {
|
||||||
KnowledgeSet knowledgeSet = requireKnowledgeSet(setId);
|
KnowledgeSet knowledgeSet = requireKnowledgeSet(setId);
|
||||||
BusinessAssert.isTrue(!isReadOnlyStatus(knowledgeSet.getStatus()),
|
BusinessAssert.isTrue(!isReadOnlyStatus(knowledgeSet.getStatus()),
|
||||||
@@ -220,6 +284,7 @@ public class KnowledgeItemApplicationService {
|
|||||||
knowledgeItem.setSourceType(KnowledgeSourceType.DATASET_FILE);
|
knowledgeItem.setSourceType(KnowledgeSourceType.DATASET_FILE);
|
||||||
knowledgeItem.setSourceDatasetId(dataset.getId());
|
knowledgeItem.setSourceDatasetId(dataset.getId());
|
||||||
knowledgeItem.setSourceFileId(datasetFile.getId());
|
knowledgeItem.setSourceFileId(datasetFile.getId());
|
||||||
|
knowledgeItem.setRelativePath(resolveDatasetFileRelativePath(dataset, datasetFile));
|
||||||
|
|
||||||
items.add(knowledgeItem);
|
items.add(knowledgeItem);
|
||||||
}
|
}
|
||||||
@@ -271,7 +336,7 @@ public class KnowledgeItemApplicationService {
|
|||||||
|
|
||||||
String relativePath = knowledgeItem.getContent();
|
String relativePath = knowledgeItem.getContent();
|
||||||
BusinessAssert.isTrue(StringUtils.isNotBlank(relativePath), CommonErrorCode.PARAM_ERROR);
|
BusinessAssert.isTrue(StringUtils.isNotBlank(relativePath), CommonErrorCode.PARAM_ERROR);
|
||||||
Path filePath = resolveKnowledgeItemStoragePath(relativePath);
|
Path filePath = resolveKnowledgeItemStoragePathWithFallback(relativePath);
|
||||||
BusinessAssert.isTrue(Files.exists(filePath) && Files.isRegularFile(filePath), CommonErrorCode.PARAM_ERROR);
|
BusinessAssert.isTrue(Files.exists(filePath) && Files.isRegularFile(filePath), CommonErrorCode.PARAM_ERROR);
|
||||||
|
|
||||||
String downloadName = StringUtils.isNotBlank(knowledgeItem.getSourceFileId())
|
String downloadName = StringUtils.isNotBlank(knowledgeItem.getSourceFileId())
|
||||||
@@ -304,12 +369,32 @@ public class KnowledgeItemApplicationService {
|
|||||||
|
|
||||||
String relativePath = knowledgeItem.getContent();
|
String relativePath = knowledgeItem.getContent();
|
||||||
BusinessAssert.isTrue(StringUtils.isNotBlank(relativePath), CommonErrorCode.PARAM_ERROR);
|
BusinessAssert.isTrue(StringUtils.isNotBlank(relativePath), CommonErrorCode.PARAM_ERROR);
|
||||||
Path filePath = resolveKnowledgeItemStoragePath(relativePath);
|
|
||||||
BusinessAssert.isTrue(Files.exists(filePath) && Files.isRegularFile(filePath), CommonErrorCode.PARAM_ERROR);
|
|
||||||
|
|
||||||
String previewName = StringUtils.isNotBlank(knowledgeItem.getSourceFileId())
|
String previewName = StringUtils.isNotBlank(knowledgeItem.getSourceFileId())
|
||||||
? knowledgeItem.getSourceFileId()
|
? knowledgeItem.getSourceFileId()
|
||||||
: filePath.getFileName().toString();
|
: Paths.get(relativePath).getFileName().toString();
|
||||||
|
|
||||||
|
if (knowledgeItemPreviewService.isOfficeDocument(previewName)) {
|
||||||
|
KnowledgeItemPreviewService.PreviewFile previewFile = knowledgeItemPreviewService.resolveReadyPreviewFile(setId, knowledgeItem);
|
||||||
|
if (previewFile == null) {
|
||||||
|
response.setStatus(HttpServletResponse.SC_CONFLICT);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.setContentType(PREVIEW_PDF_CONTENT_TYPE);
|
||||||
|
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
|
||||||
|
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
|
||||||
|
"inline; filename=\"" + URLEncoder.encode(previewFile.fileName(), StandardCharsets.UTF_8) + "\"");
|
||||||
|
try (InputStream inputStream = Files.newInputStream(previewFile.filePath())) {
|
||||||
|
inputStream.transferTo(response.getOutputStream());
|
||||||
|
response.flushBuffer();
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("preview knowledge item pdf error, itemId: {}", itemId, e);
|
||||||
|
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path filePath = resolveKnowledgeItemStoragePathWithFallback(relativePath);
|
||||||
|
BusinessAssert.isTrue(Files.exists(filePath) && Files.isRegularFile(filePath), CommonErrorCode.PARAM_ERROR);
|
||||||
|
|
||||||
String contentType = null;
|
String contentType = null;
|
||||||
try {
|
try {
|
||||||
@@ -382,7 +467,10 @@ public class KnowledgeItemApplicationService {
|
|||||||
knowledgeItem.setContentType(KnowledgeContentType.FILE);
|
knowledgeItem.setContentType(KnowledgeContentType.FILE);
|
||||||
knowledgeItem.setSourceType(KnowledgeSourceType.FILE_UPLOAD);
|
knowledgeItem.setSourceType(KnowledgeSourceType.FILE_UPLOAD);
|
||||||
knowledgeItem.setSourceFileId(sourceFileId);
|
knowledgeItem.setSourceFileId(sourceFileId);
|
||||||
|
knowledgeItem.setRelativePath(resolveReplacedRelativePath(knowledgeItem.getRelativePath(), sourceFileId));
|
||||||
|
knowledgeItem.setMetadata(knowledgeItemPreviewService.clearPreviewMetadata(knowledgeItem.getMetadata()));
|
||||||
knowledgeItemRepository.updateById(knowledgeItem);
|
knowledgeItemRepository.updateById(knowledgeItem);
|
||||||
|
knowledgeItemPreviewService.deletePreviewFileQuietly(setId, knowledgeItem.getId());
|
||||||
deleteFile(oldFilePath);
|
deleteFile(oldFilePath);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
deleteFileQuietly(targetPath);
|
deleteFileQuietly(targetPath);
|
||||||
@@ -447,11 +535,221 @@ public class KnowledgeItemApplicationService {
|
|||||||
return target;
|
return target;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Path resolveKnowledgeItemStoragePathWithFallback(String relativePath) {
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(relativePath), CommonErrorCode.PARAM_ERROR);
|
||||||
|
String normalizedInput = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
Path root = resolveUploadRootPath();
|
||||||
|
java.util.LinkedHashSet<Path> candidates = new java.util.LinkedHashSet<>();
|
||||||
|
|
||||||
|
Path inputPath = Paths.get(normalizedInput.replace(PATH_SEPARATOR, File.separator));
|
||||||
|
if (inputPath.isAbsolute()) {
|
||||||
|
Path normalizedAbsolute = inputPath.toAbsolutePath().normalize();
|
||||||
|
if (normalizedAbsolute.startsWith(root)) {
|
||||||
|
candidates.add(normalizedAbsolute);
|
||||||
|
}
|
||||||
|
String segmentRelativePath = extractRelativePathFromSegment(normalizedInput, KNOWLEDGE_ITEM_UPLOAD_DIR);
|
||||||
|
if (StringUtils.isNotBlank(segmentRelativePath)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, segmentRelativePath));
|
||||||
|
}
|
||||||
|
BusinessAssert.isTrue(!candidates.isEmpty(), CommonErrorCode.PARAM_ERROR);
|
||||||
|
} else {
|
||||||
|
String normalizedRelative = normalizeRelativePathValue(normalizedInput);
|
||||||
|
if (StringUtils.isNotBlank(normalizedRelative)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, normalizedRelative));
|
||||||
|
}
|
||||||
|
String segmentRelativePath = extractRelativePathFromSegment(normalizedInput, KNOWLEDGE_ITEM_UPLOAD_DIR);
|
||||||
|
if (StringUtils.isNotBlank(segmentRelativePath)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, segmentRelativePath));
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(normalizedRelative)
|
||||||
|
&& !normalizedRelative.startsWith(KNOWLEDGE_ITEM_UPLOAD_DIR + PATH_SEPARATOR)
|
||||||
|
&& !normalizedRelative.equals(KNOWLEDGE_ITEM_UPLOAD_DIR)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, KNOWLEDGE_ITEM_UPLOAD_DIR + PATH_SEPARATOR + normalizedRelative));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (root.getFileName() != null && KNOWLEDGE_ITEM_UPLOAD_DIR.equals(root.getFileName().toString())) {
|
||||||
|
String normalizedRelative = normalizeRelativePathValue(normalizedInput);
|
||||||
|
if (StringUtils.isNotBlank(normalizedRelative)
|
||||||
|
&& normalizedRelative.startsWith(KNOWLEDGE_ITEM_UPLOAD_DIR + PATH_SEPARATOR)) {
|
||||||
|
String withoutPrefix = normalizedRelative.substring(KNOWLEDGE_ITEM_UPLOAD_DIR.length() + PATH_SEPARATOR.length());
|
||||||
|
if (StringUtils.isNotBlank(withoutPrefix)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, withoutPrefix));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Path fallback = null;
|
||||||
|
for (Path candidate : candidates) {
|
||||||
|
if (fallback == null) {
|
||||||
|
fallback = candidate;
|
||||||
|
}
|
||||||
|
if (Files.exists(candidate) && Files.isRegularFile(candidate)) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BusinessAssert.notNull(fallback, CommonErrorCode.PARAM_ERROR);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path buildKnowledgeItemStoragePath(Path root, String relativePath) {
|
||||||
|
String normalizedRelativePath = StringUtils.defaultString(relativePath).replace(PATH_SEPARATOR, File.separator);
|
||||||
|
Path target = root.resolve(normalizedRelativePath).toAbsolutePath().normalize();
|
||||||
|
BusinessAssert.isTrue(target.startsWith(root), CommonErrorCode.PARAM_ERROR);
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractRelativePathFromSegment(String rawPath, String segment) {
|
||||||
|
if (StringUtils.isBlank(rawPath) || StringUtils.isBlank(segment)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String normalized = rawPath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
String segmentPrefix = segment + PATH_SEPARATOR;
|
||||||
|
int index = normalized.indexOf(segmentPrefix);
|
||||||
|
if (index < 0) {
|
||||||
|
return segment.equals(normalized) ? segment : null;
|
||||||
|
}
|
||||||
|
return normalizeRelativePathValue(normalized.substring(index));
|
||||||
|
}
|
||||||
|
|
||||||
|
private KnowledgeItemSearchResponse normalizeSearchResponse(KnowledgeItemSearchResponse item) {
|
||||||
|
BusinessAssert.notNull(item, CommonErrorCode.PARAM_ERROR);
|
||||||
|
if (item.getSourceType() == KnowledgeSourceType.FILE_UPLOAD) {
|
||||||
|
item.setFileSize(resolveUploadFileSize(item.getContent()));
|
||||||
|
if (StringUtils.isBlank(item.getFileName())) {
|
||||||
|
item.setFileName(item.getSourceFileId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (item.getSourceType() == KnowledgeSourceType.DATASET_FILE) {
|
||||||
|
if (item.getFileSize() == null) {
|
||||||
|
item.setFileSize(0L);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(item.getFileName())) {
|
||||||
|
item.setFileName(item.getSourceFileId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
item.setContent(null);
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
private long calculateUploadFileTotalSize() {
|
||||||
|
List<KnowledgeItem> items = knowledgeItemRepository.findFileUploadItems();
|
||||||
|
if (CollectionUtils.isEmpty(items)) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
long total = 0L;
|
||||||
|
for (KnowledgeItem item : items) {
|
||||||
|
total += resolveUploadFileSize(item.getContent());
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
private long resolveUploadFileSize(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Path filePath = resolveKnowledgeItemStoragePath(relativePath);
|
||||||
|
if (!Files.exists(filePath) || !Files.isRegularFile(filePath)) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
return Files.size(filePath);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("resolve knowledge item file size error, path: {}", relativePath, e);
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private long safeLong(Long value) {
|
||||||
|
return value == null ? 0L : value;
|
||||||
|
}
|
||||||
|
|
||||||
private String buildRelativeFilePath(String setId, String storedName) {
|
private String buildRelativeFilePath(String setId, String storedName) {
|
||||||
String relativePath = Paths.get(KNOWLEDGE_ITEM_UPLOAD_DIR, setId, storedName).toString();
|
String relativePath = Paths.get(KNOWLEDGE_ITEM_UPLOAD_DIR, setId, storedName).toString();
|
||||||
return relativePath.replace(File.separatorChar, '/');
|
return relativePath.replace(File.separatorChar, '/');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String buildRelativePath(String parentPrefix, String fileName) {
|
||||||
|
String safeName = sanitizeFileName(fileName);
|
||||||
|
if (StringUtils.isBlank(safeName)) {
|
||||||
|
safeName = "file";
|
||||||
|
}
|
||||||
|
String normalizedPrefix = normalizeRelativePathPrefix(parentPrefix);
|
||||||
|
return normalizedPrefix + safeName;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathPrefix(String prefix) {
|
||||||
|
if (StringUtils.isBlank(prefix)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = prefix.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
while (normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1);
|
||||||
|
}
|
||||||
|
BusinessAssert.isTrue(!normalized.contains(".."), CommonErrorCode.PARAM_ERROR);
|
||||||
|
if (StringUtils.isBlank(normalized)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return normalized + PATH_SEPARATOR;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathValue(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
while (normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveDatasetFileRelativePath(Dataset dataset, DatasetFile datasetFile) {
|
||||||
|
if (datasetFile == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String fileName = StringUtils.defaultIfBlank(datasetFile.getFileName(), datasetFile.getId());
|
||||||
|
String datasetPath = dataset == null ? null : dataset.getPath();
|
||||||
|
String filePath = datasetFile.getFilePath();
|
||||||
|
if (StringUtils.isBlank(datasetPath) || StringUtils.isBlank(filePath)) {
|
||||||
|
return buildRelativePath("", fileName);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Path datasetRoot = Paths.get(datasetPath).toAbsolutePath().normalize();
|
||||||
|
Path targetPath = Paths.get(filePath).toAbsolutePath().normalize();
|
||||||
|
if (targetPath.startsWith(datasetRoot)) {
|
||||||
|
Path relative = datasetRoot.relativize(targetPath);
|
||||||
|
String relativeValue = relative.toString().replace(File.separatorChar, '/');
|
||||||
|
String normalized = normalizeRelativePathValue(relativeValue);
|
||||||
|
if (!normalized.contains("..") && StringUtils.isNotBlank(normalized)) {
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("resolve dataset file relative path failed, fileId: {}", datasetFile.getId(), e);
|
||||||
|
}
|
||||||
|
return buildRelativePath("", fileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveReplacedRelativePath(String existingRelativePath, String newFileName) {
|
||||||
|
String normalized = normalizeRelativePathValue(existingRelativePath);
|
||||||
|
if (StringUtils.isBlank(normalized)) {
|
||||||
|
return buildRelativePath("", newFileName);
|
||||||
|
}
|
||||||
|
int lastIndex = normalized.lastIndexOf(PATH_SEPARATOR);
|
||||||
|
String parentPrefix = lastIndex >= 0 ? normalized.substring(0, lastIndex + 1) : "";
|
||||||
|
return buildRelativePath(parentPrefix, newFileName);
|
||||||
|
}
|
||||||
|
|
||||||
private void createDirectories(Path path) {
|
private void createDirectories(Path path) {
|
||||||
try {
|
try {
|
||||||
Files.createDirectories(path);
|
Files.createDirectories(path);
|
||||||
|
|||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeItemPreviewStatus;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
||||||
|
import com.datamate.datamanagement.infrastructure.config.DataManagementProperties;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.scheduling.annotation.Async;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目预览转换异步任务
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class KnowledgeItemPreviewAsyncService {
|
||||||
|
private static final Set<String> OFFICE_EXTENSIONS = Set.of("doc", "docx");
|
||||||
|
private static final String KNOWLEDGE_ITEM_UPLOAD_DIR = "knowledge-items";
|
||||||
|
private static final String PREVIEW_SUB_DIR = "preview";
|
||||||
|
private static final String PREVIEW_FILE_SUFFIX = ".pdf";
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
|
private static final int MAX_ERROR_LENGTH = 500;
|
||||||
|
private static final DateTimeFormatter PREVIEW_TIME_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
|
||||||
|
|
||||||
|
private final KnowledgeItemRepository knowledgeItemRepository;
|
||||||
|
private final DataManagementProperties dataManagementProperties;
|
||||||
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
@Async
|
||||||
|
public void convertPreviewAsync(String itemId) {
|
||||||
|
if (StringUtils.isBlank(itemId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
KnowledgeItem item = knowledgeItemRepository.getById(itemId);
|
||||||
|
if (item == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String extension = resolveFileExtension(resolveOriginalName(item));
|
||||||
|
if (!OFFICE_EXTENSIONS.contains(extension)) {
|
||||||
|
updatePreviewStatus(item, KnowledgeItemPreviewStatus.FAILED, null, "仅支持 DOC/DOCX 转换");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(item.getContent())) {
|
||||||
|
updatePreviewStatus(item, KnowledgeItemPreviewStatus.FAILED, null, "源文件路径为空");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Path sourcePath = resolveKnowledgeItemStoragePath(item.getContent());
|
||||||
|
if (!Files.exists(sourcePath) || !Files.isRegularFile(sourcePath)) {
|
||||||
|
updatePreviewStatus(item, KnowledgeItemPreviewStatus.FAILED, null, "源文件不存在");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(item.getMetadata(), objectMapper);
|
||||||
|
String previewRelativePath = StringUtils.defaultIfBlank(
|
||||||
|
previewInfo.pdfPath(),
|
||||||
|
resolvePreviewRelativePath(item.getSetId(), item.getId())
|
||||||
|
);
|
||||||
|
Path targetPath = resolvePreviewStoragePath(previewRelativePath);
|
||||||
|
ensureParentDirectory(targetPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
LibreOfficeConverter.convertToPdf(sourcePath, targetPath);
|
||||||
|
updatePreviewStatus(item, KnowledgeItemPreviewStatus.READY, previewRelativePath, null);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("preview convert failed, itemId: {}", item.getId(), e);
|
||||||
|
updatePreviewStatus(item, KnowledgeItemPreviewStatus.FAILED, previewRelativePath, trimError(e.getMessage()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updatePreviewStatus(
|
||||||
|
KnowledgeItem item,
|
||||||
|
KnowledgeItemPreviewStatus status,
|
||||||
|
String previewRelativePath,
|
||||||
|
String error
|
||||||
|
) {
|
||||||
|
if (item == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String updatedMetadata = KnowledgeItemPreviewMetadataHelper.applyPreviewInfo(
|
||||||
|
item.getMetadata(),
|
||||||
|
objectMapper,
|
||||||
|
status,
|
||||||
|
previewRelativePath,
|
||||||
|
error,
|
||||||
|
nowText()
|
||||||
|
);
|
||||||
|
item.setMetadata(updatedMetadata);
|
||||||
|
knowledgeItemRepository.updateById(item);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveOriginalName(KnowledgeItem item) {
|
||||||
|
if (item == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(item.getSourceFileId())) {
|
||||||
|
return item.getSourceFileId();
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(item.getContent())) {
|
||||||
|
return Paths.get(item.getContent()).getFileName().toString();
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveFileExtension(String fileName) {
|
||||||
|
if (StringUtils.isBlank(fileName)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
int dotIndex = fileName.lastIndexOf('.');
|
||||||
|
if (dotIndex <= 0 || dotIndex >= fileName.length() - 1) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return fileName.substring(dotIndex + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePreviewRelativePath(String setId, String itemId) {
|
||||||
|
String relativePath = Paths.get(KNOWLEDGE_ITEM_UPLOAD_DIR, setId, PREVIEW_SUB_DIR, itemId + PREVIEW_FILE_SUFFIX)
|
||||||
|
.toString();
|
||||||
|
return relativePath.replace("\\", PATH_SEPARATOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolvePreviewStoragePath(String relativePath) {
|
||||||
|
String normalizedRelativePath = StringUtils.defaultString(relativePath).replace("/", java.io.File.separator);
|
||||||
|
Path root = resolveUploadRootPath();
|
||||||
|
Path target = root.resolve(normalizedRelativePath).toAbsolutePath().normalize();
|
||||||
|
if (!target.startsWith(root)) {
|
||||||
|
throw new IllegalArgumentException("invalid preview path");
|
||||||
|
}
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolveKnowledgeItemStoragePath(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
throw new IllegalArgumentException("invalid knowledge item path");
|
||||||
|
}
|
||||||
|
String normalizedInput = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
Path root = resolveUploadRootPath();
|
||||||
|
java.util.LinkedHashSet<Path> candidates = new java.util.LinkedHashSet<>();
|
||||||
|
|
||||||
|
Path inputPath = Paths.get(normalizedInput.replace(PATH_SEPARATOR, java.io.File.separator));
|
||||||
|
if (inputPath.isAbsolute()) {
|
||||||
|
Path normalizedAbsolute = inputPath.toAbsolutePath().normalize();
|
||||||
|
if (normalizedAbsolute.startsWith(root)) {
|
||||||
|
candidates.add(normalizedAbsolute);
|
||||||
|
}
|
||||||
|
String segmentRelativePath = extractRelativePathFromSegment(normalizedInput, KNOWLEDGE_ITEM_UPLOAD_DIR);
|
||||||
|
if (StringUtils.isNotBlank(segmentRelativePath)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, segmentRelativePath));
|
||||||
|
}
|
||||||
|
if (candidates.isEmpty()) {
|
||||||
|
throw new IllegalArgumentException("invalid knowledge item path");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
String normalizedRelative = normalizeRelativePathValue(normalizedInput);
|
||||||
|
if (StringUtils.isNotBlank(normalizedRelative)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, normalizedRelative));
|
||||||
|
}
|
||||||
|
String segmentRelativePath = extractRelativePathFromSegment(normalizedInput, KNOWLEDGE_ITEM_UPLOAD_DIR);
|
||||||
|
if (StringUtils.isNotBlank(segmentRelativePath)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, segmentRelativePath));
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(normalizedRelative)
|
||||||
|
&& !normalizedRelative.startsWith(KNOWLEDGE_ITEM_UPLOAD_DIR + PATH_SEPARATOR)
|
||||||
|
&& !normalizedRelative.equals(KNOWLEDGE_ITEM_UPLOAD_DIR)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, KNOWLEDGE_ITEM_UPLOAD_DIR + PATH_SEPARATOR + normalizedRelative));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (root.getFileName() != null && KNOWLEDGE_ITEM_UPLOAD_DIR.equals(root.getFileName().toString())) {
|
||||||
|
String normalizedRelative = normalizeRelativePathValue(normalizedInput);
|
||||||
|
if (StringUtils.isNotBlank(normalizedRelative)
|
||||||
|
&& normalizedRelative.startsWith(KNOWLEDGE_ITEM_UPLOAD_DIR + PATH_SEPARATOR)) {
|
||||||
|
String withoutPrefix = normalizedRelative.substring(KNOWLEDGE_ITEM_UPLOAD_DIR.length() + PATH_SEPARATOR.length());
|
||||||
|
if (StringUtils.isNotBlank(withoutPrefix)) {
|
||||||
|
candidates.add(buildKnowledgeItemStoragePath(root, withoutPrefix));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Path fallback = null;
|
||||||
|
for (Path candidate : candidates) {
|
||||||
|
if (fallback == null) {
|
||||||
|
fallback = candidate;
|
||||||
|
}
|
||||||
|
if (Files.exists(candidate) && Files.isRegularFile(candidate)) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (fallback == null) {
|
||||||
|
throw new IllegalArgumentException("invalid knowledge item path");
|
||||||
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path buildKnowledgeItemStoragePath(Path root, String relativePath) {
|
||||||
|
String normalizedRelativePath = StringUtils.defaultString(relativePath).replace(PATH_SEPARATOR, java.io.File.separator);
|
||||||
|
Path target = root.resolve(normalizedRelativePath).toAbsolutePath().normalize();
|
||||||
|
if (!target.startsWith(root)) {
|
||||||
|
throw new IllegalArgumentException("invalid knowledge item path");
|
||||||
|
}
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractRelativePathFromSegment(String rawPath, String segment) {
|
||||||
|
if (StringUtils.isBlank(rawPath) || StringUtils.isBlank(segment)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String normalized = rawPath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
String segmentPrefix = segment + PATH_SEPARATOR;
|
||||||
|
int index = normalized.indexOf(segmentPrefix);
|
||||||
|
if (index < 0) {
|
||||||
|
return segment.equals(normalized) ? segment : null;
|
||||||
|
}
|
||||||
|
return normalizeRelativePathValue(normalized.substring(index));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathValue(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
while (normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolveUploadRootPath() {
|
||||||
|
String uploadDir = dataManagementProperties.getFileStorage().getUploadDir();
|
||||||
|
return Paths.get(uploadDir).toAbsolutePath().normalize();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ensureParentDirectory(Path targetPath) {
|
||||||
|
try {
|
||||||
|
Path parent = targetPath.getParent();
|
||||||
|
if (parent != null) {
|
||||||
|
Files.createDirectories(parent);
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new IllegalStateException("创建预览目录失败", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String trimError(String error) {
|
||||||
|
if (StringUtils.isBlank(error)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (error.length() <= MAX_ERROR_LENGTH) {
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
return error.substring(0, MAX_ERROR_LENGTH);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nowText() {
|
||||||
|
return LocalDateTime.now().format(PREVIEW_TIME_FORMATTER);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeItemPreviewStatus;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目预览元数据解析与写入辅助类
|
||||||
|
*/
|
||||||
|
public final class KnowledgeItemPreviewMetadataHelper {
|
||||||
|
public static final String PREVIEW_STATUS_KEY = "previewStatus";
|
||||||
|
public static final String PREVIEW_PDF_PATH_KEY = "previewPdfPath";
|
||||||
|
public static final String PREVIEW_ERROR_KEY = "previewError";
|
||||||
|
public static final String PREVIEW_UPDATED_AT_KEY = "previewUpdatedAt";
|
||||||
|
|
||||||
|
private KnowledgeItemPreviewMetadataHelper() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static PreviewInfo readPreviewInfo(String metadata, ObjectMapper objectMapper) {
|
||||||
|
if (StringUtils.isBlank(metadata) || objectMapper == null) {
|
||||||
|
return PreviewInfo.empty();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
JsonNode node = objectMapper.readTree(metadata);
|
||||||
|
if (node == null || !node.isObject()) {
|
||||||
|
return PreviewInfo.empty();
|
||||||
|
}
|
||||||
|
String statusText = textValue(node, PREVIEW_STATUS_KEY);
|
||||||
|
KnowledgeItemPreviewStatus status = parseStatus(statusText);
|
||||||
|
return new PreviewInfo(
|
||||||
|
status,
|
||||||
|
textValue(node, PREVIEW_PDF_PATH_KEY),
|
||||||
|
textValue(node, PREVIEW_ERROR_KEY),
|
||||||
|
textValue(node, PREVIEW_UPDATED_AT_KEY)
|
||||||
|
);
|
||||||
|
} catch (Exception ignore) {
|
||||||
|
return PreviewInfo.empty();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String applyPreviewInfo(
|
||||||
|
String metadata,
|
||||||
|
ObjectMapper objectMapper,
|
||||||
|
KnowledgeItemPreviewStatus status,
|
||||||
|
String pdfPath,
|
||||||
|
String error,
|
||||||
|
String updatedAt
|
||||||
|
) {
|
||||||
|
if (objectMapper == null) {
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
ObjectNode root = parseRoot(metadata, objectMapper);
|
||||||
|
if (status == null) {
|
||||||
|
root.remove(PREVIEW_STATUS_KEY);
|
||||||
|
} else {
|
||||||
|
root.put(PREVIEW_STATUS_KEY, status.name());
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(pdfPath)) {
|
||||||
|
root.remove(PREVIEW_PDF_PATH_KEY);
|
||||||
|
} else {
|
||||||
|
root.put(PREVIEW_PDF_PATH_KEY, pdfPath);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(error)) {
|
||||||
|
root.remove(PREVIEW_ERROR_KEY);
|
||||||
|
} else {
|
||||||
|
root.put(PREVIEW_ERROR_KEY, error);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(updatedAt)) {
|
||||||
|
root.remove(PREVIEW_UPDATED_AT_KEY);
|
||||||
|
} else {
|
||||||
|
root.put(PREVIEW_UPDATED_AT_KEY, updatedAt);
|
||||||
|
}
|
||||||
|
return root.size() == 0 ? null : root.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String clearPreviewInfo(String metadata, ObjectMapper objectMapper) {
|
||||||
|
if (objectMapper == null) {
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
ObjectNode root = parseRoot(metadata, objectMapper);
|
||||||
|
root.remove(PREVIEW_STATUS_KEY);
|
||||||
|
root.remove(PREVIEW_PDF_PATH_KEY);
|
||||||
|
root.remove(PREVIEW_ERROR_KEY);
|
||||||
|
root.remove(PREVIEW_UPDATED_AT_KEY);
|
||||||
|
return root.size() == 0 ? null : root.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ObjectNode parseRoot(String metadata, ObjectMapper objectMapper) {
|
||||||
|
if (StringUtils.isBlank(metadata)) {
|
||||||
|
return objectMapper.createObjectNode();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
JsonNode node = objectMapper.readTree(metadata);
|
||||||
|
if (node instanceof ObjectNode objectNode) {
|
||||||
|
return objectNode;
|
||||||
|
}
|
||||||
|
} catch (Exception ignore) {
|
||||||
|
return objectMapper.createObjectNode();
|
||||||
|
}
|
||||||
|
return objectMapper.createObjectNode();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String textValue(JsonNode node, String key) {
|
||||||
|
if (node == null || StringUtils.isBlank(key)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
JsonNode value = node.get(key);
|
||||||
|
return value == null || value.isNull() ? null : value.asText();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static KnowledgeItemPreviewStatus parseStatus(String statusText) {
|
||||||
|
if (StringUtils.isBlank(statusText)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return KnowledgeItemPreviewStatus.valueOf(statusText);
|
||||||
|
} catch (Exception ignore) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public record PreviewInfo(
|
||||||
|
KnowledgeItemPreviewStatus status,
|
||||||
|
String pdfPath,
|
||||||
|
String error,
|
||||||
|
String updatedAt
|
||||||
|
) {
|
||||||
|
public static PreviewInfo empty() {
|
||||||
|
return new PreviewInfo(null, null, null, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,244 @@
|
|||||||
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import com.datamate.common.infrastructure.exception.BusinessAssert;
|
||||||
|
import com.datamate.common.infrastructure.exception.CommonErrorCode;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeContentType;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeItemPreviewStatus;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeSourceType;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
||||||
|
import com.datamate.datamanagement.infrastructure.config.DataManagementProperties;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPreviewStatusResponse;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目预览转换服务
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class KnowledgeItemPreviewService {
|
||||||
|
private static final Set<String> OFFICE_EXTENSIONS = Set.of("doc", "docx");
|
||||||
|
private static final String KNOWLEDGE_ITEM_UPLOAD_DIR = "knowledge-items";
|
||||||
|
private static final String PREVIEW_SUB_DIR = "preview";
|
||||||
|
private static final String PREVIEW_FILE_SUFFIX = ".pdf";
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
|
private static final DateTimeFormatter PREVIEW_TIME_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
|
||||||
|
|
||||||
|
private final KnowledgeItemRepository knowledgeItemRepository;
|
||||||
|
private final DataManagementProperties dataManagementProperties;
|
||||||
|
private final KnowledgeItemPreviewAsyncService knowledgeItemPreviewAsyncService;
|
||||||
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
public KnowledgeItemPreviewStatusResponse getPreviewStatus(String setId, String itemId) {
|
||||||
|
KnowledgeItem item = requireKnowledgeItem(setId, itemId);
|
||||||
|
assertOfficeDocument(item);
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(item.getMetadata(), objectMapper);
|
||||||
|
|
||||||
|
if (previewInfo.status() == KnowledgeItemPreviewStatus.READY && !previewPdfExists(item, previewInfo)) {
|
||||||
|
previewInfo = markPreviewFailed(item, previewInfo, "预览文件不存在");
|
||||||
|
}
|
||||||
|
|
||||||
|
return buildResponse(previewInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public KnowledgeItemPreviewStatusResponse ensurePreview(String setId, String itemId) {
|
||||||
|
KnowledgeItem item = requireKnowledgeItem(setId, itemId);
|
||||||
|
assertOfficeDocument(item);
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(item.getMetadata(), objectMapper);
|
||||||
|
|
||||||
|
if (previewInfo.status() == KnowledgeItemPreviewStatus.READY && previewPdfExists(item, previewInfo)) {
|
||||||
|
return buildResponse(previewInfo);
|
||||||
|
}
|
||||||
|
if (previewInfo.status() == KnowledgeItemPreviewStatus.PROCESSING) {
|
||||||
|
return buildResponse(previewInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
String previewRelativePath = resolvePreviewRelativePath(item.getSetId(), item.getId());
|
||||||
|
String updatedMetadata = KnowledgeItemPreviewMetadataHelper.applyPreviewInfo(
|
||||||
|
item.getMetadata(),
|
||||||
|
objectMapper,
|
||||||
|
KnowledgeItemPreviewStatus.PROCESSING,
|
||||||
|
previewRelativePath,
|
||||||
|
null,
|
||||||
|
nowText()
|
||||||
|
);
|
||||||
|
item.setMetadata(updatedMetadata);
|
||||||
|
knowledgeItemRepository.updateById(item);
|
||||||
|
knowledgeItemPreviewAsyncService.convertPreviewAsync(item.getId());
|
||||||
|
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo refreshed = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(updatedMetadata, objectMapper);
|
||||||
|
return buildResponse(refreshed);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isOfficeDocument(String fileName) {
|
||||||
|
String extension = resolveFileExtension(fileName);
|
||||||
|
return StringUtils.isNotBlank(extension) && OFFICE_EXTENSIONS.contains(extension.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
public PreviewFile resolveReadyPreviewFile(String setId, KnowledgeItem item) {
|
||||||
|
if (item == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo = KnowledgeItemPreviewMetadataHelper
|
||||||
|
.readPreviewInfo(item.getMetadata(), objectMapper);
|
||||||
|
if (previewInfo.status() != KnowledgeItemPreviewStatus.READY) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String relativePath = StringUtils.defaultIfBlank(previewInfo.pdfPath(), resolvePreviewRelativePath(setId, item.getId()));
|
||||||
|
Path filePath = resolvePreviewStoragePath(relativePath);
|
||||||
|
if (!Files.exists(filePath) || !Files.isRegularFile(filePath)) {
|
||||||
|
markPreviewFailed(item, previewInfo, "预览文件不存在");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String previewName = resolvePreviewPdfName(item);
|
||||||
|
return new PreviewFile(filePath, previewName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String clearPreviewMetadata(String metadata) {
|
||||||
|
return KnowledgeItemPreviewMetadataHelper.clearPreviewInfo(metadata, objectMapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deletePreviewFileQuietly(String setId, String itemId) {
|
||||||
|
String relativePath = resolvePreviewRelativePath(setId, itemId);
|
||||||
|
Path filePath = resolvePreviewStoragePath(relativePath);
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(filePath);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("delete preview pdf error, itemId: {}", itemId, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private KnowledgeItemPreviewStatusResponse buildResponse(KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo) {
|
||||||
|
KnowledgeItemPreviewStatusResponse response = new KnowledgeItemPreviewStatusResponse();
|
||||||
|
KnowledgeItemPreviewStatus status = previewInfo.status() == null
|
||||||
|
? KnowledgeItemPreviewStatus.PENDING
|
||||||
|
: previewInfo.status();
|
||||||
|
response.setStatus(status);
|
||||||
|
response.setPreviewError(previewInfo.error());
|
||||||
|
response.setUpdatedAt(previewInfo.updatedAt());
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
private KnowledgeItem requireKnowledgeItem(String setId, String itemId) {
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(setId), CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(itemId), CommonErrorCode.PARAM_ERROR);
|
||||||
|
KnowledgeItem knowledgeItem = knowledgeItemRepository.getById(itemId);
|
||||||
|
BusinessAssert.notNull(knowledgeItem, CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(Objects.equals(knowledgeItem.getSetId(), setId), CommonErrorCode.PARAM_ERROR);
|
||||||
|
return knowledgeItem;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertOfficeDocument(KnowledgeItem item) {
|
||||||
|
BusinessAssert.notNull(item, CommonErrorCode.PARAM_ERROR);
|
||||||
|
BusinessAssert.isTrue(
|
||||||
|
item.getContentType() == KnowledgeContentType.FILE || item.getSourceType() == KnowledgeSourceType.FILE_UPLOAD,
|
||||||
|
CommonErrorCode.PARAM_ERROR
|
||||||
|
);
|
||||||
|
String extension = resolveFileExtension(resolveOriginalName(item));
|
||||||
|
BusinessAssert.isTrue(OFFICE_EXTENSIONS.contains(extension), CommonErrorCode.PARAM_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveOriginalName(KnowledgeItem item) {
|
||||||
|
if (item == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(item.getSourceFileId())) {
|
||||||
|
return item.getSourceFileId();
|
||||||
|
}
|
||||||
|
if (StringUtils.isNotBlank(item.getContent())) {
|
||||||
|
return Paths.get(item.getContent()).getFileName().toString();
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveFileExtension(String fileName) {
|
||||||
|
if (StringUtils.isBlank(fileName)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
int dotIndex = fileName.lastIndexOf('.');
|
||||||
|
if (dotIndex <= 0 || dotIndex >= fileName.length() - 1) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return fileName.substring(dotIndex + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePreviewPdfName(KnowledgeItem item) {
|
||||||
|
String originalName = resolveOriginalName(item);
|
||||||
|
if (StringUtils.isBlank(originalName)) {
|
||||||
|
return "预览.pdf";
|
||||||
|
}
|
||||||
|
int dotIndex = originalName.lastIndexOf('.');
|
||||||
|
if (dotIndex <= 0) {
|
||||||
|
return originalName + PREVIEW_FILE_SUFFIX;
|
||||||
|
}
|
||||||
|
return originalName.substring(0, dotIndex) + PREVIEW_FILE_SUFFIX;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean previewPdfExists(KnowledgeItem item, KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo) {
|
||||||
|
String relativePath = StringUtils.defaultIfBlank(previewInfo.pdfPath(), resolvePreviewRelativePath(item.getSetId(), item.getId()));
|
||||||
|
Path filePath = resolvePreviewStoragePath(relativePath);
|
||||||
|
return Files.exists(filePath) && Files.isRegularFile(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
private KnowledgeItemPreviewMetadataHelper.PreviewInfo markPreviewFailed(
|
||||||
|
KnowledgeItem item,
|
||||||
|
KnowledgeItemPreviewMetadataHelper.PreviewInfo previewInfo,
|
||||||
|
String error
|
||||||
|
) {
|
||||||
|
String relativePath = StringUtils.defaultIfBlank(previewInfo.pdfPath(), resolvePreviewRelativePath(item.getSetId(), item.getId()));
|
||||||
|
String updatedMetadata = KnowledgeItemPreviewMetadataHelper.applyPreviewInfo(
|
||||||
|
item.getMetadata(),
|
||||||
|
objectMapper,
|
||||||
|
KnowledgeItemPreviewStatus.FAILED,
|
||||||
|
relativePath,
|
||||||
|
error,
|
||||||
|
nowText()
|
||||||
|
);
|
||||||
|
item.setMetadata(updatedMetadata);
|
||||||
|
knowledgeItemRepository.updateById(item);
|
||||||
|
return KnowledgeItemPreviewMetadataHelper.readPreviewInfo(updatedMetadata, objectMapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePreviewRelativePath(String setId, String itemId) {
|
||||||
|
String relativePath = Paths.get(KNOWLEDGE_ITEM_UPLOAD_DIR, setId, PREVIEW_SUB_DIR, itemId + PREVIEW_FILE_SUFFIX)
|
||||||
|
.toString();
|
||||||
|
return relativePath.replace("\\", PATH_SEPARATOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolvePreviewStoragePath(String relativePath) {
|
||||||
|
String normalizedRelativePath = StringUtils.defaultString(relativePath).replace("/", java.io.File.separator);
|
||||||
|
Path root = resolveUploadRootPath();
|
||||||
|
Path target = root.resolve(normalizedRelativePath).toAbsolutePath().normalize();
|
||||||
|
BusinessAssert.isTrue(target.startsWith(root), CommonErrorCode.PARAM_ERROR);
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolveUploadRootPath() {
|
||||||
|
String uploadDir = dataManagementProperties.getFileStorage().getUploadDir();
|
||||||
|
BusinessAssert.isTrue(StringUtils.isNotBlank(uploadDir), CommonErrorCode.PARAM_ERROR);
|
||||||
|
return Paths.get(uploadDir).toAbsolutePath().normalize();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nowText() {
|
||||||
|
return LocalDateTime.now().format(PREVIEW_TIME_FORMATTER);
|
||||||
|
}
|
||||||
|
|
||||||
|
public record PreviewFile(Path filePath, String fileName) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package com.datamate.datamanagement.application;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.StandardCopyOption;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* LibreOffice 文档转换工具
|
||||||
|
*/
|
||||||
|
public final class LibreOfficeConverter {
|
||||||
|
private static final String LIBREOFFICE_COMMAND = "soffice";
|
||||||
|
private static final Duration CONVERT_TIMEOUT = Duration.ofMinutes(5);
|
||||||
|
private static final int MAX_OUTPUT_LENGTH = 500;
|
||||||
|
|
||||||
|
private LibreOfficeConverter() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void convertToPdf(Path sourcePath, Path targetPath) throws Exception {
|
||||||
|
Path outputDir = targetPath.getParent();
|
||||||
|
List<String> command = List.of(
|
||||||
|
LIBREOFFICE_COMMAND,
|
||||||
|
"--headless",
|
||||||
|
"--nologo",
|
||||||
|
"--nolockcheck",
|
||||||
|
"--nodefault",
|
||||||
|
"--nofirststartwizard",
|
||||||
|
"--convert-to",
|
||||||
|
"pdf",
|
||||||
|
"--outdir",
|
||||||
|
outputDir.toString(),
|
||||||
|
sourcePath.toString()
|
||||||
|
);
|
||||||
|
ProcessBuilder processBuilder = new ProcessBuilder(command);
|
||||||
|
processBuilder.redirectErrorStream(true);
|
||||||
|
Process process = processBuilder.start();
|
||||||
|
boolean finished = process.waitFor(CONVERT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
|
||||||
|
String output = readProcessOutput(process.getInputStream());
|
||||||
|
if (!finished) {
|
||||||
|
process.destroyForcibly();
|
||||||
|
throw new IllegalStateException("LibreOffice 转换超时");
|
||||||
|
}
|
||||||
|
if (process.exitValue() != 0) {
|
||||||
|
throw new IllegalStateException("LibreOffice 转换失败: " + output);
|
||||||
|
}
|
||||||
|
Path generated = outputDir.resolve(stripExtension(sourcePath.getFileName().toString()) + ".pdf");
|
||||||
|
if (!Files.exists(generated)) {
|
||||||
|
throw new IllegalStateException("LibreOffice 输出文件不存在");
|
||||||
|
}
|
||||||
|
if (!generated.equals(targetPath)) {
|
||||||
|
Files.move(generated, targetPath, StandardCopyOption.REPLACE_EXISTING);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String readProcessOutput(InputStream inputStream) throws IOException {
|
||||||
|
if (inputStream == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
byte[] buffer = new byte[1024];
|
||||||
|
StringBuilder builder = new StringBuilder();
|
||||||
|
int total = 0;
|
||||||
|
int read;
|
||||||
|
while ((read = inputStream.read(buffer)) >= 0) {
|
||||||
|
if (read == 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int remaining = MAX_OUTPUT_LENGTH - total;
|
||||||
|
if (remaining <= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
int toAppend = Math.min(remaining, read);
|
||||||
|
builder.append(new String(buffer, 0, toAppend, StandardCharsets.UTF_8));
|
||||||
|
total += toAppend;
|
||||||
|
if (total >= MAX_OUTPUT_LENGTH) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String stripExtension(String fileName) {
|
||||||
|
if (fileName == null || fileName.isBlank()) {
|
||||||
|
return "preview";
|
||||||
|
}
|
||||||
|
int dotIndex = fileName.lastIndexOf('.');
|
||||||
|
return dotIndex <= 0 ? fileName : fileName.substring(0, dotIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package com.datamate.datamanagement.common.enums;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目预览转换状态
|
||||||
|
*/
|
||||||
|
public enum KnowledgeItemPreviewStatus {
|
||||||
|
PENDING,
|
||||||
|
PROCESSING,
|
||||||
|
READY,
|
||||||
|
FAILED
|
||||||
|
}
|
||||||
@@ -114,9 +114,9 @@ public class Dataset extends BaseEntity<String> {
|
|||||||
this.updatedAt = LocalDateTime.now();
|
this.updatedAt = LocalDateTime.now();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void initCreateParam(String datasetBasePath, String parentPath) {
|
public void initCreateParam(String datasetBasePath) {
|
||||||
this.id = UUID.randomUUID().toString();
|
this.id = UUID.randomUUID().toString();
|
||||||
String basePath = normalizeBasePath(parentPath != null && !parentPath.isBlank() ? parentPath : datasetBasePath);
|
String basePath = normalizeBasePath(datasetBasePath);
|
||||||
this.path = basePath + File.separator + this.id;
|
this.path = basePath + File.separator + this.id;
|
||||||
if (this.status == null) {
|
if (this.status == null) {
|
||||||
this.status = DatasetStatusType.DRAFT;
|
this.status = DatasetStatusType.DRAFT;
|
||||||
|
|||||||
@@ -38,4 +38,12 @@ public class KnowledgeItem extends BaseEntity<String> {
|
|||||||
* 来源文件ID
|
* 来源文件ID
|
||||||
*/
|
*/
|
||||||
private String sourceFileId;
|
private String sourceFileId;
|
||||||
|
/**
|
||||||
|
* 相对路径(用于目录展示)
|
||||||
|
*/
|
||||||
|
private String relativePath;
|
||||||
|
/**
|
||||||
|
* 扩展元数据
|
||||||
|
*/
|
||||||
|
private String metadata;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package com.datamate.datamanagement.domain.model.knowledge;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import com.datamate.common.domain.model.base.BaseEntity;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目目录实体(与数据库表 t_dm_knowledge_item_directories 对齐)
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@TableName(value = "t_dm_knowledge_item_directories", autoResultMap = true)
|
||||||
|
public class KnowledgeItemDirectory extends BaseEntity<String> {
|
||||||
|
/**
|
||||||
|
* 所属知识集ID
|
||||||
|
*/
|
||||||
|
private String setId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 目录名称
|
||||||
|
*/
|
||||||
|
private String name;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 目录相对路径
|
||||||
|
*/
|
||||||
|
private String relativePath;
|
||||||
|
}
|
||||||
@@ -42,9 +42,9 @@ public enum DataManagementErrorCode implements ErrorCode {
|
|||||||
*/
|
*/
|
||||||
DIRECTORY_NOT_FOUND("data_management.0007", "目录不存在"),
|
DIRECTORY_NOT_FOUND("data_management.0007", "目录不存在"),
|
||||||
/**
|
/**
|
||||||
* 存在子数据集
|
* 存在关联数据集
|
||||||
*/
|
*/
|
||||||
DATASET_HAS_CHILDREN("data_management.0008", "存在子数据集,禁止删除或移动"),
|
DATASET_HAS_CHILDREN("data_management.0008", "存在关联数据集,禁止删除或移动"),
|
||||||
/**
|
/**
|
||||||
* 数据集文件不存在
|
* 数据集文件不存在
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package com.datamate.datamanagement.infrastructure.persistence.mapper;
|
|||||||
|
|
||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.dto.DatasetFileCount;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
import org.apache.ibatis.annotations.Param;
|
import org.apache.ibatis.annotations.Param;
|
||||||
import org.apache.ibatis.session.RowBounds;
|
import org.apache.ibatis.session.RowBounds;
|
||||||
@@ -17,6 +18,7 @@ public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
|
|||||||
Long countByDatasetId(@Param("datasetId") String datasetId);
|
Long countByDatasetId(@Param("datasetId") String datasetId);
|
||||||
Long countCompletedByDatasetId(@Param("datasetId") String datasetId);
|
Long countCompletedByDatasetId(@Param("datasetId") String datasetId);
|
||||||
Long sumSizeByDatasetId(@Param("datasetId") String datasetId);
|
Long sumSizeByDatasetId(@Param("datasetId") String datasetId);
|
||||||
|
Long countNonDerivedByDatasetId(@Param("datasetId") String datasetId);
|
||||||
DatasetFile findByDatasetIdAndFileName(@Param("datasetId") String datasetId, @Param("fileName") String fileName);
|
DatasetFile findByDatasetIdAndFileName(@Param("datasetId") String datasetId, @Param("fileName") String fileName);
|
||||||
List<DatasetFile> findAllByDatasetId(@Param("datasetId") String datasetId);
|
List<DatasetFile> findAllByDatasetId(@Param("datasetId") String datasetId);
|
||||||
List<DatasetFile> findByCriteria(@Param("datasetId") String datasetId,
|
List<DatasetFile> findByCriteria(@Param("datasetId") String datasetId,
|
||||||
@@ -38,4 +40,12 @@ public interface DatasetFileMapper extends BaseMapper<DatasetFile> {
|
|||||||
* @return 源文件ID列表
|
* @return 源文件ID列表
|
||||||
*/
|
*/
|
||||||
List<String> findSourceFileIdsWithDerivedFiles(@Param("datasetId") String datasetId);
|
List<String> findSourceFileIdsWithDerivedFiles(@Param("datasetId") String datasetId);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量统计排除衍生文件后的文件数
|
||||||
|
*
|
||||||
|
* @param datasetIds 数据集ID列表
|
||||||
|
* @return 文件数统计列表
|
||||||
|
*/
|
||||||
|
List<DatasetFileCount> countNonDerivedByDatasetIds(@Param("datasetIds") List<String> datasetIds);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.datamate.datamanagement.infrastructure.persistence.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItemDirectory;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface KnowledgeItemDirectoryMapper extends BaseMapper<KnowledgeItemDirectory> {
|
||||||
|
}
|
||||||
@@ -1,9 +1,52 @@
|
|||||||
package com.datamate.datamanagement.infrastructure.persistence.mapper;
|
package com.datamate.datamanagement.infrastructure.persistence.mapper;
|
||||||
|
|
||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface KnowledgeItemMapper extends BaseMapper<KnowledgeItem> {
|
public interface KnowledgeItemMapper extends BaseMapper<KnowledgeItem> {
|
||||||
|
@Select("""
|
||||||
|
SELECT
|
||||||
|
ki.id AS id,
|
||||||
|
ki.set_id AS setId,
|
||||||
|
ks.name AS setName,
|
||||||
|
ki.content_type AS contentType,
|
||||||
|
ki.source_type AS sourceType,
|
||||||
|
ki.source_dataset_id AS sourceDatasetId,
|
||||||
|
ki.source_file_id AS sourceFileId,
|
||||||
|
CASE
|
||||||
|
WHEN ki.source_type = 'DATASET_FILE' THEN df.file_name
|
||||||
|
ELSE ki.source_file_id
|
||||||
|
END AS fileName,
|
||||||
|
df.file_size AS fileSize,
|
||||||
|
CASE
|
||||||
|
WHEN ki.source_type = 'FILE_UPLOAD' THEN ki.content
|
||||||
|
ELSE NULL
|
||||||
|
END AS content,
|
||||||
|
ki.relative_path AS relativePath,
|
||||||
|
ki.created_at AS createdAt,
|
||||||
|
ki.updated_at AS updatedAt
|
||||||
|
FROM t_dm_knowledge_items ki
|
||||||
|
LEFT JOIN t_dm_knowledge_sets ks ON ki.set_id = ks.id
|
||||||
|
LEFT JOIN t_dm_dataset_files df ON ki.source_file_id = df.id AND ki.source_type = 'DATASET_FILE'
|
||||||
|
WHERE (ki.source_type = 'FILE_UPLOAD' AND (ki.source_file_id LIKE CONCAT('%', #{keyword}, '%')
|
||||||
|
OR ki.relative_path LIKE CONCAT('%', #{keyword}, '%')))
|
||||||
|
OR (ki.source_type = 'DATASET_FILE' AND (df.file_name LIKE CONCAT('%', #{keyword}, '%')
|
||||||
|
OR ki.relative_path LIKE CONCAT('%', #{keyword}, '%')))
|
||||||
|
ORDER BY ki.created_at DESC
|
||||||
|
""")
|
||||||
|
IPage<KnowledgeItemSearchResponse> searchFileItems(IPage<?> page, @Param("keyword") String keyword);
|
||||||
|
|
||||||
|
@Select("""
|
||||||
|
SELECT COALESCE(SUM(df.file_size), 0)
|
||||||
|
FROM t_dm_knowledge_items ki
|
||||||
|
LEFT JOIN t_dm_dataset_files df ON ki.source_file_id = df.id
|
||||||
|
WHERE ki.source_type = 'DATASET_FILE'
|
||||||
|
""")
|
||||||
|
Long sumDatasetFileSize();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ public interface TagMapper {
|
|||||||
List<Tag> findByIdIn(@Param("ids") List<String> ids);
|
List<Tag> findByIdIn(@Param("ids") List<String> ids);
|
||||||
List<Tag> findByKeyword(@Param("keyword") String keyword);
|
List<Tag> findByKeyword(@Param("keyword") String keyword);
|
||||||
List<Tag> findAllByOrderByUsageCountDesc();
|
List<Tag> findAllByOrderByUsageCountDesc();
|
||||||
|
Long countKnowledgeSetTags();
|
||||||
|
|
||||||
int insert(Tag tag);
|
int insert(Tag tag);
|
||||||
int update(Tag tag);
|
int update(Tag tag);
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package com.datamate.datamanagement.infrastructure.persistence.repository;
|
|||||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||||
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.dto.DatasetFileCount;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@@ -15,6 +16,8 @@ import java.util.List;
|
|||||||
public interface DatasetFileRepository extends IRepository<DatasetFile> {
|
public interface DatasetFileRepository extends IRepository<DatasetFile> {
|
||||||
Long countByDatasetId(String datasetId);
|
Long countByDatasetId(String datasetId);
|
||||||
|
|
||||||
|
Long countNonDerivedByDatasetId(String datasetId);
|
||||||
|
|
||||||
Long countCompletedByDatasetId(String datasetId);
|
Long countCompletedByDatasetId(String datasetId);
|
||||||
|
|
||||||
Long sumSizeByDatasetId(String datasetId);
|
Long sumSizeByDatasetId(String datasetId);
|
||||||
@@ -36,4 +39,6 @@ public interface DatasetFileRepository extends IRepository<DatasetFile> {
|
|||||||
* @return 源文件ID列表
|
* @return 源文件ID列表
|
||||||
*/
|
*/
|
||||||
List<String> findSourceFileIdsWithDerivedFiles(String datasetId);
|
List<String> findSourceFileIdsWithDerivedFiles(String datasetId);
|
||||||
|
|
||||||
|
List<DatasetFileCount> countNonDerivedByDatasetIds(List<String> datasetIds);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package com.datamate.datamanagement.infrastructure.persistence.repository;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItemDirectory;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeDirectoryQuery;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目目录仓储接口
|
||||||
|
*/
|
||||||
|
public interface KnowledgeItemDirectoryRepository extends IRepository<KnowledgeItemDirectory> {
|
||||||
|
List<KnowledgeItemDirectory> findByCriteria(KnowledgeDirectoryQuery query);
|
||||||
|
|
||||||
|
KnowledgeItemDirectory findBySetIdAndPath(String setId, String relativePath);
|
||||||
|
|
||||||
|
int removeByRelativePathPrefix(String setId, String relativePath);
|
||||||
|
}
|
||||||
@@ -2,8 +2,11 @@ package com.datamate.datamanagement.infrastructure.persistence.repository;
|
|||||||
|
|
||||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeSourceType;
|
||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -15,4 +18,16 @@ public interface KnowledgeItemRepository extends IRepository<KnowledgeItem> {
|
|||||||
long countBySetId(String setId);
|
long countBySetId(String setId);
|
||||||
|
|
||||||
List<KnowledgeItem> findAllBySetId(String setId);
|
List<KnowledgeItem> findAllBySetId(String setId);
|
||||||
|
|
||||||
|
long countBySourceTypes(List<KnowledgeSourceType> sourceTypes);
|
||||||
|
|
||||||
|
List<KnowledgeItem> findFileUploadItems();
|
||||||
|
|
||||||
|
IPage<KnowledgeItemSearchResponse> searchFileItems(IPage<?> page, String keyword);
|
||||||
|
|
||||||
|
Long sumDatasetFileSize();
|
||||||
|
|
||||||
|
boolean existsBySetIdAndRelativePath(String setId, String relativePath);
|
||||||
|
|
||||||
|
int removeByRelativePathPrefix(String setId, String relativePath);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package com.datamate.datamanagement.infrastructure.persistence.repository.dto;
|
||||||
|
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据集文件数统计结果
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
public class DatasetFileCount {
|
||||||
|
private String datasetId;
|
||||||
|
private Long fileCount;
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
|||||||
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper;
|
import com.datamate.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetFileRepository;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.dto.DatasetFileCount;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.stereotype.Repository;
|
import org.springframework.stereotype.Repository;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
@@ -30,6 +31,11 @@ public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper,
|
|||||||
return datasetFileMapper.selectCount(new LambdaQueryWrapper<DatasetFile>().eq(DatasetFile::getDatasetId, datasetId));
|
return datasetFileMapper.selectCount(new LambdaQueryWrapper<DatasetFile>().eq(DatasetFile::getDatasetId, datasetId));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long countNonDerivedByDatasetId(String datasetId) {
|
||||||
|
return datasetFileMapper.countNonDerivedByDatasetId(datasetId);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Long countCompletedByDatasetId(String datasetId) {
|
public Long countCompletedByDatasetId(String datasetId) {
|
||||||
return datasetFileMapper.countCompletedByDatasetId(datasetId);
|
return datasetFileMapper.countCompletedByDatasetId(datasetId);
|
||||||
@@ -71,4 +77,9 @@ public class DatasetFileRepositoryImpl extends CrudRepository<DatasetFileMapper,
|
|||||||
// 使用 MyBatis 的 @Select 注解或直接调用 mapper 方法
|
// 使用 MyBatis 的 @Select 注解或直接调用 mapper 方法
|
||||||
return datasetFileMapper.findSourceFileIdsWithDerivedFiles(datasetId);
|
return datasetFileMapper.findSourceFileIdsWithDerivedFiles(datasetId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DatasetFileCount> countNonDerivedByDatasetIds(List<String> datasetIds) {
|
||||||
|
return datasetFileMapper.countNonDerivedByDatasetIds(datasetIds);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package com.datamate.datamanagement.infrastructure.persistence.repository.impl;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItemDirectory;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.mapper.KnowledgeItemDirectoryMapper;
|
||||||
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemDirectoryRepository;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeDirectoryQuery;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.stereotype.Repository;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目目录仓储实现类
|
||||||
|
*/
|
||||||
|
@Repository
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class KnowledgeItemDirectoryRepositoryImpl
|
||||||
|
extends CrudRepository<KnowledgeItemDirectoryMapper, KnowledgeItemDirectory>
|
||||||
|
implements KnowledgeItemDirectoryRepository {
|
||||||
|
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
|
private final KnowledgeItemDirectoryMapper knowledgeItemDirectoryMapper;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<KnowledgeItemDirectory> findByCriteria(KnowledgeDirectoryQuery query) {
|
||||||
|
String relativePath = normalizeRelativePathPrefix(query.getRelativePath());
|
||||||
|
LambdaQueryWrapper<KnowledgeItemDirectory> wrapper = new LambdaQueryWrapper<KnowledgeItemDirectory>()
|
||||||
|
.eq(StringUtils.isNotBlank(query.getSetId()), KnowledgeItemDirectory::getSetId, query.getSetId())
|
||||||
|
.likeRight(StringUtils.isNotBlank(relativePath), KnowledgeItemDirectory::getRelativePath, relativePath);
|
||||||
|
|
||||||
|
if (StringUtils.isNotBlank(query.getKeyword())) {
|
||||||
|
wrapper.and(w -> w.like(KnowledgeItemDirectory::getName, query.getKeyword())
|
||||||
|
.or()
|
||||||
|
.like(KnowledgeItemDirectory::getRelativePath, query.getKeyword()));
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapper.orderByAsc(KnowledgeItemDirectory::getRelativePath);
|
||||||
|
return knowledgeItemDirectoryMapper.selectList(wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public KnowledgeItemDirectory findBySetIdAndPath(String setId, String relativePath) {
|
||||||
|
return knowledgeItemDirectoryMapper.selectOne(new LambdaQueryWrapper<KnowledgeItemDirectory>()
|
||||||
|
.eq(KnowledgeItemDirectory::getSetId, setId)
|
||||||
|
.eq(KnowledgeItemDirectory::getRelativePath, relativePath));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int removeByRelativePathPrefix(String setId, String relativePath) {
|
||||||
|
String normalized = normalizeRelativePathValue(relativePath);
|
||||||
|
if (StringUtils.isBlank(normalized)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
String prefix = normalizeRelativePathPrefix(normalized);
|
||||||
|
LambdaQueryWrapper<KnowledgeItemDirectory> wrapper = new LambdaQueryWrapper<KnowledgeItemDirectory>()
|
||||||
|
.eq(KnowledgeItemDirectory::getSetId, setId)
|
||||||
|
.and(w -> w.eq(KnowledgeItemDirectory::getRelativePath, normalized)
|
||||||
|
.or()
|
||||||
|
.likeRight(KnowledgeItemDirectory::getRelativePath, prefix));
|
||||||
|
return knowledgeItemDirectoryMapper.delete(wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathPrefix(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(normalized)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (!normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized + PATH_SEPARATOR;
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathValue(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
while (normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,10 +3,12 @@ package com.datamate.datamanagement.infrastructure.persistence.repository.impl;
|
|||||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
import com.baomidou.mybatisplus.extension.repository.CrudRepository;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeSourceType;
|
||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.mapper.KnowledgeItemMapper;
|
import com.datamate.datamanagement.infrastructure.persistence.mapper.KnowledgeItemMapper;
|
||||||
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
|
import com.datamate.datamanagement.infrastructure.persistence.repository.KnowledgeItemRepository;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.stereotype.Repository;
|
import org.springframework.stereotype.Repository;
|
||||||
@@ -19,21 +21,26 @@ import java.util.List;
|
|||||||
@Repository
|
@Repository
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class KnowledgeItemRepositoryImpl extends CrudRepository<KnowledgeItemMapper, KnowledgeItem> implements KnowledgeItemRepository {
|
public class KnowledgeItemRepositoryImpl extends CrudRepository<KnowledgeItemMapper, KnowledgeItem> implements KnowledgeItemRepository {
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
private final KnowledgeItemMapper knowledgeItemMapper;
|
private final KnowledgeItemMapper knowledgeItemMapper;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IPage<KnowledgeItem> findByCriteria(IPage<KnowledgeItem> page, KnowledgeItemPagingQuery query) {
|
public IPage<KnowledgeItem> findByCriteria(IPage<KnowledgeItem> page, KnowledgeItemPagingQuery query) {
|
||||||
|
String relativePath = normalizeRelativePathPrefix(query.getRelativePath());
|
||||||
LambdaQueryWrapper<KnowledgeItem> wrapper = new LambdaQueryWrapper<KnowledgeItem>()
|
LambdaQueryWrapper<KnowledgeItem> wrapper = new LambdaQueryWrapper<KnowledgeItem>()
|
||||||
.eq(StringUtils.isNotBlank(query.getSetId()), KnowledgeItem::getSetId, query.getSetId())
|
.eq(StringUtils.isNotBlank(query.getSetId()), KnowledgeItem::getSetId, query.getSetId())
|
||||||
.eq(query.getContentType() != null, KnowledgeItem::getContentType, query.getContentType())
|
.eq(query.getContentType() != null, KnowledgeItem::getContentType, query.getContentType())
|
||||||
.eq(query.getSourceType() != null, KnowledgeItem::getSourceType, query.getSourceType())
|
.eq(query.getSourceType() != null, KnowledgeItem::getSourceType, query.getSourceType())
|
||||||
.eq(StringUtils.isNotBlank(query.getSourceDatasetId()), KnowledgeItem::getSourceDatasetId, query.getSourceDatasetId())
|
.eq(StringUtils.isNotBlank(query.getSourceDatasetId()), KnowledgeItem::getSourceDatasetId, query.getSourceDatasetId())
|
||||||
.eq(StringUtils.isNotBlank(query.getSourceFileId()), KnowledgeItem::getSourceFileId, query.getSourceFileId());
|
.eq(StringUtils.isNotBlank(query.getSourceFileId()), KnowledgeItem::getSourceFileId, query.getSourceFileId())
|
||||||
|
.likeRight(StringUtils.isNotBlank(relativePath), KnowledgeItem::getRelativePath, relativePath);
|
||||||
|
|
||||||
if (StringUtils.isNotBlank(query.getKeyword())) {
|
if (StringUtils.isNotBlank(query.getKeyword())) {
|
||||||
wrapper.and(w -> w.like(KnowledgeItem::getSourceFileId, query.getKeyword())
|
wrapper.and(w -> w.like(KnowledgeItem::getSourceFileId, query.getKeyword())
|
||||||
.or()
|
.or()
|
||||||
.like(KnowledgeItem::getContent, query.getKeyword()));
|
.like(KnowledgeItem::getContent, query.getKeyword())
|
||||||
|
.or()
|
||||||
|
.like(KnowledgeItem::getRelativePath, query.getKeyword()));
|
||||||
}
|
}
|
||||||
|
|
||||||
wrapper.orderByDesc(KnowledgeItem::getCreatedAt);
|
wrapper.orderByDesc(KnowledgeItem::getCreatedAt);
|
||||||
@@ -52,4 +59,83 @@ public class KnowledgeItemRepositoryImpl extends CrudRepository<KnowledgeItemMap
|
|||||||
.eq(KnowledgeItem::getSetId, setId)
|
.eq(KnowledgeItem::getSetId, setId)
|
||||||
.orderByDesc(KnowledgeItem::getCreatedAt));
|
.orderByDesc(KnowledgeItem::getCreatedAt));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long countBySourceTypes(List<KnowledgeSourceType> sourceTypes) {
|
||||||
|
return knowledgeItemMapper.selectCount(new LambdaQueryWrapper<KnowledgeItem>()
|
||||||
|
.in(KnowledgeItem::getSourceType, sourceTypes));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<KnowledgeItem> findFileUploadItems() {
|
||||||
|
return knowledgeItemMapper.selectList(new LambdaQueryWrapper<KnowledgeItem>()
|
||||||
|
.eq(KnowledgeItem::getSourceType, KnowledgeSourceType.FILE_UPLOAD)
|
||||||
|
.select(KnowledgeItem::getId, KnowledgeItem::getContent, KnowledgeItem::getSourceFileId));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IPage<KnowledgeItemSearchResponse> searchFileItems(IPage<?> page, String keyword) {
|
||||||
|
return knowledgeItemMapper.searchFileItems(page, keyword);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long sumDatasetFileSize() {
|
||||||
|
return knowledgeItemMapper.sumDatasetFileSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean existsBySetIdAndRelativePath(String setId, String relativePath) {
|
||||||
|
if (StringUtils.isBlank(setId) || StringUtils.isBlank(relativePath)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return knowledgeItemMapper.selectCount(new LambdaQueryWrapper<KnowledgeItem>()
|
||||||
|
.eq(KnowledgeItem::getSetId, setId)
|
||||||
|
.eq(KnowledgeItem::getRelativePath, relativePath)) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int removeByRelativePathPrefix(String setId, String relativePath) {
|
||||||
|
String normalized = normalizeRelativePathValue(relativePath);
|
||||||
|
if (StringUtils.isBlank(setId) || StringUtils.isBlank(normalized)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
String prefix = normalizeRelativePathPrefix(normalized);
|
||||||
|
LambdaQueryWrapper<KnowledgeItem> wrapper = new LambdaQueryWrapper<KnowledgeItem>()
|
||||||
|
.eq(KnowledgeItem::getSetId, setId)
|
||||||
|
.and(w -> w.eq(KnowledgeItem::getRelativePath, normalized)
|
||||||
|
.or()
|
||||||
|
.likeRight(KnowledgeItem::getRelativePath, prefix));
|
||||||
|
return knowledgeItemMapper.delete(wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathPrefix(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(normalized)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (!normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized + PATH_SEPARATOR;
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePathValue(String relativePath) {
|
||||||
|
if (StringUtils.isBlank(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
while (normalized.endsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
package com.datamate.datamanagement.interfaces.converter;
|
package com.datamate.datamanagement.interfaces.converter;
|
||||||
|
|
||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItemDirectory;
|
||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
|
||||||
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeItemRequest;
|
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeItemRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeSetRequest;
|
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeSetRequest;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeDirectoryResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemResponse;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetResponse;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetResponse;
|
||||||
import org.mapstruct.Mapper;
|
import org.mapstruct.Mapper;
|
||||||
@@ -31,4 +33,8 @@ public interface KnowledgeConverter {
|
|||||||
KnowledgeItemResponse convertToResponse(KnowledgeItem knowledgeItem);
|
KnowledgeItemResponse convertToResponse(KnowledgeItem knowledgeItem);
|
||||||
|
|
||||||
List<KnowledgeItemResponse> convertItemResponses(List<KnowledgeItem> items);
|
List<KnowledgeItemResponse> convertItemResponses(List<KnowledgeItem> items);
|
||||||
|
|
||||||
|
KnowledgeDirectoryResponse convertToResponse(KnowledgeItemDirectory directory);
|
||||||
|
|
||||||
|
List<KnowledgeDirectoryResponse> convertDirectoryResponses(List<KnowledgeItemDirectory> directories);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import jakarta.validation.constraints.NotBlank;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建知识条目目录请求
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class CreateKnowledgeDirectoryRequest {
|
||||||
|
|
||||||
|
/** 父级前缀路径,例如 "docs/",为空表示知识集根目录 */
|
||||||
|
private String parentPrefix;
|
||||||
|
|
||||||
|
/** 新建目录名称 */
|
||||||
|
@NotBlank
|
||||||
|
private String directoryName;
|
||||||
|
}
|
||||||
@@ -34,4 +34,8 @@ public class CreateKnowledgeItemRequest {
|
|||||||
* 来源文件ID(用于标注同步等场景)
|
* 来源文件ID(用于标注同步等场景)
|
||||||
*/
|
*/
|
||||||
private String sourceFileId;
|
private String sourceFileId;
|
||||||
|
/**
|
||||||
|
* 扩展元数据
|
||||||
|
*/
|
||||||
|
private String metadata;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeItemPreviewStatus;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据集文件预览状态响应
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class DatasetFilePreviewStatusResponse {
|
||||||
|
private KnowledgeItemPreviewStatus status;
|
||||||
|
private String previewError;
|
||||||
|
private String updatedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import jakarta.validation.constraints.NotEmpty;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量删除知识条目请求
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class DeleteKnowledgeItemsRequest {
|
||||||
|
/**
|
||||||
|
* 知识条目ID列表
|
||||||
|
*/
|
||||||
|
@NotEmpty(message = "知识条目ID不能为空")
|
||||||
|
private List<String> ids;
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目目录查询参数
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeDirectoryQuery {
|
||||||
|
/** 所属知识集ID */
|
||||||
|
private String setId;
|
||||||
|
|
||||||
|
/** 目录相对路径前缀 */
|
||||||
|
private String relativePath;
|
||||||
|
|
||||||
|
/** 搜索关键字 */
|
||||||
|
private String keyword;
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目目录响应
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeDirectoryResponse {
|
||||||
|
private String id;
|
||||||
|
private String setId;
|
||||||
|
private String name;
|
||||||
|
private String relativePath;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
@@ -41,4 +41,8 @@ public class KnowledgeItemPagingQuery extends PagingQuery {
|
|||||||
* 来源文件ID
|
* 来源文件ID
|
||||||
*/
|
*/
|
||||||
private String sourceFileId;
|
private String sourceFileId;
|
||||||
|
/**
|
||||||
|
* 相对路径前缀
|
||||||
|
*/
|
||||||
|
private String relativePath;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeItemPreviewStatus;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目预览状态响应
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeItemPreviewStatusResponse {
|
||||||
|
private KnowledgeItemPreviewStatus status;
|
||||||
|
private String previewError;
|
||||||
|
private String updatedAt;
|
||||||
|
}
|
||||||
@@ -20,6 +20,14 @@ public class KnowledgeItemResponse {
|
|||||||
private KnowledgeSourceType sourceType;
|
private KnowledgeSourceType sourceType;
|
||||||
private String sourceDatasetId;
|
private String sourceDatasetId;
|
||||||
private String sourceFileId;
|
private String sourceFileId;
|
||||||
|
/**
|
||||||
|
* 相对路径(用于目录展示)
|
||||||
|
*/
|
||||||
|
private String relativePath;
|
||||||
|
/**
|
||||||
|
* 扩展元数据
|
||||||
|
*/
|
||||||
|
private String metadata;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
private String createdBy;
|
private String createdBy;
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import com.datamate.common.interfaces.PagingQuery;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目文件搜索请求
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeItemSearchQuery extends PagingQuery {
|
||||||
|
/**
|
||||||
|
* 文件名关键词
|
||||||
|
*/
|
||||||
|
private String keyword;
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeContentType;
|
||||||
|
import com.datamate.datamanagement.common.enums.KnowledgeSourceType;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目文件搜索响应
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeItemSearchResponse {
|
||||||
|
private String id;
|
||||||
|
private String setId;
|
||||||
|
private String setName;
|
||||||
|
private KnowledgeContentType contentType;
|
||||||
|
private KnowledgeSourceType sourceType;
|
||||||
|
private String sourceDatasetId;
|
||||||
|
private String sourceFileId;
|
||||||
|
private String fileName;
|
||||||
|
private Long fileSize;
|
||||||
|
/**
|
||||||
|
* 相对路径(用于目录展示)
|
||||||
|
*/
|
||||||
|
private String relativePath;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
|
||||||
|
@JsonIgnore
|
||||||
|
private String content;
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识管理统计响应
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeManagementStatisticsResponse {
|
||||||
|
private Long totalKnowledgeSets = 0L;
|
||||||
|
private Long totalFiles = 0L;
|
||||||
|
private Long totalSize = 0L;
|
||||||
|
private Long totalTags = 0L;
|
||||||
|
}
|
||||||
@@ -1,8 +1,10 @@
|
|||||||
package com.datamate.datamanagement.interfaces.dto;
|
package com.datamate.datamanagement.interfaces.dto;
|
||||||
|
|
||||||
import com.datamate.datamanagement.common.enums.DatasetStatusType;
|
import com.datamate.datamanagement.common.enums.DatasetStatusType;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
import jakarta.validation.constraints.NotBlank;
|
import jakarta.validation.constraints.NotBlank;
|
||||||
import jakarta.validation.constraints.Size;
|
import jakarta.validation.constraints.Size;
|
||||||
|
import lombok.AccessLevel;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
import lombok.Setter;
|
import lombok.Setter;
|
||||||
|
|
||||||
@@ -24,9 +26,18 @@ public class UpdateDatasetRequest {
|
|||||||
/** 归集任务id */
|
/** 归集任务id */
|
||||||
private String dataSource;
|
private String dataSource;
|
||||||
/** 父数据集ID */
|
/** 父数据集ID */
|
||||||
|
@Setter(AccessLevel.NONE)
|
||||||
private String parentDatasetId;
|
private String parentDatasetId;
|
||||||
|
@JsonIgnore
|
||||||
|
@Setter(AccessLevel.NONE)
|
||||||
|
private boolean parentDatasetIdProvided;
|
||||||
/** 标签列表 */
|
/** 标签列表 */
|
||||||
private List<String> tags;
|
private List<String> tags;
|
||||||
/** 数据集状态 */
|
/** 数据集状态 */
|
||||||
private DatasetStatusType status;
|
private DatasetStatusType status;
|
||||||
|
|
||||||
|
public void setParentDatasetId(String parentDatasetId) {
|
||||||
|
this.parentDatasetIdProvided = true;
|
||||||
|
this.parentDatasetId = parentDatasetId;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,4 +18,8 @@ public class UpdateKnowledgeItemRequest {
|
|||||||
* 内容类型
|
* 内容类型
|
||||||
*/
|
*/
|
||||||
private KnowledgeContentType contentType;
|
private KnowledgeContentType contentType;
|
||||||
|
/**
|
||||||
|
* 扩展元数据
|
||||||
|
*/
|
||||||
|
private String metadata;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,4 +17,8 @@ public class UploadKnowledgeItemsRequest {
|
|||||||
*/
|
*/
|
||||||
@NotEmpty(message = "文件列表不能为空")
|
@NotEmpty(message = "文件列表不能为空")
|
||||||
private List<MultipartFile> files;
|
private List<MultipartFile> files;
|
||||||
|
/**
|
||||||
|
* 目录前缀(用于目录上传)
|
||||||
|
*/
|
||||||
|
private String parentPrefix;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,20 +5,23 @@ import com.datamate.common.infrastructure.common.Response;
|
|||||||
import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
||||||
import com.datamate.common.interfaces.PagedResponse;
|
import com.datamate.common.interfaces.PagedResponse;
|
||||||
import com.datamate.common.interfaces.PagingQuery;
|
import com.datamate.common.interfaces.PagingQuery;
|
||||||
import com.datamate.datamanagement.application.DatasetFileApplicationService;
|
import com.datamate.datamanagement.application.DatasetFileApplicationService;
|
||||||
|
import com.datamate.datamanagement.application.DatasetFilePreviewService;
|
||||||
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
|
||||||
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
|
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
|
||||||
import com.datamate.datamanagement.interfaces.dto.AddFilesRequest;
|
import com.datamate.datamanagement.interfaces.dto.AddFilesRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.CopyFilesRequest;
|
import com.datamate.datamanagement.interfaces.dto.CopyFilesRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.CreateDirectoryRequest;
|
import com.datamate.datamanagement.interfaces.dto.CreateDirectoryRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
|
import com.datamate.datamanagement.interfaces.dto.DatasetFilePreviewStatusResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
|
import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
|
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
import jakarta.validation.Valid;
|
import jakarta.validation.Valid;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.core.io.Resource;
|
import org.springframework.core.io.Resource;
|
||||||
|
import org.springframework.core.io.UrlResource;
|
||||||
import org.springframework.http.HttpHeaders;
|
import org.springframework.http.HttpHeaders;
|
||||||
import org.springframework.http.HttpStatus;
|
import org.springframework.http.HttpStatus;
|
||||||
import org.springframework.http.MediaType;
|
import org.springframework.http.MediaType;
|
||||||
@@ -36,32 +39,41 @@ import java.util.List;
|
|||||||
@RequestMapping("/data-management/datasets/{datasetId}/files")
|
@RequestMapping("/data-management/datasets/{datasetId}/files")
|
||||||
public class DatasetFileController {
|
public class DatasetFileController {
|
||||||
|
|
||||||
private final DatasetFileApplicationService datasetFileApplicationService;
|
private final DatasetFileApplicationService datasetFileApplicationService;
|
||||||
|
private final DatasetFilePreviewService datasetFilePreviewService;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
public DatasetFileController(DatasetFileApplicationService datasetFileApplicationService) {
|
public DatasetFileController(DatasetFileApplicationService datasetFileApplicationService,
|
||||||
this.datasetFileApplicationService = datasetFileApplicationService;
|
DatasetFilePreviewService datasetFilePreviewService) {
|
||||||
}
|
this.datasetFileApplicationService = datasetFileApplicationService;
|
||||||
|
this.datasetFilePreviewService = datasetFilePreviewService;
|
||||||
|
}
|
||||||
|
|
||||||
@GetMapping
|
@GetMapping
|
||||||
public Response<PagedResponse<DatasetFile>> getDatasetFiles(
|
public Response<PagedResponse<DatasetFile>> getDatasetFiles(
|
||||||
@PathVariable("datasetId") String datasetId,
|
@PathVariable("datasetId") String datasetId,
|
||||||
@RequestParam(value = "isWithDirectory", required = false) boolean isWithDirectory,
|
@RequestParam(value = "isWithDirectory", required = false) boolean isWithDirectory,
|
||||||
@RequestParam(value = "page", required = false, defaultValue = "0") Integer page,
|
@RequestParam(value = "page", required = false, defaultValue = "0") Integer page,
|
||||||
@RequestParam(value = "size", required = false, defaultValue = "20") Integer size,
|
@RequestParam(value = "size", required = false, defaultValue = "20") Integer size,
|
||||||
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix,
|
@RequestParam(value = "prefix", required = false, defaultValue = "") String prefix,
|
||||||
@RequestParam(value = "status", required = false) String status,
|
@RequestParam(value = "status", required = false) String status,
|
||||||
@RequestParam(value = "hasAnnotation", required = false) Boolean hasAnnotation,
|
@RequestParam(value = "hasAnnotation", required = false) Boolean hasAnnotation,
|
||||||
@RequestParam(value = "excludeSourceDocuments", required = false, defaultValue = "false") Boolean excludeSourceDocuments) {
|
@RequestParam(value = "excludeSourceDocuments", required = false, defaultValue = "false") Boolean excludeSourceDocuments,
|
||||||
PagingQuery pagingQuery = new PagingQuery(page, size);
|
@RequestParam(value = "excludeDerivedFiles", required = false, defaultValue = "false") Boolean excludeDerivedFiles) {
|
||||||
PagedResponse<DatasetFile> filesPage;
|
PagingQuery pagingQuery = new PagingQuery(page, size);
|
||||||
if (isWithDirectory) {
|
PagedResponse<DatasetFile> filesPage;
|
||||||
filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(datasetId, prefix, pagingQuery);
|
if (isWithDirectory) {
|
||||||
} else {
|
filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(
|
||||||
filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, status, null, hasAnnotation,
|
datasetId,
|
||||||
Boolean.TRUE.equals(excludeSourceDocuments), pagingQuery);
|
prefix,
|
||||||
}
|
Boolean.TRUE.equals(excludeDerivedFiles),
|
||||||
return Response.ok(filesPage);
|
pagingQuery
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
filesPage = datasetFileApplicationService.getDatasetFiles(datasetId, null, status, null, hasAnnotation,
|
||||||
|
Boolean.TRUE.equals(excludeSourceDocuments), pagingQuery);
|
||||||
|
}
|
||||||
|
return Response.ok(filesPage);
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/{fileId}")
|
@GetMapping("/{fileId}")
|
||||||
@@ -108,15 +120,28 @@ public class DatasetFileController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@IgnoreResponseWrap
|
@IgnoreResponseWrap
|
||||||
@GetMapping(value = "/{fileId}/preview", produces = MediaType.ALL_VALUE)
|
@GetMapping(value = "/{fileId}/preview", produces = MediaType.ALL_VALUE)
|
||||||
public ResponseEntity<Resource> previewDatasetFileById(@PathVariable("datasetId") String datasetId,
|
public ResponseEntity<Resource> previewDatasetFileById(@PathVariable("datasetId") String datasetId,
|
||||||
@PathVariable("fileId") String fileId) {
|
@PathVariable("fileId") String fileId) {
|
||||||
try {
|
try {
|
||||||
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
|
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
|
||||||
Resource resource = datasetFileApplicationService.downloadFile(datasetId, fileId);
|
if (datasetFilePreviewService.isOfficeDocument(datasetFile.getFileName())) {
|
||||||
MediaType mediaType = MediaTypeFactory.getMediaType(resource)
|
DatasetFilePreviewService.PreviewFile previewFile = datasetFilePreviewService
|
||||||
.orElse(MediaType.APPLICATION_OCTET_STREAM);
|
.resolveReadyPreviewFile(datasetId, datasetFile);
|
||||||
|
if (previewFile == null) {
|
||||||
|
return ResponseEntity.status(HttpStatus.CONFLICT).build();
|
||||||
|
}
|
||||||
|
Resource previewResource = new UrlResource(previewFile.filePath().toUri());
|
||||||
|
return ResponseEntity.ok()
|
||||||
|
.contentType(MediaType.APPLICATION_PDF)
|
||||||
|
.header(HttpHeaders.CONTENT_DISPOSITION,
|
||||||
|
"inline; filename=\"" + previewFile.fileName() + "\"")
|
||||||
|
.body(previewResource);
|
||||||
|
}
|
||||||
|
Resource resource = datasetFileApplicationService.downloadFile(datasetId, fileId);
|
||||||
|
MediaType mediaType = MediaTypeFactory.getMediaType(resource)
|
||||||
|
.orElse(MediaType.APPLICATION_OCTET_STREAM);
|
||||||
|
|
||||||
return ResponseEntity.ok()
|
return ResponseEntity.ok()
|
||||||
.contentType(mediaType)
|
.contentType(mediaType)
|
||||||
@@ -127,8 +152,20 @@ public class DatasetFileController {
|
|||||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
|
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping("/{fileId}/preview/status")
|
||||||
|
public DatasetFilePreviewStatusResponse getDatasetFilePreviewStatus(@PathVariable("datasetId") String datasetId,
|
||||||
|
@PathVariable("fileId") String fileId) {
|
||||||
|
return datasetFilePreviewService.getPreviewStatus(datasetId, fileId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/{fileId}/preview/convert")
|
||||||
|
public DatasetFilePreviewStatusResponse convertDatasetFilePreview(@PathVariable("datasetId") String datasetId,
|
||||||
|
@PathVariable("fileId") String fileId) {
|
||||||
|
return datasetFilePreviewService.ensurePreview(datasetId, fileId);
|
||||||
|
}
|
||||||
|
|
||||||
@IgnoreResponseWrap
|
@IgnoreResponseWrap
|
||||||
@GetMapping(value = "/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
|
@GetMapping(value = "/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ public class DatasetTypeController {
|
|||||||
public List<DatasetTypeResponse> getDatasetTypes() {
|
public List<DatasetTypeResponse> getDatasetTypes() {
|
||||||
return Arrays.asList(
|
return Arrays.asList(
|
||||||
createDatasetType("IMAGE", "图像数据集", "用于机器学习的图像数据集", Arrays.asList("jpg", "jpeg", "png", "bmp", "gif")),
|
createDatasetType("IMAGE", "图像数据集", "用于机器学习的图像数据集", Arrays.asList("jpg", "jpeg", "png", "bmp", "gif")),
|
||||||
createDatasetType("TEXT", "文本数据集", "用于文本分析的文本数据集", Arrays.asList("txt", "csv", "json", "xml")),
|
createDatasetType("TEXT", "文本数据集", "用于文本分析的文本数据集", Arrays.asList("txt", "csv", "xls", "xlsx", "json", "xml")),
|
||||||
createDatasetType("AUDIO", "音频数据集", "用于音频处理的音频数据集", Arrays.asList("wav", "mp3", "flac", "aac")),
|
createDatasetType("AUDIO", "音频数据集", "用于音频处理的音频数据集", Arrays.asList("wav", "mp3", "flac", "aac")),
|
||||||
createDatasetType("VIDEO", "视频数据集", "用于视频分析的视频数据集", Arrays.asList("mp4", "avi", "mov", "mkv")),
|
createDatasetType("VIDEO", "视频数据集", "用于视频分析的视频数据集", Arrays.asList("mp4", "avi", "mov", "mkv")),
|
||||||
createDatasetType("MULTIMODAL", "多模态数据集", "包含多种数据类型的数据集", List.of("*"))
|
createDatasetType("MULTIMODAL", "多模态数据集", "包含多种数据类型的数据集", List.of("*"))
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.rest;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.application.DatasetFileApplicationService;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
|
import org.springframework.web.bind.annotation.PutMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据集上传控制器
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@RequestMapping("/data-management/datasets/upload")
|
||||||
|
public class DatasetUploadController {
|
||||||
|
|
||||||
|
private final DatasetFileApplicationService datasetFileApplicationService;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 取消上传
|
||||||
|
*
|
||||||
|
* @param reqId 预上传请求ID
|
||||||
|
*/
|
||||||
|
@PutMapping("/cancel-upload/{reqId}")
|
||||||
|
public ResponseEntity<Void> cancelUpload(@PathVariable("reqId") String reqId) {
|
||||||
|
datasetFileApplicationService.cancelUpload(reqId);
|
||||||
|
return ResponseEntity.ok().build();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.rest;
|
||||||
|
|
||||||
|
import com.datamate.datamanagement.application.KnowledgeDirectoryApplicationService;
|
||||||
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItemDirectory;
|
||||||
|
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeDirectoryRequest;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeDirectoryQuery;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeDirectoryResponse;
|
||||||
|
import jakarta.validation.Valid;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目目录 REST 控制器
|
||||||
|
*/
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@RequestMapping("/data-management/knowledge-sets/{setId}/directories")
|
||||||
|
public class KnowledgeDirectoryController {
|
||||||
|
private final KnowledgeDirectoryApplicationService knowledgeDirectoryApplicationService;
|
||||||
|
|
||||||
|
@GetMapping
|
||||||
|
public List<KnowledgeDirectoryResponse> getKnowledgeDirectories(@PathVariable("setId") String setId,
|
||||||
|
KnowledgeDirectoryQuery query) {
|
||||||
|
List<KnowledgeItemDirectory> directories = knowledgeDirectoryApplicationService.getKnowledgeDirectories(setId, query);
|
||||||
|
return KnowledgeConverter.INSTANCE.convertDirectoryResponses(directories);
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping
|
||||||
|
public KnowledgeDirectoryResponse createKnowledgeDirectory(@PathVariable("setId") String setId,
|
||||||
|
@RequestBody @Valid CreateKnowledgeDirectoryRequest request) {
|
||||||
|
KnowledgeItemDirectory directory = knowledgeDirectoryApplicationService.createKnowledgeDirectory(setId, request);
|
||||||
|
return KnowledgeConverter.INSTANCE.convertToResponse(directory);
|
||||||
|
}
|
||||||
|
|
||||||
|
@DeleteMapping
|
||||||
|
public void deleteKnowledgeDirectory(@PathVariable("setId") String setId,
|
||||||
|
@RequestParam("relativePath") String relativePath) {
|
||||||
|
knowledgeDirectoryApplicationService.deleteKnowledgeDirectory(setId, relativePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,11 +3,14 @@ package com.datamate.datamanagement.interfaces.rest;
|
|||||||
import com.datamate.common.infrastructure.common.IgnoreResponseWrap;
|
import com.datamate.common.infrastructure.common.IgnoreResponseWrap;
|
||||||
import com.datamate.common.interfaces.PagedResponse;
|
import com.datamate.common.interfaces.PagedResponse;
|
||||||
import com.datamate.datamanagement.application.KnowledgeItemApplicationService;
|
import com.datamate.datamanagement.application.KnowledgeItemApplicationService;
|
||||||
|
import com.datamate.datamanagement.application.KnowledgeItemPreviewService;
|
||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeItem;
|
||||||
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
|
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
|
||||||
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeItemRequest;
|
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeItemRequest;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.DeleteKnowledgeItemsRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.ImportKnowledgeItemsRequest;
|
import com.datamate.datamanagement.interfaces.dto.ImportKnowledgeItemsRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPagingQuery;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemPreviewStatusResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemResponse;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.ReplaceKnowledgeItemFileRequest;
|
import com.datamate.datamanagement.interfaces.dto.ReplaceKnowledgeItemFileRequest;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeItemRequest;
|
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeItemRequest;
|
||||||
@@ -30,6 +33,7 @@ import java.util.List;
|
|||||||
@RequestMapping("/data-management/knowledge-sets/{setId}/items")
|
@RequestMapping("/data-management/knowledge-sets/{setId}/items")
|
||||||
public class KnowledgeItemController {
|
public class KnowledgeItemController {
|
||||||
private final KnowledgeItemApplicationService knowledgeItemApplicationService;
|
private final KnowledgeItemApplicationService knowledgeItemApplicationService;
|
||||||
|
private final KnowledgeItemPreviewService knowledgeItemPreviewService;
|
||||||
|
|
||||||
@GetMapping
|
@GetMapping
|
||||||
public PagedResponse<KnowledgeItemResponse> getKnowledgeItems(@PathVariable("setId") String setId,
|
public PagedResponse<KnowledgeItemResponse> getKnowledgeItems(@PathVariable("setId") String setId,
|
||||||
@@ -80,6 +84,18 @@ public class KnowledgeItemController {
|
|||||||
knowledgeItemApplicationService.previewKnowledgeItemFile(setId, itemId, response);
|
knowledgeItemApplicationService.previewKnowledgeItemFile(setId, itemId, response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping("/{itemId}/preview/status")
|
||||||
|
public KnowledgeItemPreviewStatusResponse getKnowledgeItemPreviewStatus(@PathVariable("setId") String setId,
|
||||||
|
@PathVariable("itemId") String itemId) {
|
||||||
|
return knowledgeItemPreviewService.getPreviewStatus(setId, itemId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/{itemId}/preview/convert")
|
||||||
|
public KnowledgeItemPreviewStatusResponse convertKnowledgeItemPreview(@PathVariable("setId") String setId,
|
||||||
|
@PathVariable("itemId") String itemId) {
|
||||||
|
return knowledgeItemPreviewService.ensurePreview(setId, itemId);
|
||||||
|
}
|
||||||
|
|
||||||
@GetMapping("/{itemId}")
|
@GetMapping("/{itemId}")
|
||||||
public KnowledgeItemResponse getKnowledgeItemById(@PathVariable("setId") String setId,
|
public KnowledgeItemResponse getKnowledgeItemById(@PathVariable("setId") String setId,
|
||||||
@PathVariable("itemId") String itemId) {
|
@PathVariable("itemId") String itemId) {
|
||||||
@@ -108,4 +124,10 @@ public class KnowledgeItemController {
|
|||||||
@PathVariable("itemId") String itemId) {
|
@PathVariable("itemId") String itemId) {
|
||||||
knowledgeItemApplicationService.deleteKnowledgeItem(setId, itemId);
|
knowledgeItemApplicationService.deleteKnowledgeItem(setId, itemId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("/batch-delete")
|
||||||
|
public void deleteKnowledgeItems(@PathVariable("setId") String setId,
|
||||||
|
@RequestBody @Valid DeleteKnowledgeItemsRequest request) {
|
||||||
|
knowledgeItemApplicationService.deleteKnowledgeItems(setId, request);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package com.datamate.datamanagement.interfaces.rest;
|
||||||
|
|
||||||
|
import com.datamate.common.interfaces.PagedResponse;
|
||||||
|
import com.datamate.datamanagement.application.KnowledgeItemApplicationService;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchQuery;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeItemSearchResponse;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识条目搜索控制器
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@RequestMapping("/data-management/knowledge-items")
|
||||||
|
public class KnowledgeItemSearchController {
|
||||||
|
private final KnowledgeItemApplicationService knowledgeItemApplicationService;
|
||||||
|
|
||||||
|
@GetMapping("/search")
|
||||||
|
public PagedResponse<KnowledgeItemSearchResponse> search(KnowledgeItemSearchQuery query) {
|
||||||
|
return knowledgeItemApplicationService.searchKnowledgeItems(query);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,10 +1,12 @@
|
|||||||
package com.datamate.datamanagement.interfaces.rest;
|
package com.datamate.datamanagement.interfaces.rest;
|
||||||
|
|
||||||
import com.datamate.common.interfaces.PagedResponse;
|
import com.datamate.common.interfaces.PagedResponse;
|
||||||
|
import com.datamate.datamanagement.application.KnowledgeItemApplicationService;
|
||||||
import com.datamate.datamanagement.application.KnowledgeSetApplicationService;
|
import com.datamate.datamanagement.application.KnowledgeSetApplicationService;
|
||||||
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
|
import com.datamate.datamanagement.domain.model.knowledge.KnowledgeSet;
|
||||||
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
|
import com.datamate.datamanagement.interfaces.converter.KnowledgeConverter;
|
||||||
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeSetRequest;
|
import com.datamate.datamanagement.interfaces.dto.CreateKnowledgeSetRequest;
|
||||||
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeManagementStatisticsResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetPagingQuery;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetPagingQuery;
|
||||||
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetResponse;
|
import com.datamate.datamanagement.interfaces.dto.KnowledgeSetResponse;
|
||||||
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeSetRequest;
|
import com.datamate.datamanagement.interfaces.dto.UpdateKnowledgeSetRequest;
|
||||||
@@ -22,6 +24,7 @@ import org.springframework.web.bind.annotation.*;
|
|||||||
@RequestMapping("/data-management/knowledge-sets")
|
@RequestMapping("/data-management/knowledge-sets")
|
||||||
public class KnowledgeSetController {
|
public class KnowledgeSetController {
|
||||||
private final KnowledgeSetApplicationService knowledgeSetApplicationService;
|
private final KnowledgeSetApplicationService knowledgeSetApplicationService;
|
||||||
|
private final KnowledgeItemApplicationService knowledgeItemApplicationService;
|
||||||
|
|
||||||
@GetMapping
|
@GetMapping
|
||||||
public PagedResponse<KnowledgeSetResponse> getKnowledgeSets(KnowledgeSetPagingQuery query) {
|
public PagedResponse<KnowledgeSetResponse> getKnowledgeSets(KnowledgeSetPagingQuery query) {
|
||||||
@@ -51,4 +54,9 @@ public class KnowledgeSetController {
|
|||||||
public void deleteKnowledgeSet(@PathVariable("setId") String setId) {
|
public void deleteKnowledgeSet(@PathVariable("setId") String setId) {
|
||||||
knowledgeSetApplicationService.deleteKnowledgeSet(setId);
|
knowledgeSetApplicationService.deleteKnowledgeSet(setId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping("/statistics")
|
||||||
|
public KnowledgeManagementStatisticsResponse getKnowledgeManagementStatistics() {
|
||||||
|
return knowledgeItemApplicationService.getKnowledgeManagementStatistics();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,6 +42,13 @@
|
|||||||
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
|
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId}
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
|
<select id="countNonDerivedByDatasetId" parameterType="string" resultType="long">
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM t_dm_dataset_files
|
||||||
|
WHERE dataset_id = #{datasetId}
|
||||||
|
AND (metadata IS NULL OR JSON_EXTRACT(metadata, '$.derived_from_file_id') IS NULL)
|
||||||
|
</select>
|
||||||
|
|
||||||
<select id="countCompletedByDatasetId" parameterType="string" resultType="long">
|
<select id="countCompletedByDatasetId" parameterType="string" resultType="long">
|
||||||
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId} AND status = 'COMPLETED'
|
SELECT COUNT(*) FROM t_dm_dataset_files WHERE dataset_id = #{datasetId} AND status = 'COMPLETED'
|
||||||
</select>
|
</select>
|
||||||
@@ -110,4 +117,16 @@
|
|||||||
AND metadata IS NOT NULL
|
AND metadata IS NOT NULL
|
||||||
AND JSON_EXTRACT(metadata, '$.derived_from_file_id') IS NOT NULL
|
AND JSON_EXTRACT(metadata, '$.derived_from_file_id') IS NOT NULL
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
|
<select id="countNonDerivedByDatasetIds" resultType="com.datamate.datamanagement.infrastructure.persistence.repository.dto.DatasetFileCount">
|
||||||
|
SELECT dataset_id AS datasetId,
|
||||||
|
COUNT(*) AS fileCount
|
||||||
|
FROM t_dm_dataset_files
|
||||||
|
WHERE dataset_id IN
|
||||||
|
<foreach collection="datasetIds" item="datasetId" open="(" separator="," close=")">
|
||||||
|
#{datasetId}
|
||||||
|
</foreach>
|
||||||
|
AND (metadata IS NULL OR JSON_EXTRACT(metadata, '$.derived_from_file_id') IS NULL)
|
||||||
|
GROUP BY dataset_id
|
||||||
|
</select>
|
||||||
</mapper>
|
</mapper>
|
||||||
|
|||||||
@@ -145,9 +145,10 @@
|
|||||||
|
|
||||||
<select id="getAllDatasetStatistics" resultType="com.datamate.datamanagement.interfaces.dto.AllDatasetStatisticsResponse">
|
<select id="getAllDatasetStatistics" resultType="com.datamate.datamanagement.interfaces.dto.AllDatasetStatisticsResponse">
|
||||||
SELECT
|
SELECT
|
||||||
COUNT(*) AS total_datasets,
|
(SELECT COUNT(*) FROM t_dm_datasets) AS total_datasets,
|
||||||
SUM(size_bytes) AS total_size,
|
(SELECT COALESCE(SUM(size_bytes), 0) FROM t_dm_datasets) AS total_size,
|
||||||
SUM(file_count) AS total_files
|
(SELECT COUNT(*)
|
||||||
FROM t_dm_datasets;
|
FROM t_dm_dataset_files
|
||||||
|
WHERE metadata IS NULL OR JSON_EXTRACT(metadata, '$.derived_from_file_id') IS NULL) AS total_files
|
||||||
</select>
|
</select>
|
||||||
</mapper>
|
</mapper>
|
||||||
|
|||||||
@@ -53,6 +53,19 @@
|
|||||||
ORDER BY usage_count DESC, name ASC
|
ORDER BY usage_count DESC, name ASC
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
|
<select id="countKnowledgeSetTags" resultType="long">
|
||||||
|
SELECT COUNT(DISTINCT t.id)
|
||||||
|
FROM t_dm_tags t
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM t_dm_knowledge_sets ks
|
||||||
|
WHERE ks.tags IS NOT NULL
|
||||||
|
AND JSON_VALID(ks.tags) = 1
|
||||||
|
AND JSON_LENGTH(ks.tags) > 0
|
||||||
|
AND JSON_SEARCH(ks.tags, 'one', t.name, NULL, '$[*].name') IS NOT NULL
|
||||||
|
)
|
||||||
|
</select>
|
||||||
|
|
||||||
<insert id="insert" parameterType="com.datamate.datamanagement.domain.model.dataset.Tag">
|
<insert id="insert" parameterType="com.datamate.datamanagement.domain.model.dataset.Tag">
|
||||||
INSERT INTO t_dm_tags (id, name, description, category, color, usage_count)
|
INSERT INTO t_dm_tags (id, name, description, category, color, usage_count)
|
||||||
VALUES (#{id}, #{name}, #{description}, #{category}, #{color}, #{usageCount})
|
VALUES (#{id}, #{name}, #{description}, #{category}, #{color}, #{usageCount})
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ public class SecurityConfig {
|
|||||||
@Bean
|
@Bean
|
||||||
public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
|
public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
|
||||||
http.csrf(csrf -> csrf.disable())
|
http.csrf(csrf -> csrf.disable())
|
||||||
|
.headers(headers -> headers.frameOptions(frameOptions -> frameOptions.disable()))
|
||||||
.authorizeHttpRequests(authz -> authz
|
.authorizeHttpRequests(authz -> authz
|
||||||
.anyRequest().permitAll() // 允许所有请求无需认证
|
.anyRequest().permitAll() // 允许所有请求无需认证
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -36,7 +36,9 @@ import org.springframework.util.StringUtils;
|
|||||||
|
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 知识库服务类
|
* 知识库服务类
|
||||||
@@ -47,6 +49,7 @@ import java.util.Optional;
|
|||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class KnowledgeBaseService {
|
public class KnowledgeBaseService {
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
private final KnowledgeBaseRepository knowledgeBaseRepository;
|
private final KnowledgeBaseRepository knowledgeBaseRepository;
|
||||||
private final RagFileRepository ragFileRepository;
|
private final RagFileRepository ragFileRepository;
|
||||||
private final ApplicationEventPublisher eventPublisher;
|
private final ApplicationEventPublisher eventPublisher;
|
||||||
@@ -137,6 +140,7 @@ public class KnowledgeBaseService {
|
|||||||
return PagedResponse.of(respList, page.getCurrent(), page.getTotal(), page.getPages());
|
return PagedResponse.of(respList, page.getCurrent(), page.getTotal(), page.getPages());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Transactional(rollbackFor = Exception.class)
|
@Transactional(rollbackFor = Exception.class)
|
||||||
public void addFiles(AddFilesReq request) {
|
public void addFiles(AddFilesReq request) {
|
||||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(request.getKnowledgeBaseId()))
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(request.getKnowledgeBaseId()))
|
||||||
@@ -146,6 +150,7 @@ public class KnowledgeBaseService {
|
|||||||
ragFile.setKnowledgeBaseId(knowledgeBase.getId());
|
ragFile.setKnowledgeBaseId(knowledgeBase.getId());
|
||||||
ragFile.setFileId(fileInfo.id());
|
ragFile.setFileId(fileInfo.id());
|
||||||
ragFile.setFileName(fileInfo.fileName());
|
ragFile.setFileName(fileInfo.fileName());
|
||||||
|
ragFile.setRelativePath(normalizeRelativePath(fileInfo.relativePath()));
|
||||||
ragFile.setStatus(FileStatus.UNPROCESSED);
|
ragFile.setStatus(FileStatus.UNPROCESSED);
|
||||||
return ragFile;
|
return ragFile;
|
||||||
}).toList();
|
}).toList();
|
||||||
@@ -153,6 +158,17 @@ public class KnowledgeBaseService {
|
|||||||
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request));
|
eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase, request));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePath(String relativePath) {
|
||||||
|
if (!StringUtils.hasText(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {
|
public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {
|
||||||
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
||||||
request.setKnowledgeBaseId(knowledgeBaseId);
|
request.setKnowledgeBaseId(knowledgeBaseId);
|
||||||
@@ -160,6 +176,41 @@ public class KnowledgeBaseService {
|
|||||||
return PagedResponse.of(page.getRecords(), page.getCurrent(), page.getTotal(), page.getPages());
|
return PagedResponse.of(page.getRecords(), page.getCurrent(), page.getTotal(), page.getPages());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public PagedResponse<KnowledgeBaseFileSearchResp> searchFiles(KnowledgeBaseFileSearchReq request) {
|
||||||
|
IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
|
||||||
|
page = ragFileRepository.searchPage(page, request);
|
||||||
|
List<RagFile> records = page.getRecords();
|
||||||
|
if (records.isEmpty()) {
|
||||||
|
return PagedResponse.of(Collections.emptyList(), page.getCurrent(), page.getTotal(), page.getPages());
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> knowledgeBaseIds = records.stream()
|
||||||
|
.map(RagFile::getKnowledgeBaseId)
|
||||||
|
.filter(StringUtils::hasText)
|
||||||
|
.distinct()
|
||||||
|
.toList();
|
||||||
|
Map<String, String> knowledgeBaseNameMap = knowledgeBaseRepository.listByIds(knowledgeBaseIds).stream()
|
||||||
|
.collect(Collectors.toMap(KnowledgeBase::getId, KnowledgeBase::getName));
|
||||||
|
|
||||||
|
List<KnowledgeBaseFileSearchResp> responses = records.stream()
|
||||||
|
.map(file -> {
|
||||||
|
KnowledgeBaseFileSearchResp resp = new KnowledgeBaseFileSearchResp();
|
||||||
|
resp.setId(file.getId());
|
||||||
|
resp.setKnowledgeBaseId(file.getKnowledgeBaseId());
|
||||||
|
resp.setKnowledgeBaseName(knowledgeBaseNameMap.getOrDefault(file.getKnowledgeBaseId(), ""));
|
||||||
|
resp.setFileName(file.getFileName());
|
||||||
|
resp.setRelativePath(file.getRelativePath());
|
||||||
|
resp.setChunkCount(file.getChunkCount());
|
||||||
|
resp.setStatus(file.getStatus());
|
||||||
|
resp.setCreatedAt(file.getCreatedAt());
|
||||||
|
resp.setUpdatedAt(file.getUpdatedAt());
|
||||||
|
return resp;
|
||||||
|
})
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
return PagedResponse.of(responses, page.getCurrent(), page.getTotal(), page.getPages());
|
||||||
|
}
|
||||||
|
|
||||||
@Transactional(rollbackFor = Exception.class)
|
@Transactional(rollbackFor = Exception.class)
|
||||||
public void deleteFiles(String knowledgeBaseId, DeleteFilesReq request) {
|
public void deleteFiles(String knowledgeBaseId, DeleteFilesReq request) {
|
||||||
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
|
||||||
@@ -222,4 +273,4 @@ public class KnowledgeBaseService {
|
|||||||
});
|
});
|
||||||
return searchResults;
|
return searchResults;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,10 @@ public class RagFile extends BaseEntity<String> {
|
|||||||
* 文件名
|
* 文件名
|
||||||
*/
|
*/
|
||||||
private String fileName;
|
private String fileName;
|
||||||
|
/**
|
||||||
|
* 相对路径
|
||||||
|
*/
|
||||||
|
private String relativePath;
|
||||||
/**
|
/**
|
||||||
* 文件ID
|
* 文件ID
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package com.datamate.rag.indexer.domain.repository;
|
|||||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||||
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
import com.baomidou.mybatisplus.extension.repository.IRepository;
|
||||||
import com.datamate.rag.indexer.domain.model.RagFile;
|
import com.datamate.rag.indexer.domain.model.RagFile;
|
||||||
|
import com.datamate.rag.indexer.interfaces.dto.KnowledgeBaseFileSearchReq;
|
||||||
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
|
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -21,4 +22,6 @@ public interface RagFileRepository extends IRepository<RagFile> {
|
|||||||
List<RagFile> findAllByKnowledgeBaseId(String knowledgeBaseId);
|
List<RagFile> findAllByKnowledgeBaseId(String knowledgeBaseId);
|
||||||
|
|
||||||
IPage<RagFile> page(IPage<RagFile> page, RagFileReq request);
|
IPage<RagFile> page(IPage<RagFile> page, RagFileReq request);
|
||||||
|
|
||||||
|
IPage<RagFile> searchPage(IPage<RagFile> page, KnowledgeBaseFileSearchReq request);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.datamate.rag.indexer.domain.model.FileStatus;
|
|||||||
import com.datamate.rag.indexer.domain.model.RagFile;
|
import com.datamate.rag.indexer.domain.model.RagFile;
|
||||||
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
|
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
|
||||||
import com.datamate.rag.indexer.infrastructure.persistence.mapper.RagFileMapper;
|
import com.datamate.rag.indexer.infrastructure.persistence.mapper.RagFileMapper;
|
||||||
|
import com.datamate.rag.indexer.interfaces.dto.KnowledgeBaseFileSearchReq;
|
||||||
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
|
import com.datamate.rag.indexer.interfaces.dto.RagFileReq;
|
||||||
import org.springframework.stereotype.Repository;
|
import org.springframework.stereotype.Repository;
|
||||||
import org.springframework.util.StringUtils;
|
import org.springframework.util.StringUtils;
|
||||||
@@ -20,6 +21,7 @@ import java.util.List;
|
|||||||
*/
|
*/
|
||||||
@Repository
|
@Repository
|
||||||
public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile> implements RagFileRepository {
|
public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile> implements RagFileRepository {
|
||||||
|
private static final String PATH_SEPARATOR = "/";
|
||||||
@Override
|
@Override
|
||||||
public void removeByKnowledgeBaseId(String knowledgeBaseId) {
|
public void removeByKnowledgeBaseId(String knowledgeBaseId) {
|
||||||
lambdaUpdate().eq(RagFile::getKnowledgeBaseId, knowledgeBaseId).remove();
|
lambdaUpdate().eq(RagFile::getKnowledgeBaseId, knowledgeBaseId).remove();
|
||||||
@@ -45,6 +47,27 @@ public class RagFileRepositoryImpl extends CrudRepository<RagFileMapper, RagFile
|
|||||||
return lambdaQuery()
|
return lambdaQuery()
|
||||||
.eq(RagFile::getKnowledgeBaseId, request.getKnowledgeBaseId())
|
.eq(RagFile::getKnowledgeBaseId, request.getKnowledgeBaseId())
|
||||||
.like(StringUtils.hasText(request.getFileName()), RagFile::getFileName, request.getFileName())
|
.like(StringUtils.hasText(request.getFileName()), RagFile::getFileName, request.getFileName())
|
||||||
|
.likeRight(StringUtils.hasText(request.getRelativePath()), RagFile::getRelativePath, normalizeRelativePath(request.getRelativePath()))
|
||||||
.page(page);
|
.page(page);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IPage<RagFile> searchPage(IPage<RagFile> page, KnowledgeBaseFileSearchReq request) {
|
||||||
|
return lambdaQuery()
|
||||||
|
.eq(StringUtils.hasText(request.getKnowledgeBaseId()), RagFile::getKnowledgeBaseId, request.getKnowledgeBaseId())
|
||||||
|
.like(StringUtils.hasText(request.getFileName()), RagFile::getFileName, request.getFileName())
|
||||||
|
.likeRight(StringUtils.hasText(request.getRelativePath()), RagFile::getRelativePath, normalizeRelativePath(request.getRelativePath()))
|
||||||
|
.page(page);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeRelativePath(String relativePath) {
|
||||||
|
if (!StringUtils.hasText(relativePath)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String normalized = relativePath.replace("\\", PATH_SEPARATOR).trim();
|
||||||
|
while (normalized.startsWith(PATH_SEPARATOR)) {
|
||||||
|
normalized = normalized.substring(1);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ public class KnowledgeBaseController {
|
|||||||
return knowledgeBaseService.list(request);
|
return knowledgeBaseService.list(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 添加文件到知识库
|
* 添加文件到知识库
|
||||||
*
|
*
|
||||||
@@ -105,6 +106,17 @@ public class KnowledgeBaseController {
|
|||||||
return knowledgeBaseService.listFiles(knowledgeBaseId, request);
|
return knowledgeBaseService.listFiles(knowledgeBaseId, request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 全库检索知识库文件(跨知识库)
|
||||||
|
*
|
||||||
|
* @param request 检索请求
|
||||||
|
* @return 文件列表
|
||||||
|
*/
|
||||||
|
@GetMapping("/files/search")
|
||||||
|
public PagedResponse<KnowledgeBaseFileSearchResp> searchFiles(KnowledgeBaseFileSearchReq request) {
|
||||||
|
return knowledgeBaseService.searchFiles(request);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除知识库文件
|
* 删除知识库文件
|
||||||
*
|
*
|
||||||
@@ -141,4 +153,4 @@ public class KnowledgeBaseController {
|
|||||||
public List<SearchResp.SearchResult> retrieve(@RequestBody @Valid RetrieveReq request) {
|
public List<SearchResp.SearchResult> retrieve(@RequestBody @Valid RetrieveReq request) {
|
||||||
return knowledgeBaseService.retrieve(request);
|
return knowledgeBaseService.retrieve(request);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,6 +21,6 @@ public class AddFilesReq {
|
|||||||
private String delimiter;
|
private String delimiter;
|
||||||
private List<FileInfo> files;
|
private List<FileInfo> files;
|
||||||
|
|
||||||
public record FileInfo(String id, String fileName) {
|
public record FileInfo(String id, String fileName, String relativePath) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package com.datamate.rag.indexer.interfaces.dto;
|
||||||
|
|
||||||
|
import com.datamate.common.interfaces.PagingQuery;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识库文件全库检索请求
|
||||||
|
*
|
||||||
|
* @author dallas
|
||||||
|
* @since 2026-01-30
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeBaseFileSearchReq extends PagingQuery {
|
||||||
|
private String fileName;
|
||||||
|
private String relativePath;
|
||||||
|
private String knowledgeBaseId;
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package com.datamate.rag.indexer.interfaces.dto;
|
||||||
|
|
||||||
|
import com.datamate.rag.indexer.domain.model.FileStatus;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 知识库文件全库检索响应
|
||||||
|
*
|
||||||
|
* @author dallas
|
||||||
|
* @since 2026-01-30
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class KnowledgeBaseFileSearchResp {
|
||||||
|
private String id;
|
||||||
|
private String knowledgeBaseId;
|
||||||
|
private String knowledgeBaseName;
|
||||||
|
private String fileName;
|
||||||
|
private String relativePath;
|
||||||
|
private Integer chunkCount;
|
||||||
|
private FileStatus status;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
@@ -14,5 +14,6 @@ import lombok.Setter;
|
|||||||
@Getter
|
@Getter
|
||||||
public class RagFileReq extends PagingQuery {
|
public class RagFileReq extends PagingQuery {
|
||||||
private String fileName;
|
private String fileName;
|
||||||
|
private String relativePath;
|
||||||
private String knowledgeBaseId;
|
private String knowledgeBaseId;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import java.util.UUID;
|
|||||||
*/
|
*/
|
||||||
@Component
|
@Component
|
||||||
public class FileService {
|
public class FileService {
|
||||||
private static final int DEFAULT_TIMEOUT = 120;
|
private static final int DEFAULT_TIMEOUT = 1800;
|
||||||
|
|
||||||
private final ChunkUploadRequestMapper chunkUploadRequestMapper;
|
private final ChunkUploadRequestMapper chunkUploadRequestMapper;
|
||||||
|
|
||||||
@@ -74,6 +74,26 @@ public class FileService {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 取消上传
|
||||||
|
*/
|
||||||
|
@Transactional
|
||||||
|
public void cancelUpload(String reqId) {
|
||||||
|
if (reqId == null || reqId.isBlank()) {
|
||||||
|
throw BusinessException.of(CommonErrorCode.PARAM_ERROR);
|
||||||
|
}
|
||||||
|
ChunkUploadPreRequest preRequest = chunkUploadRequestMapper.findById(reqId);
|
||||||
|
if (preRequest == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String uploadPath = preRequest.getUploadPath();
|
||||||
|
if (uploadPath != null && !uploadPath.isBlank()) {
|
||||||
|
File tempDir = new File(uploadPath, String.format(ChunksSaver.TEMP_DIR_NAME_FORMAT, preRequest.getId()));
|
||||||
|
ChunksSaver.deleteFolder(tempDir.getPath());
|
||||||
|
}
|
||||||
|
chunkUploadRequestMapper.deleteById(reqId);
|
||||||
|
}
|
||||||
|
|
||||||
private File uploadFile(ChunkUploadRequest fileUploadRequest, ChunkUploadPreRequest preRequest) {
|
private File uploadFile(ChunkUploadRequest fileUploadRequest, ChunkUploadPreRequest preRequest) {
|
||||||
File savedFile = ChunksSaver.saveFile(fileUploadRequest, preRequest);
|
File savedFile = ChunksSaver.saveFile(fileUploadRequest, preRequest);
|
||||||
preRequest.setTimeout(LocalDateTime.now().plusSeconds(DEFAULT_TIMEOUT));
|
preRequest.setTimeout(LocalDateTime.now().plusSeconds(DEFAULT_TIMEOUT));
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ server {
|
|||||||
access_log /var/log/datamate/frontend/access.log main;
|
access_log /var/log/datamate/frontend/access.log main;
|
||||||
error_log /var/log/datamate/frontend/error.log notice;
|
error_log /var/log/datamate/frontend/error.log notice;
|
||||||
|
|
||||||
client_max_body_size 1024M;
|
client_max_body_size 0;
|
||||||
|
|
||||||
add_header Set-Cookie "NEXT_LOCALE=zh";
|
add_header Set-Cookie "NEXT_LOCALE=zh";
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ services:
|
|||||||
- log_volume:/var/log/datamate
|
- log_volume:/var/log/datamate
|
||||||
- operator-upload-volume:/operators/upload
|
- operator-upload-volume:/operators/upload
|
||||||
- operator-runtime-volume:/operators/extract
|
- operator-runtime-volume:/operators/extract
|
||||||
|
- uploads_volume:/uploads
|
||||||
networks: [ datamate ]
|
networks: [ datamate ]
|
||||||
depends_on:
|
depends_on:
|
||||||
- datamate-database
|
- datamate-database
|
||||||
@@ -154,6 +155,8 @@ services:
|
|||||||
profiles: [ data-juicer ]
|
profiles: [ data-juicer ]
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
|
uploads_volume:
|
||||||
|
name: datamate-uploads-volume
|
||||||
dataset_volume:
|
dataset_volume:
|
||||||
name: datamate-dataset-volume
|
name: datamate-dataset-volume
|
||||||
flow_volume:
|
flow_volume:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
services:
|
services:
|
||||||
etcd:
|
etcd:
|
||||||
container_name: milvus-etcd
|
container_name: milvus-etcd
|
||||||
image: quay.io/coreos/etcd:v3.5.18
|
image: quay.nju.edu.cn/coreos/etcd:v3.5.18
|
||||||
environment:
|
environment:
|
||||||
- ETCD_AUTO_COMPACTION_MODE=revision
|
- ETCD_AUTO_COMPACTION_MODE=revision
|
||||||
- ETCD_AUTO_COMPACTION_RETENTION=1000
|
- ETCD_AUTO_COMPACTION_RETENTION=1000
|
||||||
|
|||||||
@@ -169,6 +169,33 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAnnotationObject(value) {
|
||||||
|
if (!value || typeof value !== "object") return false;
|
||||||
|
return typeof value.serializeAnnotation === "function" || typeof value.serialize === "function";
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveSelectedAnnotation(store) {
|
||||||
|
if (!store) return null;
|
||||||
|
const annotations = Array.isArray(store.annotations) ? store.annotations : [];
|
||||||
|
if (isAnnotationObject(store.selectedAnnotation)) {
|
||||||
|
return store.selectedAnnotation;
|
||||||
|
}
|
||||||
|
if (isAnnotationObject(store.selected)) {
|
||||||
|
return store.selected;
|
||||||
|
}
|
||||||
|
const selectedId = store.selected;
|
||||||
|
if (selectedId !== undefined && selectedId !== null && annotations.length) {
|
||||||
|
const matched = annotations.find((ann) => ann && String(ann.id) === String(selectedId));
|
||||||
|
if (isAnnotationObject(matched)) {
|
||||||
|
return matched;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (annotations.length && isAnnotationObject(annotations[0])) {
|
||||||
|
return annotations[0];
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
function exportSelectedAnnotation() {
|
function exportSelectedAnnotation() {
|
||||||
if (!lsInstance) {
|
if (!lsInstance) {
|
||||||
throw new Error("LabelStudio 未初始化");
|
throw new Error("LabelStudio 未初始化");
|
||||||
@@ -179,10 +206,10 @@
|
|||||||
throw new Error("无法访问 annotationStore");
|
throw new Error("无法访问 annotationStore");
|
||||||
}
|
}
|
||||||
|
|
||||||
const selected =
|
const selected = resolveSelectedAnnotation(store);
|
||||||
store.selected ||
|
if (!selected) {
|
||||||
store.selectedAnnotation ||
|
throw new Error("未找到可导出的标注对象");
|
||||||
(Array.isArray(store.annotations) && store.annotations.length ? store.annotations[0] : null);
|
}
|
||||||
|
|
||||||
let serialized = null;
|
let serialized = null;
|
||||||
if (selected && typeof selected.serializeAnnotation === "function") {
|
if (selected && typeof selected.serializeAnnotation === "function") {
|
||||||
@@ -197,6 +224,10 @@
|
|||||||
? { id: selected?.id || serialized.id || "draft", ...serialized }
|
? { id: selected?.id || serialized.id || "draft", ...serialized }
|
||||||
: { id: selected?.id || "draft", result: (selected && selected.result) || [] };
|
: { id: selected?.id || "draft", result: (selected && selected.result) || [] };
|
||||||
|
|
||||||
|
if (!Array.isArray(annotationPayload.result) && Array.isArray(annotationPayload.results)) {
|
||||||
|
annotationPayload.result = annotationPayload.results;
|
||||||
|
}
|
||||||
|
|
||||||
// 最小化对齐 Label Studio Server 的字段(DataMate 侧会原样存储)
|
// 最小化对齐 Label Studio Server 的字段(DataMate 侧会原样存储)
|
||||||
const taskId = typeof currentTask?.id === "number" ? currentTask.id : Number(currentTask?.id) || null;
|
const taskId = typeof currentTask?.id === "number" ? currentTask.id : Number(currentTask?.id) || null;
|
||||||
const fileId = currentTask?.data?.file_id || currentTask?.data?.fileId || null;
|
const fileId = currentTask?.data?.file_id || currentTask?.data?.fileId || null;
|
||||||
@@ -226,6 +257,52 @@
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isSaveAndNextShortcut(event) {
|
||||||
|
if (!event || event.defaultPrevented || event.isComposing) return false;
|
||||||
|
const key = event.key;
|
||||||
|
const code = event.code;
|
||||||
|
const isEnter = key === "Enter" || code === "Enter" || code === "NumpadEnter";
|
||||||
|
if (!isEnter) return false;
|
||||||
|
if (!(event.ctrlKey || event.metaKey)) return false;
|
||||||
|
if (event.shiftKey || event.altKey) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSaveShortcut(event) {
|
||||||
|
if (!event || event.defaultPrevented || event.isComposing) return false;
|
||||||
|
const key = event.key;
|
||||||
|
const code = event.code;
|
||||||
|
const isS = key === "s" || key === "S" || code === "KeyS";
|
||||||
|
if (!isS) return false;
|
||||||
|
if (!(event.ctrlKey || event.metaKey)) return false;
|
||||||
|
if (event.shiftKey || event.altKey) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleSaveAndNextShortcut(event) {
|
||||||
|
if (!isSaveAndNextShortcut(event) || event.repeat) return;
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
try {
|
||||||
|
const raw = exportSelectedAnnotation();
|
||||||
|
postToParent("LS_SAVE_AND_NEXT", raw);
|
||||||
|
} catch (e) {
|
||||||
|
postToParent("LS_ERROR", { message: e?.message || String(e) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleSaveShortcut(event) {
|
||||||
|
if (!isSaveShortcut(event) || event.repeat) return;
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
try {
|
||||||
|
const raw = exportSelectedAnnotation();
|
||||||
|
postToParent("LS_EXPORT_RESULT", raw);
|
||||||
|
} catch (e) {
|
||||||
|
postToParent("LS_ERROR", { message: e?.message || String(e) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function initLabelStudio(payload) {
|
function initLabelStudio(payload) {
|
||||||
if (!window.LabelStudio) {
|
if (!window.LabelStudio) {
|
||||||
throw new Error("LabelStudio 未加载(请检查静态资源/网络)");
|
throw new Error("LabelStudio 未加载(请检查静态资源/网络)");
|
||||||
@@ -296,6 +373,9 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
window.addEventListener("keydown", handleSaveAndNextShortcut);
|
||||||
|
window.addEventListener("keydown", handleSaveShortcut);
|
||||||
|
|
||||||
window.addEventListener("message", (event) => {
|
window.addEventListener("message", (event) => {
|
||||||
if (event.origin !== ORIGIN) return;
|
if (event.origin !== ORIGIN) return;
|
||||||
|
|
||||||
|
|||||||
@@ -1,17 +1,17 @@
|
|||||||
import { Button, Input, Popover, theme, Tag, Empty } from "antd";
|
import { Button, Input, Popover, theme, Tag, Empty } from "antd";
|
||||||
import { PlusOutlined } from "@ant-design/icons";
|
import { PlusOutlined } from "@ant-design/icons";
|
||||||
import { useEffect, useMemo, useState } from "react";
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
|
|
||||||
interface Tag {
|
interface Tag {
|
||||||
id: number;
|
id?: string | number;
|
||||||
name: string;
|
name: string;
|
||||||
color: string;
|
color?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AddTagPopoverProps {
|
interface AddTagPopoverProps {
|
||||||
tags: Tag[];
|
tags: Tag[];
|
||||||
onFetchTags?: () => Promise<Tag[]>;
|
onFetchTags?: () => Promise<Tag[]>;
|
||||||
onAddTag?: (tag: Tag) => void;
|
onAddTag?: (tagName: string) => void;
|
||||||
onCreateAndTag?: (tagName: string) => void;
|
onCreateAndTag?: (tagName: string) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -27,20 +27,23 @@ export default function AddTagPopover({
|
|||||||
const [newTag, setNewTag] = useState("");
|
const [newTag, setNewTag] = useState("");
|
||||||
const [allTags, setAllTags] = useState<Tag[]>([]);
|
const [allTags, setAllTags] = useState<Tag[]>([]);
|
||||||
|
|
||||||
const tagsSet = useMemo(() => new Set(tags.map((tag) => tag.id)), [tags]);
|
const tagsSet = useMemo(
|
||||||
|
() => new Set(tags.map((tag) => (tag.id ?? tag.name))),
|
||||||
|
[tags]
|
||||||
|
);
|
||||||
|
|
||||||
const fetchTags = async () => {
|
const fetchTags = useCallback(async () => {
|
||||||
if (onFetchTags && showPopover) {
|
if (onFetchTags && showPopover) {
|
||||||
const data = await onFetchTags?.();
|
const data = await onFetchTags?.();
|
||||||
setAllTags(data || []);
|
setAllTags(data || []);
|
||||||
}
|
}
|
||||||
};
|
}, [onFetchTags, showPopover]);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
fetchTags();
|
fetchTags();
|
||||||
}, [showPopover]);
|
}, [fetchTags]);
|
||||||
|
|
||||||
const availableTags = useMemo(() => {
|
const availableTags = useMemo(() => {
|
||||||
return allTags.filter((tag) => !tagsSet.has(tag.id));
|
return allTags.filter((tag) => !tagsSet.has(tag.id ?? tag.name));
|
||||||
}, [allTags, tagsSet]);
|
}, [allTags, tagsSet]);
|
||||||
|
|
||||||
const handleCreateAndAddTag = () => {
|
const handleCreateAndAddTag = () => {
|
||||||
|
|||||||
@@ -276,7 +276,7 @@ function CardView<T extends BaseCardDataType>(props: CardViewProps<T>) {
|
|||||||
{formatDateTime(item?.updatedAt)}
|
{formatDateTime(item?.updatedAt)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{operations && (
|
{operations && ops(item).length > 0 && (
|
||||||
<ActionDropdown
|
<ActionDropdown
|
||||||
actions={ops(item)}
|
actions={ops(item)}
|
||||||
onAction={(key) => {
|
onAction={(key) => {
|
||||||
|
|||||||
@@ -22,44 +22,51 @@ interface OperationItem {
|
|||||||
danger?: boolean;
|
danger?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface TagConfig {
|
interface TagConfig {
|
||||||
showAdd: boolean;
|
showAdd: boolean;
|
||||||
tags: { id: number; name: string; color: string }[];
|
tags: { id?: string | number; name: string; color?: string }[];
|
||||||
onFetchTags?: () => Promise<{
|
onFetchTags?: () => Promise<{ id?: string | number; name: string; color?: string }[]>;
|
||||||
data: { id: number; name: string; color: string }[];
|
onAddTag?: (tagName: string) => void;
|
||||||
}>;
|
onCreateAndTag?: (tagName: string) => void;
|
||||||
onAddTag?: (tag: { id: number; name: string; color: string }) => void;
|
}
|
||||||
onCreateAndTag?: (tagName: string) => void;
|
interface DetailHeaderData {
|
||||||
}
|
name?: string;
|
||||||
interface DetailHeaderProps<T> {
|
description?: string;
|
||||||
data: T;
|
status?: { color?: string; icon?: React.ReactNode; label?: string };
|
||||||
statistics: StatisticItem[];
|
tags?: { id?: string | number; name?: string }[];
|
||||||
operations: OperationItem[];
|
icon?: React.ReactNode;
|
||||||
tagConfig?: TagConfig;
|
iconColor?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function DetailHeader<T>({
|
interface DetailHeaderProps<T extends DetailHeaderData> {
|
||||||
data = {} as T,
|
data: T;
|
||||||
statistics,
|
statistics: StatisticItem[];
|
||||||
operations,
|
operations: OperationItem[];
|
||||||
tagConfig,
|
tagConfig?: TagConfig;
|
||||||
}: DetailHeaderProps<T>): React.ReactNode {
|
}
|
||||||
|
|
||||||
|
function DetailHeader<T extends DetailHeaderData>({
|
||||||
|
data = {} as T,
|
||||||
|
statistics,
|
||||||
|
operations,
|
||||||
|
tagConfig,
|
||||||
|
}: DetailHeaderProps<T>): React.ReactNode {
|
||||||
return (
|
return (
|
||||||
<Card>
|
<Card>
|
||||||
<div className="flex items-start justify-between">
|
<div className="flex items-start justify-between">
|
||||||
<div className="flex items-start gap-4 flex-1">
|
<div className="flex items-start gap-4 flex-1">
|
||||||
<div
|
<div
|
||||||
className={`w-16 h-16 text-white rounded-lg flex-center shadow-lg ${
|
className={`w-16 h-16 text-white rounded-lg flex-center shadow-lg ${
|
||||||
(data as any)?.iconColor
|
data?.iconColor
|
||||||
? ""
|
? ""
|
||||||
: "bg-gradient-to-br from-sky-300 to-blue-500 text-white"
|
: "bg-gradient-to-br from-sky-300 to-blue-500 text-white"
|
||||||
}`}
|
}`}
|
||||||
style={(data as any)?.iconColor ? { backgroundColor: (data as any).iconColor } : undefined}
|
style={data?.iconColor ? { backgroundColor: data.iconColor } : undefined}
|
||||||
>
|
>
|
||||||
{<div className="w-[2.8rem] h-[2.8rem] text-gray-50">{(data as any)?.icon}</div> || (
|
{<div className="w-[2.8rem] h-[2.8rem] text-gray-50">{data?.icon}</div> || (
|
||||||
<Database className="w-8 h-8 text-white" />
|
<Database className="w-8 h-8 text-white" />
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
<div className="flex-1">
|
<div className="flex-1">
|
||||||
<div className="flex items-center gap-3 mb-2">
|
<div className="flex items-center gap-3 mb-2">
|
||||||
<h1 className="text-lg font-bold text-gray-900">{data?.name}</h1>
|
<h1 className="text-lg font-bold text-gray-900">{data?.name}</h1>
|
||||||
|
|||||||
21
frontend/src/components/ProtectedRoute.tsx
Normal file
21
frontend/src/components/ProtectedRoute.tsx
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
import React from 'react';
|
||||||
|
import { Navigate, useLocation, Outlet } from 'react-router';
|
||||||
|
import { useAppSelector } from '@/store/hooks';
|
||||||
|
|
||||||
|
interface ProtectedRouteProps {
|
||||||
|
children?: React.ReactNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ProtectedRoute: React.FC<ProtectedRouteProps> = ({ children }) => {
|
||||||
|
const { isAuthenticated } = useAppSelector((state) => state.auth);
|
||||||
|
const location = useLocation();
|
||||||
|
|
||||||
|
if (!isAuthenticated) {
|
||||||
|
// Redirect to the login page, but save the current location they were trying to go to
|
||||||
|
return <Navigate to="/login" state={{ from: location }} replace />;
|
||||||
|
}
|
||||||
|
|
||||||
|
return children ? <>{children}</> : <Outlet />;
|
||||||
|
};
|
||||||
|
|
||||||
|
export default ProtectedRoute;
|
||||||
@@ -4,6 +4,7 @@ const TopLoadingBar = () => {
|
|||||||
const [isVisible, setIsVisible] = useState(false);
|
const [isVisible, setIsVisible] = useState(false);
|
||||||
const [progress, setProgress] = useState(0);
|
const [progress, setProgress] = useState(0);
|
||||||
const intervalRef = useRef(null);
|
const intervalRef = useRef(null);
|
||||||
|
const timeoutRef = useRef(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
// 监听全局事件
|
// 监听全局事件
|
||||||
@@ -33,8 +34,13 @@ const TopLoadingBar = () => {
|
|||||||
clearInterval(intervalRef.current);
|
clearInterval(intervalRef.current);
|
||||||
intervalRef.current = null;
|
intervalRef.current = null;
|
||||||
}
|
}
|
||||||
|
// 清除旧的timeout
|
||||||
|
if (timeoutRef.current) {
|
||||||
|
clearTimeout(timeoutRef.current);
|
||||||
|
timeoutRef.current = null;
|
||||||
|
}
|
||||||
setProgress(100);
|
setProgress(100);
|
||||||
setTimeout(() => {
|
timeoutRef.current = setTimeout(() => {
|
||||||
setIsVisible(false);
|
setIsVisible(false);
|
||||||
setProgress(0);
|
setProgress(0);
|
||||||
}, 300);
|
}, 300);
|
||||||
@@ -49,6 +55,9 @@ const TopLoadingBar = () => {
|
|||||||
if (intervalRef.current) {
|
if (intervalRef.current) {
|
||||||
clearInterval(intervalRef.current);
|
clearInterval(intervalRef.current);
|
||||||
}
|
}
|
||||||
|
if (timeoutRef.current) {
|
||||||
|
clearTimeout(timeoutRef.current);
|
||||||
|
}
|
||||||
window.removeEventListener("loading:show", handleShow);
|
window.removeEventListener("loading:show", handleShow);
|
||||||
window.removeEventListener("loading:hide", handleHide);
|
window.removeEventListener("loading:hide", handleHide);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -22,11 +22,10 @@ interface DatasetFileTransferProps
|
|||||||
onDatasetSelect?: (dataset: Dataset | null) => void;
|
onDatasetSelect?: (dataset: Dataset | null) => void;
|
||||||
datasetTypeFilter?: DatasetType;
|
datasetTypeFilter?: DatasetType;
|
||||||
hasAnnotationFilter?: boolean;
|
hasAnnotationFilter?: boolean;
|
||||||
/**
|
/**
|
||||||
* 是否排除已被转换为TXT的源文档文件(PDF/DOC/DOCX)
|
* 是否排除源文档文件(PDF/DOC/DOCX/XLS/XLSX),文本标注默认启用
|
||||||
* 默认为 true,当 datasetTypeFilter 为 TEXT 时自动启用
|
*/
|
||||||
*/
|
excludeSourceDocuments?: boolean;
|
||||||
excludeSourceDocuments?: boolean;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const fileCols = [
|
const fileCols = [
|
||||||
|
|||||||
@@ -1,198 +1,383 @@
|
|||||||
import { TaskItem } from "@/pages/DataManagement/dataset.model";
|
import { TaskItem } from "@/pages/DataManagement/dataset.model";
|
||||||
import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util";
|
import { calculateSHA256, checkIsFilesExist, streamSplitAndUpload, StreamUploadResult } from "@/utils/file.util";
|
||||||
import { App } from "antd";
|
import { App } from "antd";
|
||||||
import { useRef, useState } from "react";
|
import { useRef, useState } from "react";
|
||||||
|
|
||||||
export function useFileSliceUpload(
|
export function useFileSliceUpload(
|
||||||
{
|
{
|
||||||
preUpload,
|
preUpload,
|
||||||
uploadChunk,
|
uploadChunk,
|
||||||
cancelUpload,
|
cancelUpload,
|
||||||
}: {
|
}: {
|
||||||
preUpload: (id: string, params: any) => Promise<{ data: number }>;
|
preUpload: (id: string, params: Record<string, unknown>) => Promise<{ data: number }>;
|
||||||
uploadChunk: (id: string, formData: FormData, config: any) => Promise<any>;
|
uploadChunk: (id: string, formData: FormData, config: Record<string, unknown>) => Promise<unknown>;
|
||||||
cancelUpload: ((reqId: number) => Promise<any>) | null;
|
cancelUpload: ((reqId: number) => Promise<unknown>) | null;
|
||||||
},
|
},
|
||||||
showTaskCenter = true // 上传时是否显示任务中心
|
showTaskCenter = true, // 上传时是否显示任务中心
|
||||||
) {
|
enableStreamUpload = true // 是否启用流式分割上传
|
||||||
const { message } = App.useApp();
|
) {
|
||||||
const [taskList, setTaskList] = useState<TaskItem[]>([]);
|
const { message } = App.useApp();
|
||||||
const taskListRef = useRef<TaskItem[]>([]); // 用于固定任务顺序
|
const [taskList, setTaskList] = useState<TaskItem[]>([]);
|
||||||
|
const taskListRef = useRef<TaskItem[]>([]); // 用于固定任务顺序
|
||||||
const createTask = (detail: any = {}) => {
|
|
||||||
const { dataset } = detail;
|
const createTask = (detail: Record<string, unknown> = {}) => {
|
||||||
const title = `上传数据集: ${dataset.name} `;
|
const { dataset } = detail;
|
||||||
const controller = new AbortController();
|
const title = `上传数据集: ${dataset.name} `;
|
||||||
const task: TaskItem = {
|
const controller = new AbortController();
|
||||||
key: dataset.id,
|
const task: TaskItem = {
|
||||||
title,
|
key: dataset.id,
|
||||||
percent: 0,
|
title,
|
||||||
reqId: -1,
|
percent: 0,
|
||||||
controller,
|
reqId: -1,
|
||||||
size: 0,
|
controller,
|
||||||
updateEvent: detail.updateEvent,
|
size: 0,
|
||||||
hasArchive: detail.hasArchive,
|
updateEvent: detail.updateEvent,
|
||||||
prefix: detail.prefix,
|
hasArchive: detail.hasArchive,
|
||||||
};
|
prefix: detail.prefix,
|
||||||
taskListRef.current = [task, ...taskListRef.current];
|
};
|
||||||
|
taskListRef.current = [task, ...taskListRef.current];
|
||||||
setTaskList(taskListRef.current);
|
|
||||||
return task;
|
setTaskList(taskListRef.current);
|
||||||
};
|
|
||||||
|
// 立即显示任务中心,让用户感知上传已开始
|
||||||
const updateTaskList = (task: TaskItem) => {
|
if (showTaskCenter) {
|
||||||
taskListRef.current = taskListRef.current.map((item) =>
|
window.dispatchEvent(
|
||||||
item.key === task.key ? task : item
|
new CustomEvent("show:task-popover", { detail: { show: true } })
|
||||||
);
|
);
|
||||||
setTaskList(taskListRef.current);
|
}
|
||||||
};
|
|
||||||
|
return task;
|
||||||
const removeTask = (task: TaskItem) => {
|
};
|
||||||
const { key } = task;
|
|
||||||
taskListRef.current = taskListRef.current.filter(
|
const updateTaskList = (task: TaskItem) => {
|
||||||
(item) => item.key !== key
|
taskListRef.current = taskListRef.current.map((item) =>
|
||||||
);
|
item.key === task.key ? task : item
|
||||||
setTaskList(taskListRef.current);
|
);
|
||||||
if (task.isCancel && task.cancelFn) {
|
setTaskList(taskListRef.current);
|
||||||
task.cancelFn();
|
};
|
||||||
}
|
|
||||||
if (task.updateEvent) {
|
const removeTask = (task: TaskItem) => {
|
||||||
// 携带前缀信息,便于刷新后仍停留在当前目录
|
const { key } = task;
|
||||||
window.dispatchEvent(
|
taskListRef.current = taskListRef.current.filter(
|
||||||
new CustomEvent(task.updateEvent, {
|
(item) => item.key !== key
|
||||||
detail: { prefix: (task as any).prefix },
|
);
|
||||||
})
|
setTaskList(taskListRef.current);
|
||||||
);
|
if (task.isCancel && task.cancelFn) {
|
||||||
}
|
task.cancelFn();
|
||||||
if (showTaskCenter) {
|
}
|
||||||
window.dispatchEvent(
|
if (task.updateEvent) {
|
||||||
new CustomEvent("show:task-popover", { detail: { show: false } })
|
// 携带前缀信息,便于刷新后仍停留在当前目录
|
||||||
);
|
window.dispatchEvent(
|
||||||
}
|
new CustomEvent(task.updateEvent, {
|
||||||
};
|
detail: { prefix: task.prefix },
|
||||||
|
})
|
||||||
async function buildFormData({ file, reqId, i, j }) {
|
);
|
||||||
const formData = new FormData();
|
}
|
||||||
const { slices, name, size } = file;
|
if (showTaskCenter) {
|
||||||
const checkSum = await calculateSHA256(slices[j]);
|
window.dispatchEvent(
|
||||||
formData.append("file", slices[j]);
|
new CustomEvent("show:task-popover", { detail: { show: false } })
|
||||||
formData.append("reqId", reqId.toString());
|
);
|
||||||
formData.append("fileNo", (i + 1).toString());
|
}
|
||||||
formData.append("chunkNo", (j + 1).toString());
|
};
|
||||||
formData.append("fileName", name);
|
|
||||||
formData.append("fileSize", size.toString());
|
async function buildFormData({ file, reqId, i, j }: { file: { slices: Blob[]; name: string; size: number }; reqId: number; i: number; j: number }) {
|
||||||
formData.append("totalChunkNum", slices.length.toString());
|
const formData = new FormData();
|
||||||
formData.append("checkSumHex", checkSum);
|
const { slices, name, size } = file;
|
||||||
return formData;
|
const checkSum = await calculateSHA256(slices[j]);
|
||||||
}
|
formData.append("file", slices[j]);
|
||||||
|
formData.append("reqId", reqId.toString());
|
||||||
async function uploadSlice(task: TaskItem, fileInfo) {
|
formData.append("fileNo", (i + 1).toString());
|
||||||
if (!task) {
|
formData.append("chunkNo", (j + 1).toString());
|
||||||
return;
|
formData.append("fileName", name);
|
||||||
}
|
formData.append("fileSize", size.toString());
|
||||||
const { reqId, key } = task;
|
formData.append("totalChunkNum", slices.length.toString());
|
||||||
const { loaded, i, j, files, totalSize } = fileInfo;
|
formData.append("checkSumHex", checkSum);
|
||||||
const formData = await buildFormData({
|
return formData;
|
||||||
file: files[i],
|
}
|
||||||
i,
|
|
||||||
j,
|
async function uploadSlice(task: TaskItem, fileInfo: { loaded: number; i: number; j: number; files: { slices: Blob[]; name: string; size: number }[]; totalSize: number }) {
|
||||||
reqId,
|
if (!task) {
|
||||||
});
|
return;
|
||||||
|
}
|
||||||
let newTask = { ...task };
|
const { reqId, key, controller } = task;
|
||||||
await uploadChunk(key, formData, {
|
const { loaded, i, j, files, totalSize } = fileInfo;
|
||||||
onUploadProgress: (e) => {
|
|
||||||
const loadedSize = loaded + e.loaded;
|
// 检查是否已取消
|
||||||
const curPercent = Number((loadedSize / totalSize) * 100).toFixed(2);
|
if (controller.signal.aborted) {
|
||||||
|
throw new Error("Upload cancelled");
|
||||||
newTask = {
|
}
|
||||||
...newTask,
|
|
||||||
...taskListRef.current.find((item) => item.key === key),
|
const formData = await buildFormData({
|
||||||
size: loadedSize,
|
file: files[i],
|
||||||
percent: curPercent >= 100 ? 99.99 : curPercent,
|
i,
|
||||||
};
|
j,
|
||||||
updateTaskList(newTask);
|
reqId,
|
||||||
},
|
});
|
||||||
});
|
|
||||||
}
|
let newTask = { ...task };
|
||||||
|
await uploadChunk(key, formData, {
|
||||||
async function uploadFile({ task, files, totalSize }) {
|
signal: controller.signal,
|
||||||
console.log('[useSliceUpload] Calling preUpload with prefix:', task.prefix);
|
onUploadProgress: (e) => {
|
||||||
const { data: reqId } = await preUpload(task.key, {
|
const loadedSize = loaded + e.loaded;
|
||||||
totalFileNum: files.length,
|
const curPercent = Number((loadedSize / totalSize) * 100).toFixed(2);
|
||||||
totalSize,
|
|
||||||
datasetId: task.key,
|
newTask = {
|
||||||
hasArchive: task.hasArchive,
|
...newTask,
|
||||||
prefix: task.prefix,
|
...taskListRef.current.find((item) => item.key === key),
|
||||||
});
|
size: loadedSize,
|
||||||
console.log('[useSliceUpload] PreUpload response reqId:', reqId);
|
percent: curPercent >= 100 ? 99.99 : curPercent,
|
||||||
|
};
|
||||||
const newTask: TaskItem = {
|
updateTaskList(newTask);
|
||||||
...task,
|
},
|
||||||
reqId,
|
});
|
||||||
isCancel: false,
|
}
|
||||||
cancelFn: () => {
|
|
||||||
task.controller.abort();
|
async function uploadFile({ task, files, totalSize }: { task: TaskItem; files: { slices: Blob[]; name: string; size: number; originFile: Blob }[]; totalSize: number }) {
|
||||||
cancelUpload?.(reqId);
|
console.log('[useSliceUpload] Calling preUpload with prefix:', task.prefix);
|
||||||
if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
|
const { data: reqId } = await preUpload(task.key, {
|
||||||
},
|
totalFileNum: files.length,
|
||||||
};
|
totalSize,
|
||||||
updateTaskList(newTask);
|
datasetId: task.key,
|
||||||
if (showTaskCenter) {
|
hasArchive: task.hasArchive,
|
||||||
window.dispatchEvent(
|
prefix: task.prefix,
|
||||||
new CustomEvent("show:task-popover", { detail: { show: true } })
|
});
|
||||||
);
|
console.log('[useSliceUpload] PreUpload response reqId:', reqId);
|
||||||
}
|
|
||||||
// // 更新数据状态
|
const newTask: TaskItem = {
|
||||||
if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
|
...task,
|
||||||
|
reqId,
|
||||||
let loaded = 0;
|
isCancel: false,
|
||||||
for (let i = 0; i < files.length; i++) {
|
cancelFn: () => {
|
||||||
const { slices } = files[i];
|
// 使用 newTask 的 controller 确保一致性
|
||||||
for (let j = 0; j < slices.length; j++) {
|
newTask.controller.abort();
|
||||||
await uploadSlice(newTask, {
|
cancelUpload?.(reqId);
|
||||||
loaded,
|
if (newTask.updateEvent) window.dispatchEvent(new Event(newTask.updateEvent));
|
||||||
i,
|
},
|
||||||
j,
|
};
|
||||||
files,
|
updateTaskList(newTask);
|
||||||
totalSize,
|
// 注意:show:task-popover 事件已在 createTask 中触发,此处不再重复触发
|
||||||
});
|
// // 更新数据状态
|
||||||
loaded += slices[j].size;
|
if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
|
||||||
}
|
|
||||||
}
|
let loaded = 0;
|
||||||
removeTask(newTask);
|
for (let i = 0; i < files.length; i++) {
|
||||||
}
|
// 检查是否已取消
|
||||||
|
if (newTask.controller.signal.aborted) {
|
||||||
const handleUpload = async ({ task, files }) => {
|
throw new Error("Upload cancelled");
|
||||||
const isErrorFile = await checkIsFilesExist(files);
|
}
|
||||||
if (isErrorFile) {
|
const { slices } = files[i];
|
||||||
message.error("文件被修改或删除,请重新选择文件上传");
|
for (let j = 0; j < slices.length; j++) {
|
||||||
removeTask({
|
// 检查是否已取消
|
||||||
...task,
|
if (newTask.controller.signal.aborted) {
|
||||||
isCancel: false,
|
throw new Error("Upload cancelled");
|
||||||
...taskListRef.current.find((item) => item.key === task.key),
|
}
|
||||||
});
|
await uploadSlice(newTask, {
|
||||||
return;
|
loaded,
|
||||||
}
|
i,
|
||||||
|
j,
|
||||||
try {
|
files,
|
||||||
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
|
totalSize,
|
||||||
await uploadFile({ task, files, totalSize });
|
});
|
||||||
} catch (err) {
|
loaded += slices[j].size;
|
||||||
console.error(err);
|
}
|
||||||
message.error("文件上传失败,请稍后重试");
|
}
|
||||||
removeTask({
|
removeTask(newTask);
|
||||||
...task,
|
}
|
||||||
isCancel: true,
|
|
||||||
...taskListRef.current.find((item) => item.key === task.key),
|
const handleUpload = async ({ task, files }: { task: TaskItem; files: { slices: Blob[]; name: string; size: number; originFile: Blob }[] }) => {
|
||||||
});
|
const isErrorFile = await checkIsFilesExist(files);
|
||||||
}
|
if (isErrorFile) {
|
||||||
};
|
message.error("文件被修改或删除,请重新选择文件上传");
|
||||||
|
removeTask({
|
||||||
return {
|
...task,
|
||||||
taskList,
|
isCancel: false,
|
||||||
createTask,
|
...taskListRef.current.find((item) => item.key === task.key),
|
||||||
removeTask,
|
});
|
||||||
handleUpload,
|
return;
|
||||||
};
|
}
|
||||||
}
|
|
||||||
|
try {
|
||||||
|
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
|
||||||
|
await uploadFile({ task, files, totalSize });
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
message.error("文件上传失败,请稍后重试");
|
||||||
|
removeTask({
|
||||||
|
...task,
|
||||||
|
isCancel: true,
|
||||||
|
...taskListRef.current.find((item) => item.key === task.key),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 流式分割上传处理
|
||||||
|
* 用于大文件按行分割并立即上传的场景
|
||||||
|
*/
|
||||||
|
const handleStreamUpload = async ({ task, files }: { task: TaskItem; files: File[] }) => {
|
||||||
|
try {
|
||||||
|
console.log('[useSliceUpload] Starting stream upload for', files.length, 'files');
|
||||||
|
|
||||||
|
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
|
||||||
|
|
||||||
|
// 存储所有文件的 reqId,用于取消上传
|
||||||
|
const reqIds: number[] = [];
|
||||||
|
|
||||||
|
const newTask: TaskItem = {
|
||||||
|
...task,
|
||||||
|
reqId: -1,
|
||||||
|
isCancel: false,
|
||||||
|
cancelFn: () => {
|
||||||
|
// 使用 newTask 的 controller 确保一致性
|
||||||
|
newTask.controller.abort();
|
||||||
|
// 取消所有文件的预上传请求
|
||||||
|
reqIds.forEach(id => cancelUpload?.(id));
|
||||||
|
if (newTask.updateEvent) window.dispatchEvent(new Event(newTask.updateEvent));
|
||||||
|
},
|
||||||
|
};
|
||||||
|
updateTaskList(newTask);
|
||||||
|
|
||||||
|
let totalUploadedLines = 0;
|
||||||
|
let totalProcessedBytes = 0;
|
||||||
|
const results: StreamUploadResult[] = [];
|
||||||
|
|
||||||
|
// 逐个处理文件,每个文件单独调用 preUpload
|
||||||
|
for (let i = 0; i < files.length; i++) {
|
||||||
|
// 检查是否已取消
|
||||||
|
if (newTask.controller.signal.aborted) {
|
||||||
|
throw new Error("Upload cancelled");
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = files[i];
|
||||||
|
console.log(`[useSliceUpload] Processing file ${i + 1}/${files.length}: ${file.name}`);
|
||||||
|
|
||||||
|
const result = await streamSplitAndUpload(
|
||||||
|
file,
|
||||||
|
(formData, config) => uploadChunk(task.key, formData, {
|
||||||
|
...config,
|
||||||
|
signal: newTask.controller.signal,
|
||||||
|
}),
|
||||||
|
(currentBytes, totalBytes, uploadedLines) => {
|
||||||
|
// 检查是否已取消
|
||||||
|
if (newTask.controller.signal.aborted) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 更新进度
|
||||||
|
const overallBytes = totalProcessedBytes + currentBytes;
|
||||||
|
const curPercent = Number((overallBytes / totalSize) * 100).toFixed(2);
|
||||||
|
|
||||||
|
const updatedTask: TaskItem = {
|
||||||
|
...newTask,
|
||||||
|
...taskListRef.current.find((item) => item.key === task.key),
|
||||||
|
size: overallBytes,
|
||||||
|
percent: curPercent >= 100 ? 99.99 : curPercent,
|
||||||
|
streamUploadInfo: {
|
||||||
|
currentFile: file.name,
|
||||||
|
fileIndex: i + 1,
|
||||||
|
totalFiles: files.length,
|
||||||
|
uploadedLines: totalUploadedLines + uploadedLines,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
updateTaskList(updatedTask);
|
||||||
|
},
|
||||||
|
1024 * 1024, // 1MB chunk size
|
||||||
|
{
|
||||||
|
resolveReqId: async ({ totalFileNum, totalSize }) => {
|
||||||
|
const { data: reqId } = await preUpload(task.key, {
|
||||||
|
totalFileNum,
|
||||||
|
totalSize,
|
||||||
|
datasetId: task.key,
|
||||||
|
hasArchive: task.hasArchive,
|
||||||
|
prefix: task.prefix,
|
||||||
|
});
|
||||||
|
console.log(`[useSliceUpload] File ${file.name} preUpload response reqId:`, reqId);
|
||||||
|
reqIds.push(reqId);
|
||||||
|
return reqId;
|
||||||
|
},
|
||||||
|
hasArchive: newTask.hasArchive,
|
||||||
|
prefix: newTask.prefix,
|
||||||
|
signal: newTask.controller.signal,
|
||||||
|
maxConcurrency: 3,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
results.push(result);
|
||||||
|
totalUploadedLines += result.uploadedCount;
|
||||||
|
totalProcessedBytes += file.size;
|
||||||
|
|
||||||
|
console.log(`[useSliceUpload] File ${file.name} processed, uploaded ${result.uploadedCount} lines`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('[useSliceUpload] Stream upload completed, total lines:', totalUploadedLines);
|
||||||
|
removeTask(newTask);
|
||||||
|
|
||||||
|
message.success(`成功上传 ${totalUploadedLines} 个文件(按行分割)`);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[useSliceUpload] Stream upload error:', err);
|
||||||
|
if (err.message === "Upload cancelled") {
|
||||||
|
message.info("上传已取消");
|
||||||
|
} else {
|
||||||
|
message.error("文件上传失败,请稍后重试");
|
||||||
|
}
|
||||||
|
removeTask({
|
||||||
|
...task,
|
||||||
|
isCancel: true,
|
||||||
|
...taskListRef.current.find((item) => item.key === task.key),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 注册流式上传事件监听
|
||||||
|
* 返回注销函数
|
||||||
|
*/
|
||||||
|
const registerStreamUploadListener = () => {
|
||||||
|
if (!enableStreamUpload) return () => {};
|
||||||
|
|
||||||
|
const streamUploadHandler = async (e: Event) => {
|
||||||
|
const customEvent = e as CustomEvent;
|
||||||
|
const { dataset, files, updateEvent, hasArchive, prefix } = customEvent.detail;
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const task: TaskItem = {
|
||||||
|
key: dataset.id,
|
||||||
|
title: `上传数据集: ${dataset.name} (按行分割)`,
|
||||||
|
percent: 0,
|
||||||
|
reqId: -1,
|
||||||
|
controller,
|
||||||
|
size: 0,
|
||||||
|
updateEvent,
|
||||||
|
hasArchive,
|
||||||
|
prefix,
|
||||||
|
};
|
||||||
|
|
||||||
|
taskListRef.current = [task, ...taskListRef.current];
|
||||||
|
setTaskList(taskListRef.current);
|
||||||
|
|
||||||
|
// 显示任务中心
|
||||||
|
if (showTaskCenter) {
|
||||||
|
window.dispatchEvent(
|
||||||
|
new CustomEvent("show:task-popover", { detail: { show: true } })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await handleStreamUpload({ task, files });
|
||||||
|
};
|
||||||
|
|
||||||
|
window.addEventListener("upload:dataset-stream", streamUploadHandler);
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
window.removeEventListener("upload:dataset-stream", streamUploadHandler);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
taskList,
|
||||||
|
createTask,
|
||||||
|
removeTask,
|
||||||
|
handleUpload,
|
||||||
|
handleStreamUpload,
|
||||||
|
registerStreamUploadListener,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -151,6 +151,15 @@ export default function AgentPage() {
|
|||||||
const [isTyping, setIsTyping] = useState(false);
|
const [isTyping, setIsTyping] = useState(false);
|
||||||
const messagesEndRef = useRef<HTMLDivElement>(null);
|
const messagesEndRef = useRef<HTMLDivElement>(null);
|
||||||
const inputRef = useRef<any>(null);
|
const inputRef = useRef<any>(null);
|
||||||
|
const timeoutRef = useRef<NodeJS.Timeout | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
if (timeoutRef.current) {
|
||||||
|
clearTimeout(timeoutRef.current);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
const scrollToBottom = () => {
|
const scrollToBottom = () => {
|
||||||
messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
|
messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
|
||||||
@@ -174,8 +183,13 @@ export default function AgentPage() {
|
|||||||
setInputValue("");
|
setInputValue("");
|
||||||
setIsTyping(true);
|
setIsTyping(true);
|
||||||
|
|
||||||
|
// 清理旧的 timeout
|
||||||
|
if (timeoutRef.current) {
|
||||||
|
clearTimeout(timeoutRef.current);
|
||||||
|
}
|
||||||
|
|
||||||
// 模拟AI响应
|
// 模拟AI响应
|
||||||
setTimeout(() => {
|
timeoutRef.current = setTimeout(() => {
|
||||||
const response = generateResponse(content);
|
const response = generateResponse(content);
|
||||||
const assistantMessage: Message = {
|
const assistantMessage: Message = {
|
||||||
id: (Date.now() + 1).toString(),
|
id: (Date.now() + 1).toString(),
|
||||||
|
|||||||
@@ -3,7 +3,9 @@
|
|||||||
* 通过 iframe 加载外部页面
|
* 通过 iframe 加载外部页面
|
||||||
*/
|
*/
|
||||||
export default function ContentGenerationPage() {
|
export default function ContentGenerationPage() {
|
||||||
const iframeUrl = "http://192.168.0.8:3000";
|
const iframeUrl = "/api#/meeting";
|
||||||
|
|
||||||
|
window.localStorage.setItem("geeker-user", '{"token":"123","userInfo":{"name":"xteam"},"loginFrom":null,"loginData":null}');
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="h-full w-full flex flex-col">
|
<div className="h-full w-full flex flex-col">
|
||||||
@@ -16,6 +18,11 @@ export default function ContentGenerationPage() {
|
|||||||
className="w-full h-full border-0"
|
className="w-full h-full border-0"
|
||||||
title="内容生成"
|
title="内容生成"
|
||||||
sandbox="allow-same-origin allow-scripts allow-popups allow-forms allow-downloads"
|
sandbox="allow-same-origin allow-scripts allow-popups allow-forms allow-downloads"
|
||||||
|
style={{marginLeft: "-220px",
|
||||||
|
marginTop: "-66px",
|
||||||
|
width: "calc(100% + 233px)",
|
||||||
|
height: "calc(100% + 108px)"
|
||||||
|
}}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -6,9 +6,11 @@ import { useNavigate, useParams } from "react-router";
|
|||||||
import {
|
import {
|
||||||
getEditorProjectInfoUsingGet,
|
getEditorProjectInfoUsingGet,
|
||||||
getEditorTaskUsingGet,
|
getEditorTaskUsingGet,
|
||||||
|
getEditorTaskSegmentsUsingGet,
|
||||||
listEditorTasksUsingGet,
|
listEditorTasksUsingGet,
|
||||||
upsertEditorAnnotationUsingPut,
|
upsertEditorAnnotationUsingPut,
|
||||||
} from "../annotation.api";
|
} from "../annotation.api";
|
||||||
|
import { AnnotationResultStatus } from "../annotation.model";
|
||||||
|
|
||||||
type EditorProjectInfo = {
|
type EditorProjectInfo = {
|
||||||
projectId: string;
|
projectId: string;
|
||||||
@@ -26,6 +28,8 @@ type EditorTaskListItem = {
|
|||||||
fileType?: string | null;
|
fileType?: string | null;
|
||||||
hasAnnotation: boolean;
|
hasAnnotation: boolean;
|
||||||
annotationUpdatedAt?: string | null;
|
annotationUpdatedAt?: string | null;
|
||||||
|
annotationStatus?: AnnotationResultStatus | null;
|
||||||
|
segmentStats?: SegmentStats;
|
||||||
};
|
};
|
||||||
|
|
||||||
type LsfMessage = {
|
type LsfMessage = {
|
||||||
@@ -35,14 +39,16 @@ type LsfMessage = {
|
|||||||
|
|
||||||
type SegmentInfo = {
|
type SegmentInfo = {
|
||||||
idx: number;
|
idx: number;
|
||||||
text: string;
|
|
||||||
start: number;
|
|
||||||
end: number;
|
|
||||||
hasAnnotation: boolean;
|
hasAnnotation: boolean;
|
||||||
lineIndex: number;
|
lineIndex: number;
|
||||||
chunkIndex: number;
|
chunkIndex: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type SegmentStats = {
|
||||||
|
done: number;
|
||||||
|
total: number;
|
||||||
|
};
|
||||||
|
|
||||||
type ApiResponse<T> = {
|
type ApiResponse<T> = {
|
||||||
code?: number;
|
code?: number;
|
||||||
message?: string;
|
message?: string;
|
||||||
@@ -58,10 +64,16 @@ type EditorTaskPayload = {
|
|||||||
type EditorTaskResponse = {
|
type EditorTaskResponse = {
|
||||||
task?: EditorTaskPayload;
|
task?: EditorTaskPayload;
|
||||||
segmented?: boolean;
|
segmented?: boolean;
|
||||||
segments?: SegmentInfo[];
|
totalSegments?: number;
|
||||||
currentSegmentIndex?: number;
|
currentSegmentIndex?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type EditorTaskSegmentsResponse = {
|
||||||
|
segmented?: boolean;
|
||||||
|
segments?: SegmentInfo[];
|
||||||
|
totalSegments?: number;
|
||||||
|
};
|
||||||
|
|
||||||
type EditorTaskListResponse = {
|
type EditorTaskListResponse = {
|
||||||
content?: EditorTaskListItem[];
|
content?: EditorTaskListItem[];
|
||||||
totalElements?: number;
|
totalElements?: number;
|
||||||
@@ -88,6 +100,13 @@ type SwitchDecision = "save" | "discard" | "cancel";
|
|||||||
const LSF_IFRAME_SRC = "/lsf/lsf.html";
|
const LSF_IFRAME_SRC = "/lsf/lsf.html";
|
||||||
const TASK_PAGE_START = 0;
|
const TASK_PAGE_START = 0;
|
||||||
const TASK_PAGE_SIZE = 200;
|
const TASK_PAGE_SIZE = 200;
|
||||||
|
const NO_ANNOTATION_LABEL = "无标注";
|
||||||
|
const NOT_APPLICABLE_LABEL = "不适用";
|
||||||
|
const NO_ANNOTATION_CONFIRM_TITLE = "没有标注任何内容";
|
||||||
|
const NO_ANNOTATION_CONFIRM_OK_TEXT = "设为无标注并保存";
|
||||||
|
const NOT_APPLICABLE_CONFIRM_TEXT = "设为不适用并保存";
|
||||||
|
const NO_ANNOTATION_CONFIRM_CANCEL_TEXT = "继续标注";
|
||||||
|
const SAVE_AND_NEXT_LABEL = "保存并跳转到下一段/下一条";
|
||||||
|
|
||||||
type NormalizedTaskList = {
|
type NormalizedTaskList = {
|
||||||
items: EditorTaskListItem[];
|
items: EditorTaskListItem[];
|
||||||
@@ -103,6 +122,17 @@ const resolveSegmentIndex = (value: unknown) => {
|
|||||||
return Number.isFinite(parsed) ? parsed : undefined;
|
return Number.isFinite(parsed) ? parsed : undefined;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const isSaveShortcut = (event: KeyboardEvent) => {
|
||||||
|
if (event.defaultPrevented || event.isComposing) return false;
|
||||||
|
const key = event.key;
|
||||||
|
const code = event.code;
|
||||||
|
const isS = key === "s" || key === "S" || code === "KeyS";
|
||||||
|
if (!isS) return false;
|
||||||
|
if (!(event.ctrlKey || event.metaKey)) return false;
|
||||||
|
if (event.shiftKey || event.altKey) return false;
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
const normalizePayload = (payload: unknown): ExportPayload | undefined => {
|
const normalizePayload = (payload: unknown): ExportPayload | undefined => {
|
||||||
if (!payload || typeof payload !== "object") return undefined;
|
if (!payload || typeof payload !== "object") return undefined;
|
||||||
return payload as ExportPayload;
|
return payload as ExportPayload;
|
||||||
@@ -119,6 +149,40 @@ const resolvePayloadMessage = (payload: unknown) => {
|
|||||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||||
!!value && typeof value === "object" && !Array.isArray(value);
|
!!value && typeof value === "object" && !Array.isArray(value);
|
||||||
|
|
||||||
|
const isAnnotationResultEmpty = (annotation?: Record<string, unknown>) => {
|
||||||
|
if (!annotation) return true;
|
||||||
|
if (!("result" in annotation)) return true;
|
||||||
|
const result = (annotation as { result?: unknown }).result;
|
||||||
|
if (!Array.isArray(result)) return false;
|
||||||
|
return result.length === 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
const resolveTaskStatusMeta = (item: EditorTaskListItem) => {
|
||||||
|
const segmentSummary = resolveSegmentSummary(item);
|
||||||
|
if (segmentSummary) {
|
||||||
|
if (segmentSummary.done >= segmentSummary.total) {
|
||||||
|
return { text: "已标注", type: "success" as const };
|
||||||
|
}
|
||||||
|
if (segmentSummary.done > 0) {
|
||||||
|
return { text: "标注中", type: "warning" as const };
|
||||||
|
}
|
||||||
|
return { text: "未标注", type: "secondary" as const };
|
||||||
|
}
|
||||||
|
if (!item.hasAnnotation) {
|
||||||
|
return { text: "未标注", type: "secondary" as const };
|
||||||
|
}
|
||||||
|
if (item.annotationStatus === AnnotationResultStatus.NO_ANNOTATION) {
|
||||||
|
return { text: NO_ANNOTATION_LABEL, type: "warning" as const };
|
||||||
|
}
|
||||||
|
if (item.annotationStatus === AnnotationResultStatus.NOT_APPLICABLE) {
|
||||||
|
return { text: NOT_APPLICABLE_LABEL, type: "warning" as const };
|
||||||
|
}
|
||||||
|
if (item.annotationStatus === AnnotationResultStatus.IN_PROGRESS) {
|
||||||
|
return { text: "标注中", type: "warning" as const };
|
||||||
|
}
|
||||||
|
return { text: "已标注", type: "success" as const };
|
||||||
|
};
|
||||||
|
|
||||||
const normalizeSnapshotValue = (value: unknown, seen: WeakSet<object>): unknown => {
|
const normalizeSnapshotValue = (value: unknown, seen: WeakSet<object>): unknown => {
|
||||||
if (!value || typeof value !== "object") return value;
|
if (!value || typeof value !== "object") return value;
|
||||||
const obj = value as object;
|
const obj = value as object;
|
||||||
@@ -144,6 +208,7 @@ const stableStringify = (value: unknown) => {
|
|||||||
|
|
||||||
const buildAnnotationSnapshot = (annotation?: Record<string, unknown>) => {
|
const buildAnnotationSnapshot = (annotation?: Record<string, unknown>) => {
|
||||||
if (!annotation) return "";
|
if (!annotation) return "";
|
||||||
|
if (isAnnotationResultEmpty(annotation)) return "";
|
||||||
const cleaned: Record<string, unknown> = { ...annotation };
|
const cleaned: Record<string, unknown> = { ...annotation };
|
||||||
delete cleaned.updated_at;
|
delete cleaned.updated_at;
|
||||||
delete cleaned.updatedAt;
|
delete cleaned.updatedAt;
|
||||||
@@ -155,6 +220,25 @@ const buildAnnotationSnapshot = (annotation?: Record<string, unknown>) => {
|
|||||||
const buildSnapshotKey = (fileId: string, segmentIndex?: number) =>
|
const buildSnapshotKey = (fileId: string, segmentIndex?: number) =>
|
||||||
`${fileId}::${segmentIndex ?? "full"}`;
|
`${fileId}::${segmentIndex ?? "full"}`;
|
||||||
|
|
||||||
|
const buildSegmentStats = (segmentList?: SegmentInfo[] | null): SegmentStats | null => {
|
||||||
|
if (!Array.isArray(segmentList) || segmentList.length === 0) return null;
|
||||||
|
const total = segmentList.length;
|
||||||
|
const done = segmentList.reduce((count, seg) => count + (seg.hasAnnotation ? 1 : 0), 0);
|
||||||
|
return { done, total };
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalizeSegmentStats = (stats?: SegmentStats | null): SegmentStats | null => {
|
||||||
|
if (!stats) return null;
|
||||||
|
const total = Number(stats.total);
|
||||||
|
const done = Number(stats.done);
|
||||||
|
if (!Number.isFinite(total) || total <= 0) return null;
|
||||||
|
const safeDone = Math.min(Math.max(done, 0), total);
|
||||||
|
return { done: safeDone, total };
|
||||||
|
};
|
||||||
|
|
||||||
|
const resolveSegmentSummary = (item: EditorTaskListItem) =>
|
||||||
|
normalizeSegmentStats(item.segmentStats);
|
||||||
|
|
||||||
const mergeTaskItems = (base: EditorTaskListItem[], next: EditorTaskListItem[]) => {
|
const mergeTaskItems = (base: EditorTaskListItem[], next: EditorTaskListItem[]) => {
|
||||||
if (next.length === 0) return base;
|
if (next.length === 0) return base;
|
||||||
const seen = new Set(base.map((item) => item.fileId));
|
const seen = new Set(base.map((item) => item.fileId));
|
||||||
@@ -205,6 +289,10 @@ export default function LabelStudioTextEditor() {
|
|||||||
const exportCheckSeqRef = useRef(0);
|
const exportCheckSeqRef = useRef(0);
|
||||||
const savedSnapshotsRef = useRef<Record<string, string>>({});
|
const savedSnapshotsRef = useRef<Record<string, string>>({});
|
||||||
const pendingAutoAdvanceRef = useRef(false);
|
const pendingAutoAdvanceRef = useRef(false);
|
||||||
|
const segmentStatsCacheRef = useRef<Record<string, SegmentStats>>({});
|
||||||
|
const segmentStatsSeqRef = useRef(0);
|
||||||
|
const segmentStatsLoadingRef = useRef<Set<string>>(new Set());
|
||||||
|
const segmentSummaryFileRef = useRef<string>("");
|
||||||
|
|
||||||
const [loadingProject, setLoadingProject] = useState(true);
|
const [loadingProject, setLoadingProject] = useState(true);
|
||||||
const [loadingTasks, setLoadingTasks] = useState(false);
|
const [loadingTasks, setLoadingTasks] = useState(false);
|
||||||
@@ -247,6 +335,98 @@ export default function LabelStudioTextEditor() {
|
|||||||
win.postMessage({ type, payload }, origin);
|
win.postMessage({ type, payload }, origin);
|
||||||
}, [origin]);
|
}, [origin]);
|
||||||
|
|
||||||
|
const applySegmentStats = useCallback((fileId: string, stats: SegmentStats | null) => {
|
||||||
|
if (!fileId) return;
|
||||||
|
const normalized = normalizeSegmentStats(stats);
|
||||||
|
setTasks((prev) =>
|
||||||
|
prev.map((item) =>
|
||||||
|
item.fileId === fileId
|
||||||
|
? { ...item, segmentStats: normalized || undefined }
|
||||||
|
: item
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const updateSegmentStatsCache = useCallback((fileId: string, stats: SegmentStats | null) => {
|
||||||
|
if (!fileId) return;
|
||||||
|
const normalized = normalizeSegmentStats(stats);
|
||||||
|
if (normalized) {
|
||||||
|
segmentStatsCacheRef.current[fileId] = normalized;
|
||||||
|
} else {
|
||||||
|
delete segmentStatsCacheRef.current[fileId];
|
||||||
|
}
|
||||||
|
applySegmentStats(fileId, normalized);
|
||||||
|
}, [applySegmentStats]);
|
||||||
|
|
||||||
|
const fetchSegmentStatsForFile = useCallback(async (fileId: string, seq: number) => {
|
||||||
|
if (!projectId || !fileId) return;
|
||||||
|
if (segmentStatsCacheRef.current[fileId] || segmentStatsLoadingRef.current.has(fileId)) return;
|
||||||
|
segmentStatsLoadingRef.current.add(fileId);
|
||||||
|
try {
|
||||||
|
const resp = (await getEditorTaskSegmentsUsingGet(projectId, fileId)) as ApiResponse<EditorTaskSegmentsResponse>;
|
||||||
|
if (segmentStatsSeqRef.current !== seq) return;
|
||||||
|
const data = resp?.data;
|
||||||
|
if (!data?.segmented) return;
|
||||||
|
const stats = buildSegmentStats(data.segments);
|
||||||
|
if (!stats) return;
|
||||||
|
segmentStatsCacheRef.current[fileId] = stats;
|
||||||
|
applySegmentStats(fileId, stats);
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
} finally {
|
||||||
|
segmentStatsLoadingRef.current.delete(fileId);
|
||||||
|
}
|
||||||
|
}, [applySegmentStats, projectId]);
|
||||||
|
|
||||||
|
const prefetchSegmentStats = useCallback((items: EditorTaskListItem[]) => {
|
||||||
|
if (!projectId) return;
|
||||||
|
const fileIds = items
|
||||||
|
.map((item) => item.fileId)
|
||||||
|
.filter((fileId) => fileId && !segmentStatsCacheRef.current[fileId]);
|
||||||
|
if (fileIds.length === 0) return;
|
||||||
|
const seq = segmentStatsSeqRef.current;
|
||||||
|
let cursor = 0;
|
||||||
|
const workerCount = Math.min(3, fileIds.length);
|
||||||
|
const runWorker = async () => {
|
||||||
|
while (cursor < fileIds.length && segmentStatsSeqRef.current === seq) {
|
||||||
|
const fileId = fileIds[cursor];
|
||||||
|
cursor += 1;
|
||||||
|
await fetchSegmentStatsForFile(fileId, seq);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
void Promise.all(Array.from({ length: workerCount }, () => runWorker()));
|
||||||
|
}, [fetchSegmentStatsForFile, projectId]);
|
||||||
|
|
||||||
|
const confirmEmptyAnnotationStatus = useCallback(() => {
|
||||||
|
return new Promise<AnnotationResultStatus | null>((resolve) => {
|
||||||
|
let resolved = false;
|
||||||
|
let modalInstance: { destroy: () => void } | null = null;
|
||||||
|
const settle = (value: AnnotationResultStatus | null) => {
|
||||||
|
if (resolved) return;
|
||||||
|
resolved = true;
|
||||||
|
resolve(value);
|
||||||
|
if (modalInstance) modalInstance.destroy();
|
||||||
|
};
|
||||||
|
const handleNotApplicable = () => settle(AnnotationResultStatus.NOT_APPLICABLE);
|
||||||
|
modalInstance = modal.confirm({
|
||||||
|
title: NO_ANNOTATION_CONFIRM_TITLE,
|
||||||
|
content: (
|
||||||
|
<div className="flex flex-col gap-2">
|
||||||
|
<Typography.Text>当前未发现任何标注内容。</Typography.Text>
|
||||||
|
<Typography.Text type="secondary">如确认为无标注或不适用,可继续保存。</Typography.Text>
|
||||||
|
<Button type="link" style={{ padding: 0, height: "auto" }} onClick={handleNotApplicable}>
|
||||||
|
{NOT_APPLICABLE_CONFIRM_TEXT}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
okText: NO_ANNOTATION_CONFIRM_OK_TEXT,
|
||||||
|
cancelText: NO_ANNOTATION_CONFIRM_CANCEL_TEXT,
|
||||||
|
onOk: () => settle(AnnotationResultStatus.NO_ANNOTATION),
|
||||||
|
onCancel: () => settle(null),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}, [modal]);
|
||||||
|
|
||||||
const loadProject = useCallback(async () => {
|
const loadProject = useCallback(async () => {
|
||||||
setLoadingProject(true);
|
setLoadingProject(true);
|
||||||
try {
|
try {
|
||||||
@@ -268,8 +448,13 @@ export default function LabelStudioTextEditor() {
|
|||||||
}, [message, projectId]);
|
}, [message, projectId]);
|
||||||
|
|
||||||
const updateTaskSelection = useCallback((items: EditorTaskListItem[]) => {
|
const updateTaskSelection = useCallback((items: EditorTaskListItem[]) => {
|
||||||
|
const isCompleted = (item: EditorTaskListItem) => {
|
||||||
|
const summary = resolveSegmentSummary(item);
|
||||||
|
if (summary) return summary.done >= summary.total;
|
||||||
|
return item.hasAnnotation;
|
||||||
|
};
|
||||||
const defaultFileId =
|
const defaultFileId =
|
||||||
items.find((item) => !item.hasAnnotation)?.fileId || items[0]?.fileId || "";
|
items.find((item) => !isCompleted(item))?.fileId || items[0]?.fileId || "";
|
||||||
setSelectedFileId((prev) => {
|
setSelectedFileId((prev) => {
|
||||||
if (prev && items.some((item) => item.fileId === prev)) return prev;
|
if (prev && items.some((item) => item.fileId === prev)) return prev;
|
||||||
return defaultFileId;
|
return defaultFileId;
|
||||||
@@ -326,6 +511,9 @@ export default function LabelStudioTextEditor() {
|
|||||||
if (mode === "reset") {
|
if (mode === "reset") {
|
||||||
prefetchSeqRef.current += 1;
|
prefetchSeqRef.current += 1;
|
||||||
setPrefetching(false);
|
setPrefetching(false);
|
||||||
|
segmentStatsSeqRef.current += 1;
|
||||||
|
segmentStatsCacheRef.current = {};
|
||||||
|
segmentStatsLoadingRef.current = new Set();
|
||||||
}
|
}
|
||||||
if (mode === "append") {
|
if (mode === "append") {
|
||||||
setLoadingMore(true);
|
setLoadingMore(true);
|
||||||
@@ -406,17 +594,38 @@ export default function LabelStudioTextEditor() {
|
|||||||
if (seq !== initSeqRef.current) return;
|
if (seq !== initSeqRef.current) return;
|
||||||
|
|
||||||
// 更新分段状态
|
// 更新分段状态
|
||||||
const segmentIndex = data?.segmented
|
const isSegmented = !!data?.segmented;
|
||||||
|
const segmentIndex = isSegmented
|
||||||
? resolveSegmentIndex(data.currentSegmentIndex) ?? 0
|
? resolveSegmentIndex(data.currentSegmentIndex) ?? 0
|
||||||
: undefined;
|
: undefined;
|
||||||
if (data?.segmented) {
|
if (isSegmented) {
|
||||||
|
let nextSegments: SegmentInfo[] = [];
|
||||||
|
if (segmentSummaryFileRef.current === fileId && segments.length > 0) {
|
||||||
|
nextSegments = segments;
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
const segmentResp = (await getEditorTaskSegmentsUsingGet(projectId, fileId)) as ApiResponse<EditorTaskSegmentsResponse>;
|
||||||
|
if (seq !== initSeqRef.current) return;
|
||||||
|
const segmentData = segmentResp?.data;
|
||||||
|
if (segmentData?.segmented) {
|
||||||
|
nextSegments = Array.isArray(segmentData.segments) ? segmentData.segments : [];
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const stats = buildSegmentStats(nextSegments);
|
||||||
setSegmented(true);
|
setSegmented(true);
|
||||||
setSegments(data.segments || []);
|
setSegments(nextSegments);
|
||||||
setCurrentSegmentIndex(segmentIndex ?? 0);
|
setCurrentSegmentIndex(segmentIndex ?? 0);
|
||||||
|
updateSegmentStatsCache(fileId, stats);
|
||||||
|
segmentSummaryFileRef.current = fileId;
|
||||||
} else {
|
} else {
|
||||||
setSegmented(false);
|
setSegmented(false);
|
||||||
setSegments([]);
|
setSegments([]);
|
||||||
setCurrentSegmentIndex(0);
|
setCurrentSegmentIndex(0);
|
||||||
|
updateSegmentStatsCache(fileId, null);
|
||||||
|
segmentSummaryFileRef.current = fileId;
|
||||||
}
|
}
|
||||||
|
|
||||||
const taskData = {
|
const taskData = {
|
||||||
@@ -476,7 +685,7 @@ export default function LabelStudioTextEditor() {
|
|||||||
} finally {
|
} finally {
|
||||||
if (seq === initSeqRef.current) setLoadingTaskDetail(false);
|
if (seq === initSeqRef.current) setLoadingTaskDetail(false);
|
||||||
}
|
}
|
||||||
}, [iframeReady, message, postToIframe, project, projectId]);
|
}, [iframeReady, message, postToIframe, project, projectId, segments, updateSegmentStatsCache]);
|
||||||
|
|
||||||
const advanceAfterSave = useCallback(async (fileId: string, segmentIndex?: number) => {
|
const advanceAfterSave = useCallback(async (fileId: string, segmentIndex?: number) => {
|
||||||
if (!fileId) return;
|
if (!fileId) return;
|
||||||
@@ -539,11 +748,31 @@ export default function LabelStudioTextEditor() {
|
|||||||
? currentSegmentIndex
|
? currentSegmentIndex
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
|
const annotationRecord = annotation as Record<string, unknown>;
|
||||||
|
const currentTask = tasks.find((item) => item.fileId === String(fileId));
|
||||||
|
const currentStatus = currentTask?.annotationStatus;
|
||||||
|
let resolvedStatus: AnnotationResultStatus;
|
||||||
|
if (isAnnotationResultEmpty(annotationRecord)) {
|
||||||
|
if (
|
||||||
|
currentStatus === AnnotationResultStatus.NO_ANNOTATION ||
|
||||||
|
currentStatus === AnnotationResultStatus.NOT_APPLICABLE
|
||||||
|
) {
|
||||||
|
resolvedStatus = currentStatus;
|
||||||
|
} else {
|
||||||
|
const selectedStatus = await confirmEmptyAnnotationStatus();
|
||||||
|
if (!selectedStatus) return false;
|
||||||
|
resolvedStatus = selectedStatus;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
resolvedStatus = AnnotationResultStatus.ANNOTATED;
|
||||||
|
}
|
||||||
|
|
||||||
setSaving(true);
|
setSaving(true);
|
||||||
try {
|
try {
|
||||||
const resp = (await upsertEditorAnnotationUsingPut(projectId, String(fileId), {
|
const resp = (await upsertEditorAnnotationUsingPut(projectId, String(fileId), {
|
||||||
annotation,
|
annotation,
|
||||||
segmentIndex,
|
segmentIndex,
|
||||||
|
annotationStatus: resolvedStatus,
|
||||||
})) as ApiResponse<UpsertAnnotationResponse>;
|
})) as ApiResponse<UpsertAnnotationResponse>;
|
||||||
const updatedAt = resp?.data?.updatedAt;
|
const updatedAt = resp?.data?.updatedAt;
|
||||||
message.success("标注已保存");
|
message.success("标注已保存");
|
||||||
@@ -553,6 +782,7 @@ export default function LabelStudioTextEditor() {
|
|||||||
? {
|
? {
|
||||||
...item,
|
...item,
|
||||||
hasAnnotation: true,
|
hasAnnotation: true,
|
||||||
|
annotationStatus: resolvedStatus,
|
||||||
annotationUpdatedAt: updatedAt || item.annotationUpdatedAt,
|
annotationUpdatedAt: updatedAt || item.annotationUpdatedAt,
|
||||||
}
|
}
|
||||||
: item
|
: item
|
||||||
@@ -565,13 +795,13 @@ export default function LabelStudioTextEditor() {
|
|||||||
|
|
||||||
// 分段模式下更新当前段落的标注状态
|
// 分段模式下更新当前段落的标注状态
|
||||||
if (segmented && segmentIndex !== undefined) {
|
if (segmented && segmentIndex !== undefined) {
|
||||||
setSegments((prev) =>
|
const nextSegments = segments.map((seg) =>
|
||||||
prev.map((seg) =>
|
seg.idx === segmentIndex
|
||||||
seg.idx === segmentIndex
|
? { ...seg, hasAnnotation: true }
|
||||||
? { ...seg, hasAnnotation: true }
|
: seg
|
||||||
: seg
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
|
setSegments(nextSegments);
|
||||||
|
updateSegmentStatsCache(String(fileId), buildSegmentStats(nextSegments));
|
||||||
}
|
}
|
||||||
if (options?.autoAdvance) {
|
if (options?.autoAdvance) {
|
||||||
await advanceAfterSave(String(fileId), segmentIndex);
|
await advanceAfterSave(String(fileId), segmentIndex);
|
||||||
@@ -586,11 +816,15 @@ export default function LabelStudioTextEditor() {
|
|||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
advanceAfterSave,
|
advanceAfterSave,
|
||||||
|
confirmEmptyAnnotationStatus,
|
||||||
currentSegmentIndex,
|
currentSegmentIndex,
|
||||||
message,
|
message,
|
||||||
projectId,
|
projectId,
|
||||||
segmented,
|
segmented,
|
||||||
|
segments,
|
||||||
selectedFileId,
|
selectedFileId,
|
||||||
|
tasks,
|
||||||
|
updateSegmentStatsCache,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const requestExportForCheck = useCallback(() => {
|
const requestExportForCheck = useCallback(() => {
|
||||||
@@ -650,14 +884,27 @@ export default function LabelStudioTextEditor() {
|
|||||||
});
|
});
|
||||||
}, [modal]);
|
}, [modal]);
|
||||||
|
|
||||||
const requestExport = () => {
|
const requestExport = useCallback((autoAdvance: boolean) => {
|
||||||
if (!selectedFileId) {
|
if (!selectedFileId) {
|
||||||
message.warning("请先选择文件");
|
message.warning("请先选择文件");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
pendingAutoAdvanceRef.current = true;
|
pendingAutoAdvanceRef.current = autoAdvance;
|
||||||
postToIframe("LS_EXPORT", {});
|
postToIframe("LS_EXPORT", {});
|
||||||
};
|
}, [message, postToIframe, selectedFileId]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const handleSaveShortcut = (event: KeyboardEvent) => {
|
||||||
|
if (!isSaveShortcut(event) || event.repeat) return;
|
||||||
|
if (saving || loadingTaskDetail || segmentSwitching) return;
|
||||||
|
if (!iframeReady || !lsReady) return;
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
requestExport(false);
|
||||||
|
};
|
||||||
|
window.addEventListener("keydown", handleSaveShortcut);
|
||||||
|
return () => window.removeEventListener("keydown", handleSaveShortcut);
|
||||||
|
}, [iframeReady, loadingTaskDetail, lsReady, requestExport, saving, segmentSwitching]);
|
||||||
|
|
||||||
// 段落切换处理
|
// 段落切换处理
|
||||||
const handleSegmentChange = useCallback(async (newIndex: number) => {
|
const handleSegmentChange = useCallback(async (newIndex: number) => {
|
||||||
@@ -753,7 +1000,11 @@ export default function LabelStudioTextEditor() {
|
|||||||
setSegmented(false);
|
setSegmented(false);
|
||||||
setSegments([]);
|
setSegments([]);
|
||||||
setCurrentSegmentIndex(0);
|
setCurrentSegmentIndex(0);
|
||||||
|
segmentSummaryFileRef.current = "";
|
||||||
savedSnapshotsRef.current = {};
|
savedSnapshotsRef.current = {};
|
||||||
|
segmentStatsSeqRef.current += 1;
|
||||||
|
segmentStatsCacheRef.current = {};
|
||||||
|
segmentStatsLoadingRef.current = new Set();
|
||||||
if (exportCheckRef.current?.timer) {
|
if (exportCheckRef.current?.timer) {
|
||||||
window.clearTimeout(exportCheckRef.current.timer);
|
window.clearTimeout(exportCheckRef.current.timer);
|
||||||
}
|
}
|
||||||
@@ -767,6 +1018,12 @@ export default function LabelStudioTextEditor() {
|
|||||||
loadTasks({ mode: "reset" });
|
loadTasks({ mode: "reset" });
|
||||||
}, [project?.supported, loadTasks]);
|
}, [project?.supported, loadTasks]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!segmented) return;
|
||||||
|
if (tasks.length === 0) return;
|
||||||
|
prefetchSegmentStats(tasks);
|
||||||
|
}, [prefetchSegmentStats, segmented, tasks]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!selectedFileId) return;
|
if (!selectedFileId) return;
|
||||||
initEditorForFile(selectedFileId);
|
initEditorForFile(selectedFileId);
|
||||||
@@ -826,6 +1083,15 @@ export default function LabelStudioTextEditor() {
|
|||||||
[segmentTreeData]
|
[segmentTreeData]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const inProgressSegmentedCount = useMemo(() => {
|
||||||
|
if (tasks.length === 0) return 0;
|
||||||
|
return tasks.reduce((count, item) => {
|
||||||
|
const summary = resolveSegmentSummary(item);
|
||||||
|
if (!summary) return count;
|
||||||
|
return summary.done < summary.total ? count + 1 : count;
|
||||||
|
}, 0);
|
||||||
|
}, [tasks]);
|
||||||
|
|
||||||
const handleSegmentSelect = useCallback((keys: Array<string | number>) => {
|
const handleSegmentSelect = useCallback((keys: Array<string | number>) => {
|
||||||
const [first] = keys;
|
const [first] = keys;
|
||||||
if (first === undefined || first === null) return;
|
if (first === undefined || first === null) return;
|
||||||
@@ -865,6 +1131,12 @@ export default function LabelStudioTextEditor() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (msg.type === "LS_SAVE_AND_NEXT") {
|
||||||
|
pendingAutoAdvanceRef.current = false;
|
||||||
|
saveFromExport(payload, { autoAdvance: true });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (msg.type === "LS_EXPORT_CHECK_RESULT") {
|
if (msg.type === "LS_EXPORT_CHECK_RESULT") {
|
||||||
const pending = exportCheckRef.current;
|
const pending = exportCheckRef.current;
|
||||||
if (!pending) return;
|
if (!pending) return;
|
||||||
@@ -897,6 +1169,8 @@ export default function LabelStudioTextEditor() {
|
|||||||
}, [message, origin, saveFromExport]);
|
}, [message, origin, saveFromExport]);
|
||||||
|
|
||||||
const canLoadMore = taskTotalPages > 0 && taskPage + 1 < taskTotalPages;
|
const canLoadMore = taskTotalPages > 0 && taskPage + 1 < taskTotalPages;
|
||||||
|
const saveDisabled =
|
||||||
|
!iframeReady || !selectedFileId || saving || segmentSwitching || loadingTaskDetail;
|
||||||
const loadMoreNode = canLoadMore ? (
|
const loadMoreNode = canLoadMore ? (
|
||||||
<div className="p-2 text-center">
|
<div className="p-2 text-center">
|
||||||
<Button
|
<Button
|
||||||
@@ -960,7 +1234,7 @@ export default function LabelStudioTextEditor() {
|
|||||||
return (
|
return (
|
||||||
<div className="h-full flex flex-col">
|
<div className="h-full flex flex-col">
|
||||||
{/* 顶部工具栏 */}
|
{/* 顶部工具栏 */}
|
||||||
<div className="flex items-center justify-between px-3 py-2 border-b border-gray-200 bg-white">
|
<div className="grid grid-cols-[1fr_auto_1fr] items-center px-3 py-2 border-b border-gray-200 bg-white">
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex items-center gap-2">
|
||||||
<Button icon={<LeftOutlined />} onClick={() => navigate("/data/annotation")}>
|
<Button icon={<LeftOutlined />} onClick={() => navigate("/data/annotation")}>
|
||||||
返回
|
返回
|
||||||
@@ -974,7 +1248,18 @@ export default function LabelStudioTextEditor() {
|
|||||||
标注编辑器
|
标注编辑器
|
||||||
</Typography.Title>
|
</Typography.Title>
|
||||||
</div>
|
</div>
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex items-center justify-center">
|
||||||
|
<Button
|
||||||
|
type="primary"
|
||||||
|
icon={<SaveOutlined />}
|
||||||
|
loading={saving}
|
||||||
|
disabled={saveDisabled}
|
||||||
|
onClick={() => requestExport(true)}
|
||||||
|
>
|
||||||
|
{SAVE_AND_NEXT_LABEL}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2 justify-end">
|
||||||
<Button
|
<Button
|
||||||
icon={<ReloadOutlined />}
|
icon={<ReloadOutlined />}
|
||||||
loading={loadingTasks}
|
loading={loadingTasks}
|
||||||
@@ -983,11 +1268,10 @@ export default function LabelStudioTextEditor() {
|
|||||||
刷新
|
刷新
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
type="primary"
|
|
||||||
icon={<SaveOutlined />}
|
icon={<SaveOutlined />}
|
||||||
loading={saving}
|
loading={saving}
|
||||||
disabled={!iframeReady || !selectedFileId}
|
disabled={saveDisabled}
|
||||||
onClick={requestExport}
|
onClick={() => requestExport(false)}
|
||||||
>
|
>
|
||||||
保存
|
保存
|
||||||
</Button>
|
</Button>
|
||||||
@@ -1001,8 +1285,13 @@ export default function LabelStudioTextEditor() {
|
|||||||
className="border-r border-gray-200 bg-gray-50 flex flex-col transition-all duration-200 min-h-0"
|
className="border-r border-gray-200 bg-gray-50 flex flex-col transition-all duration-200 min-h-0"
|
||||||
style={{ width: sidebarCollapsed ? 0 : 240, overflow: "hidden" }}
|
style={{ width: sidebarCollapsed ? 0 : 240, overflow: "hidden" }}
|
||||||
>
|
>
|
||||||
<div className="px-3 py-2 border-b border-gray-200 bg-white font-medium text-sm">
|
<div className="px-3 py-2 border-b border-gray-200 bg-white font-medium text-sm flex items-center justify-between gap-2">
|
||||||
文件列表
|
<span>文件列表</span>
|
||||||
|
{segmented && (
|
||||||
|
<Tag color="orange" style={{ margin: 0 }}>
|
||||||
|
标注中 {inProgressSegmentedCount}
|
||||||
|
</Tag>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
<div className="flex-1 min-h-0 overflow-auto">
|
<div className="flex-1 min-h-0 overflow-auto">
|
||||||
<List
|
<List
|
||||||
@@ -1010,37 +1299,45 @@ export default function LabelStudioTextEditor() {
|
|||||||
size="small"
|
size="small"
|
||||||
dataSource={tasks}
|
dataSource={tasks}
|
||||||
loadMore={loadMoreNode}
|
loadMore={loadMoreNode}
|
||||||
renderItem={(item) => (
|
renderItem={(item) => {
|
||||||
<List.Item
|
const segmentSummary = resolveSegmentSummary(item);
|
||||||
key={item.fileId}
|
const statusMeta = resolveTaskStatusMeta(item);
|
||||||
className="cursor-pointer hover:bg-blue-50"
|
return (
|
||||||
style={{
|
<List.Item
|
||||||
background: item.fileId === selectedFileId ? "#e6f4ff" : undefined,
|
key={item.fileId}
|
||||||
padding: "8px 12px",
|
className="cursor-pointer hover:bg-blue-50"
|
||||||
borderBottom: "1px solid #f0f0f0",
|
style={{
|
||||||
}}
|
background: item.fileId === selectedFileId ? "#e6f4ff" : undefined,
|
||||||
onClick={() => setSelectedFileId(item.fileId)}
|
padding: "8px 12px",
|
||||||
>
|
borderBottom: "1px solid #f0f0f0",
|
||||||
<div className="flex flex-col w-full gap-1">
|
}}
|
||||||
<Typography.Text ellipsis style={{ fontSize: 13 }}>
|
onClick={() => setSelectedFileId(item.fileId)}
|
||||||
{item.fileName}
|
>
|
||||||
</Typography.Text>
|
<div className="flex flex-col w-full gap-1">
|
||||||
<div className="flex items-center justify-between">
|
<Typography.Text ellipsis style={{ fontSize: 13 }}>
|
||||||
<Typography.Text
|
{item.fileName}
|
||||||
type={item.hasAnnotation ? "success" : "secondary"}
|
|
||||||
style={{ fontSize: 11 }}
|
|
||||||
>
|
|
||||||
{item.hasAnnotation ? "已标注" : "未标注"}
|
|
||||||
</Typography.Text>
|
|
||||||
{item.annotationUpdatedAt && (
|
|
||||||
<Typography.Text type="secondary" style={{ fontSize: 10 }}>
|
|
||||||
{item.annotationUpdatedAt}
|
|
||||||
</Typography.Text>
|
</Typography.Text>
|
||||||
)}
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Typography.Text type={statusMeta.type} style={{ fontSize: 11 }}>
|
||||||
|
{statusMeta.text}
|
||||||
|
</Typography.Text>
|
||||||
|
{segmentSummary && (
|
||||||
|
<Typography.Text type="secondary" style={{ fontSize: 10 }}>
|
||||||
|
已标注 {segmentSummary.done}/{segmentSummary.total}
|
||||||
|
</Typography.Text>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
{item.annotationUpdatedAt && (
|
||||||
|
<Typography.Text type="secondary" style={{ fontSize: 10 }}>
|
||||||
|
{item.annotationUpdatedAt}
|
||||||
|
</Typography.Text>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</List.Item>
|
||||||
</List.Item>
|
);
|
||||||
)}
|
}}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
{segmented && (
|
{segmented && (
|
||||||
|
|||||||
@@ -6,6 +6,12 @@ import TextArea from "antd/es/input/TextArea";
|
|||||||
import { useEffect, useMemo, useState } from "react";
|
import { useEffect, useMemo, useState } from "react";
|
||||||
import type { ReactNode } from "react";
|
import type { ReactNode } from "react";
|
||||||
import { Eye } from "lucide-react";
|
import { Eye } from "lucide-react";
|
||||||
|
import {
|
||||||
|
PREVIEW_TEXT_MAX_LENGTH,
|
||||||
|
resolvePreviewFileType,
|
||||||
|
truncatePreviewText,
|
||||||
|
type PreviewFileType,
|
||||||
|
} from "@/utils/filePreview";
|
||||||
import {
|
import {
|
||||||
createAnnotationTaskUsingPost,
|
createAnnotationTaskUsingPost,
|
||||||
getAnnotationTaskByIdUsingGet,
|
getAnnotationTaskByIdUsingGet,
|
||||||
@@ -13,7 +19,8 @@ import {
|
|||||||
queryAnnotationTemplatesUsingGet,
|
queryAnnotationTemplatesUsingGet,
|
||||||
} from "../../annotation.api";
|
} from "../../annotation.api";
|
||||||
import { DatasetType, type Dataset } from "@/pages/DataManagement/dataset.model";
|
import { DatasetType, type Dataset } from "@/pages/DataManagement/dataset.model";
|
||||||
import { DataType, type AnnotationTemplate, type AnnotationTask } from "../../annotation.model";
|
import { DataType, type AnnotationTemplate } from "../../annotation.model";
|
||||||
|
import type { AnnotationTaskListItem } from "../../annotation.const";
|
||||||
import LabelStudioEmbed from "@/components/business/LabelStudioEmbed";
|
import LabelStudioEmbed from "@/components/business/LabelStudioEmbed";
|
||||||
import TemplateConfigurationTreeEditor from "../../components/TemplateConfigurationTreeEditor";
|
import TemplateConfigurationTreeEditor from "../../components/TemplateConfigurationTreeEditor";
|
||||||
import { useTagConfig } from "@/hooks/useTagConfig";
|
import { useTagConfig } from "@/hooks/useTagConfig";
|
||||||
@@ -23,7 +30,7 @@ interface AnnotationTaskDialogProps {
|
|||||||
onClose: () => void;
|
onClose: () => void;
|
||||||
onRefresh: () => void;
|
onRefresh: () => void;
|
||||||
/** 编辑模式:传入要编辑的任务数据 */
|
/** 编辑模式:传入要编辑的任务数据 */
|
||||||
editTask?: AnnotationTask | null;
|
editTask?: AnnotationTaskListItem | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
type DatasetOption = Dataset & { icon?: ReactNode };
|
type DatasetOption = Dataset & { icon?: ReactNode };
|
||||||
@@ -53,6 +60,8 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|||||||
!!value && typeof value === "object" && !Array.isArray(value);
|
!!value && typeof value === "object" && !Array.isArray(value);
|
||||||
|
|
||||||
const DEFAULT_SEGMENTATION_ENABLED = true;
|
const DEFAULT_SEGMENTATION_ENABLED = true;
|
||||||
|
const FILE_PREVIEW_MAX_HEIGHT = 500;
|
||||||
|
const PREVIEW_MODAL_WIDTH = "80vw";
|
||||||
const SEGMENTATION_OPTIONS = [
|
const SEGMENTATION_OPTIONS = [
|
||||||
{ label: "需要切片段", value: true },
|
{ label: "需要切片段", value: true },
|
||||||
{ label: "不需要切片段", value: false },
|
{ label: "不需要切片段", value: false },
|
||||||
@@ -116,7 +125,7 @@ export default function CreateAnnotationTask({
|
|||||||
const [fileContent, setFileContent] = useState("");
|
const [fileContent, setFileContent] = useState("");
|
||||||
const [fileContentLoading, setFileContentLoading] = useState(false);
|
const [fileContentLoading, setFileContentLoading] = useState(false);
|
||||||
const [previewFileName, setPreviewFileName] = useState("");
|
const [previewFileName, setPreviewFileName] = useState("");
|
||||||
const [previewFileType, setPreviewFileType] = useState<"text" | "image" | "video" | "audio">("text");
|
const [previewFileType, setPreviewFileType] = useState<PreviewFileType>("text");
|
||||||
const [previewMediaUrl, setPreviewMediaUrl] = useState("");
|
const [previewMediaUrl, setPreviewMediaUrl] = useState("");
|
||||||
|
|
||||||
// 任务详情加载状态(编辑模式)
|
// 任务详情加载状态(编辑模式)
|
||||||
@@ -275,7 +284,7 @@ export default function CreateAnnotationTask({
|
|||||||
}
|
}
|
||||||
setDatasetPreviewLoading(true);
|
setDatasetPreviewLoading(true);
|
||||||
try {
|
try {
|
||||||
// 对于文本数据集,排除已被转换为TXT的源文档文件(PDF/DOC/DOCX)
|
// 对于文本数据集,排除源文档文件(PDF/DOC/DOCX/XLS/XLSX)
|
||||||
const params: { page: number; size: number; excludeSourceDocuments?: boolean } = { page: 0, size: 10 };
|
const params: { page: number; size: number; excludeSourceDocuments?: boolean } = { page: 0, size: 10 };
|
||||||
if (isTextDataset) {
|
if (isTextDataset) {
|
||||||
params.excludeSourceDocuments = true;
|
params.excludeSourceDocuments = true;
|
||||||
@@ -297,57 +306,32 @@ export default function CreateAnnotationTask({
|
|||||||
|
|
||||||
// 预览文件内容
|
// 预览文件内容
|
||||||
const handlePreviewFileContent = async (file: DatasetPreviewFile) => {
|
const handlePreviewFileContent = async (file: DatasetPreviewFile) => {
|
||||||
const fileName = file.fileName?.toLowerCase() || '';
|
const fileType = resolvePreviewFileType(file.fileName);
|
||||||
|
if (!fileType) {
|
||||||
// 文件类型扩展名映射
|
|
||||||
const textExtensions = ['.json', '.jsonl', '.txt', '.csv', '.tsv', '.xml', '.md', '.yaml', '.yml'];
|
|
||||||
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.svg'];
|
|
||||||
const videoExtensions = ['.mp4', '.webm', '.ogg', '.mov', '.avi'];
|
|
||||||
const audioExtensions = ['.mp3', '.wav', '.ogg', '.aac', '.flac', '.m4a'];
|
|
||||||
|
|
||||||
const isTextFile = textExtensions.some(ext => fileName.endsWith(ext));
|
|
||||||
const isImageFile = imageExtensions.some(ext => fileName.endsWith(ext));
|
|
||||||
const isVideoFile = videoExtensions.some(ext => fileName.endsWith(ext));
|
|
||||||
const isAudioFile = audioExtensions.some(ext => fileName.endsWith(ext));
|
|
||||||
|
|
||||||
if (!isTextFile && !isImageFile && !isVideoFile && !isAudioFile) {
|
|
||||||
message.warning("不支持预览该文件类型");
|
message.warning("不支持预览该文件类型");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
setFileContentLoading(true);
|
setFileContentLoading(true);
|
||||||
setPreviewFileName(file.fileName);
|
setPreviewFileName(file.fileName);
|
||||||
|
setPreviewFileType(fileType);
|
||||||
|
setFileContent("");
|
||||||
|
setPreviewMediaUrl("");
|
||||||
|
|
||||||
const fileUrl = `/api/data-management/datasets/${selectedDatasetId}/files/${file.id}/download`;
|
const previewUrl = `/api/data-management/datasets/${selectedDatasetId}/files/${file.id}/preview`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (isTextFile) {
|
if (fileType === "text") {
|
||||||
// 文本文件:获取内容
|
// 文本文件:获取内容
|
||||||
const response = await fetch(fileUrl);
|
const response = await fetch(previewUrl);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error('下载失败');
|
throw new Error('下载失败');
|
||||||
}
|
}
|
||||||
const text = await response.text();
|
const text = await response.text();
|
||||||
// 限制预览内容长度
|
setFileContent(truncatePreviewText(text, PREVIEW_TEXT_MAX_LENGTH));
|
||||||
const maxLength = 50000;
|
} else {
|
||||||
if (text.length > maxLength) {
|
// 媒体/PDF 文件:直接使用预览地址
|
||||||
setFileContent(text.substring(0, maxLength) + '\n\n... (内容过长,仅显示前 50000 字符)');
|
setPreviewMediaUrl(previewUrl);
|
||||||
} else {
|
|
||||||
setFileContent(text);
|
|
||||||
}
|
|
||||||
setPreviewFileType("text");
|
|
||||||
} else if (isImageFile) {
|
|
||||||
// 图片文件:直接使用 URL
|
|
||||||
setPreviewMediaUrl(fileUrl);
|
|
||||||
setPreviewFileType("image");
|
|
||||||
} else if (isVideoFile) {
|
|
||||||
// 视频文件:使用 URL
|
|
||||||
setPreviewMediaUrl(fileUrl);
|
|
||||||
setPreviewFileType("video");
|
|
||||||
} else if (isAudioFile) {
|
|
||||||
// 音频文件:使用 URL
|
|
||||||
setPreviewMediaUrl(fileUrl);
|
|
||||||
setPreviewFileType("audio");
|
|
||||||
}
|
}
|
||||||
setFileContentVisible(true);
|
setFileContentVisible(true);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -846,7 +830,7 @@ export default function CreateAnnotationTask({
|
|||||||
open={showPreview}
|
open={showPreview}
|
||||||
onCancel={() => setShowPreview(false)}
|
onCancel={() => setShowPreview(false)}
|
||||||
title="标注界面预览"
|
title="标注界面预览"
|
||||||
width={1000}
|
width={PREVIEW_MODAL_WIDTH}
|
||||||
footer={[
|
footer={[
|
||||||
<Button key="close" onClick={() => setShowPreview(false)}>
|
<Button key="close" onClick={() => setShowPreview(false)}>
|
||||||
关闭
|
关闭
|
||||||
@@ -871,14 +855,14 @@ export default function CreateAnnotationTask({
|
|||||||
open={datasetPreviewVisible}
|
open={datasetPreviewVisible}
|
||||||
onCancel={() => setDatasetPreviewVisible(false)}
|
onCancel={() => setDatasetPreviewVisible(false)}
|
||||||
title="数据集预览(前10条文件)"
|
title="数据集预览(前10条文件)"
|
||||||
width={700}
|
width={PREVIEW_MODAL_WIDTH}
|
||||||
footer={[
|
footer={[
|
||||||
<Button key="close" onClick={() => setDatasetPreviewVisible(false)}>
|
<Button key="close" onClick={() => setDatasetPreviewVisible(false)}>
|
||||||
关闭
|
关闭
|
||||||
</Button>
|
</Button>
|
||||||
]}
|
]}
|
||||||
>
|
>
|
||||||
<div className="mb-2 text-xs text-gray-500">点击文件名可预览文件内容(支持文本、图片、音频、视频)</div>
|
<div className="mb-2 text-xs text-gray-500">点击文件名可预览文件内容(支持文本、图片、音频、视频、PDF)</div>
|
||||||
<Table
|
<Table
|
||||||
dataSource={datasetPreviewData}
|
dataSource={datasetPreviewData}
|
||||||
columns={[
|
columns={[
|
||||||
@@ -928,7 +912,7 @@ export default function CreateAnnotationTask({
|
|||||||
setFileContent("");
|
setFileContent("");
|
||||||
}}
|
}}
|
||||||
title={`文件预览:${previewFileName}`}
|
title={`文件预览:${previewFileName}`}
|
||||||
width={previewFileType === "text" ? 800 : 700}
|
width={PREVIEW_MODAL_WIDTH}
|
||||||
footer={[
|
footer={[
|
||||||
<Button key="close" onClick={() => {
|
<Button key="close" onClick={() => {
|
||||||
setFileContentVisible(false);
|
setFileContentVisible(false);
|
||||||
@@ -942,7 +926,7 @@ export default function CreateAnnotationTask({
|
|||||||
{previewFileType === "text" && (
|
{previewFileType === "text" && (
|
||||||
<pre
|
<pre
|
||||||
style={{
|
style={{
|
||||||
maxHeight: '500px',
|
maxHeight: `${FILE_PREVIEW_MAX_HEIGHT}px`,
|
||||||
overflow: 'auto',
|
overflow: 'auto',
|
||||||
backgroundColor: '#f5f5f5',
|
backgroundColor: '#f5f5f5',
|
||||||
padding: '12px',
|
padding: '12px',
|
||||||
@@ -960,16 +944,23 @@ export default function CreateAnnotationTask({
|
|||||||
<img
|
<img
|
||||||
src={previewMediaUrl}
|
src={previewMediaUrl}
|
||||||
alt={previewFileName}
|
alt={previewFileName}
|
||||||
style={{ maxWidth: '100%', maxHeight: '500px', objectFit: 'contain' }}
|
style={{ maxWidth: '100%', maxHeight: `${FILE_PREVIEW_MAX_HEIGHT}px`, objectFit: 'contain' }}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
{previewFileType === "pdf" && (
|
||||||
|
<iframe
|
||||||
|
src={previewMediaUrl}
|
||||||
|
title={previewFileName || "PDF 预览"}
|
||||||
|
style={{ width: '100%', height: `${FILE_PREVIEW_MAX_HEIGHT}px`, border: 'none' }}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
{previewFileType === "video" && (
|
{previewFileType === "video" && (
|
||||||
<div style={{ textAlign: 'center' }}>
|
<div style={{ textAlign: 'center' }}>
|
||||||
<video
|
<video
|
||||||
src={previewMediaUrl}
|
src={previewMediaUrl}
|
||||||
controls
|
controls
|
||||||
style={{ maxWidth: '100%', maxHeight: '500px' }}
|
style={{ maxWidth: '100%', maxHeight: `${FILE_PREVIEW_MAX_HEIGHT}px` }}
|
||||||
>
|
>
|
||||||
您的浏览器不支持视频播放
|
您的浏览器不支持视频播放
|
||||||
</video>
|
</video>
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { useState } from "react";
|
import { useState } from "react";
|
||||||
import { Card, Button, Table, message, Modal, Tabs } from "antd";
|
import { Card, Button, Table, Tag, message, Modal, Tabs } from "antd";
|
||||||
import {
|
import {
|
||||||
PlusOutlined,
|
PlusOutlined,
|
||||||
EditOutlined,
|
EditOutlined,
|
||||||
@@ -10,27 +10,39 @@ import {
|
|||||||
import { useNavigate } from "react-router";
|
import { useNavigate } from "react-router";
|
||||||
import { SearchControls } from "@/components/SearchControls";
|
import { SearchControls } from "@/components/SearchControls";
|
||||||
import CardView from "@/components/CardView";
|
import CardView from "@/components/CardView";
|
||||||
import type { AnnotationTask } from "../annotation.model";
|
|
||||||
import useFetchData from "@/hooks/useFetchData";
|
import useFetchData from "@/hooks/useFetchData";
|
||||||
import {
|
import {
|
||||||
deleteAnnotationTaskByIdUsingDelete,
|
deleteAnnotationTaskByIdUsingDelete,
|
||||||
queryAnnotationTasksUsingGet,
|
queryAnnotationTasksUsingGet,
|
||||||
} from "../annotation.api";
|
} from "../annotation.api";
|
||||||
import { mapAnnotationTask } from "../annotation.const";
|
import {
|
||||||
|
AnnotationTypeMap,
|
||||||
|
mapAnnotationTask,
|
||||||
|
type AnnotationTaskListItem,
|
||||||
|
} from "../annotation.const";
|
||||||
import CreateAnnotationTask from "../Create/components/CreateAnnotationTaskDialog";
|
import CreateAnnotationTask from "../Create/components/CreateAnnotationTaskDialog";
|
||||||
import ExportAnnotationDialog from "./ExportAnnotationDialog";
|
import ExportAnnotationDialog from "./ExportAnnotationDialog";
|
||||||
import { ColumnType } from "antd/es/table";
|
import { ColumnType } from "antd/es/table";
|
||||||
import { TemplateList } from "../Template";
|
import { TemplateList } from "../Template";
|
||||||
// Note: DevelopmentInProgress intentionally not used here
|
// Note: DevelopmentInProgress intentionally not used here
|
||||||
|
|
||||||
|
type AnnotationTaskRowKey = string | number;
|
||||||
|
type AnnotationTaskOperation = {
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
icon: JSX.Element;
|
||||||
|
danger?: boolean;
|
||||||
|
onClick: (task: AnnotationTaskListItem) => void;
|
||||||
|
};
|
||||||
|
|
||||||
export default function DataAnnotation() {
|
export default function DataAnnotation() {
|
||||||
// return <DevelopmentInProgress showTime="2025.10.30" />;
|
// return <DevelopmentInProgress showTime="2025.10.30" />;
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
const [activeTab, setActiveTab] = useState("tasks");
|
const [activeTab, setActiveTab] = useState("tasks");
|
||||||
const [viewMode, setViewMode] = useState<"list" | "card">("list");
|
const [viewMode, setViewMode] = useState<"list" | "card">("list");
|
||||||
const [showCreateDialog, setShowCreateDialog] = useState(false);
|
const [showCreateDialog, setShowCreateDialog] = useState(false);
|
||||||
const [exportTask, setExportTask] = useState<AnnotationTask | null>(null);
|
const [exportTask, setExportTask] = useState<AnnotationTaskListItem | null>(null);
|
||||||
const [editTask, setEditTask] = useState<AnnotationTask | null>(null);
|
const [editTask, setEditTask] = useState<AnnotationTaskListItem | null>(null);
|
||||||
|
|
||||||
const {
|
const {
|
||||||
loading,
|
loading,
|
||||||
@@ -40,13 +52,16 @@ export default function DataAnnotation() {
|
|||||||
fetchData,
|
fetchData,
|
||||||
handleFiltersChange,
|
handleFiltersChange,
|
||||||
handleKeywordChange,
|
handleKeywordChange,
|
||||||
} = useFetchData(queryAnnotationTasksUsingGet, mapAnnotationTask, 30000, true, [], 0);
|
} = useFetchData<AnnotationTaskListItem>(queryAnnotationTasksUsingGet, mapAnnotationTask, 30000, true, [], 0);
|
||||||
|
|
||||||
const [selectedRowKeys, setSelectedRowKeys] = useState<(string | number)[]>([]);
|
const [selectedRowKeys, setSelectedRowKeys] = useState<AnnotationTaskRowKey[]>([]);
|
||||||
const [selectedRows, setSelectedRows] = useState<any[]>([]);
|
const [selectedRows, setSelectedRows] = useState<AnnotationTaskListItem[]>([]);
|
||||||
|
|
||||||
const handleAnnotate = (task: AnnotationTask) => {
|
const toSafeCount = (value: unknown) =>
|
||||||
const projectId = (task as any)?.id;
|
typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||||
|
|
||||||
|
const handleAnnotate = (task: AnnotationTaskListItem) => {
|
||||||
|
const projectId = task.id;
|
||||||
if (!projectId) {
|
if (!projectId) {
|
||||||
message.error("无法进入标注:缺少标注项目ID");
|
message.error("无法进入标注:缺少标注项目ID");
|
||||||
return;
|
return;
|
||||||
@@ -54,15 +69,15 @@ export default function DataAnnotation() {
|
|||||||
navigate(`/data/annotation/annotate/${projectId}`);
|
navigate(`/data/annotation/annotate/${projectId}`);
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleExport = (task: AnnotationTask) => {
|
const handleExport = (task: AnnotationTaskListItem) => {
|
||||||
setExportTask(task);
|
setExportTask(task);
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleEdit = (task: AnnotationTask) => {
|
const handleEdit = (task: AnnotationTaskListItem) => {
|
||||||
setEditTask(task);
|
setEditTask(task);
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDelete = (task: AnnotationTask) => {
|
const handleDelete = (task: AnnotationTaskListItem) => {
|
||||||
Modal.confirm({
|
Modal.confirm({
|
||||||
title: `确认删除标注任务「${task.name}」吗?`,
|
title: `确认删除标注任务「${task.name}」吗?`,
|
||||||
content: "删除标注任务不会删除对应数据集,但会删除该任务的所有标注结果。",
|
content: "删除标注任务不会删除对应数据集,但会删除该任务的所有标注结果。",
|
||||||
@@ -110,7 +125,7 @@ export default function DataAnnotation() {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const operations = [
|
const operations: AnnotationTaskOperation[] = [
|
||||||
{
|
{
|
||||||
key: "annotate",
|
key: "annotate",
|
||||||
label: "标注",
|
label: "标注",
|
||||||
@@ -142,24 +157,45 @@ export default function DataAnnotation() {
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const columns: ColumnType<any>[] = [
|
const columns: ColumnType<AnnotationTaskListItem>[] = [
|
||||||
|
{
|
||||||
|
title: "序号",
|
||||||
|
key: "index",
|
||||||
|
width: 80,
|
||||||
|
align: "center" as const,
|
||||||
|
render: (_value: unknown, _record: AnnotationTaskListItem, index: number) => {
|
||||||
|
const current = pagination.current ?? 1;
|
||||||
|
const pageSize = pagination.pageSize ?? tableData.length ?? 0;
|
||||||
|
return (current - 1) * pageSize + index + 1;
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
title: "任务名称",
|
title: "任务名称",
|
||||||
dataIndex: "name",
|
dataIndex: "name",
|
||||||
key: "name",
|
key: "name",
|
||||||
fixed: "left" as const,
|
fixed: "left" as const,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
title: "任务ID",
|
|
||||||
dataIndex: "id",
|
|
||||||
key: "id",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
title: "数据集",
|
title: "数据集",
|
||||||
dataIndex: "datasetName",
|
dataIndex: "datasetName",
|
||||||
key: "datasetName",
|
key: "datasetName",
|
||||||
width: 180,
|
width: 180,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
title: "标注类型",
|
||||||
|
dataIndex: "labelingType",
|
||||||
|
key: "labelingType",
|
||||||
|
width: 160,
|
||||||
|
render: (value?: string) => {
|
||||||
|
if (!value) {
|
||||||
|
return "-";
|
||||||
|
}
|
||||||
|
const label =
|
||||||
|
AnnotationTypeMap[value as keyof typeof AnnotationTypeMap]?.label ||
|
||||||
|
value;
|
||||||
|
return <Tag color="geekblue">{label}</Tag>;
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
title: "数据量",
|
title: "数据量",
|
||||||
dataIndex: "totalCount",
|
dataIndex: "totalCount",
|
||||||
@@ -173,9 +209,21 @@ export default function DataAnnotation() {
|
|||||||
key: "annotatedCount",
|
key: "annotatedCount",
|
||||||
width: 100,
|
width: 100,
|
||||||
align: "center" as const,
|
align: "center" as const,
|
||||||
render: (value: number, record: any) => {
|
render: (value: number, record: AnnotationTaskListItem) => {
|
||||||
const total = record.totalCount || 0;
|
const total = toSafeCount(record.totalCount ?? record.total_count);
|
||||||
const annotated = value || 0;
|
const annotatedRaw = toSafeCount(
|
||||||
|
value ?? record.annotatedCount ?? record.annotated_count
|
||||||
|
);
|
||||||
|
const segmentationEnabled =
|
||||||
|
record.segmentationEnabled ?? record.segmentation_enabled;
|
||||||
|
const inProgressRaw = segmentationEnabled
|
||||||
|
? toSafeCount(record.inProgressCount ?? record.in_progress_count)
|
||||||
|
: 0;
|
||||||
|
const shouldExcludeInProgress =
|
||||||
|
total > 0 && annotatedRaw + inProgressRaw > total;
|
||||||
|
const annotated = shouldExcludeInProgress
|
||||||
|
? Math.max(annotatedRaw - inProgressRaw, 0)
|
||||||
|
: annotatedRaw;
|
||||||
const percent = total > 0 ? Math.round((annotated / total) * 100) : 0;
|
const percent = total > 0 ? Math.round((annotated / total) * 100) : 0;
|
||||||
return (
|
return (
|
||||||
<span title={`${annotated}/${total} (${percent}%)`}>
|
<span title={`${annotated}/${total} (${percent}%)`}>
|
||||||
@@ -184,6 +232,23 @@ export default function DataAnnotation() {
|
|||||||
);
|
);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
title: "标注中",
|
||||||
|
dataIndex: "inProgressCount",
|
||||||
|
key: "inProgressCount",
|
||||||
|
width: 100,
|
||||||
|
align: "center" as const,
|
||||||
|
render: (value: number, record: AnnotationTaskListItem) => {
|
||||||
|
const segmentationEnabled =
|
||||||
|
record.segmentationEnabled ?? record.segmentation_enabled;
|
||||||
|
if (!segmentationEnabled) return "-";
|
||||||
|
const resolved =
|
||||||
|
Number.isFinite(value)
|
||||||
|
? value
|
||||||
|
: record.inProgressCount ?? record.in_progress_count ?? 0;
|
||||||
|
return resolved;
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
title: "创建时间",
|
title: "创建时间",
|
||||||
dataIndex: "createdAt",
|
dataIndex: "createdAt",
|
||||||
@@ -202,14 +267,14 @@ export default function DataAnnotation() {
|
|||||||
fixed: "right" as const,
|
fixed: "right" as const,
|
||||||
width: 150,
|
width: 150,
|
||||||
dataIndex: "actions",
|
dataIndex: "actions",
|
||||||
render: (_: any, task: any) => (
|
render: (_value: unknown, task: AnnotationTaskListItem) => (
|
||||||
<div className="flex items-center justify-center space-x-1">
|
<div className="flex items-center justify-center space-x-1">
|
||||||
{operations.map((operation) => (
|
{operations.map((operation) => (
|
||||||
<Button
|
<Button
|
||||||
key={operation.key}
|
key={operation.key}
|
||||||
type="text"
|
type="text"
|
||||||
icon={operation.icon}
|
icon={operation.icon}
|
||||||
onClick={() => (operation?.onClick as any)?.(task)}
|
onClick={() => operation.onClick(task)}
|
||||||
title={operation.label}
|
title={operation.label}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
@@ -282,9 +347,9 @@ export default function DataAnnotation() {
|
|||||||
pagination={pagination}
|
pagination={pagination}
|
||||||
rowSelection={{
|
rowSelection={{
|
||||||
selectedRowKeys,
|
selectedRowKeys,
|
||||||
onChange: (keys, rows) => {
|
onChange: (keys: AnnotationTaskRowKey[], rows: AnnotationTaskListItem[]) => {
|
||||||
setSelectedRowKeys(keys as (string | number)[]);
|
setSelectedRowKeys(keys);
|
||||||
setSelectedRows(rows as any[]);
|
setSelectedRows(rows);
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
scroll={{ x: "max-content", y: "calc(100vh - 24rem)" }}
|
scroll={{ x: "max-content", y: "calc(100vh - 24rem)" }}
|
||||||
@@ -293,7 +358,7 @@ export default function DataAnnotation() {
|
|||||||
) : (
|
) : (
|
||||||
<CardView
|
<CardView
|
||||||
data={tableData}
|
data={tableData}
|
||||||
operations={operations as any}
|
operations={operations}
|
||||||
pagination={pagination}
|
pagination={pagination}
|
||||||
loading={loading}
|
loading={loading}
|
||||||
/>
|
/>
|
||||||
@@ -327,4 +392,4 @@ export default function DataAnnotation() {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -106,13 +106,6 @@ export default function ExportAnnotationDialog({
|
|||||||
const values = await form.validateFields();
|
const values = await form.validateFields();
|
||||||
setExporting(true);
|
setExporting(true);
|
||||||
|
|
||||||
const blob = await downloadAnnotationsUsingGet(
|
|
||||||
projectId,
|
|
||||||
values.format,
|
|
||||||
values.onlyAnnotated,
|
|
||||||
values.includeData
|
|
||||||
);
|
|
||||||
|
|
||||||
// 获取文件名
|
// 获取文件名
|
||||||
const formatExt: Record<ExportFormat, string> = {
|
const formatExt: Record<ExportFormat, string> = {
|
||||||
json: "json",
|
json: "json",
|
||||||
@@ -124,15 +117,14 @@ export default function ExportAnnotationDialog({
|
|||||||
const ext = formatExt[values.format as ExportFormat] || "json";
|
const ext = formatExt[values.format as ExportFormat] || "json";
|
||||||
const filename = `${projectName}_annotations.${ext}`;
|
const filename = `${projectName}_annotations.${ext}`;
|
||||||
|
|
||||||
// 下载文件
|
// 下载文件(download函数内部已处理下载逻辑)
|
||||||
const url = window.URL.createObjectURL(blob as Blob);
|
await downloadAnnotationsUsingGet(
|
||||||
const a = document.createElement("a");
|
projectId,
|
||||||
a.href = url;
|
values.format,
|
||||||
a.download = filename;
|
values.onlyAnnotated,
|
||||||
document.body.appendChild(a);
|
values.includeData,
|
||||||
a.click();
|
filename
|
||||||
window.URL.revokeObjectURL(url);
|
);
|
||||||
document.body.removeChild(a);
|
|
||||||
|
|
||||||
message.success("导出成功");
|
message.success("导出成功");
|
||||||
onClose();
|
onClose();
|
||||||
@@ -186,14 +178,15 @@ export default function ExportAnnotationDialog({
|
|||||||
<Select
|
<Select
|
||||||
options={FORMAT_OPTIONS.map((opt) => ({
|
options={FORMAT_OPTIONS.map((opt) => ({
|
||||||
label: (
|
label: (
|
||||||
<div>
|
<div className="py-1">
|
||||||
<div className="font-medium">{opt.label}</div>
|
<div className="font-medium">{opt.label}</div>
|
||||||
<div className="text-xs text-gray-400">{opt.description}</div>
|
<div className="text-xs text-gray-400">{opt.description}</div>
|
||||||
</div>
|
</div>
|
||||||
),
|
),
|
||||||
value: opt.value,
|
value: opt.value,
|
||||||
|
simpleLabel: opt.label,
|
||||||
}))}
|
}))}
|
||||||
optionLabelProp="label"
|
optionLabelProp="simpleLabel"
|
||||||
/>
|
/>
|
||||||
</Form.Item>
|
</Form.Item>
|
||||||
|
|
||||||
|
|||||||
@@ -43,14 +43,6 @@ const TemplateDetail: React.FC<TemplateDetailProps> = ({
|
|||||||
<Descriptions.Item label="样式">
|
<Descriptions.Item label="样式">
|
||||||
{template.style}
|
{template.style}
|
||||||
</Descriptions.Item>
|
</Descriptions.Item>
|
||||||
<Descriptions.Item label="类型">
|
|
||||||
<Tag color={template.builtIn ? "gold" : "default"}>
|
|
||||||
{template.builtIn ? "系统内置" : "自定义"}
|
|
||||||
</Tag>
|
|
||||||
</Descriptions.Item>
|
|
||||||
<Descriptions.Item label="版本">
|
|
||||||
{template.version}
|
|
||||||
</Descriptions.Item>
|
|
||||||
<Descriptions.Item label="创建时间" span={2}>
|
<Descriptions.Item label="创建时间" span={2}>
|
||||||
{new Date(template.createdAt).toLocaleString()}
|
{new Date(template.createdAt).toLocaleString()}
|
||||||
</Descriptions.Item>
|
</Descriptions.Item>
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ const TemplateForm: React.FC<TemplateFormProps> = ({
|
|||||||
const [form] = Form.useForm();
|
const [form] = Form.useForm();
|
||||||
const [loading, setLoading] = useState(false);
|
const [loading, setLoading] = useState(false);
|
||||||
const [labelConfig, setLabelConfig] = useState("");
|
const [labelConfig, setLabelConfig] = useState("");
|
||||||
|
const selectedDataType = Form.useWatch("dataType", form);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (visible && template && mode === "edit") {
|
if (visible && template && mode === "edit") {
|
||||||
@@ -96,8 +97,12 @@ const TemplateForm: React.FC<TemplateFormProps> = ({
|
|||||||
} else {
|
} else {
|
||||||
message.error(response.message || `模板${mode === "create" ? "创建" : "更新"}失败`);
|
message.error(response.message || `模板${mode === "create" ? "创建" : "更新"}失败`);
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: unknown) {
|
||||||
if (error.errorFields) {
|
const hasErrorFields =
|
||||||
|
typeof error === "object" &&
|
||||||
|
error !== null &&
|
||||||
|
"errorFields" in error;
|
||||||
|
if (hasErrorFields) {
|
||||||
message.error("请填写所有必填字段");
|
message.error("请填写所有必填字段");
|
||||||
} else {
|
} else {
|
||||||
message.error(`模板${mode === "create" ? "创建" : "更新"}失败`);
|
message.error(`模板${mode === "create" ? "创建" : "更新"}失败`);
|
||||||
@@ -195,6 +200,7 @@ const TemplateForm: React.FC<TemplateFormProps> = ({
|
|||||||
value={labelConfig}
|
value={labelConfig}
|
||||||
onChange={setLabelConfig}
|
onChange={setLabelConfig}
|
||||||
height={420}
|
height={420}
|
||||||
|
dataType={selectedDataType}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Form>
|
</Form>
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import React, { useState } from "react";
|
import React, { useState, useEffect } from "react";
|
||||||
import {
|
import {
|
||||||
Button,
|
Button,
|
||||||
Table,
|
Table,
|
||||||
@@ -32,7 +32,16 @@ import {
|
|||||||
TemplateTypeMap
|
TemplateTypeMap
|
||||||
} from "@/pages/DataAnnotation/annotation.const.tsx";
|
} from "@/pages/DataAnnotation/annotation.const.tsx";
|
||||||
|
|
||||||
|
const TEMPLATE_ADMIN_KEY = "datamate_template_admin";
|
||||||
|
|
||||||
const TemplateList: React.FC = () => {
|
const TemplateList: React.FC = () => {
|
||||||
|
const [isAdmin, setIsAdmin] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
// 检查 localStorage 中是否存在特殊键
|
||||||
|
const hasAdminKey = localStorage.getItem(TEMPLATE_ADMIN_KEY) !== null;
|
||||||
|
setIsAdmin(hasAdminKey);
|
||||||
|
}, []);
|
||||||
const filterOptions = [
|
const filterOptions = [
|
||||||
{
|
{
|
||||||
key: "category",
|
key: "category",
|
||||||
@@ -225,23 +234,7 @@ const TemplateList: React.FC = () => {
|
|||||||
<Tag color={getCategoryColor(category)}>{ClassificationMap[category as keyof typeof ClassificationMap]?.label || category}</Tag>
|
<Tag color={getCategoryColor(category)}>{ClassificationMap[category as keyof typeof ClassificationMap]?.label || category}</Tag>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
{
|
|
||||||
title: "类型",
|
|
||||||
dataIndex: "builtIn",
|
|
||||||
key: "builtIn",
|
|
||||||
width: 100,
|
|
||||||
render: (builtIn: boolean) => (
|
|
||||||
<Tag color={builtIn ? "gold" : "default"}>
|
|
||||||
{builtIn ? "系统内置" : "自定义"}
|
|
||||||
</Tag>
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
title: "版本",
|
|
||||||
dataIndex: "version",
|
|
||||||
key: "version",
|
|
||||||
width: 80,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
title: "创建时间",
|
title: "创建时间",
|
||||||
dataIndex: "createdAt",
|
dataIndex: "createdAt",
|
||||||
@@ -263,29 +256,31 @@ const TemplateList: React.FC = () => {
|
|||||||
onClick={() => handleView(record)}
|
onClick={() => handleView(record)}
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
<>
|
{isAdmin && (
|
||||||
<Tooltip title="编辑">
|
<>
|
||||||
<Button
|
<Tooltip title="编辑">
|
||||||
type="link"
|
|
||||||
icon={<EditOutlined />}
|
|
||||||
onClick={() => handleEdit(record)}
|
|
||||||
/>
|
|
||||||
</Tooltip>
|
|
||||||
<Popconfirm
|
|
||||||
title="确定要删除这个模板吗?"
|
|
||||||
onConfirm={() => handleDelete(record.id)}
|
|
||||||
okText="确定"
|
|
||||||
cancelText="取消"
|
|
||||||
>
|
|
||||||
<Tooltip title="删除">
|
|
||||||
<Button
|
<Button
|
||||||
type="link"
|
type="link"
|
||||||
danger
|
icon={<EditOutlined />}
|
||||||
icon={<DeleteOutlined />}
|
onClick={() => handleEdit(record)}
|
||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
</Popconfirm>
|
<Popconfirm
|
||||||
</>
|
title="确定要删除这个模板吗?"
|
||||||
|
onConfirm={() => handleDelete(record.id)}
|
||||||
|
okText="确定"
|
||||||
|
cancelText="取消"
|
||||||
|
>
|
||||||
|
<Tooltip title="删除">
|
||||||
|
<Button
|
||||||
|
type="link"
|
||||||
|
danger
|
||||||
|
icon={<DeleteOutlined />}
|
||||||
|
/>
|
||||||
|
</Tooltip>
|
||||||
|
</Popconfirm>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</Space>
|
</Space>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -310,11 +305,13 @@ const TemplateList: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Right side: Create button */}
|
{/* Right side: Create button */}
|
||||||
<div className="flex items-center gap-2">
|
{isAdmin && (
|
||||||
<Button type="primary" icon={<PlusOutlined />} onClick={handleCreate}>
|
<div className="flex items-center gap-2">
|
||||||
创建模板
|
<Button type="primary" icon={<PlusOutlined />} onClick={handleCreate}>
|
||||||
</Button>
|
创建模板
|
||||||
</div>
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Card>
|
<Card>
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import {
|
|||||||
import { TagBrowser } from "./components";
|
import { TagBrowser } from "./components";
|
||||||
|
|
||||||
const { Paragraph } = Typography;
|
const { Paragraph } = Typography;
|
||||||
|
const PREVIEW_DRAWER_WIDTH = "80vw";
|
||||||
|
|
||||||
interface VisualTemplateBuilderProps {
|
interface VisualTemplateBuilderProps {
|
||||||
onSave?: (templateCode: string) => void;
|
onSave?: (templateCode: string) => void;
|
||||||
@@ -129,7 +130,7 @@ const VisualTemplateBuilder: React.FC<VisualTemplateBuilderProps> = ({
|
|||||||
<Drawer
|
<Drawer
|
||||||
title="模板代码预览"
|
title="模板代码预览"
|
||||||
placement="right"
|
placement="right"
|
||||||
width={600}
|
width={PREVIEW_DRAWER_WIDTH}
|
||||||
open={previewVisible}
|
open={previewVisible}
|
||||||
onClose={() => setPreviewVisible(false)}
|
onClose={() => setPreviewVisible(false)}
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -1,20 +1,23 @@
|
|||||||
import { get, post, put, del, download } from "@/utils/request";
|
import { get, post, put, del, download } from "@/utils/request";
|
||||||
|
|
||||||
// 导出格式类型
|
// 导出格式类型
|
||||||
export type ExportFormat = "json" | "jsonl" | "csv" | "coco" | "yolo";
|
export type ExportFormat = "json" | "jsonl" | "csv" | "coco" | "yolo";
|
||||||
|
|
||||||
// 标注任务管理相关接口
|
type RequestParams = Record<string, unknown>;
|
||||||
export function queryAnnotationTasksUsingGet(params?: any) {
|
type RequestPayload = Record<string, unknown>;
|
||||||
return get("/api/annotation/project", params);
|
|
||||||
}
|
// 标注任务管理相关接口
|
||||||
|
export function queryAnnotationTasksUsingGet(params?: RequestParams) {
|
||||||
export function createAnnotationTaskUsingPost(data: any) {
|
return get("/api/annotation/project", params);
|
||||||
return post("/api/annotation/project", data);
|
}
|
||||||
}
|
|
||||||
|
export function createAnnotationTaskUsingPost(data: RequestPayload) {
|
||||||
export function syncAnnotationTaskUsingPost(data: any) {
|
return post("/api/annotation/project", data);
|
||||||
return post(`/api/annotation/task/sync`, data);
|
}
|
||||||
}
|
|
||||||
|
export function syncAnnotationTaskUsingPost(data: RequestPayload) {
|
||||||
|
return post(`/api/annotation/task/sync`, data);
|
||||||
|
}
|
||||||
|
|
||||||
export function deleteAnnotationTaskByIdUsingDelete(mappingId: string) {
|
export function deleteAnnotationTaskByIdUsingDelete(mappingId: string) {
|
||||||
// Backend expects mapping UUID as path parameter
|
// Backend expects mapping UUID as path parameter
|
||||||
@@ -25,9 +28,9 @@ export function getAnnotationTaskByIdUsingGet(taskId: string) {
|
|||||||
return get(`/api/annotation/project/${taskId}`);
|
return get(`/api/annotation/project/${taskId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function updateAnnotationTaskByIdUsingPut(taskId: string, data: any) {
|
export function updateAnnotationTaskByIdUsingPut(taskId: string, data: RequestPayload) {
|
||||||
return put(`/api/annotation/project/${taskId}`, data);
|
return put(`/api/annotation/project/${taskId}`, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 标签配置管理
|
// 标签配置管理
|
||||||
export function getTagConfigUsingGet() {
|
export function getTagConfigUsingGet() {
|
||||||
@@ -35,20 +38,20 @@ export function getTagConfigUsingGet() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 标注模板管理
|
// 标注模板管理
|
||||||
export function queryAnnotationTemplatesUsingGet(params?: any) {
|
export function queryAnnotationTemplatesUsingGet(params?: RequestParams) {
|
||||||
return get("/api/annotation/template", params);
|
return get("/api/annotation/template", params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function createAnnotationTemplateUsingPost(data: RequestPayload) {
|
||||||
|
return post("/api/annotation/template", data);
|
||||||
|
}
|
||||||
|
|
||||||
export function createAnnotationTemplateUsingPost(data: any) {
|
export function updateAnnotationTemplateByIdUsingPut(
|
||||||
return post("/api/annotation/template", data);
|
templateId: string | number,
|
||||||
}
|
data: RequestPayload
|
||||||
|
) {
|
||||||
export function updateAnnotationTemplateByIdUsingPut(
|
return put(`/api/annotation/template/${templateId}`, data);
|
||||||
templateId: string | number,
|
}
|
||||||
data: any
|
|
||||||
) {
|
|
||||||
return put(`/api/annotation/template/${templateId}`, data);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function deleteAnnotationTemplateByIdUsingDelete(
|
export function deleteAnnotationTemplateByIdUsingDelete(
|
||||||
templateId: string | number
|
templateId: string | number
|
||||||
@@ -65,27 +68,31 @@ export function getEditorProjectInfoUsingGet(projectId: string) {
|
|||||||
return get(`/api/annotation/editor/projects/${projectId}`);
|
return get(`/api/annotation/editor/projects/${projectId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function listEditorTasksUsingGet(projectId: string, params?: any) {
|
export function listEditorTasksUsingGet(projectId: string, params?: RequestParams) {
|
||||||
return get(`/api/annotation/editor/projects/${projectId}/tasks`, params);
|
return get(`/api/annotation/editor/projects/${projectId}/tasks`, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getEditorTaskUsingGet(
|
export function getEditorTaskUsingGet(
|
||||||
projectId: string,
|
projectId: string,
|
||||||
fileId: string,
|
fileId: string,
|
||||||
params?: { segmentIndex?: number }
|
params?: { segmentIndex?: number }
|
||||||
) {
|
) {
|
||||||
return get(`/api/annotation/editor/projects/${projectId}/tasks/${fileId}`, params);
|
return get(`/api/annotation/editor/projects/${projectId}/tasks/${fileId}`, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getEditorTaskSegmentsUsingGet(projectId: string, fileId: string) {
|
||||||
|
return get(`/api/annotation/editor/projects/${projectId}/tasks/${fileId}/segments`);
|
||||||
|
}
|
||||||
|
|
||||||
export function upsertEditorAnnotationUsingPut(
|
export function upsertEditorAnnotationUsingPut(
|
||||||
projectId: string,
|
projectId: string,
|
||||||
fileId: string,
|
fileId: string,
|
||||||
data: {
|
data: {
|
||||||
annotation: any;
|
annotation: Record<string, unknown>;
|
||||||
expectedUpdatedAt?: string;
|
expectedUpdatedAt?: string;
|
||||||
segmentIndex?: number;
|
segmentIndex?: number;
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
return put(`/api/annotation/editor/projects/${projectId}/tasks/${fileId}/annotation`, data);
|
return put(`/api/annotation/editor/projects/${projectId}/tasks/${fileId}/annotation`, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,12 +116,13 @@ export function downloadAnnotationsUsingGet(
|
|||||||
projectId: string,
|
projectId: string,
|
||||||
format: ExportFormat = "json",
|
format: ExportFormat = "json",
|
||||||
onlyAnnotated: boolean = true,
|
onlyAnnotated: boolean = true,
|
||||||
includeData: boolean = false
|
includeData: boolean = false,
|
||||||
|
filename?: string
|
||||||
) {
|
) {
|
||||||
const params = new URLSearchParams({
|
const params = new URLSearchParams({
|
||||||
format,
|
format,
|
||||||
only_annotated: String(onlyAnnotated),
|
only_annotated: String(onlyAnnotated),
|
||||||
include_data: String(includeData),
|
include_data: String(includeData),
|
||||||
});
|
});
|
||||||
return download(`/api/annotation/export/projects/${projectId}/download?${params.toString()}`);
|
return download(`/api/annotation/export/projects/${projectId}/download?${params.toString()}`, null, filename);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,71 @@ import {
|
|||||||
CloseCircleOutlined,
|
CloseCircleOutlined,
|
||||||
} from "@ant-design/icons";
|
} from "@ant-design/icons";
|
||||||
|
|
||||||
|
type AnnotationTaskStatistics = {
|
||||||
|
accuracy?: number | string;
|
||||||
|
averageTime?: number | string;
|
||||||
|
reviewCount?: number | string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type AnnotationTaskPayload = {
|
||||||
|
id?: string;
|
||||||
|
labelingProjId?: string;
|
||||||
|
labelingProjectId?: string;
|
||||||
|
projId?: string;
|
||||||
|
labeling_project_id?: string;
|
||||||
|
name?: string;
|
||||||
|
description?: string;
|
||||||
|
datasetId?: string;
|
||||||
|
datasetName?: string;
|
||||||
|
dataset_name?: string;
|
||||||
|
labelingType?: string;
|
||||||
|
labeling_type?: string;
|
||||||
|
template?: {
|
||||||
|
labelingType?: string;
|
||||||
|
labeling_type?: string;
|
||||||
|
};
|
||||||
|
totalCount?: number;
|
||||||
|
total_count?: number;
|
||||||
|
annotatedCount?: number;
|
||||||
|
annotated_count?: number;
|
||||||
|
inProgressCount?: number;
|
||||||
|
in_progress_count?: number;
|
||||||
|
segmentationEnabled?: boolean;
|
||||||
|
segmentation_enabled?: boolean;
|
||||||
|
createdAt?: string;
|
||||||
|
created_at?: string;
|
||||||
|
updatedAt?: string;
|
||||||
|
updated_at?: string;
|
||||||
|
status?: string;
|
||||||
|
statistics?: AnnotationTaskStatistics;
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type AnnotationTaskListItem = {
|
||||||
|
id?: string;
|
||||||
|
labelingProjId?: string;
|
||||||
|
projId?: string;
|
||||||
|
name?: string;
|
||||||
|
description?: string;
|
||||||
|
datasetId?: string;
|
||||||
|
datasetName?: string;
|
||||||
|
labelingType?: string;
|
||||||
|
totalCount?: number;
|
||||||
|
annotatedCount?: number;
|
||||||
|
inProgressCount?: number;
|
||||||
|
segmentationEnabled?: boolean;
|
||||||
|
createdAt?: string;
|
||||||
|
updatedAt?: string;
|
||||||
|
icon?: JSX.Element;
|
||||||
|
iconColor?: string;
|
||||||
|
status?: {
|
||||||
|
label: string;
|
||||||
|
color: string;
|
||||||
|
};
|
||||||
|
statistics?: { label: string; value: string | number }[];
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
export const AnnotationTaskStatusMap = {
|
export const AnnotationTaskStatusMap = {
|
||||||
[AnnotationTaskStatus.ACTIVE]: {
|
[AnnotationTaskStatus.ACTIVE]: {
|
||||||
label: "活跃",
|
label: "活跃",
|
||||||
@@ -27,9 +92,16 @@ export const AnnotationTaskStatusMap = {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
export function mapAnnotationTask(task: any) {
|
export function mapAnnotationTask(task: AnnotationTaskPayload): AnnotationTaskListItem {
|
||||||
// Normalize labeling project id from possible backend field names
|
// Normalize labeling project id from possible backend field names
|
||||||
const labelingProjId = task?.labelingProjId || task?.labelingProjectId || task?.projId || task?.labeling_project_id || "";
|
const labelingProjId = task?.labelingProjId || task?.labelingProjectId || task?.projId || task?.labeling_project_id || "";
|
||||||
|
const segmentationEnabled = task?.segmentationEnabled ?? task?.segmentation_enabled ?? false;
|
||||||
|
const inProgressCount = task?.inProgressCount ?? task?.in_progress_count ?? 0;
|
||||||
|
const labelingType =
|
||||||
|
task?.labelingType ||
|
||||||
|
task?.labeling_type ||
|
||||||
|
task?.template?.labelingType ||
|
||||||
|
task?.template?.labeling_type;
|
||||||
|
|
||||||
const statsArray = task?.statistics
|
const statsArray = task?.statistics
|
||||||
? [
|
? [
|
||||||
@@ -45,6 +117,9 @@ export function mapAnnotationTask(task: any) {
|
|||||||
// provide consistent field for components
|
// provide consistent field for components
|
||||||
labelingProjId,
|
labelingProjId,
|
||||||
projId: labelingProjId,
|
projId: labelingProjId,
|
||||||
|
segmentationEnabled,
|
||||||
|
inProgressCount,
|
||||||
|
labelingType,
|
||||||
name: task.name,
|
name: task.name,
|
||||||
description: task.description || "",
|
description: task.description || "",
|
||||||
datasetName: task.datasetName || task.dataset_name || "-",
|
datasetName: task.datasetName || task.dataset_name || "-",
|
||||||
@@ -478,4 +553,4 @@ export const TemplateTypeMap = {
|
|||||||
label: "自定义",
|
label: "自定义",
|
||||||
value: TemplateType.CUSTOM
|
value: TemplateType.CUSTOM
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,13 @@ export enum AnnotationTaskStatus {
|
|||||||
SKIPPED = "skipped",
|
SKIPPED = "skipped",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export enum AnnotationResultStatus {
|
||||||
|
ANNOTATED = "ANNOTATED",
|
||||||
|
IN_PROGRESS = "IN_PROGRESS",
|
||||||
|
NO_ANNOTATION = "NO_ANNOTATION",
|
||||||
|
NOT_APPLICABLE = "NOT_APPLICABLE",
|
||||||
|
}
|
||||||
|
|
||||||
export interface AnnotationTask {
|
export interface AnnotationTask {
|
||||||
id: string;
|
id: string;
|
||||||
name: string;
|
name: string;
|
||||||
@@ -52,7 +59,7 @@ export interface ObjectDefinition {
|
|||||||
export interface TemplateConfiguration {
|
export interface TemplateConfiguration {
|
||||||
labels: LabelDefinition[];
|
labels: LabelDefinition[];
|
||||||
objects: ObjectDefinition[];
|
objects: ObjectDefinition[];
|
||||||
metadata?: Record<string, any>;
|
metadata?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AnnotationTemplate {
|
export interface AnnotationTemplate {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import {
|
|||||||
getObjectDisplayName,
|
getObjectDisplayName,
|
||||||
type LabelStudioTagConfig,
|
type LabelStudioTagConfig,
|
||||||
} from "../annotation.tagconfig";
|
} from "../annotation.tagconfig";
|
||||||
|
import { DataType } from "../annotation.model";
|
||||||
|
|
||||||
const { Text, Title } = Typography;
|
const { Text, Title } = Typography;
|
||||||
|
|
||||||
@@ -44,10 +45,22 @@ interface TemplateConfigurationTreeEditorProps {
|
|||||||
readOnly?: boolean;
|
readOnly?: boolean;
|
||||||
readOnlyStructure?: boolean;
|
readOnlyStructure?: boolean;
|
||||||
height?: number | string;
|
height?: number | string;
|
||||||
|
dataType?: DataType;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_ROOT_TAG = "View";
|
const DEFAULT_ROOT_TAG = "View";
|
||||||
const CHILD_TAGS = ["Label", "Choice", "Relation", "Item", "Path", "Channel"];
|
const CHILD_TAGS = ["Label", "Choice", "Relation", "Item", "Path", "Channel"];
|
||||||
|
const OBJECT_TAGS_BY_DATA_TYPE: Record<DataType, string[]> = {
|
||||||
|
[DataType.TEXT]: ["Text", "Paragraphs", "Markdown"],
|
||||||
|
[DataType.IMAGE]: ["Image", "Bitmask"],
|
||||||
|
[DataType.AUDIO]: ["Audio", "AudioPlus"],
|
||||||
|
[DataType.VIDEO]: ["Video"],
|
||||||
|
[DataType.PDF]: ["PDF"],
|
||||||
|
[DataType.TIMESERIES]: ["Timeseries", "TimeSeries", "Vector"],
|
||||||
|
[DataType.CHAT]: ["Chat"],
|
||||||
|
[DataType.HTML]: ["HyperText", "Markdown"],
|
||||||
|
[DataType.TABLE]: ["Table", "Vector"],
|
||||||
|
};
|
||||||
|
|
||||||
const createId = () =>
|
const createId = () =>
|
||||||
`node_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
`node_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
||||||
@@ -247,18 +260,34 @@ const createNode = (
|
|||||||
attrs[attr] = "";
|
attrs[attr] = "";
|
||||||
});
|
});
|
||||||
|
|
||||||
if (objectConfig && attrs.name !== undefined) {
|
if (objectConfig) {
|
||||||
const name = getDefaultName(tag);
|
const name = getDefaultName(tag);
|
||||||
attrs.name = name;
|
if (!attrs.name) {
|
||||||
if (attrs.value !== undefined) {
|
attrs.name = name;
|
||||||
attrs.value = `$${name}`;
|
}
|
||||||
|
if (!attrs.value) {
|
||||||
|
attrs.value = `$${attrs.name}`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (controlConfig && attrs.name !== undefined) {
|
if (controlConfig) {
|
||||||
attrs.name = getDefaultName(tag);
|
const isLabeling = controlConfig.category === "labeling";
|
||||||
if (attrs.toName !== undefined) {
|
|
||||||
attrs.toName = objectNames[0] || "";
|
if (isLabeling) {
|
||||||
|
if (!attrs.name) {
|
||||||
|
attrs.name = getDefaultName(tag);
|
||||||
|
}
|
||||||
|
if (!attrs.toName) {
|
||||||
|
attrs.toName = objectNames[0] || "";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// For layout controls, only fill if required
|
||||||
|
if (attrs.name !== undefined && !attrs.name) {
|
||||||
|
attrs.name = getDefaultName(tag);
|
||||||
|
}
|
||||||
|
if (attrs.toName !== undefined && !attrs.toName) {
|
||||||
|
attrs.toName = objectNames[0] || "";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -420,14 +449,13 @@ const TemplateConfigurationTreeEditor = ({
|
|||||||
readOnly = false,
|
readOnly = false,
|
||||||
readOnlyStructure = false,
|
readOnlyStructure = false,
|
||||||
height = 420,
|
height = 420,
|
||||||
|
dataType,
|
||||||
}: TemplateConfigurationTreeEditorProps) => {
|
}: TemplateConfigurationTreeEditorProps) => {
|
||||||
const { config } = useTagConfig(false);
|
const { config } = useTagConfig(false);
|
||||||
const [tree, setTree] = useState<XmlNode>(() => createEmptyTree());
|
const [tree, setTree] = useState<XmlNode>(() => createEmptyTree());
|
||||||
const [selectedId, setSelectedId] = useState<string>(tree.id);
|
const [selectedId, setSelectedId] = useState<string>(tree.id);
|
||||||
const [parseError, setParseError] = useState<string | null>(null);
|
const [parseError, setParseError] = useState<string | null>(null);
|
||||||
const lastSerialized = useRef<string>("");
|
const lastSerialized = useRef<string>("");
|
||||||
const [addChildTag, setAddChildTag] = useState<string | undefined>();
|
|
||||||
const [addSiblingTag, setAddSiblingTag] = useState<string | undefined>();
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!value) {
|
if (!value) {
|
||||||
@@ -498,11 +526,17 @@ const TemplateConfigurationTreeEditor = ({
|
|||||||
|
|
||||||
const objectOptions = useMemo(() => {
|
const objectOptions = useMemo(() => {
|
||||||
if (!config?.objects) return [];
|
if (!config?.objects) return [];
|
||||||
return Object.keys(config.objects).map((tag) => ({
|
const options = Object.keys(config.objects).map((tag) => ({
|
||||||
value: tag,
|
value: tag,
|
||||||
label: getObjectDisplayName(tag),
|
label: getObjectDisplayName(tag),
|
||||||
}));
|
}));
|
||||||
}, [config]);
|
if (!dataType) return options;
|
||||||
|
const allowedTags = OBJECT_TAGS_BY_DATA_TYPE[dataType];
|
||||||
|
if (!allowedTags) return options;
|
||||||
|
const allowedSet = new Set(allowedTags);
|
||||||
|
const filtered = options.filter((option) => allowedSet.has(option.value));
|
||||||
|
return filtered.length > 0 ? filtered : options;
|
||||||
|
}, [config, dataType]);
|
||||||
|
|
||||||
const tagOptions = useMemo(() => {
|
const tagOptions = useMemo(() => {
|
||||||
const options = [] as {
|
const options = [] as {
|
||||||
@@ -763,9 +797,8 @@ const TemplateConfigurationTreeEditor = ({
|
|||||||
<Select
|
<Select
|
||||||
placeholder="添加子节点"
|
placeholder="添加子节点"
|
||||||
options={tagOptions}
|
options={tagOptions}
|
||||||
value={addChildTag}
|
value={null}
|
||||||
onChange={(value) => {
|
onChange={(value) => {
|
||||||
setAddChildTag(undefined);
|
|
||||||
handleAddNode(value, "child");
|
handleAddNode(value, "child");
|
||||||
}}
|
}}
|
||||||
disabled={isStructureLocked}
|
disabled={isStructureLocked}
|
||||||
@@ -773,9 +806,8 @@ const TemplateConfigurationTreeEditor = ({
|
|||||||
<Select
|
<Select
|
||||||
placeholder="添加同级节点"
|
placeholder="添加同级节点"
|
||||||
options={tagOptions}
|
options={tagOptions}
|
||||||
value={addSiblingTag}
|
value={null}
|
||||||
onChange={(value) => {
|
onChange={(value) => {
|
||||||
setAddSiblingTag(undefined);
|
|
||||||
handleAddNode(value, "sibling");
|
handleAddNode(value, "sibling");
|
||||||
}}
|
}}
|
||||||
disabled={isStructureLocked || selectedNode.id === tree.id}
|
disabled={isStructureLocked || selectedNode.id === tree.id}
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ interface PreviewPromptModalProps {
|
|||||||
evaluationPrompt: string;
|
evaluationPrompt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PREVIEW_MODAL_WIDTH = "80vw";
|
||||||
|
|
||||||
const PreviewPromptModal: React.FC<PreviewPromptModalProps> = ({ previewVisible, onCancel, evaluationPrompt }) => {
|
const PreviewPromptModal: React.FC<PreviewPromptModalProps> = ({ previewVisible, onCancel, evaluationPrompt }) => {
|
||||||
return (
|
return (
|
||||||
<Modal
|
<Modal
|
||||||
@@ -24,7 +26,7 @@ const PreviewPromptModal: React.FC<PreviewPromptModalProps> = ({ previewVisible,
|
|||||||
关闭
|
关闭
|
||||||
</Button>
|
</Button>
|
||||||
]}
|
]}
|
||||||
width={800}
|
width={PREVIEW_MODAL_WIDTH}
|
||||||
>
|
>
|
||||||
<div style={{
|
<div style={{
|
||||||
background: '#f5f5f5',
|
background: '#f5f5f5',
|
||||||
|
|||||||
@@ -78,7 +78,11 @@ export default function DatasetCreate() {
|
|||||||
onValuesChange={handleValuesChange}
|
onValuesChange={handleValuesChange}
|
||||||
layout="vertical"
|
layout="vertical"
|
||||||
>
|
>
|
||||||
<BasicInformation data={newDataset} setData={setNewDataset} />
|
<BasicInformation
|
||||||
|
data={newDataset}
|
||||||
|
setData={setNewDataset}
|
||||||
|
hidden={["dataSource"]}
|
||||||
|
/>
|
||||||
</Form>
|
</Form>
|
||||||
</div>
|
</div>
|
||||||
<div className="flex gap-2 justify-end p-6 border-top">
|
<div className="flex gap-2 justify-end p-6 border-top">
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ export default function EditDataset({
|
|||||||
<BasicInformation
|
<BasicInformation
|
||||||
data={newDataset}
|
data={newDataset}
|
||||||
setData={setNewDataset}
|
setData={setNewDataset}
|
||||||
hidden={["datasetType"]}
|
hidden={["datasetType", "dataSource"]}
|
||||||
/>
|
/>
|
||||||
</Form>
|
</Form>
|
||||||
</Modal>
|
</Modal>
|
||||||
|
|||||||
@@ -11,10 +11,12 @@ export default function BasicInformation({
|
|||||||
data,
|
data,
|
||||||
setData,
|
setData,
|
||||||
hidden = [],
|
hidden = [],
|
||||||
|
datasetTypeOptions = datasetTypes,
|
||||||
}: {
|
}: {
|
||||||
data: DatasetFormData;
|
data: DatasetFormData;
|
||||||
setData: Dispatch<SetStateAction<DatasetFormData>>;
|
setData: Dispatch<SetStateAction<DatasetFormData>>;
|
||||||
hidden?: string[];
|
hidden?: string[];
|
||||||
|
datasetTypeOptions?: DatasetTypeOption[];
|
||||||
}) {
|
}) {
|
||||||
const [tagOptions, setTagOptions] = useState<DatasetTagOption[]>([]);
|
const [tagOptions, setTagOptions] = useState<DatasetTagOption[]>([]);
|
||||||
const [collectionOptions, setCollectionOptions] = useState<SelectOption[]>([]);
|
const [collectionOptions, setCollectionOptions] = useState<SelectOption[]>([]);
|
||||||
@@ -39,6 +41,7 @@ export default function BasicInformation({
|
|||||||
|
|
||||||
// 获取归集任务
|
// 获取归集任务
|
||||||
const fetchCollectionTasks = useCallback(async () => {
|
const fetchCollectionTasks = useCallback(async () => {
|
||||||
|
if (hidden.includes("dataSource")) return;
|
||||||
try {
|
try {
|
||||||
const res = await queryTasksUsingGet({ page: 0, size: 100 });
|
const res = await queryTasksUsingGet({ page: 0, size: 100 });
|
||||||
const tasks = Array.isArray(res?.data?.content)
|
const tasks = Array.isArray(res?.data?.content)
|
||||||
@@ -52,7 +55,7 @@ export default function BasicInformation({
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error fetching collection tasks:", error);
|
console.error("Error fetching collection tasks:", error);
|
||||||
}
|
}
|
||||||
}, []);
|
}, [hidden]);
|
||||||
|
|
||||||
const fetchParentDatasets = useCallback(async () => {
|
const fetchParentDatasets = useCallback(async () => {
|
||||||
if (hidden.includes("parentDatasetId")) return;
|
if (hidden.includes("parentDatasetId")) return;
|
||||||
@@ -73,7 +76,7 @@ export default function BasicInformation({
|
|||||||
value: dataset.id,
|
value: dataset.id,
|
||||||
}));
|
}));
|
||||||
setParentDatasetOptions([
|
setParentDatasetOptions([
|
||||||
{ label: "根数据集", value: "" },
|
{ label: "无关联数据集", value: "" },
|
||||||
...options,
|
...options,
|
||||||
]);
|
]);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -101,11 +104,11 @@ export default function BasicInformation({
|
|||||||
</Form.Item>
|
</Form.Item>
|
||||||
)}
|
)}
|
||||||
{!hidden.includes("parentDatasetId") && (
|
{!hidden.includes("parentDatasetId") && (
|
||||||
<Form.Item name="parentDatasetId" label="父数据集">
|
<Form.Item name="parentDatasetId" label="关联数据集">
|
||||||
<Select
|
<Select
|
||||||
className="w-full"
|
className="w-full"
|
||||||
options={parentDatasetOptions}
|
options={parentDatasetOptions}
|
||||||
placeholder="选择父数据集(仅支持一层)"
|
placeholder="选择关联数据集(仅支持一层)"
|
||||||
/>
|
/>
|
||||||
</Form.Item>
|
</Form.Item>
|
||||||
)}
|
)}
|
||||||
@@ -118,7 +121,7 @@ export default function BasicInformation({
|
|||||||
rules={[{ required: true, message: "请选择数据集类型" }]}
|
rules={[{ required: true, message: "请选择数据集类型" }]}
|
||||||
>
|
>
|
||||||
<RadioCard
|
<RadioCard
|
||||||
options={datasetTypes}
|
options={datasetTypeOptions}
|
||||||
value={data.type}
|
value={data.type}
|
||||||
onChange={(datasetType) => setData({ ...data, datasetType })}
|
onChange={(datasetType) => setData({ ...data, datasetType })}
|
||||||
/>
|
/>
|
||||||
@@ -148,6 +151,8 @@ type DatasetFormData = Partial<Dataset> & {
|
|||||||
parentDatasetId?: string;
|
parentDatasetId?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type DatasetTypeOption = (typeof datasetTypes)[number];
|
||||||
|
|
||||||
type DatasetTagOption = {
|
type DatasetTagOption = {
|
||||||
label: string;
|
label: string;
|
||||||
value: string;
|
value: string;
|
||||||
|
|||||||
@@ -1,195 +1,216 @@
|
|||||||
import { useEffect, useMemo, useRef, useState } from "react";
|
import { useEffect, useMemo, useRef, useState } from "react";
|
||||||
import { Breadcrumb, App, Tabs, Table, Tag } from "antd";
|
import { Breadcrumb, App, Tabs, Table, Tag } from "antd";
|
||||||
import {
|
import {
|
||||||
ReloadOutlined,
|
ReloadOutlined,
|
||||||
DownloadOutlined,
|
DownloadOutlined,
|
||||||
EditOutlined,
|
EditOutlined,
|
||||||
DeleteOutlined,
|
DeleteOutlined,
|
||||||
PlusOutlined,
|
PlusOutlined,
|
||||||
} from "@ant-design/icons";
|
} from "@ant-design/icons";
|
||||||
import DetailHeader from "@/components/DetailHeader";
|
import DetailHeader from "@/components/DetailHeader";
|
||||||
import { mapDataset, datasetTypeMap } from "../dataset.const";
|
import { mapDataset, datasetTypeMap } from "../dataset.const";
|
||||||
import type { Dataset } from "@/pages/DataManagement/dataset.model";
|
import type { Dataset } from "@/pages/DataManagement/dataset.model";
|
||||||
import { Link, useNavigate, useParams } from "react-router";
|
import { Link, useNavigate, useParams } from "react-router";
|
||||||
import { useFilesOperation } from "./useFilesOperation";
|
import { useFilesOperation } from "./useFilesOperation";
|
||||||
import {
|
import {
|
||||||
createDatasetTagUsingPost,
|
createDatasetTagUsingPost,
|
||||||
deleteDatasetByIdUsingDelete,
|
deleteDatasetByIdUsingDelete,
|
||||||
downloadDatasetUsingGet,
|
downloadDatasetUsingGet,
|
||||||
queryDatasetByIdUsingGet,
|
queryDatasetByIdUsingGet,
|
||||||
queryDatasetsUsingGet,
|
queryDatasetsUsingGet,
|
||||||
queryDatasetTagsUsingGet,
|
queryDatasetTagsUsingGet,
|
||||||
querySimilarDatasetsUsingGet,
|
querySimilarDatasetsUsingGet,
|
||||||
updateDatasetByIdUsingPut,
|
updateDatasetByIdUsingPut,
|
||||||
} from "../dataset.api";
|
} from "../dataset.api";
|
||||||
import DataQuality from "./components/DataQuality";
|
import DataQuality from "./components/DataQuality";
|
||||||
import DataLineageFlow from "./components/DataLineageFlow";
|
import DataLineageFlow from "./components/DataLineageFlow";
|
||||||
import Overview from "./components/Overview";
|
import Overview from "./components/Overview";
|
||||||
import { Activity, Clock, File, FileType } from "lucide-react";
|
import { Activity, Clock, File, FileType } from "lucide-react";
|
||||||
import EditDataset from "../Create/EditDataset";
|
import EditDataset from "../Create/EditDataset";
|
||||||
import ImportConfiguration from "./components/ImportConfiguration";
|
import ImportConfiguration from "./components/ImportConfiguration";
|
||||||
|
import CardView from "@/components/CardView";
|
||||||
const SIMILAR_DATASET_LIMIT = 4;
|
|
||||||
|
|
||||||
export default function DatasetDetail() {
|
|
||||||
const { id } = useParams(); // 获取动态路由参数
|
|
||||||
const navigate = useNavigate();
|
|
||||||
const [activeTab, setActiveTab] = useState("overview");
|
|
||||||
const { message } = App.useApp();
|
|
||||||
const [showEditDialog, setShowEditDialog] = useState(false);
|
|
||||||
|
|
||||||
const [dataset, setDataset] = useState<Dataset>({} as Dataset);
|
|
||||||
const [parentDataset, setParentDataset] = useState<Dataset | null>(null);
|
|
||||||
const [childDatasets, setChildDatasets] = useState<Dataset[]>([]);
|
|
||||||
const [childDatasetsLoading, setChildDatasetsLoading] = useState(false);
|
|
||||||
const [similarDatasets, setSimilarDatasets] = useState<Dataset[]>([]);
|
|
||||||
const [similarDatasetsLoading, setSimilarDatasetsLoading] = useState(false);
|
|
||||||
const [similarTagNames, setSimilarTagNames] = useState<string[]>([]);
|
|
||||||
const similarRequestRef = useRef(0);
|
|
||||||
const filesOperation = useFilesOperation(dataset);
|
|
||||||
|
|
||||||
const [showUploadDialog, setShowUploadDialog] = useState(false);
|
const SIMILAR_DATASET_LIMIT = 4;
|
||||||
const normalizeTagNames = (
|
const SIMILAR_TAGS_PREVIEW_LIMIT = 3;
|
||||||
tags?: Array<string | { name?: string | null } | null>
|
|
||||||
) => {
|
|
||||||
if (!tags || tags.length === 0) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
const names = tags
|
|
||||||
.map((tag) => (typeof tag === "string" ? tag : tag?.name))
|
|
||||||
.filter((name): name is string => !!name && name.trim().length > 0)
|
|
||||||
.map((name) => name.trim());
|
|
||||||
return Array.from(new Set(names));
|
|
||||||
};
|
|
||||||
const fetchSimilarDatasets = async (currentDataset: Dataset) => {
|
|
||||||
const requestId = similarRequestRef.current + 1;
|
|
||||||
similarRequestRef.current = requestId;
|
|
||||||
if (!currentDataset?.id) {
|
|
||||||
setSimilarDatasets([]);
|
|
||||||
setSimilarTagNames([]);
|
|
||||||
setSimilarDatasetsLoading(false);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const tagNames = normalizeTagNames(
|
|
||||||
currentDataset.tags as Array<string | { name?: string }>
|
|
||||||
);
|
|
||||||
setSimilarTagNames(tagNames);
|
|
||||||
setSimilarDatasets([]);
|
|
||||||
if (tagNames.length === 0) {
|
|
||||||
setSimilarDatasetsLoading(false);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
setSimilarDatasetsLoading(true);
|
|
||||||
try {
|
|
||||||
const { data } = await querySimilarDatasetsUsingGet(currentDataset.id, {
|
|
||||||
limit: SIMILAR_DATASET_LIMIT,
|
|
||||||
});
|
|
||||||
if (similarRequestRef.current !== requestId) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const list = Array.isArray(data) ? data : [];
|
|
||||||
setSimilarDatasets(list.map((item) => mapDataset(item)));
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Failed to fetch similar datasets:", error);
|
|
||||||
} finally {
|
|
||||||
if (similarRequestRef.current === requestId) {
|
|
||||||
setSimilarDatasetsLoading(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
const navigateItems = useMemo(() => {
|
|
||||||
const items = [
|
|
||||||
{
|
|
||||||
title: <Link to="/data/management">数据管理</Link>,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
if (parentDataset) {
|
|
||||||
items.push({
|
|
||||||
title: (
|
|
||||||
<Link to={`/data/management/detail/${parentDataset.id}`}>
|
|
||||||
{parentDataset.name}
|
|
||||||
</Link>
|
|
||||||
),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
items.push({
|
|
||||||
title: dataset.name || "数据集详情",
|
|
||||||
});
|
|
||||||
return items;
|
|
||||||
}, [dataset, parentDataset]);
|
|
||||||
const tabList = useMemo(() => {
|
|
||||||
const items = [
|
|
||||||
{
|
|
||||||
key: "overview",
|
|
||||||
label: "概览",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
if (!dataset?.parentDatasetId) {
|
|
||||||
items.push({
|
|
||||||
key: "children",
|
|
||||||
label: "子数据集",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return items;
|
|
||||||
}, [dataset?.parentDatasetId]);
|
|
||||||
const handleCreateChildDataset = () => {
|
|
||||||
if (!dataset?.id) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
navigate("/data/management/create", {
|
|
||||||
state: { parentDatasetId: dataset.id },
|
|
||||||
});
|
|
||||||
};
|
|
||||||
const fetchChildDatasets = async (parentId?: string) => {
|
|
||||||
if (!parentId) {
|
|
||||||
setChildDatasets([]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
setChildDatasetsLoading(true);
|
|
||||||
try {
|
|
||||||
const { data: res } = await queryDatasetsUsingGet({
|
|
||||||
parentDatasetId: parentId,
|
|
||||||
page: 1,
|
|
||||||
size: 1000,
|
|
||||||
});
|
|
||||||
const list = res?.content || res?.data || [];
|
|
||||||
setChildDatasets(list.map((item) => mapDataset(item)));
|
|
||||||
} finally {
|
|
||||||
setChildDatasetsLoading(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
const fetchDataset = async () => {
|
|
||||||
if (!id) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const { data } = await queryDatasetByIdUsingGet(id);
|
|
||||||
const mapped = mapDataset(data);
|
|
||||||
setDataset(mapped);
|
|
||||||
fetchSimilarDatasets(mapped);
|
|
||||||
if (data?.parentDatasetId) {
|
|
||||||
const { data: parentData } = await queryDatasetByIdUsingGet(
|
|
||||||
data.parentDatasetId
|
|
||||||
);
|
|
||||||
setParentDataset(mapDataset(parentData));
|
|
||||||
setChildDatasets([]);
|
|
||||||
} else {
|
|
||||||
setParentDataset(null);
|
|
||||||
await fetchChildDatasets(data?.id);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
useEffect(() => {
|
export default function DatasetDetail() {
|
||||||
if (!id) {
|
const { id } = useParams(); // 获取动态路由参数
|
||||||
return;
|
const navigate = useNavigate();
|
||||||
}
|
const [activeTab, setActiveTab] = useState("overview");
|
||||||
fetchDataset();
|
const { message } = App.useApp();
|
||||||
filesOperation.fetchFiles("", 1, 10); // 从根目录开始,第一页
|
const [showEditDialog, setShowEditDialog] = useState(false);
|
||||||
}, [id]);
|
|
||||||
useEffect(() => {
|
const [dataset, setDataset] = useState<Dataset>({} as Dataset);
|
||||||
if (dataset?.parentDatasetId && activeTab === "children") {
|
const [parentDataset, setParentDataset] = useState<Dataset | null>(null);
|
||||||
setActiveTab("overview");
|
const [childDatasets, setChildDatasets] = useState<Dataset[]>([]);
|
||||||
}
|
const [childDatasetsLoading, setChildDatasetsLoading] = useState(false);
|
||||||
}, [activeTab, dataset?.parentDatasetId]);
|
const [similarDatasets, setSimilarDatasets] = useState<Dataset[]>([]);
|
||||||
|
const [similarDatasetsLoading, setSimilarDatasetsLoading] = useState(false);
|
||||||
|
const [similarTagNames, setSimilarTagNames] = useState<string[]>([]);
|
||||||
|
const similarRequestRef = useRef(0);
|
||||||
|
const filesOperation = useFilesOperation(dataset);
|
||||||
|
|
||||||
|
const [showUploadDialog, setShowUploadDialog] = useState(false);
|
||||||
|
const normalizeTagNames = (
|
||||||
|
tags?: Array<string | { name?: string | null } | null>
|
||||||
|
) => {
|
||||||
|
if (!tags || tags.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const names = tags
|
||||||
|
.map((tag) => (typeof tag === "string" ? tag : tag?.name))
|
||||||
|
.filter((name): name is string => !!name && name.trim().length > 0)
|
||||||
|
.map((name) => name.trim());
|
||||||
|
return Array.from(new Set(names));
|
||||||
|
};
|
||||||
|
const fetchSimilarDatasets = async (currentDataset: Dataset) => {
|
||||||
|
const requestId = similarRequestRef.current + 1;
|
||||||
|
similarRequestRef.current = requestId;
|
||||||
|
if (!currentDataset?.id) {
|
||||||
|
setSimilarDatasets([]);
|
||||||
|
setSimilarTagNames([]);
|
||||||
|
setSimilarDatasetsLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const tagNames = normalizeTagNames(
|
||||||
|
currentDataset.tags as Array<string | { name?: string }>
|
||||||
|
);
|
||||||
|
setSimilarTagNames(tagNames);
|
||||||
|
setSimilarDatasets([]);
|
||||||
|
if (tagNames.length === 0) {
|
||||||
|
setSimilarDatasetsLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setSimilarDatasetsLoading(true);
|
||||||
|
try {
|
||||||
|
const { data } = await querySimilarDatasetsUsingGet(currentDataset.id, {
|
||||||
|
limit: SIMILAR_DATASET_LIMIT,
|
||||||
|
});
|
||||||
|
if (similarRequestRef.current !== requestId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const list = Array.isArray(data) ? data : [];
|
||||||
|
setSimilarDatasets(list.map((item) => mapDataset(item)));
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to fetch similar datasets:", error);
|
||||||
|
} finally {
|
||||||
|
if (similarRequestRef.current === requestId) {
|
||||||
|
setSimilarDatasetsLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const similarTagsSummary = useMemo(() => {
|
||||||
|
if (!similarTagNames || similarTagNames.length === 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
const visibleTags = similarTagNames.slice(0, SIMILAR_TAGS_PREVIEW_LIMIT);
|
||||||
|
const hiddenCount = similarTagNames.length - visibleTags.length;
|
||||||
|
if (hiddenCount > 0) {
|
||||||
|
return `${visibleTags.join("、")} 等 ${similarTagNames.length} 个`;
|
||||||
|
}
|
||||||
|
return visibleTags.join("、");
|
||||||
|
}, [similarTagNames]);
|
||||||
|
|
||||||
|
const navigateItems = useMemo(() => {
|
||||||
|
const items = [
|
||||||
|
{
|
||||||
|
title: <Link to="/data/management">数据管理</Link>,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
if (parentDataset) {
|
||||||
|
items.push({
|
||||||
|
title: (
|
||||||
|
<Link to={`/data/management/detail/${parentDataset.id}`}>
|
||||||
|
{parentDataset.name}
|
||||||
|
</Link>
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
items.push({
|
||||||
|
title: dataset.name || "数据集详情",
|
||||||
|
});
|
||||||
|
return items;
|
||||||
|
}, [dataset, parentDataset]);
|
||||||
|
const tabList = useMemo(() => {
|
||||||
|
const items = [
|
||||||
|
{
|
||||||
|
key: "overview",
|
||||||
|
label: "概览",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
if (!dataset?.parentDatasetId) {
|
||||||
|
items.push({
|
||||||
|
key: "children",
|
||||||
|
label: "关联数据集",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return items;
|
||||||
|
}, [dataset?.parentDatasetId]);
|
||||||
|
const handleCreateChildDataset = () => {
|
||||||
|
if (!dataset?.id) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
navigate("/data/management/create", {
|
||||||
|
state: { parentDatasetId: dataset.id },
|
||||||
|
});
|
||||||
|
};
|
||||||
|
const fetchChildDatasets = async (parentId?: string) => {
|
||||||
|
if (!parentId) {
|
||||||
|
setChildDatasets([]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setChildDatasetsLoading(true);
|
||||||
|
try {
|
||||||
|
const { data: res } = await queryDatasetsUsingGet({
|
||||||
|
parentDatasetId: parentId,
|
||||||
|
page: 1,
|
||||||
|
size: 1000,
|
||||||
|
});
|
||||||
|
const list = res?.content || res?.data || [];
|
||||||
|
setChildDatasets(list.map((item) => mapDataset(item)));
|
||||||
|
} finally {
|
||||||
|
setChildDatasetsLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const fetchDataset = async () => {
|
||||||
|
if (!id) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { data } = await queryDatasetByIdUsingGet(id);
|
||||||
|
const mapped = mapDataset(data);
|
||||||
|
setDataset(mapped);
|
||||||
|
fetchSimilarDatasets(mapped);
|
||||||
|
if (data?.parentDatasetId) {
|
||||||
|
const { data: parentData } = await queryDatasetByIdUsingGet(
|
||||||
|
data.parentDatasetId
|
||||||
|
);
|
||||||
|
setParentDataset(mapDataset(parentData));
|
||||||
|
setChildDatasets([]);
|
||||||
|
} else {
|
||||||
|
setParentDataset(null);
|
||||||
|
await fetchChildDatasets(data?.id);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!id) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
fetchDataset();
|
||||||
|
}, [id]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (dataset?.id) {
|
||||||
|
filesOperation.fetchFiles("", 1, 10); // 从根目录开始,第一页
|
||||||
|
}
|
||||||
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
|
}, [dataset?.id]);
|
||||||
|
useEffect(() => {
|
||||||
|
if (dataset?.parentDatasetId && activeTab === "children") {
|
||||||
|
setActiveTab("overview");
|
||||||
|
}
|
||||||
|
}, [activeTab, dataset?.parentDatasetId]);
|
||||||
|
|
||||||
const handleRefresh = async (showMessage = true, prefixOverride?: string) => {
|
const handleRefresh = async (showMessage = true, prefixOverride?: string) => {
|
||||||
fetchDataset();
|
fetchDataset();
|
||||||
@@ -261,22 +282,22 @@ export default function DatasetDetail() {
|
|||||||
];
|
];
|
||||||
|
|
||||||
// 数据集操作列表
|
// 数据集操作列表
|
||||||
const operations = [
|
const operations = [
|
||||||
...(dataset?.id && !dataset.parentDatasetId
|
...(dataset?.id && !dataset.parentDatasetId
|
||||||
? [
|
? [
|
||||||
{
|
{
|
||||||
key: "create-child",
|
key: "create-child",
|
||||||
label: "创建子数据集",
|
label: "创建关联数据集",
|
||||||
icon: <PlusOutlined />,
|
icon: <PlusOutlined />,
|
||||||
onClick: handleCreateChildDataset,
|
onClick: handleCreateChildDataset,
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
: []),
|
: []),
|
||||||
{
|
{
|
||||||
key: "edit",
|
key: "edit",
|
||||||
label: "编辑",
|
label: "编辑",
|
||||||
icon: <EditOutlined />,
|
icon: <EditOutlined />,
|
||||||
onClick: () => {
|
onClick: () => {
|
||||||
setShowEditDialog(true);
|
setShowEditDialog(true);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -314,55 +335,55 @@ export default function DatasetDetail() {
|
|||||||
icon: <DeleteOutlined />,
|
icon: <DeleteOutlined />,
|
||||||
onClick: handleDeleteDataset,
|
onClick: handleDeleteDataset,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const childColumns = [
|
const childColumns = [
|
||||||
{
|
{
|
||||||
title: "名称",
|
title: "名称",
|
||||||
dataIndex: "name",
|
dataIndex: "name",
|
||||||
key: "name",
|
key: "name",
|
||||||
render: (_: string, record: Dataset) => (
|
render: (_: string, record: Dataset) => (
|
||||||
<Link to={`/data/management/detail/${record.id}`}>{record.name}</Link>
|
<Link to={`/data/management/detail/${record.id}`}>{record.name}</Link>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: "类型",
|
title: "类型",
|
||||||
dataIndex: "datasetType",
|
dataIndex: "datasetType",
|
||||||
key: "datasetType",
|
key: "datasetType",
|
||||||
width: 120,
|
width: 120,
|
||||||
render: (value: string) => datasetTypeMap[value]?.label || "未知",
|
render: (value: string) => datasetTypeMap[value]?.label || "未知",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: "状态",
|
title: "状态",
|
||||||
dataIndex: "status",
|
dataIndex: "status",
|
||||||
key: "status",
|
key: "status",
|
||||||
width: 120,
|
width: 120,
|
||||||
render: (status) =>
|
render: (status) =>
|
||||||
status ? <Tag color={status.color}>{status.label}</Tag> : "-",
|
status ? <Tag color={status.color}>{status.label}</Tag> : "-",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: "文件数",
|
title: "文件数",
|
||||||
dataIndex: "fileCount",
|
dataIndex: "fileCount",
|
||||||
key: "fileCount",
|
key: "fileCount",
|
||||||
width: 120,
|
width: 120,
|
||||||
render: (value?: number) => value ?? 0,
|
render: (value?: number) => value ?? 0,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: "大小",
|
title: "大小",
|
||||||
dataIndex: "size",
|
dataIndex: "size",
|
||||||
key: "size",
|
key: "size",
|
||||||
width: 140,
|
width: 140,
|
||||||
render: (value?: string) => value || "0 B",
|
render: (value?: string) => value || "0 B",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: "更新时间",
|
title: "更新时间",
|
||||||
dataIndex: "updatedAt",
|
dataIndex: "updatedAt",
|
||||||
key: "updatedAt",
|
key: "updatedAt",
|
||||||
width: 180,
|
width: 180,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="h-full flex flex-col gap-4">
|
<div className="h-full flex flex-col gap-4 overflow-hidden">
|
||||||
<Breadcrumb items={navigateItems} />
|
<Breadcrumb items={navigateItems} />
|
||||||
{/* Header */}
|
{/* Header */}
|
||||||
<DetailHeader
|
<DetailHeader
|
||||||
@@ -398,42 +419,67 @@ export default function DatasetDetail() {
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<div className="flex-overflow-auto p-6 pt-2 bg-white rounded-md shadow">
|
<div className="flex-1 overflow-auto">
|
||||||
<Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
|
<div className="p-6 pt-2 bg-white rounded-md shadow mb-4">
|
||||||
<div className="h-full overflow-auto">
|
<Tabs activeKey={activeTab} items={tabList} onChange={setActiveTab} />
|
||||||
{activeTab === "overview" && (
|
<div className="">
|
||||||
<Overview
|
{activeTab === "overview" && (
|
||||||
dataset={dataset}
|
<Overview
|
||||||
filesOperation={filesOperation}
|
dataset={dataset}
|
||||||
fetchDataset={fetchDataset}
|
filesOperation={filesOperation}
|
||||||
onUpload={() => setShowUploadDialog(true)}
|
fetchDataset={fetchDataset}
|
||||||
similarDatasets={similarDatasets}
|
onUpload={() => setShowUploadDialog(true)}
|
||||||
similarDatasetsLoading={similarDatasetsLoading}
|
/>
|
||||||
similarTags={similarTagNames}
|
)}
|
||||||
/>
|
{activeTab === "children" && (
|
||||||
)}
|
<div className="pt-4">
|
||||||
{activeTab === "children" && (
|
<div className="flex items-center justify-between mb-3">
|
||||||
<div className="pt-4">
|
<h2 className="text-base font-semibold">关联数据集</h2>
|
||||||
<div className="flex items-center justify-between mb-3">
|
<span className="text-xs text-gray-500">
|
||||||
<h2 className="text-base font-semibold">子数据集</h2>
|
共 {childDatasets.length} 个
|
||||||
<span className="text-xs text-gray-500">
|
</span>
|
||||||
共 {childDatasets.length} 个
|
</div>
|
||||||
</span>
|
<Table
|
||||||
</div>
|
rowKey="id"
|
||||||
<Table
|
columns={childColumns}
|
||||||
rowKey="id"
|
dataSource={childDatasets}
|
||||||
columns={childColumns}
|
loading={childDatasetsLoading}
|
||||||
dataSource={childDatasets}
|
pagination={false}
|
||||||
loading={childDatasetsLoading}
|
locale={{ emptyText: "暂无关联数据集" }}
|
||||||
pagination={false}
|
/>
|
||||||
locale={{ emptyText: "暂无子数据集" }}
|
</div>
|
||||||
/>
|
)}
|
||||||
</div>
|
{activeTab === "lineage" && <DataLineageFlow dataset={dataset} />}
|
||||||
)}
|
{activeTab === "quality" && <DataQuality />}
|
||||||
{activeTab === "lineage" && <DataLineageFlow dataset={dataset} />}
|
</div>
|
||||||
{activeTab === "quality" && <DataQuality />}
|
</div>
|
||||||
</div>
|
|
||||||
</div>
|
{/* 相似数据集 */}
|
||||||
|
<div className="bg-white rounded-md shadow p-6 mb-4">
|
||||||
|
<div className="flex items-center justify-between mb-3">
|
||||||
|
<h2 className="text-base font-semibold">相似数据集</h2>
|
||||||
|
{similarTagsSummary && (
|
||||||
|
<span className="text-xs text-gray-500">
|
||||||
|
匹配标签:{similarTagsSummary}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<CardView
|
||||||
|
data={similarDatasets}
|
||||||
|
loading={similarDatasetsLoading}
|
||||||
|
operations={[]}
|
||||||
|
pagination={{
|
||||||
|
current: 1,
|
||||||
|
pageSize: similarDatasets.length || 10,
|
||||||
|
total: similarDatasets.length || 0,
|
||||||
|
style: { display: "none" },
|
||||||
|
}}
|
||||||
|
onView={(item) => {
|
||||||
|
navigate(`/data/management/detail/${item.id}`);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<ImportConfiguration
|
<ImportConfiguration
|
||||||
data={dataset}
|
data={dataset}
|
||||||
open={showUploadDialog}
|
open={showUploadDialog}
|
||||||
|
|||||||
@@ -1,351 +1,530 @@
|
|||||||
import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch, Tooltip } from "antd";
|
import { Select, Input, Form, Radio, Modal, Button, UploadFile, Switch, Tooltip } from "antd";
|
||||||
import { InboxOutlined, QuestionCircleOutlined } from "@ant-design/icons";
|
import { InboxOutlined, QuestionCircleOutlined } from "@ant-design/icons";
|
||||||
import { dataSourceOptions } from "../../dataset.const";
|
import { dataSourceOptions } from "../../dataset.const";
|
||||||
import { Dataset, DataSource } from "../../dataset.model";
|
import { Dataset, DatasetType, DataSource } from "../../dataset.model";
|
||||||
import { useEffect, useState } from "react";
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis";
|
import { queryTasksUsingGet } from "@/pages/DataCollection/collection.apis";
|
||||||
import { updateDatasetByIdUsingPut } from "../../dataset.api";
|
import { updateDatasetByIdUsingPut } from "../../dataset.api";
|
||||||
import { sliceFile } from "@/utils/file.util";
|
import { sliceFile, shouldStreamUpload } from "@/utils/file.util";
|
||||||
import Dragger from "antd/es/upload/Dragger";
|
import Dragger from "antd/es/upload/Dragger";
|
||||||
|
|
||||||
/**
|
const TEXT_FILE_MIME_PREFIX = "text/";
|
||||||
* 按行分割文件
|
const TEXT_FILE_MIME_TYPES = new Set([
|
||||||
* @param file 原始文件
|
"application/json",
|
||||||
* @returns 分割后的文件列表,每行一个文件
|
"application/xml",
|
||||||
*/
|
"application/csv",
|
||||||
async function splitFileByLines(file: UploadFile): Promise<UploadFile[]> {
|
"application/ndjson",
|
||||||
const originFile = (file as any).originFileObj || file;
|
"application/x-ndjson",
|
||||||
if (!originFile || typeof originFile.text !== "function") {
|
"application/x-yaml",
|
||||||
return [file];
|
"application/yaml",
|
||||||
}
|
"application/javascript",
|
||||||
|
"application/x-javascript",
|
||||||
const text = await originFile.text();
|
"application/sql",
|
||||||
if (!text) return [file];
|
]);
|
||||||
|
const TEXT_FILE_EXTENSIONS = new Set([
|
||||||
// 按行分割并过滤空行
|
".txt",
|
||||||
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
|
".md",
|
||||||
if (lines.length === 0) return [];
|
".csv",
|
||||||
|
".tsv",
|
||||||
// 生成文件名:原文件名_序号.扩展名
|
".json",
|
||||||
const nameParts = file.name.split(".");
|
".jsonl",
|
||||||
const ext = nameParts.length > 1 ? "." + nameParts.pop() : "";
|
".ndjson",
|
||||||
const baseName = nameParts.join(".");
|
".log",
|
||||||
const padLength = String(lines.length).length;
|
".xml",
|
||||||
|
".yaml",
|
||||||
return lines.map((line: string, index: number) => {
|
".yml",
|
||||||
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}${ext}`;
|
".sql",
|
||||||
const blob = new Blob([line], { type: "text/plain" });
|
]);
|
||||||
const newFile = new File([blob], newFileName, { type: "text/plain" });
|
|
||||||
return {
|
function getUploadFileName(file: UploadFile): string {
|
||||||
uid: `${file.uid}-${index}`,
|
if (file.name) return file.name;
|
||||||
name: newFileName,
|
const originFile = file.originFileObj;
|
||||||
size: newFile.size,
|
if (originFile instanceof File && originFile.name) {
|
||||||
type: "text/plain",
|
return originFile.name;
|
||||||
originFileObj: newFile as any,
|
}
|
||||||
} as UploadFile;
|
return "";
|
||||||
});
|
}
|
||||||
}
|
|
||||||
|
function getUploadFileType(file: UploadFile): string {
|
||||||
export default function ImportConfiguration({
|
if (file.type) return file.type;
|
||||||
data,
|
const originFile = file.originFileObj;
|
||||||
open,
|
if (originFile instanceof File && typeof originFile.type === "string") {
|
||||||
onClose,
|
return originFile.type;
|
||||||
updateEvent = "update:dataset",
|
}
|
||||||
prefix,
|
return "";
|
||||||
}: {
|
}
|
||||||
data: Dataset | null;
|
|
||||||
open: boolean;
|
function isTextUploadFile(file: UploadFile): boolean {
|
||||||
onClose: () => void;
|
const mimeType = getUploadFileType(file).toLowerCase();
|
||||||
updateEvent?: string;
|
if (mimeType) {
|
||||||
prefix?: string;
|
if (mimeType.startsWith(TEXT_FILE_MIME_PREFIX)) return true;
|
||||||
}) {
|
if (TEXT_FILE_MIME_TYPES.has(mimeType)) return true;
|
||||||
const [form] = Form.useForm();
|
}
|
||||||
const [collectionOptions, setCollectionOptions] = useState([]);
|
|
||||||
const [importConfig, setImportConfig] = useState<any>({
|
const fileName = getUploadFileName(file);
|
||||||
source: DataSource.UPLOAD,
|
const dotIndex = fileName.lastIndexOf(".");
|
||||||
hasArchive: true,
|
if (dotIndex < 0) return false;
|
||||||
splitByLine: false,
|
const ext = fileName.slice(dotIndex).toLowerCase();
|
||||||
});
|
return TEXT_FILE_EXTENSIONS.has(ext);
|
||||||
const [currentPrefix, setCurrentPrefix] = useState<string>("");
|
}
|
||||||
|
|
||||||
// 本地上传文件相关逻辑
|
/**
|
||||||
|
* 按行分割文件
|
||||||
const handleUpload = async (dataset: Dataset) => {
|
* @param file 原始文件
|
||||||
let filesToUpload = form.getFieldValue("files") || [];
|
* @returns 分割后的文件列表,每行一个文件
|
||||||
|
*/
|
||||||
// 如果启用分行分割,处理文件
|
async function splitFileByLines(file: UploadFile): Promise<UploadFile[]> {
|
||||||
if (importConfig.splitByLine) {
|
if (!isTextUploadFile(file)) {
|
||||||
const splitResults = await Promise.all(
|
return [file];
|
||||||
filesToUpload.map((file) => splitFileByLines(file))
|
}
|
||||||
);
|
|
||||||
filesToUpload = splitResults.flat();
|
const originFile = file.originFileObj ?? file;
|
||||||
}
|
if (!(originFile instanceof File) || typeof originFile.text !== "function") {
|
||||||
|
return [file];
|
||||||
// 计算分片列表
|
}
|
||||||
const sliceList = filesToUpload.map((file) => {
|
|
||||||
const originFile = (file as any).originFileObj || file;
|
const text = await originFile.text();
|
||||||
const slices = sliceFile(originFile);
|
if (!text) return [file];
|
||||||
return {
|
|
||||||
originFile: originFile, // 传入真正的 File/Blob 对象
|
// 按行分割并过滤空行
|
||||||
slices,
|
const lines = text.split(/\r?\n/).filter((line: string) => line.trim() !== "");
|
||||||
name: file.name,
|
if (lines.length === 0) return [];
|
||||||
size: originFile.size || 0,
|
|
||||||
};
|
// 生成文件名:原文件名_序号(不保留后缀)
|
||||||
});
|
const nameParts = file.name.split(".");
|
||||||
|
if (nameParts.length > 1) {
|
||||||
console.log("[ImportConfiguration] Uploading with currentPrefix:", currentPrefix);
|
nameParts.pop();
|
||||||
window.dispatchEvent(
|
}
|
||||||
new CustomEvent("upload:dataset", {
|
const baseName = nameParts.join(".");
|
||||||
detail: {
|
const padLength = String(lines.length).length;
|
||||||
dataset,
|
|
||||||
files: sliceList,
|
return lines.map((line: string, index: number) => {
|
||||||
updateEvent,
|
const newFileName = `${baseName}_${String(index + 1).padStart(padLength, "0")}`;
|
||||||
hasArchive: importConfig.hasArchive,
|
const blob = new Blob([line], { type: "text/plain" });
|
||||||
prefix: currentPrefix,
|
const newFile = new File([blob], newFileName, { type: "text/plain" });
|
||||||
},
|
return {
|
||||||
})
|
uid: `${file.uid}-${index}`,
|
||||||
);
|
name: newFileName,
|
||||||
};
|
size: newFile.size,
|
||||||
|
type: "text/plain",
|
||||||
const fetchCollectionTasks = async () => {
|
originFileObj: newFile as UploadFile["originFileObj"],
|
||||||
if (importConfig.source !== DataSource.COLLECTION) return;
|
} as UploadFile;
|
||||||
try {
|
});
|
||||||
const res = await queryTasksUsingGet({ page: 0, size: 100 });
|
}
|
||||||
const options = res.data.content.map((task: any) => ({
|
|
||||||
label: task.name,
|
type SelectOption = {
|
||||||
value: task.id,
|
label: string;
|
||||||
}));
|
value: string;
|
||||||
setCollectionOptions(options);
|
};
|
||||||
} catch (error) {
|
|
||||||
console.error("Error fetching collection tasks:", error);
|
type CollectionTask = {
|
||||||
}
|
id: string;
|
||||||
};
|
name: string;
|
||||||
|
};
|
||||||
const resetState = () => {
|
|
||||||
console.log('[ImportConfiguration] resetState called, preserving currentPrefix:', currentPrefix);
|
type ImportConfig = {
|
||||||
form.resetFields();
|
source: DataSource;
|
||||||
form.setFieldsValue({ files: null });
|
hasArchive: boolean;
|
||||||
setImportConfig({
|
splitByLine: boolean;
|
||||||
source: importConfig.source ? importConfig.source : DataSource.UPLOAD,
|
files?: UploadFile[];
|
||||||
hasArchive: true,
|
dataSource?: string;
|
||||||
splitByLine: false,
|
target?: DataSource;
|
||||||
});
|
[key: string]: unknown;
|
||||||
console.log('[ImportConfiguration] resetState done, currentPrefix still:', currentPrefix);
|
};
|
||||||
};
|
|
||||||
|
export default function ImportConfiguration({
|
||||||
const handleImportData = async () => {
|
data,
|
||||||
if (!data) return;
|
open,
|
||||||
console.log('[ImportConfiguration] handleImportData called, currentPrefix:', currentPrefix);
|
onClose,
|
||||||
if (importConfig.source === DataSource.UPLOAD) {
|
updateEvent = "update:dataset",
|
||||||
await handleUpload(data);
|
prefix,
|
||||||
} else if (importConfig.source === DataSource.COLLECTION) {
|
}: {
|
||||||
await updateDatasetByIdUsingPut(data.id, {
|
data: Dataset | null;
|
||||||
...importConfig,
|
open: boolean;
|
||||||
});
|
onClose: () => void;
|
||||||
}
|
updateEvent?: string;
|
||||||
onClose();
|
prefix?: string;
|
||||||
};
|
}) {
|
||||||
|
const [form] = Form.useForm();
|
||||||
useEffect(() => {
|
const [collectionOptions, setCollectionOptions] = useState<SelectOption[]>([]);
|
||||||
if (open) {
|
const availableSourceOptions = dataSourceOptions.filter(
|
||||||
setCurrentPrefix(prefix || "");
|
(option) => option.value !== DataSource.COLLECTION
|
||||||
console.log('[ImportConfiguration] Modal opened with prefix:', prefix);
|
);
|
||||||
resetState();
|
const [importConfig, setImportConfig] = useState<ImportConfig>({
|
||||||
fetchCollectionTasks();
|
source: DataSource.UPLOAD,
|
||||||
}
|
hasArchive: true,
|
||||||
}, [open]);
|
splitByLine: false,
|
||||||
|
});
|
||||||
// Separate effect for fetching collection tasks when source changes
|
const [currentPrefix, setCurrentPrefix] = useState<string>("");
|
||||||
useEffect(() => {
|
const hasNonTextFile = useMemo(() => {
|
||||||
if (open && importConfig.source === DataSource.COLLECTION) {
|
const files = importConfig.files ?? [];
|
||||||
fetchCollectionTasks();
|
if (files.length === 0) return false;
|
||||||
}
|
return files.some((file) => !isTextUploadFile(file));
|
||||||
}, [importConfig.source]);
|
}, [importConfig.files]);
|
||||||
|
const isTextDataset = data?.datasetType === DatasetType.TEXT;
|
||||||
return (
|
|
||||||
<Modal
|
// 本地上传文件相关逻辑
|
||||||
title="导入数据"
|
|
||||||
open={open}
|
const handleUpload = async (dataset: Dataset) => {
|
||||||
width={600}
|
const filesToUpload =
|
||||||
onCancel={() => {
|
(form.getFieldValue("files") as UploadFile[] | undefined) || [];
|
||||||
onClose();
|
|
||||||
resetState();
|
// 如果启用分行分割,对大文件使用流式处理
|
||||||
}}
|
if (importConfig.splitByLine && !hasNonTextFile) {
|
||||||
maskClosable={false}
|
// 检查是否有大文件需要流式分割上传
|
||||||
footer={
|
const filesForStreamUpload: File[] = [];
|
||||||
<>
|
const filesForNormalUpload: UploadFile[] = [];
|
||||||
<Button onClick={onClose}>取消</Button>
|
|
||||||
<Button
|
for (const file of filesToUpload) {
|
||||||
type="primary"
|
const originFile = file.originFileObj ?? file;
|
||||||
disabled={!importConfig?.files?.length && !importConfig.dataSource}
|
if (originFile instanceof File && shouldStreamUpload(originFile)) {
|
||||||
onClick={handleImportData}
|
filesForStreamUpload.push(originFile);
|
||||||
>
|
} else {
|
||||||
确定
|
filesForNormalUpload.push(file);
|
||||||
</Button>
|
}
|
||||||
</>
|
}
|
||||||
}
|
|
||||||
>
|
// 大文件使用流式分割上传
|
||||||
<Form
|
if (filesForStreamUpload.length > 0) {
|
||||||
form={form}
|
window.dispatchEvent(
|
||||||
layout="vertical"
|
new CustomEvent("upload:dataset-stream", {
|
||||||
initialValues={importConfig || {}}
|
detail: {
|
||||||
onValuesChange={(_, allValues) => setImportConfig(allValues)}
|
dataset,
|
||||||
>
|
files: filesForStreamUpload,
|
||||||
<Form.Item
|
updateEvent,
|
||||||
label="数据源"
|
hasArchive: importConfig.hasArchive,
|
||||||
name="source"
|
prefix: currentPrefix,
|
||||||
rules={[{ required: true, message: "请选择数据源" }]}
|
},
|
||||||
>
|
})
|
||||||
<Radio.Group
|
);
|
||||||
buttonStyle="solid"
|
}
|
||||||
options={dataSourceOptions}
|
|
||||||
optionType="button"
|
// 小文件使用传统分割方式
|
||||||
/>
|
if (filesForNormalUpload.length > 0) {
|
||||||
</Form.Item>
|
const splitResults = await Promise.all(
|
||||||
{importConfig?.source === DataSource.COLLECTION && (
|
filesForNormalUpload.map((file) => splitFileByLines(file))
|
||||||
<Form.Item name="dataSource" label="归集任务" required>
|
);
|
||||||
<Select placeholder="请选择归集任务" options={collectionOptions} />
|
const smallFilesToUpload = splitResults.flat();
|
||||||
</Form.Item>
|
|
||||||
)}
|
// 计算分片列表
|
||||||
|
const sliceList = smallFilesToUpload.map((file) => {
|
||||||
{/* obs import */}
|
const originFile = (file.originFileObj ?? file) as Blob;
|
||||||
{importConfig?.source === DataSource.OBS && (
|
const slices = sliceFile(originFile);
|
||||||
<div className="grid grid-cols-2 gap-3 p-4 bg-blue-50 rounded-lg">
|
return {
|
||||||
<Form.Item
|
originFile: originFile,
|
||||||
name="endpoint"
|
slices,
|
||||||
rules={[{ required: true }]}
|
name: file.name,
|
||||||
label="Endpoint"
|
size: originFile.size || 0,
|
||||||
>
|
};
|
||||||
<Input
|
});
|
||||||
className="h-8 text-xs"
|
|
||||||
placeholder="obs.cn-north-4.myhuaweicloud.com"
|
console.log("[ImportConfiguration] Uploading small files with currentPrefix:", currentPrefix);
|
||||||
/>
|
window.dispatchEvent(
|
||||||
</Form.Item>
|
new CustomEvent("upload:dataset", {
|
||||||
<Form.Item
|
detail: {
|
||||||
name="bucket"
|
dataset,
|
||||||
rules={[{ required: true }]}
|
files: sliceList,
|
||||||
label="Bucket"
|
updateEvent,
|
||||||
>
|
hasArchive: importConfig.hasArchive,
|
||||||
<Input className="h-8 text-xs" placeholder="my-bucket" />
|
prefix: currentPrefix,
|
||||||
</Form.Item>
|
},
|
||||||
<Form.Item
|
})
|
||||||
name="accessKey"
|
);
|
||||||
rules={[{ required: true }]}
|
}
|
||||||
label="Access Key"
|
return;
|
||||||
>
|
}
|
||||||
<Input className="h-8 text-xs" placeholder="Access Key" />
|
|
||||||
</Form.Item>
|
// 未启用分行分割,使用普通上传
|
||||||
<Form.Item
|
// 计算分片列表
|
||||||
name="secretKey"
|
const sliceList = filesToUpload.map((file) => {
|
||||||
rules={[{ required: true }]}
|
const originFile = (file.originFileObj ?? file) as Blob;
|
||||||
label="Secret Key"
|
const slices = sliceFile(originFile);
|
||||||
>
|
return {
|
||||||
<Input
|
originFile: originFile, // 传入真正的 File/Blob 对象
|
||||||
type="password"
|
slices,
|
||||||
className="h-8 text-xs"
|
name: file.name,
|
||||||
placeholder="Secret Key"
|
size: originFile.size || 0,
|
||||||
/>
|
};
|
||||||
</Form.Item>
|
});
|
||||||
</div>
|
|
||||||
)}
|
console.log("[ImportConfiguration] Uploading with currentPrefix:", currentPrefix);
|
||||||
|
window.dispatchEvent(
|
||||||
{/* Local Upload Component */}
|
new CustomEvent("upload:dataset", {
|
||||||
{importConfig?.source === DataSource.UPLOAD && (
|
detail: {
|
||||||
<>
|
dataset,
|
||||||
<Form.Item
|
files: sliceList,
|
||||||
label="自动解压上传的压缩包"
|
updateEvent,
|
||||||
name="hasArchive"
|
hasArchive: importConfig.hasArchive,
|
||||||
valuePropName="checked"
|
prefix: currentPrefix,
|
||||||
>
|
},
|
||||||
<Switch />
|
})
|
||||||
</Form.Item>
|
);
|
||||||
<Form.Item
|
};
|
||||||
label={
|
|
||||||
<span>
|
const fetchCollectionTasks = useCallback(async () => {
|
||||||
按分行分割{" "}
|
if (importConfig.source !== DataSource.COLLECTION) return;
|
||||||
<Tooltip title="选中后,文本文件的每一行将被分割成独立文件">
|
try {
|
||||||
<QuestionCircleOutlined style={{ color: "#999" }} />
|
const res = await queryTasksUsingGet({ page: 0, size: 100 });
|
||||||
</Tooltip>
|
const tasks = Array.isArray(res?.data?.content)
|
||||||
</span>
|
? (res.data.content as CollectionTask[])
|
||||||
}
|
: [];
|
||||||
name="splitByLine"
|
const options = tasks.map((task) => ({
|
||||||
valuePropName="checked"
|
label: task.name,
|
||||||
>
|
value: task.id,
|
||||||
<Switch />
|
}));
|
||||||
</Form.Item>
|
setCollectionOptions(options);
|
||||||
<Form.Item
|
} catch (error) {
|
||||||
label="上传文件"
|
console.error("Error fetching collection tasks:", error);
|
||||||
name="files"
|
}
|
||||||
valuePropName="fileList"
|
}, [importConfig.source]);
|
||||||
getValueFromEvent={(e: any) => {
|
|
||||||
if (Array.isArray(e)) {
|
const resetState = useCallback(() => {
|
||||||
return e;
|
console.log('[ImportConfiguration] resetState called, preserving currentPrefix:', currentPrefix);
|
||||||
}
|
form.resetFields();
|
||||||
return e && e.fileList;
|
form.setFieldsValue({ files: null });
|
||||||
}}
|
setImportConfig({
|
||||||
rules={[
|
source: DataSource.UPLOAD,
|
||||||
{
|
hasArchive: true,
|
||||||
required: true,
|
splitByLine: false,
|
||||||
message: "请上传文件",
|
});
|
||||||
},
|
console.log('[ImportConfiguration] resetState done, currentPrefix still:', currentPrefix);
|
||||||
]}
|
}, [currentPrefix, form]);
|
||||||
>
|
|
||||||
<Dragger
|
const handleImportData = async () => {
|
||||||
className="w-full"
|
if (!data) return;
|
||||||
beforeUpload={() => false}
|
console.log('[ImportConfiguration] handleImportData called, currentPrefix:', currentPrefix);
|
||||||
multiple
|
if (importConfig.source === DataSource.UPLOAD) {
|
||||||
>
|
// 立即显示任务中心,让用户感知上传已开始(在文件分割等耗时操作之前)
|
||||||
<p className="ant-upload-drag-icon">
|
window.dispatchEvent(
|
||||||
<InboxOutlined />
|
new CustomEvent("show:task-popover", { detail: { show: true } })
|
||||||
</p>
|
);
|
||||||
<p className="ant-upload-text">本地文件上传</p>
|
await handleUpload(data);
|
||||||
<p className="ant-upload-hint">拖拽文件到此处或点击选择文件</p>
|
} else if (importConfig.source === DataSource.COLLECTION) {
|
||||||
</Dragger>
|
await updateDatasetByIdUsingPut(data.id, {
|
||||||
</Form.Item>
|
...importConfig,
|
||||||
</>
|
});
|
||||||
)}
|
}
|
||||||
|
onClose();
|
||||||
{/* Target Configuration */}
|
};
|
||||||
{importConfig?.target && importConfig?.target !== DataSource.UPLOAD && (
|
|
||||||
<div className="space-y-3 p-4 bg-blue-50 rounded-lg">
|
useEffect(() => {
|
||||||
{importConfig?.target === DataSource.DATABASE && (
|
if (open) {
|
||||||
<div className="grid grid-cols-2 gap-3">
|
setCurrentPrefix(prefix || "");
|
||||||
<Form.Item
|
console.log('[ImportConfiguration] Modal opened with prefix:', prefix);
|
||||||
name="databaseType"
|
resetState();
|
||||||
rules={[{ required: true }]}
|
fetchCollectionTasks();
|
||||||
label="数据库类型"
|
}
|
||||||
>
|
}, [fetchCollectionTasks, open, prefix, resetState]);
|
||||||
<Select
|
|
||||||
className="w-full"
|
useEffect(() => {
|
||||||
options={[
|
if (!importConfig.files?.length) return;
|
||||||
{ label: "MySQL", value: "mysql" },
|
if (!importConfig.splitByLine) return;
|
||||||
{ label: "PostgreSQL", value: "postgresql" },
|
if (!hasNonTextFile) return;
|
||||||
{ label: "MongoDB", value: "mongodb" },
|
form.setFieldsValue({ splitByLine: false });
|
||||||
]}
|
setImportConfig((prev) => ({ ...prev, splitByLine: false }));
|
||||||
></Select>
|
}, [form, hasNonTextFile, importConfig.files, importConfig.splitByLine]);
|
||||||
</Form.Item>
|
|
||||||
<Form.Item
|
// Separate effect for fetching collection tasks when source changes
|
||||||
name="tableName"
|
useEffect(() => {
|
||||||
rules={[{ required: true }]}
|
if (open && importConfig.source === DataSource.COLLECTION) {
|
||||||
label="表名"
|
fetchCollectionTasks();
|
||||||
>
|
}
|
||||||
<Input className="h-8 text-xs" placeholder="dataset_table" />
|
}, [fetchCollectionTasks, importConfig.source, open]);
|
||||||
</Form.Item>
|
|
||||||
<Form.Item
|
return (
|
||||||
name="connectionString"
|
<Modal
|
||||||
rules={[{ required: true }]}
|
title="导入数据"
|
||||||
label="连接字符串"
|
open={open}
|
||||||
>
|
width={600}
|
||||||
<Input
|
onCancel={() => {
|
||||||
className="h-8 text-xs col-span-2"
|
onClose();
|
||||||
placeholder="数据库连接字符串"
|
resetState();
|
||||||
/>
|
}}
|
||||||
</Form.Item>
|
maskClosable={false}
|
||||||
</div>
|
footer={
|
||||||
)}
|
<>
|
||||||
</div>
|
<Button onClick={onClose}>取消</Button>
|
||||||
)}
|
<Button
|
||||||
</Form>
|
type="primary"
|
||||||
</Modal>
|
disabled={!importConfig?.files?.length && !importConfig.dataSource}
|
||||||
);
|
onClick={handleImportData}
|
||||||
}
|
>
|
||||||
|
确定
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<Form
|
||||||
|
form={form}
|
||||||
|
layout="vertical"
|
||||||
|
initialValues={importConfig || {}}
|
||||||
|
onValuesChange={(_, allValues) => setImportConfig(allValues)}
|
||||||
|
>
|
||||||
|
<Form.Item
|
||||||
|
label="数据源"
|
||||||
|
name="source"
|
||||||
|
rules={[{ required: true, message: "请选择数据源" }]}
|
||||||
|
>
|
||||||
|
<Radio.Group
|
||||||
|
buttonStyle="solid"
|
||||||
|
options={availableSourceOptions}
|
||||||
|
optionType="button"
|
||||||
|
/>
|
||||||
|
</Form.Item>
|
||||||
|
{importConfig?.source === DataSource.COLLECTION && (
|
||||||
|
<Form.Item name="dataSource" label="归集任务" required>
|
||||||
|
<Select placeholder="请选择归集任务" options={collectionOptions} />
|
||||||
|
</Form.Item>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* obs import */}
|
||||||
|
{importConfig?.source === DataSource.OBS && (
|
||||||
|
<div className="grid grid-cols-2 gap-3 p-4 bg-blue-50 rounded-lg">
|
||||||
|
<Form.Item
|
||||||
|
name="endpoint"
|
||||||
|
rules={[{ required: true }]}
|
||||||
|
label="Endpoint"
|
||||||
|
>
|
||||||
|
<Input
|
||||||
|
className="h-8 text-xs"
|
||||||
|
placeholder="obs.cn-north-4.myhuaweicloud.com"
|
||||||
|
/>
|
||||||
|
</Form.Item>
|
||||||
|
<Form.Item
|
||||||
|
name="bucket"
|
||||||
|
rules={[{ required: true }]}
|
||||||
|
label="Bucket"
|
||||||
|
>
|
||||||
|
<Input className="h-8 text-xs" placeholder="my-bucket" />
|
||||||
|
</Form.Item>
|
||||||
|
<Form.Item
|
||||||
|
name="accessKey"
|
||||||
|
rules={[{ required: true }]}
|
||||||
|
label="Access Key"
|
||||||
|
>
|
||||||
|
<Input className="h-8 text-xs" placeholder="Access Key" />
|
||||||
|
</Form.Item>
|
||||||
|
<Form.Item
|
||||||
|
name="secretKey"
|
||||||
|
rules={[{ required: true }]}
|
||||||
|
label="Secret Key"
|
||||||
|
>
|
||||||
|
<Input
|
||||||
|
type="password"
|
||||||
|
className="h-8 text-xs"
|
||||||
|
placeholder="Secret Key"
|
||||||
|
/>
|
||||||
|
</Form.Item>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Local Upload Component */}
|
||||||
|
{importConfig?.source === DataSource.UPLOAD && (
|
||||||
|
<>
|
||||||
|
<Form.Item
|
||||||
|
label="自动解压上传的压缩包"
|
||||||
|
name="hasArchive"
|
||||||
|
valuePropName="checked"
|
||||||
|
>
|
||||||
|
<Switch />
|
||||||
|
</Form.Item>
|
||||||
|
{isTextDataset && (
|
||||||
|
<Form.Item
|
||||||
|
label={
|
||||||
|
<span>
|
||||||
|
按分行分割{" "}
|
||||||
|
<Tooltip
|
||||||
|
title={
|
||||||
|
hasNonTextFile
|
||||||
|
? "已选择非文本文件,无法按行分割"
|
||||||
|
: "选中后,文本文件的每一行将被分割成独立文件"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<QuestionCircleOutlined style={{ color: "#999" }} />
|
||||||
|
</Tooltip>
|
||||||
|
</span>
|
||||||
|
}
|
||||||
|
name="splitByLine"
|
||||||
|
valuePropName="checked"
|
||||||
|
>
|
||||||
|
<Switch disabled={hasNonTextFile} />
|
||||||
|
</Form.Item>
|
||||||
|
)}
|
||||||
|
<Form.Item
|
||||||
|
label="上传文件"
|
||||||
|
name="files"
|
||||||
|
valuePropName="fileList"
|
||||||
|
getValueFromEvent={(
|
||||||
|
event: { fileList?: UploadFile[] } | UploadFile[]
|
||||||
|
) => {
|
||||||
|
if (Array.isArray(event)) {
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
return event?.fileList;
|
||||||
|
}}
|
||||||
|
rules={[
|
||||||
|
{
|
||||||
|
required: true,
|
||||||
|
message: "请上传文件",
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
>
|
||||||
|
<Dragger
|
||||||
|
className="w-full"
|
||||||
|
beforeUpload={() => false}
|
||||||
|
multiple
|
||||||
|
>
|
||||||
|
<p className="ant-upload-drag-icon">
|
||||||
|
<InboxOutlined />
|
||||||
|
</p>
|
||||||
|
<p className="ant-upload-text">本地文件上传</p>
|
||||||
|
<p className="ant-upload-hint">拖拽文件到此处或点击选择文件</p>
|
||||||
|
</Dragger>
|
||||||
|
</Form.Item>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Target Configuration */}
|
||||||
|
{importConfig?.target && importConfig?.target !== DataSource.UPLOAD && (
|
||||||
|
<div className="space-y-3 p-4 bg-blue-50 rounded-lg">
|
||||||
|
{importConfig?.target === DataSource.DATABASE && (
|
||||||
|
<div className="grid grid-cols-2 gap-3">
|
||||||
|
<Form.Item
|
||||||
|
name="databaseType"
|
||||||
|
rules={[{ required: true }]}
|
||||||
|
label="数据库类型"
|
||||||
|
>
|
||||||
|
<Select
|
||||||
|
className="w-full"
|
||||||
|
options={[
|
||||||
|
{ label: "MySQL", value: "mysql" },
|
||||||
|
{ label: "PostgreSQL", value: "postgresql" },
|
||||||
|
{ label: "MongoDB", value: "mongodb" },
|
||||||
|
]}
|
||||||
|
></Select>
|
||||||
|
</Form.Item>
|
||||||
|
<Form.Item
|
||||||
|
name="tableName"
|
||||||
|
rules={[{ required: true }]}
|
||||||
|
label="表名"
|
||||||
|
>
|
||||||
|
<Input className="h-8 text-xs" placeholder="dataset_table" />
|
||||||
|
</Form.Item>
|
||||||
|
<Form.Item
|
||||||
|
name="connectionString"
|
||||||
|
rules={[{ required: true }]}
|
||||||
|
label="连接字符串"
|
||||||
|
>
|
||||||
|
<Input
|
||||||
|
className="h-8 text-xs col-span-2"
|
||||||
|
placeholder="数据库连接字符串"
|
||||||
|
/>
|
||||||
|
</Form.Item>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</Form>
|
||||||
|
</Modal>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,152 +4,69 @@ import {
|
|||||||
Descriptions,
|
Descriptions,
|
||||||
DescriptionsProps,
|
DescriptionsProps,
|
||||||
Modal,
|
Modal,
|
||||||
|
Spin,
|
||||||
Table,
|
Table,
|
||||||
Input,
|
Input,
|
||||||
Tag,
|
|
||||||
} from "antd";
|
} from "antd";
|
||||||
import { formatBytes, formatDateTime } from "@/utils/unit";
|
import { formatBytes, formatDateTime } from "@/utils/unit";
|
||||||
import { Download, Trash2, Folder, File } from "lucide-react";
|
import { Download, Trash2, Folder, File } from "lucide-react";
|
||||||
import { datasetTypeMap } from "../../dataset.const";
|
import { datasetTypeMap } from "../../dataset.const";
|
||||||
import type { Dataset, DatasetFile } from "@/pages/DataManagement/dataset.model";
|
import type { Dataset, DatasetFile } from "@/pages/DataManagement/dataset.model";
|
||||||
import { Link } from "react-router";
|
import type { useFilesOperation } from "../useFilesOperation";
|
||||||
import type { useFilesOperation } from "../useFilesOperation";
|
|
||||||
|
type DatasetFileRow = DatasetFile & {
|
||||||
type DatasetFileRow = DatasetFile & {
|
fileSize?: number;
|
||||||
fileSize?: number;
|
fileCount?: number;
|
||||||
fileCount?: number;
|
uploadTime?: string;
|
||||||
uploadTime?: string;
|
};
|
||||||
};
|
|
||||||
|
const PREVIEW_MAX_HEIGHT = 500;
|
||||||
const PREVIEW_MAX_HEIGHT = 500;
|
|
||||||
const PREVIEW_MODAL_WIDTH = {
|
const PREVIEW_MODAL_WIDTH = {
|
||||||
text: 800,
|
text: "80vw",
|
||||||
media: 700,
|
media: "80vw",
|
||||||
};
|
};
|
||||||
const PREVIEW_TEXT_FONT_SIZE = 12;
|
const PREVIEW_TEXT_FONT_SIZE = 12;
|
||||||
const PREVIEW_TEXT_PADDING = 12;
|
const PREVIEW_TEXT_PADDING = 12;
|
||||||
const PREVIEW_AUDIO_PADDING = 40;
|
const PREVIEW_AUDIO_PADDING = 40;
|
||||||
const SIMILAR_TAGS_PREVIEW_LIMIT = 3;
|
|
||||||
const SIMILAR_DATASET_TAG_PREVIEW_LIMIT = 4;
|
type OverviewProps = {
|
||||||
|
dataset: Dataset;
|
||||||
type OverviewProps = {
|
filesOperation: ReturnType<typeof useFilesOperation>;
|
||||||
dataset: Dataset;
|
fetchDataset: () => void;
|
||||||
filesOperation: ReturnType<typeof useFilesOperation>;
|
onUpload?: () => void;
|
||||||
fetchDataset: () => void;
|
};
|
||||||
onUpload?: () => void;
|
|
||||||
similarDatasets: Dataset[];
|
export default function Overview({
|
||||||
similarDatasetsLoading: boolean;
|
dataset,
|
||||||
similarTags: string[];
|
filesOperation,
|
||||||
};
|
fetchDataset,
|
||||||
|
onUpload,
|
||||||
export default function Overview({
|
}: OverviewProps) {
|
||||||
dataset,
|
const { modal, message } = App.useApp();
|
||||||
filesOperation,
|
|
||||||
fetchDataset,
|
|
||||||
onUpload,
|
|
||||||
similarDatasets,
|
|
||||||
similarDatasetsLoading,
|
|
||||||
similarTags,
|
|
||||||
}: OverviewProps) {
|
|
||||||
const { modal, message } = App.useApp();
|
|
||||||
const {
|
const {
|
||||||
fileList,
|
fileList,
|
||||||
pagination,
|
pagination,
|
||||||
selectedFiles,
|
selectedFiles,
|
||||||
previewVisible,
|
previewVisible,
|
||||||
previewFileName,
|
previewFileName,
|
||||||
previewContent,
|
previewContent,
|
||||||
previewFileType,
|
previewFileType,
|
||||||
previewMediaUrl,
|
previewMediaUrl,
|
||||||
previewLoading,
|
previewLoading,
|
||||||
|
officePreviewStatus,
|
||||||
|
officePreviewError,
|
||||||
closePreview,
|
closePreview,
|
||||||
handleDeleteFile,
|
handleDeleteFile,
|
||||||
handleDownloadFile,
|
handleDownloadFile,
|
||||||
handleBatchDeleteFiles,
|
handleBatchDeleteFiles,
|
||||||
handleBatchExport,
|
handleBatchExport,
|
||||||
handleCreateDirectory,
|
handleCreateDirectory,
|
||||||
handleDownloadDirectory,
|
handleDownloadDirectory,
|
||||||
handleDeleteDirectory,
|
handleDeleteDirectory,
|
||||||
handlePreviewFile,
|
handlePreviewFile,
|
||||||
} = filesOperation;
|
} = filesOperation;
|
||||||
const similarTagsSummary = (() => {
|
|
||||||
if (!similarTags || similarTags.length === 0) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
const visibleTags = similarTags.slice(0, SIMILAR_TAGS_PREVIEW_LIMIT);
|
|
||||||
const hiddenCount = similarTags.length - visibleTags.length;
|
|
||||||
if (hiddenCount > 0) {
|
|
||||||
return `${visibleTags.join("、")} 等 ${similarTags.length} 个`;
|
|
||||||
}
|
|
||||||
return visibleTags.join("、");
|
|
||||||
})();
|
|
||||||
const renderDatasetTags = (
|
|
||||||
tags?: Array<string | { name?: string; color?: string } | null>
|
|
||||||
) => {
|
|
||||||
if (!tags || tags.length === 0) {
|
|
||||||
return "-";
|
|
||||||
}
|
|
||||||
const visibleTags = tags.slice(0, SIMILAR_DATASET_TAG_PREVIEW_LIMIT);
|
|
||||||
const hiddenCount = tags.length - visibleTags.length;
|
|
||||||
return (
|
|
||||||
<div className="flex flex-wrap gap-1">
|
|
||||||
{visibleTags.map((tag, index) => {
|
|
||||||
const tagName = typeof tag === "string" ? tag : tag?.name;
|
|
||||||
if (!tagName) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const tagColor = typeof tag === "string" ? undefined : tag?.color;
|
|
||||||
return (
|
|
||||||
<Tag key={`${tagName}-${index}`} color={tagColor}>
|
|
||||||
{tagName}
|
|
||||||
</Tag>
|
|
||||||
);
|
|
||||||
})}
|
|
||||||
{hiddenCount > 0 && <Tag>+{hiddenCount}</Tag>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
};
|
|
||||||
const similarColumns = [
|
|
||||||
{
|
|
||||||
title: "名称",
|
|
||||||
dataIndex: "name",
|
|
||||||
key: "name",
|
|
||||||
render: (_: string, record: Dataset) => (
|
|
||||||
<Link to={`/data/management/detail/${record.id}`}>{record.name}</Link>
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
title: "标签",
|
|
||||||
dataIndex: "tags",
|
|
||||||
key: "tags",
|
|
||||||
render: (tags: Array<string | { name?: string; color?: string }>) =>
|
|
||||||
renderDatasetTags(tags),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
title: "类型",
|
|
||||||
dataIndex: "datasetType",
|
|
||||||
key: "datasetType",
|
|
||||||
width: 120,
|
|
||||||
render: (_: string, record: Dataset) =>
|
|
||||||
datasetTypeMap[record.datasetType as keyof typeof datasetTypeMap]?.label ||
|
|
||||||
"未知",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
title: "文件数",
|
|
||||||
dataIndex: "fileCount",
|
|
||||||
key: "fileCount",
|
|
||||||
width: 120,
|
|
||||||
render: (value?: number) => value ?? 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
title: "更新时间",
|
|
||||||
dataIndex: "updatedAt",
|
|
||||||
key: "updatedAt",
|
|
||||||
width: 180,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
// 基本信息
|
// 基本信息
|
||||||
const items: DescriptionsProps["items"] = [
|
const items: DescriptionsProps["items"] = [
|
||||||
{
|
{
|
||||||
key: "id",
|
key: "id",
|
||||||
@@ -211,7 +128,7 @@ export default function Overview({
|
|||||||
dataIndex: "fileName",
|
dataIndex: "fileName",
|
||||||
key: "fileName",
|
key: "fileName",
|
||||||
fixed: "left",
|
fixed: "left",
|
||||||
render: (text: string, record: DatasetFileRow) => {
|
render: (text: string, record: DatasetFileRow) => {
|
||||||
const isDirectory = record.id.startsWith('directory-');
|
const isDirectory = record.id.startsWith('directory-');
|
||||||
const iconSize = 16;
|
const iconSize = 16;
|
||||||
|
|
||||||
@@ -230,35 +147,35 @@ export default function Overview({
|
|||||||
return (
|
return (
|
||||||
<Button
|
<Button
|
||||||
type="link"
|
type="link"
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
const currentPath = filesOperation.pagination.prefix || '';
|
const currentPath = filesOperation.pagination.prefix || '';
|
||||||
// 文件夹路径必须以斜杠结尾
|
// 文件夹路径必须以斜杠结尾
|
||||||
const newPath = `${currentPath}${record.fileName}/`;
|
const newPath = `${currentPath}${record.fileName}/`;
|
||||||
filesOperation.fetchFiles(newPath, 1, filesOperation.pagination.pageSize);
|
filesOperation.fetchFiles(newPath, 1, filesOperation.pagination.pageSize);
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{content}
|
{content}
|
||||||
</Button>
|
</Button>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Button
|
<Button
|
||||||
type="link"
|
type="link"
|
||||||
loading={previewLoading && previewFileName === record.fileName}
|
loading={previewLoading && previewFileName === record.fileName}
|
||||||
onClick={() => handlePreviewFile(record)}
|
onClick={() => handlePreviewFile(record)}
|
||||||
>
|
>
|
||||||
{content}
|
{content}
|
||||||
</Button>
|
</Button>
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: "大小",
|
title: "大小",
|
||||||
dataIndex: "fileSize",
|
dataIndex: "fileSize",
|
||||||
key: "fileSize",
|
key: "fileSize",
|
||||||
width: 150,
|
width: 150,
|
||||||
render: (text: number, record: DatasetFileRow) => {
|
render: (text: number, record: DatasetFileRow) => {
|
||||||
const isDirectory = record.id.startsWith('directory-');
|
const isDirectory = record.id.startsWith('directory-');
|
||||||
if (isDirectory) {
|
if (isDirectory) {
|
||||||
return formatBytes(record.fileSize || 0);
|
return formatBytes(record.fileSize || 0);
|
||||||
@@ -271,7 +188,7 @@ export default function Overview({
|
|||||||
dataIndex: "fileCount",
|
dataIndex: "fileCount",
|
||||||
key: "fileCount",
|
key: "fileCount",
|
||||||
width: 120,
|
width: 120,
|
||||||
render: (text: number, record: DatasetFileRow) => {
|
render: (text: number, record: DatasetFileRow) => {
|
||||||
const isDirectory = record.id.startsWith('directory-');
|
const isDirectory = record.id.startsWith('directory-');
|
||||||
if (!isDirectory) {
|
if (!isDirectory) {
|
||||||
return "-";
|
return "-";
|
||||||
@@ -291,7 +208,7 @@ export default function Overview({
|
|||||||
key: "action",
|
key: "action",
|
||||||
width: 180,
|
width: 180,
|
||||||
fixed: "right",
|
fixed: "right",
|
||||||
render: (_, record: DatasetFileRow) => {
|
render: (_, record: DatasetFileRow) => {
|
||||||
const isDirectory = record.id.startsWith('directory-');
|
const isDirectory = record.id.startsWith('directory-');
|
||||||
|
|
||||||
if (isDirectory) {
|
if (isDirectory) {
|
||||||
@@ -332,6 +249,14 @@ export default function Overview({
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex">
|
<div className="flex">
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
type="link"
|
||||||
|
loading={previewLoading && previewFileName === record.fileName}
|
||||||
|
onClick={() => handlePreviewFile(record)}
|
||||||
|
>
|
||||||
|
预览
|
||||||
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
size="small"
|
size="small"
|
||||||
type="link"
|
type="link"
|
||||||
@@ -367,70 +292,45 @@ export default function Overview({
|
|||||||
column={5}
|
column={5}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
{/* 相似数据集 */}
|
{/* 文件列表 */}
|
||||||
<div className="mt-8">
|
<div className="flex items-center justify-between mt-8 mb-2">
|
||||||
<div className="flex items-center justify-between mb-3">
|
<h2 className="text-base font-semibold">文件列表</h2>
|
||||||
<h2 className="text-base font-semibold">相似数据集</h2>
|
<div className="flex items-center gap-2">
|
||||||
{similarTagsSummary && (
|
<Button size="small" onClick={() => onUpload?.()}>
|
||||||
<span className="text-xs text-gray-500">
|
文件上传
|
||||||
匹配标签:{similarTagsSummary}
|
</Button>
|
||||||
</span>
|
<Button
|
||||||
)}
|
type="primary"
|
||||||
</div>
|
size="small"
|
||||||
<Table
|
onClick={() => {
|
||||||
size="small"
|
let dirName = "";
|
||||||
rowKey="id"
|
modal.confirm({
|
||||||
columns={similarColumns}
|
title: "新建文件夹",
|
||||||
dataSource={similarDatasets}
|
content: (
|
||||||
loading={similarDatasetsLoading}
|
<Input
|
||||||
pagination={false}
|
autoFocus
|
||||||
locale={{
|
placeholder="请输入文件夹名称"
|
||||||
emptyText: similarTags?.length
|
onChange={(e) => {
|
||||||
? "暂无相似数据集"
|
dirName = e.target.value?.trim();
|
||||||
: "当前数据集未设置标签",
|
}}
|
||||||
}}
|
/>
|
||||||
/>
|
),
|
||||||
</div>
|
okText: "确定",
|
||||||
|
cancelText: "取消",
|
||||||
{/* 文件列表 */}
|
onOk: async () => {
|
||||||
<div className="flex items-center justify-between mt-8 mb-2">
|
if (!dirName) {
|
||||||
<h2 className="text-base font-semibold">文件列表</h2>
|
message.warning("请输入文件夹名称");
|
||||||
<div className="flex items-center gap-2">
|
return Promise.reject();
|
||||||
<Button size="small" onClick={() => onUpload?.()}>
|
}
|
||||||
文件上传
|
await handleCreateDirectory(dirName);
|
||||||
</Button>
|
},
|
||||||
<Button
|
});
|
||||||
type="primary"
|
}}
|
||||||
size="small"
|
>
|
||||||
onClick={() => {
|
新建文件夹
|
||||||
let dirName = "";
|
</Button>
|
||||||
modal.confirm({
|
</div>
|
||||||
title: "新建文件夹",
|
</div>
|
||||||
content: (
|
|
||||||
<Input
|
|
||||||
autoFocus
|
|
||||||
placeholder="请输入文件夹名称"
|
|
||||||
onChange={(e) => {
|
|
||||||
dirName = e.target.value?.trim();
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
),
|
|
||||||
okText: "确定",
|
|
||||||
cancelText: "取消",
|
|
||||||
onOk: async () => {
|
|
||||||
if (!dirName) {
|
|
||||||
message.warning("请输入文件夹名称");
|
|
||||||
return Promise.reject();
|
|
||||||
}
|
|
||||||
await handleCreateDirectory(dirName);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
新建文件夹
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{selectedFiles.length > 0 && (
|
{selectedFiles.length > 0 && (
|
||||||
<div className="flex items-center gap-2 p-3 bg-blue-50 rounded-lg border border-blue-200">
|
<div className="flex items-center gap-2 p-3 bg-blue-50 rounded-lg border border-blue-200">
|
||||||
<span className="text-sm text-blue-700 font-medium">
|
<span className="text-sm text-blue-700 font-medium">
|
||||||
@@ -511,63 +411,98 @@ export default function Overview({
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{/* 文件预览弹窗 */}
|
{/* 文件预览弹窗 */}
|
||||||
<Modal
|
<Modal
|
||||||
title={`文件预览:${previewFileName}`}
|
title={`文件预览:${previewFileName}`}
|
||||||
open={previewVisible}
|
open={previewVisible}
|
||||||
onCancel={closePreview}
|
onCancel={closePreview}
|
||||||
footer={[
|
footer={[
|
||||||
<Button key="close" onClick={closePreview}>
|
<Button key="close" onClick={closePreview}>
|
||||||
关闭
|
关闭
|
||||||
</Button>,
|
</Button>,
|
||||||
]}
|
]}
|
||||||
width={previewFileType === "text" ? PREVIEW_MODAL_WIDTH.text : PREVIEW_MODAL_WIDTH.media}
|
width={previewFileType === "text" ? PREVIEW_MODAL_WIDTH.text : PREVIEW_MODAL_WIDTH.media}
|
||||||
>
|
>
|
||||||
{previewFileType === "text" && (
|
{previewFileType === "text" && (
|
||||||
<pre
|
<pre
|
||||||
style={{
|
style={{
|
||||||
maxHeight: `${PREVIEW_MAX_HEIGHT}px`,
|
maxHeight: `${PREVIEW_MAX_HEIGHT}px`,
|
||||||
overflow: "auto",
|
overflow: "auto",
|
||||||
whiteSpace: "pre-wrap",
|
whiteSpace: "pre-wrap",
|
||||||
wordBreak: "break-all",
|
wordBreak: "break-all",
|
||||||
fontSize: PREVIEW_TEXT_FONT_SIZE,
|
fontSize: PREVIEW_TEXT_FONT_SIZE,
|
||||||
color: "#222",
|
color: "#222",
|
||||||
backgroundColor: "#f5f5f5",
|
backgroundColor: "#f5f5f5",
|
||||||
padding: `${PREVIEW_TEXT_PADDING}px`,
|
padding: `${PREVIEW_TEXT_PADDING}px`,
|
||||||
borderRadius: "4px",
|
borderRadius: "4px",
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{previewContent}
|
{previewContent}
|
||||||
</pre>
|
</pre>
|
||||||
|
)}
|
||||||
|
{previewFileType === "image" && (
|
||||||
|
<div style={{ textAlign: "center" }}>
|
||||||
|
<img
|
||||||
|
src={previewMediaUrl}
|
||||||
|
alt={previewFileName}
|
||||||
|
style={{ maxWidth: "100%", maxHeight: `${PREVIEW_MAX_HEIGHT}px`, objectFit: "contain" }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{previewFileType === "pdf" && (
|
||||||
|
<>
|
||||||
|
{previewMediaUrl ? (
|
||||||
|
<iframe
|
||||||
|
src={previewMediaUrl}
|
||||||
|
title={previewFileName || "PDF 预览"}
|
||||||
|
style={{ width: "100%", height: `${PREVIEW_MAX_HEIGHT}px`, border: "none" }}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
height: `${PREVIEW_MAX_HEIGHT}px`,
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
gap: 12,
|
||||||
|
color: "#666",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{officePreviewStatus === "FAILED" ? (
|
||||||
|
<>
|
||||||
|
<div>转换失败</div>
|
||||||
|
<div>{officePreviewError || "请稍后重试"}</div>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Spin />
|
||||||
|
<div>正在转换,请稍候...</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
)}
|
)}
|
||||||
{previewFileType === "image" && (
|
{previewFileType === "video" && (
|
||||||
<div style={{ textAlign: "center" }}>
|
<div style={{ textAlign: "center" }}>
|
||||||
<img
|
<video
|
||||||
src={previewMediaUrl}
|
src={previewMediaUrl}
|
||||||
alt={previewFileName}
|
controls
|
||||||
style={{ maxWidth: "100%", maxHeight: `${PREVIEW_MAX_HEIGHT}px`, objectFit: "contain" }}
|
style={{ maxWidth: "100%", maxHeight: `${PREVIEW_MAX_HEIGHT}px` }}
|
||||||
/>
|
>
|
||||||
</div>
|
您的浏览器不支持视频播放
|
||||||
)}
|
</video>
|
||||||
{previewFileType === "video" && (
|
</div>
|
||||||
<div style={{ textAlign: "center" }}>
|
)}
|
||||||
<video
|
{previewFileType === "audio" && (
|
||||||
src={previewMediaUrl}
|
<div style={{ textAlign: "center", padding: `${PREVIEW_AUDIO_PADDING}px 0` }}>
|
||||||
controls
|
<audio src={previewMediaUrl} controls style={{ width: "100%" }}>
|
||||||
style={{ maxWidth: "100%", maxHeight: `${PREVIEW_MAX_HEIGHT}px` }}
|
您的浏览器不支持音频播放
|
||||||
>
|
</audio>
|
||||||
您的浏览器不支持视频播放
|
</div>
|
||||||
</video>
|
)}
|
||||||
</div>
|
</Modal>
|
||||||
)}
|
</>
|
||||||
{previewFileType === "audio" && (
|
);
|
||||||
<div style={{ textAlign: "center", padding: `${PREVIEW_AUDIO_PADDING}px 0` }}>
|
}
|
||||||
<audio src={previewMediaUrl} controls style={{ width: "100%" }}>
|
|
||||||
您的浏览器不支持音频播放
|
|
||||||
</audio>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</Modal>
|
|
||||||
</>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,22 +1,51 @@
|
|||||||
import type {
|
import type {
|
||||||
Dataset,
|
Dataset,
|
||||||
DatasetFile,
|
DatasetFile,
|
||||||
} from "@/pages/DataManagement/dataset.model";
|
} from "@/pages/DataManagement/dataset.model";
|
||||||
import { App } from "antd";
|
import { App } from "antd";
|
||||||
import { useState } from "react";
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
import { PREVIEW_TEXT_MAX_LENGTH, resolvePreviewFileType, truncatePreviewText } from "@/utils/filePreview";
|
import {
|
||||||
import {
|
PREVIEW_TEXT_MAX_LENGTH,
|
||||||
deleteDatasetFileUsingDelete,
|
resolvePreviewFileType,
|
||||||
downloadFileByIdUsingGet,
|
truncatePreviewText,
|
||||||
exportDatasetUsingPost,
|
type PreviewFileType,
|
||||||
queryDatasetFilesUsingGet,
|
} from "@/utils/filePreview";
|
||||||
createDatasetDirectoryUsingPost,
|
import {
|
||||||
downloadDirectoryUsingGet,
|
deleteDatasetFileUsingDelete,
|
||||||
deleteDirectoryUsingDelete,
|
downloadFileByIdUsingGet,
|
||||||
} from "../dataset.api";
|
exportDatasetUsingPost,
|
||||||
|
queryDatasetFilesUsingGet,
|
||||||
|
createDatasetDirectoryUsingPost,
|
||||||
|
downloadDirectoryUsingGet,
|
||||||
|
deleteDirectoryUsingDelete,
|
||||||
|
queryDatasetFilePreviewStatusUsingGet,
|
||||||
|
convertDatasetFilePreviewUsingPost,
|
||||||
|
} from "../dataset.api";
|
||||||
import { useParams } from "react-router";
|
import { useParams } from "react-router";
|
||||||
|
|
||||||
|
const OFFICE_FILE_EXTENSIONS = [".doc", ".docx"];
|
||||||
|
const OFFICE_PREVIEW_POLL_INTERVAL = 2000;
|
||||||
|
const OFFICE_PREVIEW_POLL_MAX_TIMES = 60;
|
||||||
|
|
||||||
|
type OfficePreviewStatus = "UNSET" | "PENDING" | "PROCESSING" | "READY" | "FAILED";
|
||||||
|
|
||||||
|
const isOfficeFileName = (fileName?: string) => {
|
||||||
|
const lowerName = (fileName || "").toLowerCase();
|
||||||
|
return OFFICE_FILE_EXTENSIONS.some((ext) => lowerName.endsWith(ext));
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalizeOfficePreviewStatus = (status?: string): OfficePreviewStatus => {
|
||||||
|
if (!status) {
|
||||||
|
return "UNSET";
|
||||||
|
}
|
||||||
|
const upper = status.toUpperCase();
|
||||||
|
if (upper === "PENDING" || upper === "PROCESSING" || upper === "READY" || upper === "FAILED") {
|
||||||
|
return upper as OfficePreviewStatus;
|
||||||
|
}
|
||||||
|
return "UNSET";
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
export function useFilesOperation(dataset: Dataset) {
|
export function useFilesOperation(dataset: Dataset) {
|
||||||
const { message } = App.useApp();
|
const { message } = App.useApp();
|
||||||
const { id } = useParams(); // 获取动态路由参数
|
const { id } = useParams(); // 获取动态路由参数
|
||||||
@@ -35,9 +64,26 @@ export function useFilesOperation(dataset: Dataset) {
|
|||||||
const [previewVisible, setPreviewVisible] = useState(false);
|
const [previewVisible, setPreviewVisible] = useState(false);
|
||||||
const [previewContent, setPreviewContent] = useState("");
|
const [previewContent, setPreviewContent] = useState("");
|
||||||
const [previewFileName, setPreviewFileName] = useState("");
|
const [previewFileName, setPreviewFileName] = useState("");
|
||||||
const [previewFileType, setPreviewFileType] = useState<"text" | "image" | "video" | "audio">("text");
|
const [previewFileType, setPreviewFileType] = useState<PreviewFileType>("text");
|
||||||
const [previewMediaUrl, setPreviewMediaUrl] = useState("");
|
const [previewMediaUrl, setPreviewMediaUrl] = useState("");
|
||||||
const [previewLoading, setPreviewLoading] = useState(false);
|
const [previewLoading, setPreviewLoading] = useState(false);
|
||||||
|
const [officePreviewStatus, setOfficePreviewStatus] = useState<OfficePreviewStatus | null>(null);
|
||||||
|
const [officePreviewError, setOfficePreviewError] = useState("");
|
||||||
|
const officePreviewPollingRef = useRef<number | null>(null);
|
||||||
|
const officePreviewFileRef = useRef<string | null>(null);
|
||||||
|
|
||||||
|
const clearOfficePreviewPolling = useCallback(() => {
|
||||||
|
if (officePreviewPollingRef.current) {
|
||||||
|
window.clearTimeout(officePreviewPollingRef.current);
|
||||||
|
officePreviewPollingRef.current = null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
clearOfficePreviewPolling();
|
||||||
|
};
|
||||||
|
}, [clearOfficePreviewPolling]);
|
||||||
|
|
||||||
const fetchFiles = async (
|
const fetchFiles = async (
|
||||||
prefix?: string,
|
prefix?: string,
|
||||||
@@ -52,6 +98,7 @@ export function useFilesOperation(dataset: Dataset) {
|
|||||||
size: pageSize !== undefined ? pageSize : pagination.pageSize,
|
size: pageSize !== undefined ? pageSize : pagination.pageSize,
|
||||||
isWithDirectory: true,
|
isWithDirectory: true,
|
||||||
prefix: targetPrefix,
|
prefix: targetPrefix,
|
||||||
|
excludeDerivedFiles: true,
|
||||||
};
|
};
|
||||||
|
|
||||||
const { data } = await queryDatasetFilesUsingGet(id!, params);
|
const { data } = await queryDatasetFilesUsingGet(id!, params);
|
||||||
@@ -105,22 +152,66 @@ export function useFilesOperation(dataset: Dataset) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const previewUrl = `/api/data-management/datasets/${datasetId}/files/${file.id}/preview`;
|
||||||
|
setPreviewFileName(file.fileName);
|
||||||
|
setPreviewContent("");
|
||||||
|
setPreviewMediaUrl("");
|
||||||
|
|
||||||
|
if (isOfficeFileName(file?.fileName)) {
|
||||||
|
setPreviewFileType("pdf");
|
||||||
|
setPreviewVisible(true);
|
||||||
|
setPreviewLoading(true);
|
||||||
|
setOfficePreviewStatus("PROCESSING");
|
||||||
|
setOfficePreviewError("");
|
||||||
|
officePreviewFileRef.current = file.id;
|
||||||
|
try {
|
||||||
|
const { data: statusData } = await queryDatasetFilePreviewStatusUsingGet(datasetId, file.id);
|
||||||
|
const currentStatus = normalizeOfficePreviewStatus(statusData?.status);
|
||||||
|
if (currentStatus === "READY") {
|
||||||
|
setPreviewMediaUrl(previewUrl);
|
||||||
|
setOfficePreviewStatus("READY");
|
||||||
|
setPreviewLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (currentStatus === "PROCESSING") {
|
||||||
|
pollOfficePreviewStatus(datasetId, file.id, 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { data } = await convertDatasetFilePreviewUsingPost(datasetId, file.id);
|
||||||
|
const status = normalizeOfficePreviewStatus(data?.status);
|
||||||
|
if (status === "READY") {
|
||||||
|
setPreviewMediaUrl(previewUrl);
|
||||||
|
setOfficePreviewStatus("READY");
|
||||||
|
} else if (status === "FAILED") {
|
||||||
|
setOfficePreviewStatus("FAILED");
|
||||||
|
setOfficePreviewError(data?.previewError || "转换失败,请稍后重试");
|
||||||
|
} else {
|
||||||
|
setOfficePreviewStatus("PROCESSING");
|
||||||
|
pollOfficePreviewStatus(datasetId, file.id, 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error("触发预览转换失败", error);
|
||||||
|
message.error({ content: "触发预览转换失败" });
|
||||||
|
setOfficePreviewStatus("FAILED");
|
||||||
|
setOfficePreviewError("触发预览转换失败");
|
||||||
|
} finally {
|
||||||
|
setPreviewLoading(false);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const fileType = resolvePreviewFileType(file?.fileName);
|
const fileType = resolvePreviewFileType(file?.fileName);
|
||||||
if (!fileType) {
|
if (!fileType) {
|
||||||
message.warning({ content: "不支持预览该文件类型" });
|
message.warning({ content: "不支持预览该文件类型" });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const fileUrl = `/api/data-management/datasets/${datasetId}/files/${file.id}/download`;
|
|
||||||
setPreviewFileName(file.fileName);
|
|
||||||
setPreviewFileType(fileType);
|
setPreviewFileType(fileType);
|
||||||
setPreviewContent("");
|
|
||||||
setPreviewMediaUrl("");
|
|
||||||
|
|
||||||
if (fileType === "text") {
|
if (fileType === "text") {
|
||||||
setPreviewLoading(true);
|
setPreviewLoading(true);
|
||||||
try {
|
try {
|
||||||
const response = await fetch(fileUrl);
|
const response = await fetch(previewUrl);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error("下载失败");
|
throw new Error("下载失败");
|
||||||
}
|
}
|
||||||
@@ -136,18 +227,67 @@ export function useFilesOperation(dataset: Dataset) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
setPreviewMediaUrl(fileUrl);
|
setPreviewMediaUrl(previewUrl);
|
||||||
setPreviewVisible(true);
|
setPreviewVisible(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const closePreview = () => {
|
const closePreview = () => {
|
||||||
|
clearOfficePreviewPolling();
|
||||||
|
officePreviewFileRef.current = null;
|
||||||
setPreviewVisible(false);
|
setPreviewVisible(false);
|
||||||
setPreviewContent("");
|
setPreviewContent("");
|
||||||
setPreviewMediaUrl("");
|
setPreviewMediaUrl("");
|
||||||
setPreviewFileName("");
|
setPreviewFileName("");
|
||||||
setPreviewFileType("text");
|
setPreviewFileType("text");
|
||||||
|
setOfficePreviewStatus(null);
|
||||||
|
setOfficePreviewError("");
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const pollOfficePreviewStatus = useCallback(
|
||||||
|
async (datasetId: string, fileId: string, attempt: number) => {
|
||||||
|
clearOfficePreviewPolling();
|
||||||
|
officePreviewPollingRef.current = window.setTimeout(async () => {
|
||||||
|
if (officePreviewFileRef.current !== fileId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const { data } = await queryDatasetFilePreviewStatusUsingGet(datasetId, fileId);
|
||||||
|
const status = normalizeOfficePreviewStatus(data?.status);
|
||||||
|
if (status === "READY") {
|
||||||
|
setPreviewMediaUrl(`/api/data-management/datasets/${datasetId}/files/${fileId}/preview`);
|
||||||
|
setOfficePreviewStatus("READY");
|
||||||
|
setOfficePreviewError("");
|
||||||
|
setPreviewLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (status === "FAILED") {
|
||||||
|
setOfficePreviewStatus("FAILED");
|
||||||
|
setOfficePreviewError(data?.previewError || "转换失败,请稍后重试");
|
||||||
|
setPreviewLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (attempt >= OFFICE_PREVIEW_POLL_MAX_TIMES - 1) {
|
||||||
|
setOfficePreviewStatus("FAILED");
|
||||||
|
setOfficePreviewError("转换超时,请稍后重试");
|
||||||
|
setPreviewLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pollOfficePreviewStatus(datasetId, fileId, attempt + 1);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("轮询预览状态失败", error);
|
||||||
|
if (attempt >= OFFICE_PREVIEW_POLL_MAX_TIMES - 1) {
|
||||||
|
setOfficePreviewStatus("FAILED");
|
||||||
|
setOfficePreviewError("转换超时,请稍后重试");
|
||||||
|
setPreviewLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pollOfficePreviewStatus(datasetId, fileId, attempt + 1);
|
||||||
|
}
|
||||||
|
}, OFFICE_PREVIEW_POLL_INTERVAL);
|
||||||
|
},
|
||||||
|
[clearOfficePreviewPolling]
|
||||||
|
);
|
||||||
|
|
||||||
const handleDeleteFile = async (file: DatasetFile) => {
|
const handleDeleteFile = async (file: DatasetFile) => {
|
||||||
try {
|
try {
|
||||||
await deleteDatasetFileUsingDelete(dataset.id, file.id);
|
await deleteDatasetFileUsingDelete(dataset.id, file.id);
|
||||||
@@ -190,6 +330,8 @@ export function useFilesOperation(dataset: Dataset) {
|
|||||||
previewFileType,
|
previewFileType,
|
||||||
previewMediaUrl,
|
previewMediaUrl,
|
||||||
previewLoading,
|
previewLoading,
|
||||||
|
officePreviewStatus,
|
||||||
|
officePreviewError,
|
||||||
closePreview,
|
closePreview,
|
||||||
fetchFiles,
|
fetchFiles,
|
||||||
setFileList,
|
setFileList,
|
||||||
@@ -240,4 +382,5 @@ interface DatasetFilesQueryParams {
|
|||||||
size: number;
|
size: number;
|
||||||
isWithDirectory: boolean;
|
isWithDirectory: boolean;
|
||||||
prefix: string;
|
prefix: string;
|
||||||
|
excludeDerivedFiles?: boolean;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ import {
|
|||||||
} from "@ant-design/icons";
|
} from "@ant-design/icons";
|
||||||
import TagManager from "@/components/business/TagManagement";
|
import TagManager from "@/components/business/TagManagement";
|
||||||
import { Link, useNavigate } from "react-router";
|
import { Link, useNavigate } from "react-router";
|
||||||
import { useEffect, useMemo, useState } from "react";
|
import { useEffect, useMemo, useState } from "react";
|
||||||
import type { ReactNode } from "react";
|
import type { ReactNode } from "react";
|
||||||
import { SearchControls } from "@/components/SearchControls";
|
import { SearchControls } from "@/components/SearchControls";
|
||||||
import CardView from "@/components/CardView";
|
import CardView from "@/components/CardView";
|
||||||
import type { Dataset } from "@/pages/DataManagement/dataset.model";
|
import type { Dataset } from "@/pages/DataManagement/dataset.model";
|
||||||
@@ -36,19 +36,19 @@ export default function DatasetManagementPage() {
|
|||||||
const [editDatasetOpen, setEditDatasetOpen] = useState(false);
|
const [editDatasetOpen, setEditDatasetOpen] = useState(false);
|
||||||
const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null);
|
const [currentDataset, setCurrentDataset] = useState<Dataset | null>(null);
|
||||||
const [showUploadDialog, setShowUploadDialog] = useState(false);
|
const [showUploadDialog, setShowUploadDialog] = useState(false);
|
||||||
const [statisticsData, setStatisticsData] = useState<StatisticsData>({
|
const [statisticsData, setStatisticsData] = useState<StatisticsData>({
|
||||||
count: [],
|
count: [],
|
||||||
size: [],
|
size: [],
|
||||||
});
|
});
|
||||||
|
|
||||||
async function fetchStatistics() {
|
async function fetchStatistics() {
|
||||||
const { data } = await getDatasetStatisticsUsingGet();
|
const { data } = await getDatasetStatisticsUsingGet();
|
||||||
|
|
||||||
const statistics: StatisticsData = {
|
const statistics: StatisticsData = {
|
||||||
size: [
|
size: [
|
||||||
{
|
{
|
||||||
title: "数据集总数",
|
title: "数据集总数",
|
||||||
value: data?.totalDatasets || 0,
|
value: data?.totalDatasets || 0,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: "文件总数",
|
title: "文件总数",
|
||||||
@@ -76,10 +76,10 @@ export default function DatasetManagementPage() {
|
|||||||
title: "视频",
|
title: "视频",
|
||||||
value: data?.count?.video || 0,
|
value: data?.count?.video || 0,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
setStatisticsData(statistics);
|
setStatisticsData(statistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [tags, setTags] = useState<string[]>([]);
|
const [tags, setTags] = useState<string[]>([]);
|
||||||
|
|
||||||
@@ -136,9 +136,9 @@ export default function DatasetManagementPage() {
|
|||||||
message.success("数据集下载成功");
|
message.success("数据集下载成功");
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDeleteDataset = async (id: string) => {
|
const handleDeleteDataset = async (id: string) => {
|
||||||
if (!id) return;
|
if (!id) return;
|
||||||
await deleteDatasetByIdUsingDelete(id);
|
await deleteDatasetByIdUsingDelete(id);
|
||||||
fetchData({ pageOffset: 0 });
|
fetchData({ pageOffset: 0 });
|
||||||
message.success("数据删除成功");
|
message.success("数据删除成功");
|
||||||
};
|
};
|
||||||
@@ -223,12 +223,12 @@ export default function DatasetManagementPage() {
|
|||||||
title: "状态",
|
title: "状态",
|
||||||
dataIndex: "status",
|
dataIndex: "status",
|
||||||
key: "status",
|
key: "status",
|
||||||
render: (status: DatasetStatusMeta) => {
|
render: (status: DatasetStatusMeta) => {
|
||||||
return (
|
return (
|
||||||
<Tag icon={status?.icon} color={status?.color}>
|
<Tag icon={status?.icon} color={status?.color}>
|
||||||
{status?.label}
|
{status?.label}
|
||||||
</Tag>
|
</Tag>
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
width: 120,
|
width: 120,
|
||||||
},
|
},
|
||||||
@@ -274,10 +274,10 @@ export default function DatasetManagementPage() {
|
|||||||
key: "actions",
|
key: "actions",
|
||||||
width: 200,
|
width: 200,
|
||||||
fixed: "right",
|
fixed: "right",
|
||||||
render: (_: unknown, record: Dataset) => (
|
render: (_: unknown, record: Dataset) => (
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex items-center gap-2">
|
||||||
{operations.map((op) => (
|
{operations.map((op) => (
|
||||||
<Tooltip key={op.key} title={op.label}>
|
<Tooltip key={op.key} title={op.label}>
|
||||||
<Button
|
<Button
|
||||||
type="text"
|
type="text"
|
||||||
icon={op.icon}
|
icon={op.icon}
|
||||||
@@ -329,7 +329,7 @@ export default function DatasetManagementPage() {
|
|||||||
<div className="gap-4 h-full flex flex-col">
|
<div className="gap-4 h-full flex flex-col">
|
||||||
{/* Header */}
|
{/* Header */}
|
||||||
<div className="flex items-center justify-between">
|
<div className="flex items-center justify-between">
|
||||||
<h1 className="text-xl font-bold">数据管理</h1>
|
<h1 className="text-xl font-bold">数据集统计</h1>
|
||||||
<div className="flex gap-2 items-center">
|
<div className="flex gap-2 items-center">
|
||||||
{/* tasks */}
|
{/* tasks */}
|
||||||
<TagManager
|
<TagManager
|
||||||
@@ -353,13 +353,13 @@ export default function DatasetManagementPage() {
|
|||||||
<div className="grid grid-cols-1 gap-4">
|
<div className="grid grid-cols-1 gap-4">
|
||||||
<Card>
|
<Card>
|
||||||
<div className="grid grid-cols-3">
|
<div className="grid grid-cols-3">
|
||||||
{statisticsData.size.map((item) => (
|
{statisticsData.size.map((item) => (
|
||||||
<Statistic
|
<Statistic
|
||||||
title={item.title}
|
title={item.title}
|
||||||
key={item.title}
|
key={item.title}
|
||||||
value={`${item.value}`}
|
value={`${item.value}`}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
</Card>
|
</Card>
|
||||||
</div>
|
</div>
|
||||||
@@ -396,22 +396,22 @@ export default function DatasetManagementPage() {
|
|||||||
updateEvent="update:datasets"
|
updateEvent="update:datasets"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
type StatisticsItem = {
|
type StatisticsItem = {
|
||||||
title: string;
|
title: string;
|
||||||
value: number | string;
|
value: number | string;
|
||||||
};
|
};
|
||||||
|
|
||||||
type StatisticsData = {
|
type StatisticsData = {
|
||||||
count: StatisticsItem[];
|
count: StatisticsItem[];
|
||||||
size: StatisticsItem[];
|
size: StatisticsItem[];
|
||||||
};
|
};
|
||||||
|
|
||||||
type DatasetStatusMeta = {
|
type DatasetStatusMeta = {
|
||||||
label: string;
|
label: string;
|
||||||
value: string;
|
value: string;
|
||||||
color: string;
|
color: string;
|
||||||
icon: ReactNode;
|
icon: ReactNode;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -107,17 +107,33 @@ export function deleteDirectoryUsingDelete(
|
|||||||
return del(`/api/data-management/datasets/${id}/files/directories?prefix=${encodeURIComponent(directoryPath)}`);
|
return del(`/api/data-management/datasets/${id}/files/directories?prefix=${encodeURIComponent(directoryPath)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function downloadFileByIdUsingGet(
|
export function downloadFileByIdUsingGet(
|
||||||
id: string | number,
|
id: string | number,
|
||||||
fileId: string | number,
|
fileId: string | number,
|
||||||
fileName: string
|
fileName: string
|
||||||
) {
|
) {
|
||||||
return download(
|
return download(
|
||||||
`/api/data-management/datasets/${id}/files/${fileId}/download`,
|
`/api/data-management/datasets/${id}/files/${fileId}/download`,
|
||||||
null,
|
null,
|
||||||
fileName
|
fileName
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 数据集文件预览状态
|
||||||
|
export function queryDatasetFilePreviewStatusUsingGet(
|
||||||
|
datasetId: string | number,
|
||||||
|
fileId: string | number
|
||||||
|
) {
|
||||||
|
return get(`/api/data-management/datasets/${datasetId}/files/${fileId}/preview/status`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 触发数据集文件预览转换
|
||||||
|
export function convertDatasetFilePreviewUsingPost(
|
||||||
|
datasetId: string | number,
|
||||||
|
fileId: string | number
|
||||||
|
) {
|
||||||
|
return post(`/api/data-management/datasets/${datasetId}/files/${fileId}/preview/convert`, {});
|
||||||
|
}
|
||||||
|
|
||||||
// 删除数据集文件
|
// 删除数据集文件
|
||||||
export function deleteDatasetFileUsingDelete(
|
export function deleteDatasetFileUsingDelete(
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user