You've already forked DataMate
Jerry Yan
05f3efc148
- 将 frontend Dockerfile 中的基础镜像从 gcr.io 切换到 gcr.nju.edu.cn - 更新 offline Dockerfile 中的 nodejs20-debian12 镜像源 - 修改 export-cache.sh 脚本中的基础镜像列表为南京大学镜像 - 更新 Makefile.offline.mk 中的镜像拉取地址为本地镜像源 - 优化 export-cache.sh 脚本的格式和输出信息 - 添加缓存导出过程中的警告处理机制
258 lines
11 KiB
Makefile
258 lines
11 KiB
Makefile
# ============================================================================
|
|
# Makefile 离线构建扩展
|
|
# 将此文件内容追加到主 Makefile 末尾，或单独包含使用
|
|
# ============================================================================
|
|
|
|
# 离线构建配置
|
|
CACHE_DIR ?= ./build-cache
|
|
OFFLINE_VERSION ?= latest
|
|
|
|
# 创建 buildx 构建器（如果不存在）
|
|
.PHONY: ensure-buildx
|
|
ensure-buildx:
|
|
@if ! docker buildx inspect offline-builder > /dev/null 2>&1; then \
|
|
echo "创建 buildx 构建器..."; \
|
|
docker buildx create --name offline-builder --driver docker-container --use 2>/dev/null || docker buildx use offline-builder; \
|
|
else \
|
|
docker buildx use offline-builder 2>/dev/null || true; \
|
|
fi
|
|
|
|
# ========== 离线缓存导出（有网环境） ==========
|
|
|
|
.PHONY: offline-export
|
|
offline-export: ensure-buildx
|
|
@echo "======================================"
|
|
@echo "导出离线构建缓存..."
|
|
@echo "======================================"
|
|
@mkdir -p $(CACHE_DIR)/buildkit $(CACHE_DIR)/images $(CACHE_DIR)/resources
|
|
@$(MAKE) _offline-export-base-images
|
|
@$(MAKE) _offline-export-cache
|
|
@$(MAKE) _offline-export-resources
|
|
@$(MAKE) _offline-package
|
|
|
|
.PHONY: _offline-export-base-images
|
|
_offline-export-base-images:
|
|
@echo ""
|
|
@echo "1. 导出基础镜像..."
|
|
@bash -c 'images=( \
|
|
"maven:3-eclipse-temurin-21" \
|
|
"maven:3-eclipse-temurin-8" \
|
|
"eclipse-temurin:21-jdk" \
|
|
"mysql:8" \
|
|
"node:20-alpine" \
|
|
"nginx:1.29" \
|
|
"ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm" \
|
|
"ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm" \
|
|
"ghcr.nju.edu.cn/astral-sh/uv:latest" \
|
|
"python:3.12-slim" \
|
|
"python:3.11-slim" \
|
|
"gcr.nju.edu.cn/distroless/nodejs20-debian12" \
|
|
); for img in "$${images[@]}"; do echo " Pulling $$img..."; docker pull "$$img" 2>/dev/null || true; done'
|
|
@echo " Saving base images..."
|
|
@docker save -o $(CACHE_DIR)/images/base-images.tar \
|
|
maven:3-eclipse-temurin-21 \
|
|
maven:3-eclipse-temurin-8 \
|
|
eclipse-temurin:21-jdk \
|
|
mysql:8 \
|
|
node:20-alpine \
|
|
nginx:1.29 \
|
|
ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm \
|
|
ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm \
|
|
ghcr.nju.edu.cn/astral-sh/uv:latest \
|
|
python:3.12-slim \
|
|
python:3.11-slim \
|
|
gcr.nju.edu.cn/distroless/nodejs20-debian12 2>/dev/null || echo " Warning: Some images may not exist"
|
|
|
|
.PHONY: _offline-export-cache
|
|
_offline-export-cache:
|
|
@echo ""
|
|
@echo "2. 导出 BuildKit 缓存..."
|
|
@echo " backend..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-cache,mode=max -f scripts/images/backend/Dockerfile -t datamate-backend:cache . 2>/dev/null || echo " Warning: backend cache export failed"
|
|
@echo " backend-python..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-python-cache,mode=max -f scripts/images/backend-python/Dockerfile -t datamate-backend-python:cache . 2>/dev/null || echo " Warning: backend-python cache export failed"
|
|
@echo " database..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/database-cache,mode=max -f scripts/images/database/Dockerfile -t datamate-database:cache . 2>/dev/null || echo " Warning: database cache export failed"
|
|
@echo " frontend..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/frontend-cache,mode=max -f scripts/images/frontend/Dockerfile -t datamate-frontend:cache . 2>/dev/null || echo " Warning: frontend cache export failed"
|
|
@echo " gateway..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/gateway-cache,mode=max -f scripts/images/gateway/Dockerfile -t datamate-gateway:cache . 2>/dev/null || echo " Warning: gateway cache export failed"
|
|
@echo " runtime..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/runtime-cache,mode=max -f scripts/images/runtime/Dockerfile -t datamate-runtime:cache . 2>/dev/null || echo " Warning: runtime cache export failed"
|
|
@echo " deer-flow-backend..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-backend-cache,mode=max -f scripts/images/deer-flow-backend/Dockerfile -t deer-flow-backend:cache . 2>/dev/null || echo " Warning: deer-flow-backend cache export failed"
|
|
@echo " deer-flow-frontend..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-frontend-cache,mode=max -f scripts/images/deer-flow-frontend/Dockerfile -t deer-flow-frontend:cache . 2>/dev/null || echo " Warning: deer-flow-frontend cache export failed"
|
|
@echo " mineru..."
|
|
@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/mineru-cache,mode=max -f scripts/images/mineru/Dockerfile -t datamate-mineru:cache . 2>/dev/null || echo " Warning: mineru cache export failed"
|
|
|
|
.PHONY: _offline-export-resources
|
|
_offline-export-resources:
|
|
@echo ""
|
|
@echo "3. 预下载外部资源..."
|
|
@mkdir -p $(CACHE_DIR)/resources/models
|
|
@echo " PaddleOCR model..."
|
|
@wget -q -O $(CACHE_DIR)/resources/models/ch_ppocr_mobile_v2.0_cls_infer.tar \
|
|
https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar 2>/dev/null || echo " Warning: PaddleOCR model download failed"
|
|
@echo " spaCy model..."
|
|
@wget -q -O $(CACHE_DIR)/resources/models/zh_core_web_sm-3.8.0-py3-none-any.whl \
|
|
https://ghproxy.net/https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl 2>/dev/null || echo " Warning: spaCy model download failed"
|
|
@echo " DataX source..."
|
|
@if [ ! -d "$(CACHE_DIR)/resources/DataX" ]; then \
|
|
git clone --depth 1 https://gitee.com/alibaba/DataX.git $(CACHE_DIR)/resources/DataX 2>/dev/null || echo " Warning: DataX clone failed"; \
|
|
fi
|
|
@echo " deer-flow source..."
|
|
@if [ ! -d "$(CACHE_DIR)/resources/deer-flow" ]; then \
|
|
git clone --depth 1 https://ghproxy.net/https://github.com/ModelEngine-Group/deer-flow.git $(CACHE_DIR)/resources/deer-flow 2>/dev/null || echo " Warning: deer-flow clone failed"; \
|
|
fi
|
|
|
|
.PHONY: _offline-package
|
|
_offline-package:
|
|
@echo ""
|
|
@echo "4. 打包缓存..."
|
|
@cd $(CACHE_DIR) && tar -czf "build-cache-$$(date +%Y%m%d).tar.gz" buildkit images resources 2>/dev/null && cd - > /dev/null
|
|
@echo ""
|
|
@echo "======================================"
|
|
@echo "✓ 缓存导出完成！"
|
|
@echo "======================================"
|
|
@echo "传输文件: $(CACHE_DIR)/build-cache-$$(date +%Y%m%d).tar.gz"
|
|
|
|
# ========== 离线构建（无网环境） ==========
|
|
|
|
.PHONY: offline-setup
|
|
offline-setup:
|
|
@echo "======================================"
|
|
@echo "设置离线构建环境..."
|
|
@echo "======================================"
|
|
@if [ ! -d "$(CACHE_DIR)" ]; then \
|
|
echo "查找并解压缓存包..."; \
|
|
cache_file=$$(ls -t build-cache-*.tar.gz 2>/dev/null | head -1); \
|
|
if [ -z "$$cache_file" ]; then \
|
|
echo "错误: 未找到缓存压缩包 (build-cache-*.tar.gz)"; \
|
|
exit 1; \
|
|
fi; \
|
|
echo "解压 $$cache_file..."; \
|
|
tar -xzf "$$cache_file"; \
|
|
else \
|
|
echo "缓存目录已存在: $(CACHE_DIR)"; \
|
|
fi
|
|
@echo ""
|
|
@echo "加载基础镜像..."
|
|
@if [ -f "$(CACHE_DIR)/images/base-images.tar" ]; then \
|
|
docker load -i $(CACHE_DIR)/images/base-images.tar; \
|
|
else \
|
|
echo "警告: 基础镜像文件不存在，假设已手动加载"; \
|
|
fi
|
|
@$(MAKE) ensure-buildx
|
|
@echo ""
|
|
@echo "✓ 离线环境准备完成"
|
|
|
|
.PHONY: offline-build
|
|
offline-build: offline-setup
|
|
@echo ""
|
|
@echo "======================================"
|
|
@echo "开始离线构建..."
|
|
@echo "======================================"
|
|
@$(MAKE) _offline-build-services
|
|
|
|
.PHONY: _offline-build-services
|
|
_offline-build-services: ensure-buildx
|
|
@echo ""
|
|
@echo "构建 datamate-database..."
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \
|
|
--network=none \
|
|
-f scripts/images/database/Dockerfile \
|
|
-t datamate-database:$(OFFLINE_VERSION) \
|
|
--load . 2>/dev/null || echo " Warning: database build may need network, retrying without --network=none..."
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \
|
|
-f scripts/images/database/Dockerfile \
|
|
-t datamate-database:$(OFFLINE_VERSION) \
|
|
--load . || echo " Failed"
|
|
|
|
@echo ""
|
|
@echo "构建 datamate-gateway..."
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/gateway-cache \
|
|
-f scripts/images/gateway/Dockerfile \
|
|
-t datamate-gateway:$(OFFLINE_VERSION) \
|
|
--load . || echo " Failed"
|
|
|
|
@echo ""
|
|
@echo "构建 datamate-backend..."
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-cache \
|
|
-f scripts/images/backend/Dockerfile \
|
|
-t datamate-backend:$(OFFLINE_VERSION) \
|
|
--load . || echo " Failed"
|
|
|
|
@echo ""
|
|
@echo "构建 datamate-frontend..."
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/frontend-cache \
|
|
-f scripts/images/frontend/Dockerfile \
|
|
-t datamate-frontend:$(OFFLINE_VERSION) \
|
|
--load . || echo " Failed"
|
|
|
|
@echo ""
|
|
@echo "构建 datamate-runtime..."
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/runtime-cache \
|
|
--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
|
|
-f scripts/images/runtime/Dockerfile \
|
|
-t datamate-runtime:$(OFFLINE_VERSION) \
|
|
--load . || echo " Failed"
|
|
|
|
@echo ""
|
|
@echo "构建 datamate-backend-python..."
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-python-cache \
|
|
--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
|
|
-f scripts/images/backend-python/Dockerfile \
|
|
-t datamate-backend-python:$(OFFLINE_VERSION) \
|
|
--load . || echo " Failed"
|
|
|
|
@echo ""
|
|
@echo "======================================"
|
|
@echo "✓ 离线构建完成"
|
|
@echo "======================================"
|
|
|
|
# 单个服务离线构建
|
|
.PHONY: %-offline-build
|
|
%-offline-build: offline-setup ensure-buildx
|
|
@echo "离线构建 $*..."
|
|
@if [ ! -d "$(CACHE_DIR)/buildkit/$*-cache" ]; then \
|
|
echo "错误: $* 的缓存不存在"; \
|
|
exit 1; \
|
|
fi
|
|
@$(eval IMAGE_NAME := $(if $(filter deer-flow%,$*),$*,datamate-$*))
|
|
@docker buildx build \
|
|
--cache-from type=local,src=$(CACHE_DIR)/buildkit/$*-cache \
|
|
$(if $(filter runtime backend-python deer-flow%,$*),--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources,) \
|
|
-f scripts/images/$*/Dockerfile \
|
|
-t $(IMAGE_NAME):$(OFFLINE_VERSION) \
|
|
--load .
|
|
|
|
# ========== 帮助 ==========
|
|
|
|
.PHONY: help-offline
|
|
help-offline:
|
|
@echo "离线构建命令:"
|
|
@echo " make offline-export [CACHE_DIR=./build-cache] - 在有网环境导出构建缓存"
|
|
@echo " make offline-setup [CACHE_DIR=./build-cache] - 解压并准备离线缓存"
|
|
@echo " make offline-build [CACHE_DIR=./build-cache] - 在无网环境构建所有服务"
|
|
@echo " make <service>-offline-build - 离线构建单个服务"
|
|
@echo " (如: make backend-offline-build)"
|
|
@echo ""
|
|
@echo "完整工作流程:"
|
|
@echo " # 1. 有网环境导出缓存"
|
|
@echo " make offline-export"
|
|
@echo ""
|
|
@echo " # 2. 传输缓存到无网环境"
|
|
@echo " scp build-cache-*.tar.gz user@offline-server:/path/to/project/"
|
|
@echo ""
|
|
@echo " # 3. 无网环境构建"
|
|
@echo " tar -xzf build-cache-*.tar.gz"
|
|
@echo " make offline-build"
|