# ============================================================================ # Makefile 离线构建扩展 # 将此文件内容追加到主 Makefile 末尾,或单独包含使用 # ============================================================================ # 离线构建配置 CACHE_DIR ?= ./build-cache OFFLINE_VERSION ?= latest # 创建 buildx 构建器(如果不存在) .PHONY: ensure-buildx ensure-buildx: @if ! docker buildx inspect offline-builder > /dev/null 2>&1; then \ echo "创建 buildx 构建器..."; \ docker buildx create --name offline-builder --driver docker-container --use 2>/dev/null || docker buildx use offline-builder; \ else \ docker buildx use offline-builder 2>/dev/null || true; \ fi # ========== 离线缓存导出(有网环境) ========== .PHONY: offline-export offline-export: ensure-buildx @echo "======================================" @echo "导出离线构建缓存..." @echo "======================================" @mkdir -p $(CACHE_DIR)/buildkit $(CACHE_DIR)/images $(CACHE_DIR)/resources @$(MAKE) _offline-export-base-images @$(MAKE) _offline-export-cache @$(MAKE) _offline-export-resources @$(MAKE) _offline-package .PHONY: _offline-export-base-images _offline-export-base-images: @echo "" @echo "1. 导出基础镜像..." @bash -c 'images=( \ "maven:3-eclipse-temurin-21" \ "maven:3-eclipse-temurin-8" \ "eclipse-temurin:21-jdk" \ "mysql:8" \ "node:20-alpine" \ "nginx:1.29" \ "ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm" \ "ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm" \ "ghcr.nju.edu.cn/astral-sh/uv:latest" \ "python:3.12-slim" \ "python:3.11-slim" \ "gcr.nju.edu.cn/distroless/nodejs20-debian12" \ ); for img in "$${images[@]}"; do echo " Pulling $$img..."; docker pull "$$img" 2>/dev/null || true; done' @echo " Saving base images..." @docker save -o $(CACHE_DIR)/images/base-images.tar \ maven:3-eclipse-temurin-21 \ maven:3-eclipse-temurin-8 \ eclipse-temurin:21-jdk \ mysql:8 \ node:20-alpine \ nginx:1.29 \ ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm \ ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm \ ghcr.nju.edu.cn/astral-sh/uv:latest \ python:3.12-slim \ python:3.11-slim \ gcr.nju.edu.cn/distroless/nodejs20-debian12 2>/dev/null || echo " Warning: Some images may not exist" .PHONY: _offline-export-cache _offline-export-cache: @echo "" @echo "2. 导出 BuildKit 缓存..." @echo " backend..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-cache,mode=max -f scripts/images/backend/Dockerfile -t datamate-backend:cache . 2>/dev/null || echo " Warning: backend cache export failed" @echo " backend-python..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-python-cache,mode=max -f scripts/images/backend-python/Dockerfile -t datamate-backend-python:cache . 2>/dev/null || echo " Warning: backend-python cache export failed" @echo " database..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/database-cache,mode=max -f scripts/images/database/Dockerfile -t datamate-database:cache . 2>/dev/null || echo " Warning: database cache export failed" @echo " frontend..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/frontend-cache,mode=max -f scripts/images/frontend/Dockerfile -t datamate-frontend:cache . 2>/dev/null || echo " Warning: frontend cache export failed" @echo " gateway..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/gateway-cache,mode=max -f scripts/images/gateway/Dockerfile -t datamate-gateway:cache . 2>/dev/null || echo " Warning: gateway cache export failed" @echo " runtime..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/runtime-cache,mode=max -f scripts/images/runtime/Dockerfile -t datamate-runtime:cache . 2>/dev/null || echo " Warning: runtime cache export failed" @echo " deer-flow-backend..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-backend-cache,mode=max -f scripts/images/deer-flow-backend/Dockerfile -t deer-flow-backend:cache . 2>/dev/null || echo " Warning: deer-flow-backend cache export failed" @echo " deer-flow-frontend..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-frontend-cache,mode=max -f scripts/images/deer-flow-frontend/Dockerfile -t deer-flow-frontend:cache . 2>/dev/null || echo " Warning: deer-flow-frontend cache export failed" @echo " mineru..." @docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/mineru-cache,mode=max -f scripts/images/mineru/Dockerfile -t datamate-mineru:cache . 2>/dev/null || echo " Warning: mineru cache export failed" .PHONY: _offline-export-resources _offline-export-resources: @echo "" @echo "3. 预下载外部资源..." @mkdir -p $(CACHE_DIR)/resources/models @echo " PaddleOCR model..." @wget -q -O $(CACHE_DIR)/resources/models/ch_ppocr_mobile_v2.0_cls_infer.tar \ https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar 2>/dev/null || echo " Warning: PaddleOCR model download failed" @echo " spaCy model..." @wget -q -O $(CACHE_DIR)/resources/models/zh_core_web_sm-3.8.0-py3-none-any.whl \ https://ghproxy.net/https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl 2>/dev/null || echo " Warning: spaCy model download failed" @echo " DataX source..." @if [ ! -d "$(CACHE_DIR)/resources/DataX" ]; then \ git clone --depth 1 https://gitee.com/alibaba/DataX.git $(CACHE_DIR)/resources/DataX 2>/dev/null || echo " Warning: DataX clone failed"; \ fi @echo " deer-flow source..." @if [ ! -d "$(CACHE_DIR)/resources/deer-flow" ]; then \ git clone --depth 1 https://ghproxy.net/https://github.com/ModelEngine-Group/deer-flow.git $(CACHE_DIR)/resources/deer-flow 2>/dev/null || echo " Warning: deer-flow clone failed"; \ fi .PHONY: _offline-package _offline-package: @echo "" @echo "4. 打包缓存..." @cd $(CACHE_DIR) && tar -czf "build-cache-$$(date +%Y%m%d).tar.gz" buildkit images resources 2>/dev/null && cd - > /dev/null @echo "" @echo "======================================" @echo "✓ 缓存导出完成!" @echo "======================================" @echo "传输文件: $(CACHE_DIR)/build-cache-$$(date +%Y%m%d).tar.gz" # ========== 离线构建(无网环境) ========== .PHONY: offline-setup offline-setup: @echo "======================================" @echo "设置离线构建环境..." @echo "======================================" @if [ ! -d "$(CACHE_DIR)" ]; then \ echo "查找并解压缓存包..."; \ cache_file=$$(ls -t build-cache-*.tar.gz 2>/dev/null | head -1); \ if [ -z "$$cache_file" ]; then \ echo "错误: 未找到缓存压缩包 (build-cache-*.tar.gz)"; \ exit 1; \ fi; \ echo "解压 $$cache_file..."; \ tar -xzf "$$cache_file"; \ else \ echo "缓存目录已存在: $(CACHE_DIR)"; \ fi @echo "" @echo "加载基础镜像..." @if [ -f "$(CACHE_DIR)/images/base-images.tar" ]; then \ docker load -i $(CACHE_DIR)/images/base-images.tar; \ else \ echo "警告: 基础镜像文件不存在,假设已手动加载"; \ fi @$(MAKE) ensure-buildx @echo "" @echo "✓ 离线环境准备完成" .PHONY: offline-build offline-build: offline-setup @echo "" @echo "======================================" @echo "开始离线构建..." @echo "======================================" @$(MAKE) _offline-build-services .PHONY: _offline-build-services _offline-build-services: ensure-buildx @echo "" @echo "构建 datamate-database..." @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \ --network=none \ -f scripts/images/database/Dockerfile \ -t datamate-database:$(OFFLINE_VERSION) \ --load . 2>/dev/null || echo " Warning: database build may need network, retrying without --network=none..." @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \ -f scripts/images/database/Dockerfile \ -t datamate-database:$(OFFLINE_VERSION) \ --load . || echo " Failed" @echo "" @echo "构建 datamate-gateway..." @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/gateway-cache \ -f scripts/images/gateway/Dockerfile \ -t datamate-gateway:$(OFFLINE_VERSION) \ --load . || echo " Failed" @echo "" @echo "构建 datamate-backend..." @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-cache \ -f scripts/images/backend/Dockerfile \ -t datamate-backend:$(OFFLINE_VERSION) \ --load . || echo " Failed" @echo "" @echo "构建 datamate-frontend..." @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/frontend-cache \ -f scripts/images/frontend/Dockerfile \ -t datamate-frontend:$(OFFLINE_VERSION) \ --load . || echo " Failed" @echo "" @echo "构建 datamate-runtime..." @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/runtime-cache \ --build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \ -f scripts/images/runtime/Dockerfile \ -t datamate-runtime:$(OFFLINE_VERSION) \ --load . || echo " Failed" @echo "" @echo "构建 datamate-backend-python..." @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-python-cache \ --build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \ -f scripts/images/backend-python/Dockerfile \ -t datamate-backend-python:$(OFFLINE_VERSION) \ --load . || echo " Failed" @echo "" @echo "======================================" @echo "✓ 离线构建完成" @echo "======================================" # 单个服务离线构建 .PHONY: %-offline-build %-offline-build: offline-setup ensure-buildx @echo "离线构建 $*..." @if [ ! -d "$(CACHE_DIR)/buildkit/$*-cache" ]; then \ echo "错误: $* 的缓存不存在"; \ exit 1; \ fi @$(eval IMAGE_NAME := $(if $(filter deer-flow%,$*),$*,datamate-$*)) @docker buildx build \ --cache-from type=local,src=$(CACHE_DIR)/buildkit/$*-cache \ $(if $(filter runtime backend-python deer-flow%,$*),--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources,) \ -f scripts/images/$*/Dockerfile \ -t $(IMAGE_NAME):$(OFFLINE_VERSION) \ --load . # ========== 帮助 ========== .PHONY: help-offline help-offline: @echo "离线构建命令:" @echo " make offline-export [CACHE_DIR=./build-cache] - 在有网环境导出构建缓存" @echo " make offline-setup [CACHE_DIR=./build-cache] - 解压并准备离线缓存" @echo " make offline-build [CACHE_DIR=./build-cache] - 在无网环境构建所有服务" @echo " make -offline-build - 离线构建单个服务" @echo " (如: make backend-offline-build)" @echo "" @echo "完整工作流程:" @echo " # 1. 有网环境导出缓存" @echo " make offline-export" @echo "" @echo " # 2. 传输缓存到无网环境" @echo " scp build-cache-*.tar.gz user@offline-server:/path/to/project/" @echo "" @echo " # 3. 无网环境构建" @echo " tar -xzf build-cache-*.tar.gz" @echo " make offline-build"