DataMate/Makefile.offline.mk

# ============================================================================
# Makefile 离线构建扩展
# 将此文件内容追加到主 Makefile 末尾，或单独包含使用
# ============================================================================

# 离线构建配置
CACHE_DIR ?= ./build-cache
OFFLINE_VERSION ?= latest

# 创建 buildx 构建器（如果不存在）
.PHONY: ensure-buildx
ensure-buildx:
	@if ! docker buildx inspect offline-builder > /dev/null 2>&1; then \
		echo "创建 buildx 构建器..."; \
		docker buildx create --name offline-builder --driver docker-container --use 2>/dev/null || docker buildx use offline-builder; \
	else \
		docker buildx use offline-builder 2>/dev/null || true; \
	fi

# ========== 离线缓存导出（有网环境） ==========

.PHONY: offline-export
offline-export: ensure-buildx
	@echo "======================================"
	@echo "导出离线构建缓存..."
	@echo "======================================"
	@mkdir -p $(CACHE_DIR)/buildkit $(CACHE_DIR)/images $(CACHE_DIR)/resources
	@$(MAKE) _offline-export-base-images
	@$(MAKE) _offline-export-cache
	@$(MAKE) _offline-export-resources
	@$(MAKE) _offline-package

.PHONY: _offline-export-base-images
_offline-export-base-images:
	@echo ""
	@echo "1. 导出基础镜像..."
	@bash -c 'images=( \
		"maven:3-eclipse-temurin-21" \
		"maven:3-eclipse-temurin-8" \
		"eclipse-temurin:21-jdk" \
		"mysql:8" \
		"node:20-alpine" \
		"nginx:1.29" \
		"ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm" \
		"ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm" \
		"ghcr.nju.edu.cn/astral-sh/uv:latest" \
		"python:3.12-slim" \
		"python:3.11-slim" \
		"gcr.nju.edu.cn/distroless/nodejs20-debian12" \
	); for img in "$${images[@]}"; do echo "  Pulling $$img..."; docker pull "$$img" 2>/dev/null || true; done'
	@echo "  Saving base images..."
	@docker save -o $(CACHE_DIR)/images/base-images.tar \
		maven:3-eclipse-temurin-21 \
		maven:3-eclipse-temurin-8 \
		eclipse-temurin:21-jdk \
		mysql:8 \
		node:20-alpine \
		nginx:1.29 \
		ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm \
		ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm \
		ghcr.nju.edu.cn/astral-sh/uv:latest \
		python:3.12-slim \
		python:3.11-slim \
		gcr.nju.edu.cn/distroless/nodejs20-debian12 2>/dev/null || echo "  Warning: Some images may not exist"

.PHONY: _offline-export-cache
_offline-export-cache:
	@echo ""
	@echo "2. 导出 BuildKit 缓存..."
	@echo "   backend..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-cache,mode=max -f scripts/images/backend/Dockerfile -t datamate-backend:cache . 2>/dev/null || echo "   Warning: backend cache export failed"
	@echo "   backend-python..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/backend-python-cache,mode=max -f scripts/images/backend-python/Dockerfile -t datamate-backend-python:cache . 2>/dev/null || echo "   Warning: backend-python cache export failed"
	@echo "   database..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/database-cache,mode=max -f scripts/images/database/Dockerfile -t datamate-database:cache . 2>/dev/null || echo "   Warning: database cache export failed"
	@echo "   frontend..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/frontend-cache,mode=max -f scripts/images/frontend/Dockerfile -t datamate-frontend:cache . 2>/dev/null || echo "   Warning: frontend cache export failed"
	@echo "   gateway..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/gateway-cache,mode=max -f scripts/images/gateway/Dockerfile -t datamate-gateway:cache . 2>/dev/null || echo "   Warning: gateway cache export failed"
	@echo "   runtime..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/runtime-cache,mode=max -f scripts/images/runtime/Dockerfile -t datamate-runtime:cache . 2>/dev/null || echo "   Warning: runtime cache export failed"
	@echo "   deer-flow-backend..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-backend-cache,mode=max -f scripts/images/deer-flow-backend/Dockerfile -t deer-flow-backend:cache . 2>/dev/null || echo "   Warning: deer-flow-backend cache export failed"
	@echo "   deer-flow-frontend..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/deer-flow-frontend-cache,mode=max -f scripts/images/deer-flow-frontend/Dockerfile -t deer-flow-frontend:cache . 2>/dev/null || echo "   Warning: deer-flow-frontend cache export failed"
	@echo "   mineru..."
	@docker buildx build --cache-to type=local,dest=$(CACHE_DIR)/buildkit/mineru-cache,mode=max -f scripts/images/mineru/Dockerfile -t datamate-mineru:cache . 2>/dev/null || echo "   Warning: mineru cache export failed"

.PHONY: _offline-export-resources
_offline-export-resources:
	@echo ""
	@echo "3. 预下载外部资源..."
	@mkdir -p $(CACHE_DIR)/resources/models
	@echo "   PaddleOCR model..."
	@wget -q -O $(CACHE_DIR)/resources/models/ch_ppocr_mobile_v2.0_cls_infer.tar \
		https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar 2>/dev/null || echo "   Warning: PaddleOCR model download failed"
	@echo "   spaCy model..."
	@wget -q -O $(CACHE_DIR)/resources/models/zh_core_web_sm-3.8.0-py3-none-any.whl \
		https://ghproxy.net/https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl 2>/dev/null || echo "   Warning: spaCy model download failed"
	@echo "   DataX source..."
	@if [ ! -d "$(CACHE_DIR)/resources/DataX" ]; then \
		git clone --depth 1 https://gitee.com/alibaba/DataX.git $(CACHE_DIR)/resources/DataX 2>/dev/null || echo "   Warning: DataX clone failed"; \
	fi
	@echo "   deer-flow source..."
	@if [ ! -d "$(CACHE_DIR)/resources/deer-flow" ]; then \
		git clone --depth 1 https://ghproxy.net/https://github.com/ModelEngine-Group/deer-flow.git $(CACHE_DIR)/resources/deer-flow 2>/dev/null || echo "   Warning: deer-flow clone failed"; \
	fi

.PHONY: _offline-package
_offline-package:
	@echo ""
	@echo "4. 打包缓存..."
	@cd $(CACHE_DIR) && tar -czf "build-cache-$$(date +%Y%m%d).tar.gz" buildkit images resources 2>/dev/null && cd - > /dev/null
	@echo ""
	@echo "======================================"
	@echo "✓ 缓存导出完成！"
	@echo "======================================"
	@echo "传输文件: $(CACHE_DIR)/build-cache-$$(date +%Y%m%d).tar.gz"

# ========== 离线构建（无网环境） ==========

.PHONY: offline-setup
offline-setup:
	@echo "======================================"
	@echo "设置离线构建环境..."
	@echo "======================================"
	@if [ ! -d "$(CACHE_DIR)" ]; then \
		echo "查找并解压缓存包..."; \
		cache_file=$$(ls -t build-cache-*.tar.gz 2>/dev/null | head -1); \
		if [ -z "$$cache_file" ]; then \
			echo "错误: 未找到缓存压缩包 (build-cache-*.tar.gz)"; \
			exit 1; \
		fi; \
		echo "解压 $$cache_file..."; \
		tar -xzf "$$cache_file"; \
	else \
		echo "缓存目录已存在: $(CACHE_DIR)"; \
	fi
	@echo ""
	@echo "加载基础镜像..."
	@if [ -f "$(CACHE_DIR)/images/base-images.tar" ]; then \
		docker load -i $(CACHE_DIR)/images/base-images.tar; \
	else \
		echo "警告: 基础镜像文件不存在，假设已手动加载"; \
	fi
	@$(MAKE) ensure-buildx
	@echo ""
	@echo "✓ 离线环境准备完成"

.PHONY: offline-build
offline-build: offline-setup
	@echo ""
	@echo "======================================"
	@echo "开始离线构建..."
	@echo "======================================"
	@$(MAKE) _offline-build-services

.PHONY: _offline-build-services
_offline-build-services: ensure-buildx
	@echo ""
	@echo "构建 datamate-database..."
	@docker buildx build \
		--cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \
		--pull=false \
		-f scripts/images/database/Dockerfile \
		-t datamate-database:$(OFFLINE_VERSION) \
		--load . || echo "  Failed"

	@echo ""
	@echo "构建 datamate-gateway..."
	@docker buildx build \
		--cache-from type=local,src=$(CACHE_DIR)/buildkit/gateway-cache \
		--pull=false \
		-f scripts/images/gateway/Dockerfile \
		-t datamate-gateway:$(OFFLINE_VERSION) \
		--load . || echo "  Failed"

	@echo ""
	@echo "构建 datamate-backend..."
	@docker buildx build \
		--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-cache \
		--pull=false \
		-f scripts/images/backend/Dockerfile \
		-t datamate-backend:$(OFFLINE_VERSION) \
		--load . || echo "  Failed"

	@echo ""
	@echo "构建 datamate-frontend..."
	@docker buildx build \
		--cache-from type=local,src=$(CACHE_DIR)/buildkit/frontend-cache \
		--pull=false \
		-f scripts/images/frontend/Dockerfile \
		-t datamate-frontend:$(OFFLINE_VERSION) \
		--load . || echo "  Failed"

	@echo ""
	@echo "构建 datamate-runtime..."
	@docker buildx build \
		--cache-from type=local,src=$(CACHE_DIR)/buildkit/runtime-cache \
		--pull=false \
		--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
		-f scripts/images/runtime/Dockerfile \
		-t datamate-runtime:$(OFFLINE_VERSION) \
		--load . || echo "  Failed"

	@echo ""
	@echo "构建 datamate-backend-python..."
	@docker buildx build \
		--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-python-cache \
		--pull=false \
		--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
		-f scripts/images/backend-python/Dockerfile \
		-t datamate-backend-python:$(OFFLINE_VERSION) \
		--load . || echo "  Failed"

	@echo ""
	@echo "======================================"
	@echo "✓ 离线构建完成"
	@echo "======================================"

# 单个服务离线构建 (BuildKit)
.PHONY: %-offline-build
%-offline-build: offline-setup ensure-buildx
	@echo "离线构建 $*..."
	@if [ ! -d "$(CACHE_DIR)/buildkit/$*-cache" ]; then \
		echo "错误: $* 的缓存不存在"; \
		exit 1; \
	fi
	@$(eval IMAGE_NAME := $(if $(filter deer-flow%,$*),$*,datamate-$*))
	@docker buildx build \
		--cache-from type=local,src=$(CACHE_DIR)/buildkit/$*-cache \
		--pull=false \
		$(if $(filter runtime backend-python deer-flow%,$*),--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources,) \
		-f scripts/images/$*/Dockerfile \
		-t $(IMAGE_NAME):$(OFFLINE_VERSION) \
		--load .

# 传统 Docker 构建（不使用 BuildKit，更稳定）
.PHONY: offline-build-classic
offline-build-classic: offline-setup
	@echo "使用传统 docker build 进行离线构建..."
	@bash scripts/offline/build-offline-classic.sh $(CACHE_DIR) $(OFFLINE_VERSION)

# 诊断离线环境
.PHONY: offline-diagnose
offline-diagnose:
	@bash scripts/offline/diagnose.sh $(CACHE_DIR)

# 构建 APT 预装基础镜像（有网环境）
.PHONY: offline-build-base-images
offline-build-base-images:
	@echo "构建 APT 预装基础镜像..."
	@bash scripts/offline/build-base-images.sh $(CACHE_DIR)

# 使用预装基础镜像进行离线构建（推荐）
.PHONY: offline-build-final
offline-build-final: offline-setup
	@echo "使用预装 APT 包的基础镜像进行离线构建..."
	@bash scripts/offline/build-offline-final.sh $(CACHE_DIR) $(OFFLINE_VERSION)

# 完整离线导出（包含 APT 预装基础镜像）
.PHONY: offline-export-full
offline-export-full:
	@echo "======================================"
	@echo "完整离线缓存导出（含 APT 预装基础镜像）"
	@echo "======================================"
	@$(MAKE) offline-build-base-images
	@$(MAKE) offline-export
	@echo ""
	@echo "导出完成！传输时请包含以下文件:"
	@echo "  - build-cache/images/base-images-with-apt.tar"
	@echo "  - build-cache-YYYYMMDD.tar.gz"

# ========== 帮助 ==========

.PHONY: help-offline
help-offline:
	@echo "离线构建命令:"
	@echo ""
	@echo "【有网环境】"
	@echo "  make offline-export [CACHE_DIR=./build-cache]       - 导出构建缓存"
	@echo "  make offline-export-full                            - 导出完整缓存（含 APT 预装基础镜像）"
	@echo "  make offline-build-base-images                      - 构建 APT 预装基础镜像"
	@echo ""
	@echo "【无网环境】"
	@echo "  make offline-setup [CACHE_DIR=./build-cache]        - 解压并准备离线缓存"
	@echo "  make offline-build-final                            - 使用预装基础镜像构建（推荐，解决 APT 问题）"
	@echo "  make offline-build-classic                          - 使用传统 docker build"
	@echo "  make offline-build                                  - 使用 BuildKit 构建"
	@echo "  make offline-diagnose                               - 诊断离线构建环境"
	@echo "  make <service>-offline-build                        - 离线构建单个服务"
	@echo ""
	@echo "【完整工作流程（推荐）】"
	@echo "  # 1. 有网环境导出完整缓存"
	@echo "  make offline-export-full"
	@echo ""
	@echo "  # 2. 传输到无网环境（需要传输两个文件）"
	@echo "  scp build-cache/images/base-images-with-apt.tar user@offline-server:/path/"
	@echo "  scp build-cache-*.tar.gz user@offline-server:/path/"
	@echo ""
	@echo "  # 3. 无网环境构建"
	@echo "  tar -xzf build-cache-*.tar.gz"
	@echo "  docker load -i build-cache/images/base-images-with-apt.tar"
	@echo "  make offline-build-final"