You've already forked DataMate
feat(build): 添加传统 Docker 构建方式和诊断功能
- 在 build-offline.sh 脚本中添加 --pull=false 参数并改进错误处理 - 为 Makefile.offline.mk 中的各个服务构建任务添加 --pull=false 参数 - 新增 build-offline-classic.sh 脚本，提供不使用 BuildKit 的传统构建方式 - 新增 build-offline-v2.sh 脚本，提供增强版 BuildKit 离线构建功能 - 新增 diagnose.sh 脚本，用于诊断离线构建环境状态 - 在 Makefile 中添加 offline-build-classic 和 offline-diagnose
This commit is contained in:
@@ -161,12 +161,7 @@ _offline-build-services: ensure-buildx
|
||||
@echo "构建 datamate-database..."
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \
|
||||
--network=none \
|
||||
-f scripts/images/database/Dockerfile \
|
||||
-t datamate-database:$(OFFLINE_VERSION) \
|
||||
--load . 2>/dev/null || echo " Warning: database build may need network, retrying without --network=none..."
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/database-cache \
|
||||
--pull=false \
|
||||
-f scripts/images/database/Dockerfile \
|
||||
-t datamate-database:$(OFFLINE_VERSION) \
|
||||
--load . || echo " Failed"
|
||||
@@ -175,6 +170,7 @@ _offline-build-services: ensure-buildx
|
||||
@echo "构建 datamate-gateway..."
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/gateway-cache \
|
||||
--pull=false \
|
||||
-f scripts/images/gateway/Dockerfile \
|
||||
-t datamate-gateway:$(OFFLINE_VERSION) \
|
||||
--load . || echo " Failed"
|
||||
@@ -183,6 +179,7 @@ _offline-build-services: ensure-buildx
|
||||
@echo "构建 datamate-backend..."
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-cache \
|
||||
--pull=false \
|
||||
-f scripts/images/backend/Dockerfile \
|
||||
-t datamate-backend:$(OFFLINE_VERSION) \
|
||||
--load . || echo " Failed"
|
||||
@@ -191,6 +188,7 @@ _offline-build-services: ensure-buildx
|
||||
@echo "构建 datamate-frontend..."
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/frontend-cache \
|
||||
--pull=false \
|
||||
-f scripts/images/frontend/Dockerfile \
|
||||
-t datamate-frontend:$(OFFLINE_VERSION) \
|
||||
--load . || echo " Failed"
|
||||
@@ -199,6 +197,7 @@ _offline-build-services: ensure-buildx
|
||||
@echo "构建 datamate-runtime..."
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/runtime-cache \
|
||||
--pull=false \
|
||||
--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
|
||||
-f scripts/images/runtime/Dockerfile \
|
||||
-t datamate-runtime:$(OFFLINE_VERSION) \
|
||||
@@ -208,6 +207,7 @@ _offline-build-services: ensure-buildx
|
||||
@echo "构建 datamate-backend-python..."
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/backend-python-cache \
|
||||
--pull=false \
|
||||
--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources \
|
||||
-f scripts/images/backend-python/Dockerfile \
|
||||
-t datamate-backend-python:$(OFFLINE_VERSION) \
|
||||
@@ -218,7 +218,7 @@ _offline-build-services: ensure-buildx
|
||||
@echo "✓ 离线构建完成"
|
||||
@echo "======================================"
|
||||
|
||||
# 单个服务离线构建
|
||||
# 单个服务离线构建 (BuildKit)
|
||||
.PHONY: %-offline-build
|
||||
%-offline-build: offline-setup ensure-buildx
|
||||
@echo "离线构建 $*..."
|
||||
@@ -229,11 +229,23 @@ _offline-build-services: ensure-buildx
|
||||
@$(eval IMAGE_NAME := $(if $(filter deer-flow%,$*),$*,datamate-$*))
|
||||
@docker buildx build \
|
||||
--cache-from type=local,src=$(CACHE_DIR)/buildkit/$*-cache \
|
||||
--pull=false \
|
||||
$(if $(filter runtime backend-python deer-flow%,$*),--build-arg RESOURCES_DIR=$(CACHE_DIR)/resources,) \
|
||||
-f scripts/images/$*/Dockerfile \
|
||||
-t $(IMAGE_NAME):$(OFFLINE_VERSION) \
|
||||
--load .
|
||||
|
||||
# 传统 Docker 构建（不使用 BuildKit，更稳定）
|
||||
.PHONY: offline-build-classic
|
||||
offline-build-classic: offline-setup
|
||||
@echo "使用传统 docker build 进行离线构建..."
|
||||
@bash scripts/offline/build-offline-classic.sh $(CACHE_DIR) $(OFFLINE_VERSION)
|
||||
|
||||
# 诊断离线环境
|
||||
.PHONY: offline-diagnose
|
||||
offline-diagnose:
|
||||
@bash scripts/offline/diagnose.sh $(CACHE_DIR)
|
||||
|
||||
# ========== 帮助 ==========
|
||||
|
||||
.PHONY: help-offline
|
||||
@@ -241,9 +253,11 @@ help-offline:
|
||||
@echo "离线构建命令:"
|
||||
@echo " make offline-export [CACHE_DIR=./build-cache] - 在有网环境导出构建缓存"
|
||||
@echo " make offline-setup [CACHE_DIR=./build-cache] - 解压并准备离线缓存"
|
||||
@echo " make offline-build [CACHE_DIR=./build-cache] - 在无网环境构建所有服务"
|
||||
@echo " make offline-build [CACHE_DIR=./build-cache] - 在无网环境构建所有服务(BuildKit)"
|
||||
@echo " make offline-build-classic - 使用传统 docker build（更稳定）"
|
||||
@echo " make <service>-offline-build - 离线构建单个服务"
|
||||
@echo " (如: make backend-offline-build)"
|
||||
@echo " make offline-diagnose - 诊断离线构建环境"
|
||||
@echo ""
|
||||
@echo "完整工作流程:"
|
||||
@echo " # 1. 有网环境导出缓存"
|
||||
@@ -252,6 +266,8 @@ help-offline:
|
||||
@echo " # 2. 传输缓存到无网环境"
|
||||
@echo " scp build-cache-*.tar.gz user@offline-server:/path/to/project/"
|
||||
@echo ""
|
||||
@echo " # 3. 无网环境构建"
|
||||
@echo " # 3. 无网环境构建（推荐先用传统方式）"
|
||||
@echo " tar -xzf build-cache-*.tar.gz"
|
||||
@echo " make offline-build"
|
||||
@echo " make offline-diagnose # 检查环境"
|
||||
@echo " make offline-build-classic # 传统构建（推荐）"
|
||||
@echo " # 或 make offline-build # BuildKit 构建"
|
||||
|
||||
206
scripts/offline/build-offline-classic.sh
Normal file
206
scripts/offline/build-offline-classic.sh
Normal file
@@ -0,0 +1,206 @@
|
||||
#!/bin/bash
|
||||
# 传统 docker build 离线构建脚本（不使用 buildx）
|
||||
# 这种方式更稳定，兼容性更好
|
||||
# Usage: ./build-offline-classic.sh [cache-dir] [version]
|
||||
|
||||
set -e
|
||||
|
||||
CACHE_DIR="${1:-./build-cache}"
|
||||
VERSION="${2:-latest}"
|
||||
IMAGES_DIR="$CACHE_DIR/images"
|
||||
RESOURCES_DIR="$CACHE_DIR/resources"
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
log_debug() { echo -e "${BLUE}[DEBUG]${NC} $1"; }
|
||||
|
||||
# 检查缓存目录
|
||||
if [ ! -d "$CACHE_DIR" ]; then
|
||||
log_error "缓存目录 $CACHE_DIR 不存在"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 加载基础镜像
|
||||
load_base_images() {
|
||||
log_info "加载基础镜像..."
|
||||
|
||||
if [ ! -f "$IMAGES_DIR/base-images.tar" ]; then
|
||||
log_warn "基础镜像 tar 包不存在，检查本地镜像..."
|
||||
return
|
||||
fi
|
||||
|
||||
log_info "从 $IMAGES_DIR/base-images.tar 加载..."
|
||||
docker load -i "$IMAGES_DIR/base-images.tar"
|
||||
log_info "✓ 基础镜像加载完成"
|
||||
}
|
||||
|
||||
# 检查镜像是否存在
|
||||
check_image() {
|
||||
docker inspect "$1" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# 构建函数
|
||||
build_service() {
|
||||
local service_name=$1
|
||||
local image_name=$2
|
||||
local dockerfile=$3
|
||||
|
||||
log_info "----------------------------------------"
|
||||
log_info "构建 $service_name"
|
||||
log_info "----------------------------------------"
|
||||
|
||||
# 检查 Dockerfile 是否存在
|
||||
if [ ! -f "$dockerfile" ]; then
|
||||
log_error "Dockerfile 不存在: $dockerfile"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 获取所需的基础镜像
|
||||
local from_images
|
||||
from_images=$(grep -E '^FROM' "$dockerfile" | sed 's/FROM //' | sed 's/ AS .*//' | sed 's/ as .*//' | awk '{print $1}' | sort -u)
|
||||
|
||||
log_info "检查基础镜像..."
|
||||
local all_exist=true
|
||||
for img in $from_images; do
|
||||
# 跳过多阶段构建的中间阶段引用
|
||||
if [[ "$img" == --from=* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if check_image "$img"; then
|
||||
log_info " ✓ $img"
|
||||
else
|
||||
log_error " ✗ $img (缺失)"
|
||||
all_exist=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$all_exist" = false ]; then
|
||||
log_error "缺少必要的基础镜像，无法构建 $service_name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 准备构建参数
|
||||
local build_args=()
|
||||
|
||||
# 根据服务类型添加特殊处理
|
||||
case "$service_name" in
|
||||
runtime)
|
||||
# runtime 需要模型文件
|
||||
if [ -d "$RESOURCES_DIR/models" ]; then
|
||||
log_info "使用本地模型文件"
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
fi
|
||||
;;
|
||||
backend-python)
|
||||
if [ -d "$RESOURCES_DIR/DataX" ]; then
|
||||
log_info "使用本地 DataX 源码"
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
build_args+=("--build-arg" "DATAX_LOCAL_PATH=$RESOURCES_DIR/DataX")
|
||||
fi
|
||||
;;
|
||||
deer-flow-backend|deer-flow-frontend)
|
||||
if [ -d "$RESOURCES_DIR/deer-flow" ]; then
|
||||
log_info "使用本地 deer-flow 源码"
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
# 执行构建
|
||||
log_info "开始构建..."
|
||||
if docker build \
|
||||
--pull=false \
|
||||
"${build_args[@]}" \
|
||||
-f "$dockerfile" \
|
||||
-t "$image_name:$VERSION" \
|
||||
. 2>&1; then
|
||||
log_info "✓ $service_name 构建成功"
|
||||
return 0
|
||||
else
|
||||
log_error "✗ $service_name 构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 主流程
|
||||
main() {
|
||||
log_info "======================================"
|
||||
log_info "传统 Docker 离线构建"
|
||||
log_info "======================================"
|
||||
|
||||
# 加载基础镜像
|
||||
load_base_images
|
||||
|
||||
# 定义要构建的服务
|
||||
declare -A SERVICES=(
|
||||
["database"]="datamate-database:scripts/images/database/Dockerfile"
|
||||
["gateway"]="datamate-gateway:scripts/images/gateway/Dockerfile"
|
||||
["backend"]="datamate-backend:scripts/images/backend/Dockerfile"
|
||||
["frontend"]="datamate-frontend:scripts/images/frontend/Dockerfile"
|
||||
["runtime"]="datamate-runtime:scripts/images/runtime/Dockerfile"
|
||||
["backend-python"]="datamate-backend-python:scripts/images/backend-python/Dockerfile"
|
||||
)
|
||||
|
||||
# deer-flow 和 mineru 是可选的
|
||||
OPTIONAL_SERVICES=(
|
||||
"deer-flow-backend:deer-flow-backend:scripts/images/deer-flow-backend/Dockerfile"
|
||||
"deer-flow-frontend:deer-flow-frontend:scripts/images/deer-flow-frontend/Dockerfile"
|
||||
"mineru:datamate-mineru:scripts/images/mineru/Dockerfile"
|
||||
)
|
||||
|
||||
log_info ""
|
||||
log_info "构建核心服务..."
|
||||
local failed=()
|
||||
local succeeded=()
|
||||
|
||||
for service_name in "${!SERVICES[@]}"; do
|
||||
IFS=':' read -r image_name dockerfile <<< "${SERVICES[$service_name]}"
|
||||
if build_service "$service_name" "$image_name" "$dockerfile"; then
|
||||
succeeded+=("$service_name")
|
||||
else
|
||||
failed+=("$service_name")
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# 尝试构建可选服务
|
||||
log_info "构建可选服务..."
|
||||
for service_config in "${OPTIONAL_SERVICES[@]}"; do
|
||||
IFS=':' read -r service_name image_name dockerfile <<< "$service_config"
|
||||
if build_service "$service_name" "$image_name" "$dockerfile"; then
|
||||
succeeded+=("$service_name")
|
||||
else
|
||||
log_warn "$service_name 构建失败（可选服务，继续）"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# 汇总
|
||||
log_info "======================================"
|
||||
log_info "构建结果"
|
||||
log_info "======================================"
|
||||
|
||||
if [ ${#succeeded[@]} -gt 0 ]; then
|
||||
log_info "成功 (${#succeeded[@]}): ${succeeded[*]}"
|
||||
fi
|
||||
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
log_error "失败 (${#failed[@]}): ${failed[*]}"
|
||||
exit 1
|
||||
else
|
||||
log_info "✓ 所有核心服务构建成功！"
|
||||
echo ""
|
||||
docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" | grep -E "(datamate-|deer-flow-)" || true
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
249
scripts/offline/build-offline-v2.sh
Normal file
249
scripts/offline/build-offline-v2.sh
Normal file
@@ -0,0 +1,249 @@
|
||||
#!/bin/bash
|
||||
# BuildKit 离线构建脚本 v2 - 增强版
|
||||
# Usage: ./build-offline-v2.sh [cache-dir] [version]
|
||||
|
||||
set -e
|
||||
|
||||
CACHE_DIR="${1:-./build-cache}"
|
||||
VERSION="${2:-latest}"
|
||||
BUILDKIT_CACHE_DIR="$CACHE_DIR/buildkit"
|
||||
IMAGES_DIR="$CACHE_DIR/images"
|
||||
RESOURCES_DIR="$CACHE_DIR/resources"
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log_info() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# 检查缓存目录
|
||||
if [ ! -d "$CACHE_DIR" ]; then
|
||||
log_error "缓存目录 $CACHE_DIR 不存在"
|
||||
log_info "请先解压缓存包: tar -xzf build-cache-*.tar.gz"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 确保 buildx 构建器存在（使用 docker 驱动以支持本地镜像）
|
||||
setup_buildx() {
|
||||
log_info "设置 BuildKit 构建器..."
|
||||
|
||||
# 删除旧的构建器
|
||||
if docker buildx inspect offline-builder > /dev/null 2>&1; then
|
||||
docker buildx rm offline-builder 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 创建新的构建器，使用 docker 驱动（支持本地镜像，不需要推送到 registry）
|
||||
docker buildx create --name offline-builder \
|
||||
--driver docker-container \
|
||||
--driver-opt image=moby/buildkit:buildx-stable-1 \
|
||||
--use
|
||||
|
||||
log_info "BuildKit 构建器创建完成"
|
||||
}
|
||||
|
||||
# 加载基础镜像
|
||||
load_base_images() {
|
||||
log_info "加载基础镜像..."
|
||||
|
||||
if [ ! -f "$IMAGES_DIR/base-images.tar" ]; then
|
||||
log_warn "基础镜像文件不存在: $IMAGES_DIR/base-images.tar"
|
||||
log_info "检查本地是否存在所需镜像..."
|
||||
|
||||
# 检查关键镜像是否存在
|
||||
required_images=(
|
||||
"maven:3-eclipse-temurin-21"
|
||||
"eclipse-temurin:21-jdk"
|
||||
"mysql:8"
|
||||
"node:20-alpine"
|
||||
"nginx:1.29"
|
||||
)
|
||||
|
||||
for img in "${required_images[@]}"; do
|
||||
if ! docker inspect "$img" > /dev/null 2>&1; then
|
||||
log_error "缺少基础镜像: $img"
|
||||
log_info "请确保基础镜像已加载: docker load -i base-images.tar"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
log_info "本地基础镜像检查通过"
|
||||
return
|
||||
fi
|
||||
|
||||
log_info "从 $IMAGES_DIR/base-images.tar 加载基础镜像..."
|
||||
docker load -i "$IMAGES_DIR/base-images.tar"
|
||||
log_info "基础镜像加载完成"
|
||||
}
|
||||
|
||||
# 验证镜像是否存在
|
||||
verify_image() {
|
||||
local image_name=$1
|
||||
if docker inspect "$image_name" > /dev/null 2>&1; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 离线构建函数
|
||||
offline_build() {
|
||||
local service_name=$1
|
||||
local image_name=$2
|
||||
local dockerfile=$3
|
||||
local cache_file="$BUILDKIT_CACHE_DIR/${service_name}-cache"
|
||||
|
||||
log_info "----------------------------------------"
|
||||
log_info "构建 [$service_name] -> $image_name:$VERSION"
|
||||
log_info "----------------------------------------"
|
||||
|
||||
if [ ! -d "$cache_file" ]; then
|
||||
log_warn "$service_name 的缓存不存在，跳过..."
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 获取 Dockerfile 中的基础镜像
|
||||
local base_images
|
||||
base_images=$(grep -E '^FROM' "$dockerfile" | awk '{print $2}' | sort -u)
|
||||
|
||||
log_info "检查基础镜像..."
|
||||
for base_img in $base_images; do
|
||||
# 跳过多阶段构建中的 AS 别名
|
||||
base_img=$(echo "$base_img" | cut -d':' -f1-2 | sed 's/AS.*//i' | tr -d ' ')
|
||||
|
||||
if [ -z "$base_img" ] || [[ "$base_img" == *"AS"* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if verify_image "$base_img"; then
|
||||
log_info " ✓ $base_img"
|
||||
else
|
||||
log_warn " ✗ $base_img (未找到)"
|
||||
# 尝试从 base-images.tar 中加载
|
||||
if [ -f "$IMAGES_DIR/base-images.tar" ]; then
|
||||
log_info " 尝试从 tar 包加载..."
|
||||
docker load -i "$IMAGES_DIR/base-images.tar" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# 执行离线构建
|
||||
log_info "开始构建..."
|
||||
|
||||
# 构建参数
|
||||
local build_args=()
|
||||
|
||||
# 为需要外部资源的服务添加 build-arg
|
||||
case "$service_name" in
|
||||
runtime|deer-flow-backend|deer-flow-frontend)
|
||||
if [ -d "$RESOURCES_DIR" ]; then
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
fi
|
||||
;;
|
||||
backend-python)
|
||||
if [ -d "$RESOURCES_DIR" ]; then
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
build_args+=("--build-arg" "DATAX_LOCAL_PATH=$RESOURCES_DIR/DataX")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
# 执行构建
|
||||
if docker buildx build \
|
||||
--builder offline-builder \
|
||||
--cache-from "type=local,src=$cache_file" \
|
||||
--pull=false \
|
||||
--output "type=docker" \
|
||||
"${build_args[@]}" \
|
||||
-f "$dockerfile" \
|
||||
-t "$image_name:$VERSION" \
|
||||
. 2>&1; then
|
||||
|
||||
log_info "✓ $service_name 构建成功"
|
||||
return 0
|
||||
else
|
||||
log_error "✗ $service_name 构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 主流程
|
||||
main() {
|
||||
log_info "======================================"
|
||||
log_info "BuildKit 离线构建"
|
||||
log_info "======================================"
|
||||
log_info "缓存目录: $CACHE_DIR"
|
||||
log_info "版本: $VERSION"
|
||||
|
||||
# 步骤 1: 设置构建器
|
||||
setup_buildx
|
||||
|
||||
# 步骤 2: 加载基础镜像
|
||||
load_base_images
|
||||
|
||||
# 步骤 3: 定义服务列表
|
||||
declare -A SERVICES=(
|
||||
["database"]="datamate-database:scripts/images/database/Dockerfile"
|
||||
["gateway"]="datamate-gateway:scripts/images/gateway/Dockerfile"
|
||||
["backend"]="datamate-backend:scripts/images/backend/Dockerfile"
|
||||
["frontend"]="datamate-frontend:scripts/images/frontend/Dockerfile"
|
||||
["runtime"]="datamate-runtime:scripts/images/runtime/Dockerfile"
|
||||
["backend-python"]="datamate-backend-python:scripts/images/backend-python/Dockerfile"
|
||||
["deer-flow-backend"]="deer-flow-backend:scripts/images/deer-flow-backend/Dockerfile"
|
||||
["deer-flow-frontend"]="deer-flow-frontend:scripts/images/deer-flow-frontend/Dockerfile"
|
||||
["mineru"]="datamate-mineru:scripts/images/mineru/Dockerfile"
|
||||
)
|
||||
|
||||
# 步骤 4: 批量构建
|
||||
log_info ""
|
||||
log_info "======================================"
|
||||
log_info "开始批量构建"
|
||||
log_info "======================================"
|
||||
|
||||
local failed=()
|
||||
local succeeded=()
|
||||
|
||||
for service_name in "${!SERVICES[@]}"; do
|
||||
IFS=':' read -r image_name dockerfile <<< "${SERVICES[$service_name]}"
|
||||
|
||||
if offline_build "$service_name" "$image_name" "$dockerfile"; then
|
||||
succeeded+=("$service_name")
|
||||
else
|
||||
failed+=("$service_name")
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# 步骤 5: 汇总结果
|
||||
log_info "======================================"
|
||||
log_info "构建完成"
|
||||
log_info "======================================"
|
||||
|
||||
if [ ${#succeeded[@]} -gt 0 ]; then
|
||||
log_info "成功 (${#succeeded[@]}): ${succeeded[*]}"
|
||||
fi
|
||||
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
log_error "失败 (${#failed[@]}): ${failed[*]}"
|
||||
exit 1
|
||||
else
|
||||
log_info "✓ 所有服务构建成功！"
|
||||
echo ""
|
||||
log_info "镜像列表:"
|
||||
docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" | grep -E "(datamate-|deer-flow-)" || true
|
||||
fi
|
||||
}
|
||||
|
||||
# 执行主流程
|
||||
main "$@"
|
||||
@@ -77,22 +77,24 @@ for service_config in "${SERVICES[@]}"; do
|
||||
fi
|
||||
|
||||
# 使用缓存进行离线构建
|
||||
# --network=none 确保不访问网络
|
||||
# --pull=false: 不尝试拉取镜像
|
||||
# --network=none: 禁用网络访问
|
||||
docker buildx build \
|
||||
--cache-from "type=local,src=$cache_file" \
|
||||
--pull=false \
|
||||
--network=none \
|
||||
-f "$dockerfile" \
|
||||
-t "$image_name:$VERSION" \
|
||||
--load \
|
||||
. || {
|
||||
echo "错误: $service_name 构建失败"
|
||||
echo "尝试不使用 --network=none 重新构建..."
|
||||
. 2>&1 || {
|
||||
echo "警告: $service_name 离线构建遇到问题，尝试仅使用缓存..."
|
||||
docker buildx build \
|
||||
--cache-from "type=local,src=$cache_file" \
|
||||
--pull=false \
|
||||
-f "$dockerfile" \
|
||||
-t "$image_name:$VERSION" \
|
||||
--load \
|
||||
.
|
||||
. 2>&1
|
||||
}
|
||||
|
||||
echo "✓ $service_name 构建完成"
|
||||
|
||||
151
scripts/offline/diagnose.sh
Normal file
151
scripts/offline/diagnose.sh
Normal file
@@ -0,0 +1,151 @@
|
||||
#!/bin/bash
|
||||
# 离线构建诊断脚本
|
||||
# Usage: ./diagnose.sh [cache-dir]
|
||||
|
||||
set -e
|
||||
|
||||
CACHE_DIR="${1:-./build-cache}"
|
||||
|
||||
echo "======================================"
|
||||
echo "离线构建环境诊断"
|
||||
echo "======================================"
|
||||
echo ""
|
||||
|
||||
# 1. 检查 Docker
|
||||
echo "1. Docker 版本:"
|
||||
docker version --format '{{.Server.Version}}' 2>/dev/null || echo " 无法获取版本"
|
||||
echo ""
|
||||
|
||||
# 2. 检查 BuildKit
|
||||
echo "2. BuildKit 状态:"
|
||||
if docker buildx version > /dev/null 2>&1; then
|
||||
docker buildx version
|
||||
echo ""
|
||||
echo "可用的构建器:"
|
||||
docker buildx ls
|
||||
else
|
||||
echo " BuildKit 不可用"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 3. 检查缓存目录
|
||||
echo "3. 缓存目录检查 ($CACHE_DIR):"
|
||||
if [ -d "$CACHE_DIR" ]; then
|
||||
echo " ✓ 缓存目录存在"
|
||||
|
||||
# 检查子目录
|
||||
for subdir in buildkit images resources; do
|
||||
if [ -d "$CACHE_DIR/$subdir" ]; then
|
||||
echo " ✓ $subdir/ 存在"
|
||||
count=$(find "$CACHE_DIR/$subdir" -type d | wc -l)
|
||||
echo " 子目录数量: $count"
|
||||
else
|
||||
echo " ✗ $subdir/ 不存在"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo " ✗ 缓存目录不存在"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 4. 检查基础镜像
|
||||
echo "4. 基础镜像检查:"
|
||||
required_images=(
|
||||
"maven:3-eclipse-temurin-21"
|
||||
"maven:3-eclipse-temurin-8"
|
||||
"eclipse-temurin:21-jdk"
|
||||
"mysql:8"
|
||||
"node:20-alpine"
|
||||
"nginx:1.29"
|
||||
"ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm"
|
||||
"python:3.12-slim"
|
||||
"gcr.io/distroless/nodejs20-debian12"
|
||||
)
|
||||
|
||||
missing_images=()
|
||||
for img in "${required_images[@]}"; do
|
||||
if docker inspect "$img" > /dev/null 2>&1; then
|
||||
size=$(docker images --format "{{.Size}}" "$img" | head -1)
|
||||
echo " ✓ $img ($size)"
|
||||
else
|
||||
echo " ✗ $img (缺失)"
|
||||
missing_images+=("$img")
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 5. 检查 BuildKit 缓存
|
||||
echo "5. BuildKit 缓存检查:"
|
||||
if [ -d "$CACHE_DIR/buildkit" ]; then
|
||||
for cache_dir in "$CACHE_DIR/buildkit"/*-cache; do
|
||||
if [ -d "$cache_dir" ]; then
|
||||
name=$(basename "$cache_dir")
|
||||
size=$(du -sh "$cache_dir" 2>/dev/null | cut -f1)
|
||||
echo " ✓ $name ($size)"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo " ✗ 缓存目录不存在"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 6. 检查资源文件
|
||||
echo "6. 外部资源检查:"
|
||||
if [ -d "$CACHE_DIR/resources" ]; then
|
||||
if [ -f "$CACHE_DIR/resources/models/ch_ppocr_mobile_v2.0_cls_infer.tar" ]; then
|
||||
size=$(du -sh "$CACHE_DIR/resources/models/ch_ppocr_mobile_v2.0_cls_infer.tar" | cut -f1)
|
||||
echo " ✓ PaddleOCR 模型 ($size)"
|
||||
else
|
||||
echo " ✗ PaddleOCR 模型缺失"
|
||||
fi
|
||||
|
||||
if [ -f "$CACHE_DIR/resources/models/zh_core_web_sm-3.8.0-py3-none-any.whl" ]; then
|
||||
size=$(du -sh "$CACHE_DIR/resources/models/zh_core_web_sm-3.8.0-py3-none-any.whl" | cut -f1)
|
||||
echo " ✓ spaCy 模型 ($size)"
|
||||
else
|
||||
echo " ✗ spaCy 模型缺失"
|
||||
fi
|
||||
|
||||
if [ -d "$CACHE_DIR/resources/DataX" ]; then
|
||||
echo " ✓ DataX 源码"
|
||||
else
|
||||
echo " ✗ DataX 源码缺失"
|
||||
fi
|
||||
|
||||
if [ -d "$CACHE_DIR/resources/deer-flow" ]; then
|
||||
echo " ✓ deer-flow 源码"
|
||||
else
|
||||
echo " ✗ deer-flow 源码缺失"
|
||||
fi
|
||||
else
|
||||
echo " ✗ 资源目录不存在"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 7. 网络检查
|
||||
echo "7. 网络检查:"
|
||||
if ping -c 1 8.8.8.8 > /dev/null 2>&1; then
|
||||
echo " ⚠ 网络可用（离线构建环境通常不需要）"
|
||||
else
|
||||
echo " ✓ 网络不可达（符合离线环境）"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 8. 总结
|
||||
echo "======================================"
|
||||
echo "诊断总结"
|
||||
echo "======================================"
|
||||
|
||||
if [ ${#missing_images[@]} -eq 0 ]; then
|
||||
echo "✓ 所有基础镜像已就绪"
|
||||
else
|
||||
echo "✗ 缺少 ${#missing_images[@]} 个基础镜像:"
|
||||
printf ' - %s\n' "${missing_images[@]}"
|
||||
echo ""
|
||||
echo "修复方法:"
|
||||
if [ -f "$CACHE_DIR/images/base-images.tar" ]; then
|
||||
echo " docker load -i $CACHE_DIR/images/base-images.tar"
|
||||
else
|
||||
echo " 请确保有网环境导出时包含所有基础镜像"
|
||||
fi
|
||||
fi
|
||||
Reference in New Issue
Block a user