You've already forked DataMate
feat(build): 添加传统 Docker 构建方式和诊断功能
- 在 build-offline.sh 脚本中添加 --pull=false 参数并改进错误处理 - 为 Makefile.offline.mk 中的各个服务构建任务添加 --pull=false 参数 - 新增 build-offline-classic.sh 脚本，提供不使用 BuildKit 的传统构建方式 - 新增 build-offline-v2.sh 脚本，提供增强版 BuildKit 离线构建功能 - 新增 diagnose.sh 脚本，用于诊断离线构建环境状态 - 在 Makefile 中添加 offline-build-classic 和 offline-diagnose
This commit is contained in:
206
scripts/offline/build-offline-classic.sh
Normal file
206
scripts/offline/build-offline-classic.sh
Normal file
@@ -0,0 +1,206 @@
|
||||
#!/bin/bash
|
||||
# 传统 docker build 离线构建脚本（不使用 buildx）
|
||||
# 这种方式更稳定，兼容性更好
|
||||
# Usage: ./build-offline-classic.sh [cache-dir] [version]
|
||||
|
||||
set -e
|
||||
|
||||
CACHE_DIR="${1:-./build-cache}"
|
||||
VERSION="${2:-latest}"
|
||||
IMAGES_DIR="$CACHE_DIR/images"
|
||||
RESOURCES_DIR="$CACHE_DIR/resources"
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
log_debug() { echo -e "${BLUE}[DEBUG]${NC} $1"; }
|
||||
|
||||
# 检查缓存目录
|
||||
if [ ! -d "$CACHE_DIR" ]; then
|
||||
log_error "缓存目录 $CACHE_DIR 不存在"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 加载基础镜像
|
||||
load_base_images() {
|
||||
log_info "加载基础镜像..."
|
||||
|
||||
if [ ! -f "$IMAGES_DIR/base-images.tar" ]; then
|
||||
log_warn "基础镜像 tar 包不存在，检查本地镜像..."
|
||||
return
|
||||
fi
|
||||
|
||||
log_info "从 $IMAGES_DIR/base-images.tar 加载..."
|
||||
docker load -i "$IMAGES_DIR/base-images.tar"
|
||||
log_info "✓ 基础镜像加载完成"
|
||||
}
|
||||
|
||||
# 检查镜像是否存在
|
||||
check_image() {
|
||||
docker inspect "$1" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# 构建函数
|
||||
build_service() {
|
||||
local service_name=$1
|
||||
local image_name=$2
|
||||
local dockerfile=$3
|
||||
|
||||
log_info "----------------------------------------"
|
||||
log_info "构建 $service_name"
|
||||
log_info "----------------------------------------"
|
||||
|
||||
# 检查 Dockerfile 是否存在
|
||||
if [ ! -f "$dockerfile" ]; then
|
||||
log_error "Dockerfile 不存在: $dockerfile"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 获取所需的基础镜像
|
||||
local from_images
|
||||
from_images=$(grep -E '^FROM' "$dockerfile" | sed 's/FROM //' | sed 's/ AS .*//' | sed 's/ as .*//' | awk '{print $1}' | sort -u)
|
||||
|
||||
log_info "检查基础镜像..."
|
||||
local all_exist=true
|
||||
for img in $from_images; do
|
||||
# 跳过多阶段构建的中间阶段引用
|
||||
if [[ "$img" == --from=* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if check_image "$img"; then
|
||||
log_info " ✓ $img"
|
||||
else
|
||||
log_error " ✗ $img (缺失)"
|
||||
all_exist=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$all_exist" = false ]; then
|
||||
log_error "缺少必要的基础镜像，无法构建 $service_name"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 准备构建参数
|
||||
local build_args=()
|
||||
|
||||
# 根据服务类型添加特殊处理
|
||||
case "$service_name" in
|
||||
runtime)
|
||||
# runtime 需要模型文件
|
||||
if [ -d "$RESOURCES_DIR/models" ]; then
|
||||
log_info "使用本地模型文件"
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
fi
|
||||
;;
|
||||
backend-python)
|
||||
if [ -d "$RESOURCES_DIR/DataX" ]; then
|
||||
log_info "使用本地 DataX 源码"
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
build_args+=("--build-arg" "DATAX_LOCAL_PATH=$RESOURCES_DIR/DataX")
|
||||
fi
|
||||
;;
|
||||
deer-flow-backend|deer-flow-frontend)
|
||||
if [ -d "$RESOURCES_DIR/deer-flow" ]; then
|
||||
log_info "使用本地 deer-flow 源码"
|
||||
build_args+=("--build-arg" "RESOURCES_DIR=$RESOURCES_DIR")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
# 执行构建
|
||||
log_info "开始构建..."
|
||||
if docker build \
|
||||
--pull=false \
|
||||
"${build_args[@]}" \
|
||||
-f "$dockerfile" \
|
||||
-t "$image_name:$VERSION" \
|
||||
. 2>&1; then
|
||||
log_info "✓ $service_name 构建成功"
|
||||
return 0
|
||||
else
|
||||
log_error "✗ $service_name 构建失败"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 主流程
|
||||
main() {
|
||||
log_info "======================================"
|
||||
log_info "传统 Docker 离线构建"
|
||||
log_info "======================================"
|
||||
|
||||
# 加载基础镜像
|
||||
load_base_images
|
||||
|
||||
# 定义要构建的服务
|
||||
declare -A SERVICES=(
|
||||
["database"]="datamate-database:scripts/images/database/Dockerfile"
|
||||
["gateway"]="datamate-gateway:scripts/images/gateway/Dockerfile"
|
||||
["backend"]="datamate-backend:scripts/images/backend/Dockerfile"
|
||||
["frontend"]="datamate-frontend:scripts/images/frontend/Dockerfile"
|
||||
["runtime"]="datamate-runtime:scripts/images/runtime/Dockerfile"
|
||||
["backend-python"]="datamate-backend-python:scripts/images/backend-python/Dockerfile"
|
||||
)
|
||||
|
||||
# deer-flow 和 mineru 是可选的
|
||||
OPTIONAL_SERVICES=(
|
||||
"deer-flow-backend:deer-flow-backend:scripts/images/deer-flow-backend/Dockerfile"
|
||||
"deer-flow-frontend:deer-flow-frontend:scripts/images/deer-flow-frontend/Dockerfile"
|
||||
"mineru:datamate-mineru:scripts/images/mineru/Dockerfile"
|
||||
)
|
||||
|
||||
log_info ""
|
||||
log_info "构建核心服务..."
|
||||
local failed=()
|
||||
local succeeded=()
|
||||
|
||||
for service_name in "${!SERVICES[@]}"; do
|
||||
IFS=':' read -r image_name dockerfile <<< "${SERVICES[$service_name]}"
|
||||
if build_service "$service_name" "$image_name" "$dockerfile"; then
|
||||
succeeded+=("$service_name")
|
||||
else
|
||||
failed+=("$service_name")
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# 尝试构建可选服务
|
||||
log_info "构建可选服务..."
|
||||
for service_config in "${OPTIONAL_SERVICES[@]}"; do
|
||||
IFS=':' read -r service_name image_name dockerfile <<< "$service_config"
|
||||
if build_service "$service_name" "$image_name" "$dockerfile"; then
|
||||
succeeded+=("$service_name")
|
||||
else
|
||||
log_warn "$service_name 构建失败（可选服务，继续）"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# 汇总
|
||||
log_info "======================================"
|
||||
log_info "构建结果"
|
||||
log_info "======================================"
|
||||
|
||||
if [ ${#succeeded[@]} -gt 0 ]; then
|
||||
log_info "成功 (${#succeeded[@]}): ${succeeded[*]}"
|
||||
fi
|
||||
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
log_error "失败 (${#failed[@]}): ${failed[*]}"
|
||||
exit 1
|
||||
else
|
||||
log_info "✓ 所有核心服务构建成功！"
|
||||
echo ""
|
||||
docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" | grep -E "(datamate-|deer-flow-)" || true
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user