You've already forked DataMate
Jerry Yan
05f3efc148
- 将 frontend Dockerfile 中的基础镜像从 gcr.io 切换到 gcr.nju.edu.cn - 更新 offline Dockerfile 中的 nodejs20-debian12 镜像源 - 修改 export-cache.sh 脚本中的基础镜像列表为南京大学镜像 - 更新 Makefile.offline.mk 中的镜像拉取地址为本地镜像源 - 优化 export-cache.sh 脚本的格式和输出信息 - 添加缓存导出过程中的警告处理机制
135 lines
4.5 KiB
Bash
135 lines
4.5 KiB
Bash
#!/bin/bash
|
|
# BuildKit 缓存导出脚本 - 在有网环境执行
|
|
# Usage: ./export-cache.sh [output-dir]
|
|
|
|
set -e
|
|
|
|
OUTPUT_DIR="${1:-./build-cache}"
|
|
BUILDKIT_CACHE_DIR="$OUTPUT_DIR/buildkit"
|
|
IMAGES_DIR="$OUTPUT_DIR/images"
|
|
RESOURCES_DIR="$OUTPUT_DIR/resources"
|
|
|
|
# 确保 buildx 构建器存在
|
|
if ! docker buildx inspect offline-builder > /dev/null 2>&1; then
|
|
echo "创建 buildx 构建器..."
|
|
docker buildx create --name offline-builder --driver docker-container --use
|
|
else
|
|
docker buildx use offline-builder
|
|
fi
|
|
|
|
mkdir -p "$BUILDKIT_CACHE_DIR" "$IMAGES_DIR" "$RESOURCES_DIR"
|
|
|
|
echo "======================================"
|
|
echo "1. 导出基础镜像"
|
|
echo "======================================"
|
|
|
|
BASE_IMAGES=(
|
|
"maven:3-eclipse-temurin-21"
|
|
"maven:3-eclipse-temurin-8"
|
|
"eclipse-temurin:21-jdk"
|
|
"mysql:8"
|
|
"node:20-alpine"
|
|
"nginx:1.29"
|
|
"ghcr.nju.edu.cn/astral-sh/uv:python3.11-bookworm"
|
|
"ghcr.nju.edu.cn/astral-sh/uv:python3.12-bookworm"
|
|
"ghcr.nju.edu.cn/astral-sh/uv:latest"
|
|
"python:3.12-slim"
|
|
"python:3.11-slim"
|
|
"gcr.nju.edu.cn/distroless/nodejs20-debian12"
|
|
)
|
|
|
|
for img in "${BASE_IMAGES[@]}"; do
|
|
echo "拉取: $img"
|
|
docker pull "$img" || echo "警告: $img 拉取失败，可能已存在"
|
|
done
|
|
|
|
echo ""
|
|
echo "保存基础镜像到 $IMAGES_DIR/base-images.tar..."
|
|
docker save -o "$IMAGES_DIR/base-images.tar" "${BASE_IMAGES[@]}"
|
|
echo "✓ 基础镜像保存完成"
|
|
|
|
echo ""
|
|
echo "======================================"
|
|
echo "2. 导出 BuildKit 构建缓存"
|
|
echo "======================================"
|
|
|
|
# 定义服务配置
|
|
SERVICES=(
|
|
"backend:datamate-backend:scripts/images/backend/Dockerfile"
|
|
"backend-python:datamate-backend-python:scripts/images/backend-python/Dockerfile"
|
|
"database:datamate-database:scripts/images/database/Dockerfile"
|
|
"frontend:datamate-frontend:scripts/images/frontend/Dockerfile"
|
|
"gateway:datamate-gateway:scripts/images/gateway/Dockerfile"
|
|
"runtime:datamate-runtime:scripts/images/runtime/Dockerfile"
|
|
"deer-flow-backend:deer-flow-backend:scripts/images/deer-flow-backend/Dockerfile"
|
|
"deer-flow-frontend:deer-flow-frontend:scripts/images/deer-flow-frontend/Dockerfile"
|
|
"mineru:datamate-mineru:scripts/images/mineru/Dockerfile"
|
|
)
|
|
|
|
for service_config in "${SERVICES[@]}"; do
|
|
IFS=':' read -r service_name image_name dockerfile <<< "$service_config"
|
|
cache_file="$BUILDKIT_CACHE_DIR/$service_name-cache"
|
|
|
|
echo ""
|
|
echo "导出 [$service_name] 缓存到 $cache_file..."
|
|
|
|
# 先正常构建以填充缓存
|
|
docker buildx build \
|
|
--cache-to "type=local,dest=$cache_file,mode=max" \
|
|
-f "$dockerfile" \
|
|
-t "$image_name:cache" \
|
|
. || echo "警告: $service_name 缓存导出失败"
|
|
|
|
echo "✓ $service_name 缓存导出完成"
|
|
done
|
|
|
|
echo ""
|
|
echo "======================================"
|
|
echo "3. 预下载外部资源"
|
|
echo "======================================"
|
|
|
|
# PaddleOCR 模型
|
|
mkdir -p "$RESOURCES_DIR/models"
|
|
if [ ! -f "$RESOURCES_DIR/models/ch_ppocr_mobile_v2.0_cls_infer.tar" ]; then
|
|
echo "下载 PaddleOCR 模型..."
|
|
wget -O "$RESOURCES_DIR/models/ch_ppocr_mobile_v2.0_cls_infer.tar" \
|
|
"https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar" || true
|
|
fi
|
|
|
|
# spaCy 模型
|
|
if [ ! -f "$RESOURCES_DIR/models/zh_core_web_sm-3.8.0-py3-none-any.whl" ]; then
|
|
echo "下载 spaCy 模型..."
|
|
wget -O "$RESOURCES_DIR/models/zh_core_web_sm-3.8.0-py3-none-any.whl" \
|
|
"https://ghproxy.net/https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl" || true
|
|
fi
|
|
|
|
# DataX 源码
|
|
if [ ! -d "$RESOURCES_DIR/DataX" ]; then
|
|
echo "克隆 DataX 源码..."
|
|
git clone --depth 1 "https://gitee.com/alibaba/DataX.git" "$RESOURCES_DIR/DataX" || true
|
|
fi
|
|
|
|
# deer-flow 源码（用于 deer-flow 构建）
|
|
if [ ! -d "$RESOURCES_DIR/deer-flow" ]; then
|
|
echo "克隆 deer-flow 源码..."
|
|
git clone --depth 1 "https://ghproxy.net/https://github.com/ModelEngine-Group/deer-flow.git" "$RESOURCES_DIR/deer-flow" || true
|
|
fi
|
|
|
|
echo ""
|
|
echo "======================================"
|
|
echo "4. 打包缓存"
|
|
echo "======================================"
|
|
|
|
cd "$OUTPUT_DIR"
|
|
tar -czf "build-cache-$(date +%Y%m%d).tar.gz" buildkit images resources
|
|
cd - > /dev/null
|
|
|
|
echo ""
|
|
echo "======================================"
|
|
echo "✓ 缓存导出完成！"
|
|
echo "======================================"
|
|
echo "缓存位置: $OUTPUT_DIR"
|
|
echo "传输文件: $OUTPUT_DIR/build-cache-$(date +%Y%m%d).tar.gz"
|
|
echo ""
|
|
echo "请将此压缩包传输到无网环境后解压使用"
|