File size: 5,325 Bytes
26d5b81 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | #!/bin/bash
# ═══════════════════════════════════════════════════════════════
# NeuroFlow v2 训练启动脚本 (双路 Xeon E5-2666 v3 优化版)
#
# 硬件: 2× Intel Xeon E5-2666 v3, 40 threads, AVX2, 64GB DDR4
# 语料: D:\语料\ (~420GB, 10个目录, 含大学215万文件)
#
# 用法:
# chmod +x scripts/train_optimized.sh
# bash scripts/train_optimized.sh
# ═══════════════════════════════════════════════════════════════
set -euo pipefail
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$PROJECT_ROOT"
# ═══ 配置 ═══════════════════════════════════════════════════
CORPUS_DIR="/mnt/d/语料"
TOKENIZED_DIR="${HOME}/neuroflow_data"
OUTPUT_DIR="${PROJECT_ROOT}/output"
CONFIG_PATH="${PROJECT_ROOT}/configs/config.json"
TOKENIZER_PATH="${PROJECT_ROOT}/configs/tokenizer_128k.json"
# 训练超参
EPOCHS=20
BATCH_SIZE=64
GRAD_ACCUM=4 # 梯度累积步数 (等效 batch=256)
LEARNING_RATE=3e-5
GRAD_CLIP=4.0
LOG_INTERVAL=50
SAVE_INTERVAL=5000
SEED=42
# 经验回放
REPLAY_BUFFER=10000
REPLAY_RATIO=0.25
# ═══ Step 1: 编译 ═══════════════════════════════════════════
echo "═════════════════════════════════════════════════════"
echo "Step 1: 编译 NeuroFlow"
echo "═════════════════════════════════════════════════════"
mkdir -p build && cd build
cmake .. \
-DCMAKE_BUILD_TYPE=Release \
-DNEUROFLOW_USE_BLAS=ON \
-DNEUROFLOW_USE_AVX2=ON \
-DNEUROFLOW_BUILD_TESTS=OFF
make -j$(nproc)
cd "$PROJECT_ROOT"
echo "编译完成."
# ═══ Step 2: 数据预处理 (首次运行) ══════════════════════════
echo ""
echo "═════════════════════════════════════════════════════"
echo "Step 2: 数据预处理"
echo "═════════════════════════════════════════════════════"
if [ ! -f "${TOKENIZED_DIR}/train.tok1" ]; then
echo "首次运行, 预处理语料到 ${TOKENIZED_DIR} ..."
echo "语料: ${CORPUS_DIR}"
echo "输出: ${TOKENIZED_DIR}"
python3 scripts/prepare_training_data.py \
--corpus "${CORPUS_DIR}" \
--tokenizer "${TOKENIZER_PATH}" \
--output "${TOKENIZED_DIR}" \
--max-seq-len 128 \
--max-samples 5000000
echo "预处理完成."
else
echo "已存在 tokenized 数据: ${TOKENIZED_DIR}/train.tok1"
ls -lh "${TOKENIZED_DIR}/train.tok1"
fi
# ═══ Step 3: 训练 ═══════════════════════════════════════════
echo ""
echo "═════════════════════════════════════════════════════"
echo "Step 3: 启动训练"
echo "═════════════════════════════════════════════════════"
# OpenBLAS 线程优化 (双路Xeon, 每socket 10物理核)
# 使用一半物理核心给BLAS, 另一半给OpenMP
NCPU=$(nproc)
BLAS_THREADS=$((NCPU / 4)) # 10 threads for BLAS (one socket)
export OPENBLAS_NUM_THREADS=${BLAS_THREADS}
export GOTO_NUM_THREADS=${BLAS_THREADS}
export OMP_NUM_THREADS=${BLAS_THREADS}
export OMP_PROC_BIND=close
export OMP_PLACES=cores
echo "硬件配置:"
echo " CPU 核心: ${NCPU}"
echo " BLAS 线程: ${BLAS_THREADS}"
echo " 批大小: ${BATCH_SIZE} × ${GRAD_ACCUM} grad_accum = $((BATCH_SIZE * GRAD_ACCUM))"
echo " 学习率: ${LEARNING_RATE}"
echo " 数据: ${TOKENIZED_DIR}/train.tok1"
mkdir -p "${OUTPUT_DIR}"
# 训练命令
./build/neuroflow_train_v2 \
--config "${CONFIG_PATH}" \
--tokenizer "${TOKENIZER_PATH}" \
--data "${TOKENIZED_DIR}" \
--output "${OUTPUT_DIR}" \
--epochs ${EPOCHS} \
--batch-size ${BATCH_SIZE} \
--lr ${LEARNING_RATE} \
--grad-clip ${GRAD_CLIP} \
--grad-accum ${GRAD_ACCUM} \
--seed ${SEED} \
--log-interval ${LOG_INTERVAL} \
--save-interval ${SAVE_INTERVAL} \
--replay-buffer ${REPLAY_BUFFER} \
--replay-ratio ${REPLAY_RATIO} \
--init-weights xavier
echo ""
echo "═════════════════════════════════════════════════════"
echo "训练完成! 模型保存于: ${OUTPUT_DIR}/model_final.nfv1"
echo "═════════════════════════════════════════════════════"
|