#!/bin/bash

ulimit -n 1048576 && source /root/miniconda3/bin/activate flagscale-train
mkdir -p /workspace/hygon/FlagScale/emu3p5_32b/checkpoints
mkdir -p /workspace/hygon/FlagScale/emu3p5_32b/checkpoints
mkdir -p /workspace/hygon/FlagScale/emu3p5_32b/logs
mkdir -p /workspace/hygon/FlagScale/emu3p5_32b/logs/pids
mkdir -p /workspace/hygon/FlagScale/emu3p5_32b/logs/details
mkdir -p /workspace/hygon/FlagScale/emu3p5_32b/tensorboard
mkdir -p /workspace/hygon/FlagScale/emu3p5_32b/wandb

cd /workspace/hygon/FlagScale

export PYTHONPATH=/workspace/hygon/FlagScale/third_party/Megatron-LM:/workspace/hygon/FlagScale:${PYTHONPATH}

cmd="LOGLEVEL=INFO CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 CUDA_DEVICE_MAX_CONNECTIONS=1 NCCL_DEBUG=INFO torchrun --rdzv_backend static --rdzv_id default --nnodes 2 --node_rank 0 --nproc_per_node 8 --rdzv_endpoint 10.6.208.15:37683 --log_dir /workspace/hygon/FlagScale/emu3p5_32b/logs/details/host_0_10.6.208.15/20250910_160211.052828 --redirects 3 --tee 3 flagscale/train/train_emu3.py --num-workers 2 --tensor-model-parallel-size 8 --pipeline-model-parallel-size 1 --expert-model-parallel-size 1 --context-parallel-size 2 --sequence-parallel --use-distributed-optimizer --overlap-grad-reduce --overlap-param-gather --distributed-timeout-minutes 40 --bf16 --attention-softmax-in-fp32 --accumulate-allreduce-grads-in-fp32 --log-interval 1 --tensorboard-log-interval 1 --wandb-project emu3p5_32b --wandb-exp-name emu3p5_32b --log-timers-to-tensorboard --log-validation-ppl-to-tensorboard --log-throughput --log-params-norm --log-num-zeros-in-grad --log-memory-to-tensorboard --tensorboard-dir /workspace/hygon/FlagScale/emu3p5_32b/tensorboard --wandb-save-dir /workspace/hygon/FlagScale/emu3p5_32b/wandb --save-interval 500 --ckpt-format torch --save /workspace/hygon/FlagScale/emu3p5_32b/checkpoints --load /workspace/hygon/FlagScale/emu3p5_32b/checkpoints --transformer-impl transformer_engine --num-layers 16 --hidden-size 5120 --ffn-hidden-size 25600 --num-attention-heads 64 --kv-channels 128 --group-query-attention --num-query-groups 8 --seq-length 32768 --max-position-embeddings 32768 --norm-epsilon 1e-06 --use-rotary-position-embeddings --rotary-base 1000000 --norm-init-weight 0.5 --swiglu --normalization RMSNorm --qk-layernorm --attention-dropout 0.1 --hidden-dropout 0.1 --untie-embeddings-and-output-weights --no-position-embedding --no-rope-fusion --disable-bias-linear --language-vocab-size 151851 --multimodal-visual-loss-weight 0.5 --multimodal-token-sup at --multimodal-visual-start-end-tokens 151851 151852 151853 --seed 19971104 --train-iters 500 --micro-batch-size 1 --global-batch-size 32 --eval-interval 3000 --eval-iters 16 --extra-eval-interval 3000 --optimizer adam --lr 0.0005 --min-lr 1e-05 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-08 --clip-grad 5.0 --lr-warmup-fraction 0.01 --lr-decay-iters 700000 --lr-decay-style cosine --reset-position-ids --reset-attention-mask --no-create-attention-mask-in-dataloader --train-data-path 500 /workspace/hygon/K72 --tokenizer-type Emu3TokenizerFS --tokenizer-path examples/emu3/tokenizer_emu3_ibq --special-tokens-file examples/emu3/tokenizer_emu3_ibq/emu3_vision_tokens.txt --vocab-size 282926 --make-vocab-size-divisible-by 64"

nohup bash -c "$cmd; sync" >> /workspace/hygon/FlagScale/emu3p5_32b/logs/host_0_10.6.208.15.output 2>&1 & echo $! > /workspace/hygon/FlagScale/emu3p5_32b/logs/pids/host_0_10.6.208.15.pid

