#!/bin/bash
export OMPI_MCA_rmaps_base_oversubscribe=1  # 允许过度订阅
export OMPI_MCA_btl_base_verbose=100       # 设置 BTL 模块的调试级别
export OMPI_MCA_plm_base_verbose=100       # 设置 PLM 模块的调试级别
export OMPI_MCA_orte_base_help_aggregate=0 # 禁用汇总帮助信息
export OMPI_MCA_odebug=1                   # 开启调试模式
export NCCL_DEBUG=INFO  # 设置 NCCL 调试级别为 INFO
export NCCL_DEBUG_SUBSYS=ALL  # 设置调试子系统为 ALL
#export NCCL_DEBUG_FILE=/share/projset/wwx/nccl-tests/nccl_debug.log  # 指定调试日志文件路径
ulimit -n 1048576
#export PATH=/usr/local/ucx/bin:$PATH
#export LD_LIBRARY_PATH=/usr/local/ucx/lib:$LD_LIBRARY_PATH
#export NCCL_IB_DISABLE=1           # 禁用 InfiniBand
#export NCCL_P2P_DISABLE=1          # 禁用 GPU Direct P2P
export NCCL_DEBUG=INFO
#export NCCL_UCX_TLS=rc,cuda,cuda_copy

mpirun --allow-run-as-root --hostfile my_hostfile -np 16 -N 8 \
	-x GLOO_SOCKET_IFNAME=bond0 \
	-x NCCL_SOCKET_IFNAME=bond0 \
	-x NCCL_IB_QPS_PER_CONNECTION=2 -x NCCL_PXN_DISABLE=1 -x NCCL_MIN_NCHANNELS=23 \
	-x NCCL_ALGO=Ring \
	-x NCCL_IB_HCA=mlx5_101,mlx5_102,mlx5_103,mlx5_104,mlx5_105,mlx5_106,mlx5_107,mlx5_100 \
	./build/all_reduce_perf -b 16G -e 16G -f 2 -g 1
