#!/bin/bash
###
 # @Author: wxwang
 # @Script name: NvidiaGPU_Check.sh
 # @Script description: 网卡、GPU等硬件健康状态检查
 # @Environmental Requirement: 适用于基于 Debian（如 Ubuntu）或 Red Hat（如 CentOS）系列的 Linux 发行版，Bash.
###


# 设置 nvidia-smi 命令的超时时间
NVSMI_TIMEOUT=15


# Define the test result colors, with PASS as green and FAIL as red
# Use 8-bit color mode
RED_COLOR='\033[1;31m'     # 红色
GREEN_COLOR='\033[1;32m'   # 绿色
YELLOW_COLOR='\033[1;33m'  # 黄色
BLUE_COLOR='\033[1;34m'    # 蓝色
PURPLE_COLOR='\033[1;35m'  # 紫色
RESET_COLOR='\033[0m'      # 重置颜色

# 创建一个临时目录
TEMP_DIR=$(mktemp -d) || { echo "Failed to create temporary directory"; exit 1; }
#echo "临时目录是：$TEMP_DIR"

# 日志文件路径定义
NVSMI_INFO_FILE="${TEMP_DIR}/gpu_info.csv"     # GPU 信息 CSV 文件
ERROR_LOG="${TEMP_DIR}/error.log"              # 错误日志文件
WARING_LOG="${TEMP_DIR}/waring.log"            # 警告日志文件
NET_ERROR_LOG="${TEMP_DIR}/net_error.log"      # 网络错误日志文件
GPU_ERROR_LOG="${TEMP_DIR}/gpu_error.log"      # GPU 错误日志文件
NET_INFO_JSON="${TEMP_DIR}/network_info.json"  # 网络信息 JSON 文件
GPU_INFO_JSON="${TEMP_DIR}/gpu_info.json"      # GPU 信息 JSON 文件
MERGE_DATA_JSON="${TEMP_DIR}/merge_data.json"  # 合并数据 JSON 文件

# CX6&7 网络固件版本信息
declare -A FW_EXPECTATION=(
    ["1021"]="28.39.1002"  # CX7 400G
    ["101b"]="20.39.1002"  # CX6 200G
    ["101d"]="22.39.1002"   # CX6 100G
)

# 不同 GPU 模型对应的 NVswitch 数量
declare -A NVSWITCH_COUNTS_EXPECTATION=(
    ["1af1"]="6"  # A 系列
    ["22a3"]="4"  # H 系列
)

# 不同 GPU 模型对应的 NVLink 数量
declare -A NVLINK_COUNTS_EXPECTATION=(
    ["20b2"]="12"  # A100
    ["20b0"]="12"  # A100
    ["20f3"]="8"   # A800
    ["2324"]="8"   # 4090
    ["2330"]="18"  # H100
    ["2335"]="18"  # H200
    ["2329"]="18"  # H20
)

# 不同 GPU 模型对应的 NVLink 带宽
declare -A NVLINK_BANDWIDTH_EXPECTATION=(
    ["20b2"]="300"      # A100 - 80G
    ["20b0"]="300"      # A100 - 40G
    ["20f3"]="200"      # A800
    ["2324"]="212.496"  # H800
    ["2330"]="478.116"  # H100
    ["2335"]="478.116"  # H200
    ["2329"]="478.116"  # H20
)

# 收集设备信息
function collect_devices() {
    local type=$1
    lspci -nn | grep -E "$type" | awk '{print $1}'
}

# 收集设备数量
function collect_device_count() {
    local type=$1
    lspci -nn | grep -cE "$type"
}

# 收集设备 DID
function collect_did() {
    local type=$1
    lspci -nn | grep -Eo "$type" | sort -u
}

## 收集设备根端口(rport)
#function collect_rport() {
#    local type=$1
#    if lspci -PP > /dev/null 2>&1;then
#        dirname $(lspci -PPnn | awk "/$type/{print \$1}") | tr '/' '\n' | sort -u
#    fi
#}
# 收集设备根端口(rport)
function collect_rport() {
    local type=$1
    if lspci -PP > /dev/null 2>&1; then
        local result=$(lspci -PPnn | awk "/$type/{print \$1}")
        if [[ -n "$result" ]]; then
            dirname "$result" | tr '/' '\n' | sort -u
        else
            echo -e "    ${YELLOW_COLOR}警告:${RESET_COLOR} 未获取到 GPU 信息，请检查 ${CYAN_COLOR}nvidia-smi${RESET_COLOR} 是否正常输出。" >> "$ERROR_LOG"
            exit 1
        fi
    else
        echo -e "    ${YELLOW_COLOR}警告:${RESET_COLOR} lspci -PP 执行失败。" >> "$ERROR_LOG"
        exit 1
    fi
}

# Mellanox 厂商 ID 是 15b3, NVIDIA 厂商 ID 是 10de
NET_VENDOR_ID='15b3'
GPU_VENDOR_ID='10de'

# 收集 CX6&7 网络设备信息
NET_TYPE_ALL="Infiniband|Ethernet"
NET_TYPE_CX6="ConnectX-6|101b|101d"
NET_TYPE_CX7="ConnectX-7|1021"
NET_TYPE_CX6_7="$NET_TYPE_CX6|$NET_TYPE_CX7"

NET_DEVICES=$(collect_devices "$NET_TYPE_ALL")
NET_COUNTS=$(collect_device_count "$NET_TYPE_ALL")

# 收集 GPU 设备
#GPU_TYPE="20b2|20f3|2324|2330|2335|2329"
GPU_TYPE="20b2|20f3|2324|2330|2335|2329|20b0"
NVIDIA_DEVICES=$(collect_devices "$GPU_TYPE")
NVIDIA_COUNTS=$(collect_device_count "$GPU_TYPE")
NVIDIA_DID=$(collect_did "$GPU_TYPE")

# 收集 NVSwitch 设备
NVSWITCH_TYPE="1af1|22a3"
NVSWITCH_DEVICES=$(collect_devices "$NVSWITCH_TYPE")
NVSWITCH_COUNTS=$(collect_device_count "$NVSWITCH_TYPE")
NVSWITCH_DID=$(collect_did "$NVSWITCH_TYPE")

# 收集 rport 总线信息，使用 lspci -PPn 命令查询所有 NVIDIA 设备 PCIe Link BUS
NET_RPORT_BUS=$(collect_rport "$NET_TYPE_CX7")
NET_RPORT_COUNTS=$(echo "$NET_RPORT_BUS" | wc -l)
NVIDIA_RPORT_BUS=$(collect_rport "$GPU_TYPE")
NVIDIA_RPORT_COUNTS=$(echo "$NVIDIA_RPORT_BUS" | wc -l)
NVSWITCH_RPORT_BUS=$(collect_rport "$NVSWITCH_TYPE")
NVSWITCH_RPORT_COUNTS=$(echo "$NVSWITCH_RPORT_BUS" | wc -l)

# 重置 gpu 变量定义
PROJECTS_REQUIRING_RESET=()
GPU_IDS=()

# 删除临时目录
trap 'rm -rf "$TEMP_DIR"' EXIT > /dev/null 2>&1

# 输出错误日志
function output_error() {
    #printf '%*s\n' 70 '' | tr ' ' -
    if [ -s "$ERROR_LOG" ];then
        echo -e "${RED_COLOR}Error Log:${RESET_COLOR}" | cat - "$ERROR_LOG" > "${ERROR_LOG}_all" 
        cat "${ERROR_LOG}_all"
    fi
}

################################ CHECK NETWORK CARDS ################################
# 收集所有网络卡信息
function collect_net_info() {
    # 遍历所有网络卡
    while read -r bus_id_net;do
        local net_did=$(lspci -nn | grep "$bus_id_net" | awk -F '[][]' "/$NET_TYPE_ALL/{print \$(NF-1)}" | awk -F: '{print $2}')               # 获取设备ID
        local eth_name=$(ls -l /sys/class/net/ | awk -F '>' "/$bus_id_net/{print $NF}" | xargs -i basename {})                                 # 获取以太网接口名称
        local mlx_name=$(ls -l /sys/class/infiniband | awk -F '>' "/$bus_id_net/{print $NF}" | xargs -i basename {})                           # 获取InfiniBand接口名称
        local fw_ver=$(cat /sys/class/infiniband/${mlx_name}/fw_ver 2> /dev/null)                                                              # 获取固件版本
        local speed_cap=$(lspci -vvs "$bus_id_net"  | awk -F"," '/LnkCap:/{print $2}' | awk '{print $2}' | tr -d ',' | tr '[A-Z]' '[a-z]')     # 获取链路速度能力
        local speed_sta=$(lspci -vvs "$bus_id_net"  | awk -F"," '/LnkSta:/{print $1}' | awk '{print $3}' | tr -d ',' | tr '[A-Z]' '[a-z]')     # 获取链路速度状态
        local width_cap=$(lspci -vvs "$bus_id_net" | awk -F"," '/LnkCap:/{print $3}' | awk '{print $2}' | tr -d ',')                           # 获取链路宽度能力
        local width_sta=$(lspci -vvs "$bus_id_net" | awk -F"," '/LnkSta:/{print $2}' | awk '{print $2}' | tr -d ',')                           # 获取链路宽度状态
        local speed=$(cat /sys/class/net/${eth_name}/speed 2> /dev/null)                                                                       # 获取网络速度
        echo "{
            \"pci_bus_id\":\"$bus_id_net\",
            \"device_id\":\"$net_did\",
            \"eth_name\":\"$eth_name\",
            \"mlx_name\":\"$mlx_name\",
            \"firmware_version\":\"$fw_ver\",
            \"link_speed_cap\":\"$speed_cap\",
            \"link_speed_sta\":\"$speed_sta\",
            \"link_width_cap\":\"$width_cap\",
            \"link_width_sta\":\"$width_sta\",
            \"net_speed\":\"$speed\"
        }" >> "${NET_INFO_JSON}_tmp1"
    done <<< "$NET_DEVICES"

    # 将所有网络卡信息合并成一个 JSON 文件
    jq -sc . "${NET_INFO_JSON}_tmp1" > "${NET_INFO_JSON}_tmp2"

    # 遍历所有 NVIDIA 设备的根端口
    if [ -n "$NET_RPORT_BUS" ];then
        while read -r bus_id_rport;do
            # 获取 Lnksta 和 Lnkcap 状态
            local speed_cap=$(lspci -vvs "$bus_id_rport"  | awk -F"," '/LnkCap:/{print $2}' | awk '{print $2}' | tr -d ',' | tr '[A-Z]' '[a-z]')
            local speed_sta=$(lspci -vvs "$bus_id_rport"  | awk -F"," '/LnkSta:/{print $1}' | awk '{print $3}' | tr -d ',' | tr '[A-Z]' '[a-z]')
            local width_cap=$(lspci -vvs "$bus_id_rport" | awk -F"," '/LnkCap:/{print $3}' | awk '{print $2}' | tr -d ',')
            local width_sta=$(lspci -vvs "$bus_id_rport" | awk -F"," '/LnkSta:/{print $2}' | awk '{print $2}' | tr -d ',')

            echo "{
                \"pci_bus_id\":\"$bus_id_rport\",
                \"link_speed_cap\":\"$speed_cap\",
                \"link_speed_sta\":\"$speed_sta\",
                \"link_width_cap\":\"$width_cap\",
                \"link_width_sta\":\"$width_sta\"
            }" >> "${NET_INFO_JSON}_net_rport1"
        done <<< "$NET_RPORT_BUS"
        # 将所有 GPU 根端口信息合并成一个 JSON 文件
        jq -sc . "${NET_INFO_JSON}_net_rport1" > "${NET_INFO_JSON}_net_rport2"
    else
        echo "null"> "${NET_INFO_JSON}_net_rport2"
    fi

    # 生成最终 JSON 输出
    echo "{
        \"net_rport\":$(cat "${NET_INFO_JSON}_net_rport2"),
        \"nets\":$(cat "${NET_INFO_JSON}_tmp2")
    }" > "$NET_INFO_JSON"

}

# 检查 CX6&7 网络卡固件版本
function check_net_fw() {
    local check_flag=0

    for ((i=0; i<"$NET_COUNTS"; i++)); do
        local device_id=$(jq -r .nets[$i].device_id "$NET_INFO_JSON")
        if [[ "$device_id" =~ $NET_TYPE_CX6_7 ]];then
            local pci_bus_id=$(jq -r .nets[$i].pci_bus_id "$NET_INFO_JSON")
            local eth_name=$(jq -r .nets[$i].eth_name "$NET_INFO_JSON")
            local fw_ver=$(jq -r .nets[$i].firmware_version "$NET_INFO_JSON")

            # 检查网络固件版本是否符合预期
            local expected_fw="${FW_EXPECTATION[$device_id]}"
            if [[ -n "$expected_fw" && "$fw_ver" != "$expected_fw" ]]; then
                echo -e "    ${pci_bus_id} , ${eth_name} , 实际: ${fw_ver}, 预期: ${expected_fw}.${RED_COLOR}固件版本不符合预期.${RESET_COLOR}" >> "${NET_ERROR_LOG}_net_fw"
                ((check_flag++))
            else
                # 确保 $expected_fw 未定义或固件版本匹配
                if [[ -z "$expected_fw" ]]; then
                    echo -e "    ${pci_bus_id} , ${eth_name} .${RED_COLOR}未找到匹配的设备ID !${RESET_COLOR}" >> "${NET_ERROR_LOG}_net_fw"
                    ((check_flag++))
                fi
            fi
        fi
    done

    local iterm="Net Firmware 版本检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${NET_ERROR_LOG}_net_fw")" >> "$ERROR_LOG"
    fi

}

# 检查网络卡 PCIe 链路状态
function check_net_link_status() {
    local check_flag=0

    for ((i=0; i<"$NET_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .nets[$i].pci_bus_id "$NET_INFO_JSON")
        local eth_name=$(jq -r .nets[$i].eth_name "$NET_INFO_JSON")
        local width_cap=$(jq -r .nets[$i].link_width_cap "$NET_INFO_JSON")
        local width_sta=$(jq -r .nets[$i].link_width_sta "$NET_INFO_JSON")
        local speed_cap=$(jq -r .nets[$i].link_speed_cap "$NET_INFO_JSON")
        local speed_sta=$(jq -r .nets[$i].link_speed_sta "$NET_INFO_JSON")

        # 跳过 eno 开头的网卡
        if [[ "$eth_name" == eno* ]]; then
            continue
        fi

        # 检查 PCIe 链路宽度是否符合预期
        if [[ -z "$width_sta" || "$width_sta" != "$width_cap" ]]; then
            echo -e "    ${pci_bus_id} , ${eth_name} , 实际: ${width_sta}, 预期: ${width_cap}.${RED_COLOR}PCIe 链路宽度不符合预期.${RESET_COLOR}" >> "${NET_ERROR_LOG}_net_link"
            ((check_flag++))
        fi

        # 检查 PCIe 链路速度是否符合预期
        if [[ -z "$speed_sta" || "$speed_sta" != "$speed_cap" ]]; then
            echo -e "    ${pci_bus_id} , ${eth_name} , 实际: ${speed_sta}, 预期: ${speed_cap}.${RED_COLOR}PCIe 链路速度不符合预期.${RESET_COLOR}" >> "${NET_ERROR_LOG}_net_link"
            ((check_flag++))
        fi
    done

    local iterm="Net PCIe Link 状态检查结果  "
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${NET_ERROR_LOG}_net_link")" >> "$ERROR_LOG"
    fi
}

# 检查网络卡根端口 PCIe 链路状态
function check_net_rport_link_status() {
    local check_flag=0

    if [ -n "$NET_RPORT_BUS" ];then
        for ((i=0; i<"$NET_RPORT_COUNTS"; i++)); do
            local pci_bus_id=$(jq -r .net_rport[$i].pci_bus_id "$NET_INFO_JSON")
            local width_cap=$(jq -r .net_rport[$i].link_width_cap "$NET_INFO_JSON")
            local width_sta=$(jq -r .net_rport[$i].link_width_sta "$NET_INFO_JSON")
            local speed_cap=$(jq -r .net_rport[$i].link_speed_cap "$NET_INFO_JSON")
            local speed_sta=$(jq -r .net_rport[$i].link_speed_sta "$NET_INFO_JSON")

            # 新增：获取根端口对应的设备名称
            local port_device=$(lspci -s "$pci_bus_id" | awk -F': ' '{print $2}')

            # 检查 PCIe 链路宽度
            if [[ -z "$width_sta" || "$width_sta" != "$width_cap" ]]; then
                echo -e "    ${pci_bus_id} (${port_device}), 实际: ${width_sta}, 预期: ${width_cap}.${RED_COLOR} PCIe 链路宽度不符合预期.${RESET_COLOR}" >> "${NET_ERROR_LOG}_netrport_link"
                ((check_flag++))
            fi

            # 检查 PCIe 链路速度
            if [[ -z "$speed_sta" || "$speed_sta" != "$speed_cap" ]]; then
                echo -e "    ${pci_bus_id} (${port_device}), 实际: ${speed_sta}, 预期: ${speed_cap}.${RED_COLOR} PCIe 链路速度不符合预期.${RESET_COLOR}" >> "${NET_ERROR_LOG}_netrport_link"
                ((check_flag++))
            fi
        done
    else
        echo -e "    ${RED_COLOR}没有 CX7 网络适配器存在或 $(lspci --version) 不支持 -PP 参数.${RESET_COLOR}" >> "${NET_ERROR_LOG}_netrport_link"
        ((check_flag++))
    fi

    local iterm="Net Root port 链路状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${NET_ERROR_LOG}_netrport_link")" >> "$ERROR_LOG"
    fi
}

# 检查网络卡的日志和状态
#function check_net_log() {
#    if [ -f "/var/log/syslog" ]; then
#        local logfile="/var/log/syslog"
#    else
#        local logfile="/var/log/messages"
#    fi
#
#    grep -aEi "i40e|mlx5_core" "$logfile" > "${NET_ERROR_LOG}_system"
#    local check_flag=0
#
#    # 遍历所有网络设备
#    for ((i=0; i<"$NET_COUNTS"; i++)); do
#        # 提取设备信息
#        local pci_bus_id=$(jq -r .nets[$i].pci_bus_id "$NET_INFO_JSON")
#        local mlx_name=$(jq -r .nets[$i].mlx_name "$NET_INFO_JSON")
#        local updown_counts=$(grep "$pci_bus_id" "${NET_ERROR_LOG}_system" | grep -ic "down")
#        local updown_time=$(grep "$pci_bus_id" "${NET_ERROR_LOG}_system" | grep -i "down" | tail -n1 | awk '{print $1,$2,$3}')
#
#        # 检查网络错误和上下线日志
#        if [[ "$updown_counts" -ne 0 ]]; then
#            echo -e "    ${RED_COLOR}网卡: ${pci_bus_id} , 抖动网口: ${mlx_name} . 抖动次数: ${updown_counts} . 最后抖动时间: ${updown_time} ." >> "${NET_ERROR_LOG}_updown"
#            ((check_flag++))
#        fi
#    done
#
#    local iterm="Net 日志检查结果"
#    if [ "$check_flag" -ne 0 ];then
#        echo -e "$iterm:\n$(cat "${NET_ERROR_LOG}_updown")" >> "$ERROR_LOG"
#    fi
#}
function check_net_log() {
    local NET_LOG_CHECK_RANGE_HOURS=24

    if [ -f "/var/log/syslog" ]; then
        local logfile="/var/log/syslog"
    else
        local logfile="/var/log/messages"
    fi

    local since_epoch=$(date -d "$NET_LOG_CHECK_RANGE_HOURS hours ago" +%s)
    local current_year=$(date +%Y)

    grep -aEi "i40e|mlx5_core" "$logfile" | awk -v since="$since_epoch" -v y="$current_year" '
    {
        # 日志时间格式通常为：Apr 14 15:40:07
        month=$1; day=$2; time=$3;

        cmd = "date -d \"" month " " day " " time " " y "\" +%s";
        cmd | getline log_ts;
        close(cmd);

        if (log_ts >= since) print $0;
    }
    ' > "${NET_ERROR_LOG}_system"

    local check_flag=0

    for ((i=0; i<"$NET_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .nets[$i].pci_bus_id "$NET_INFO_JSON")
        local mlx_name=$(jq -r .nets[$i].mlx_name "$NET_INFO_JSON")
        local updown_counts=$(grep "$pci_bus_id" "${NET_ERROR_LOG}_system" | grep -ic "down")
        local updown_time=$(grep "$pci_bus_id" "${NET_ERROR_LOG}_system" | grep -i "down" | tail -n1 | awk '{print $1, $2, $3}')

        if [[ "$updown_counts" -ne 0 ]]; then
            echo -e "    ${RED_COLOR}网卡: ${pci_bus_id} , 抖动网口: ${mlx_name} . 抖动次数: ${updown_counts} . 最后抖动时间: ${updown_time} . ${RESET_COLOR}" >> "${NET_ERROR_LOG}_updown"
            ((check_flag++))
        fi
    done

    local iterm="Net 日志检查结果（最近 ${NET_LOG_CHECK_RANGE_HOURS} 小时）"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${NET_ERROR_LOG}_updown")" >> "$ERROR_LOG"
    fi
}



################################ CHECK GPUS ################################
# 检查 GPU 是否属于指定系列
function check_gpu_series(){
    if [[ "$NVIDIA_DID" =~ $GPU_TYPE ]];then
        return 0
    else
        echo -e "    ${YELLOW_COLOR}警告:${RESET_COLOR} 不支持的 GPU 模型 ${RED_COLOR} $NVIDIA_DID ${RESET_COLOR} ，脚本退出!\n
            A100    10de:20b2
            A800    10de:20f3
            4090    10de:2324
            H100    10de:2330
            H200    10de:2335
            H20    10de:2329\n" >> "$ERROR_LOG"
        output_error
        exit 1
    fi
}

# 使用 nvidia-smi 检查并获取 GPU 信息
function check_and_get_value() {
    timeout $NVSMI_TIMEOUT nvidia-smi > /dev/null 2>&1
    local smi_code=$?
    if [ "$smi_code" -eq 0 ]; then
        local query=(
            pci.bus_id
            index
            name
            serial
            temperature.gpu
            temperature.gpu.tlimit
            power.draw
            clocks_event_reasons.hw_slowdown
            pcie.link.gen.current
            ecc.mode.current
            ecc.errors.corrected.volatile.total
            ecc.errors.uncorrected.volatile.total
        )
        local query_list=()
        for i in "${query[@]}"; do
            timeout $NVSMI_TIMEOUT nvidia-smi --query-gpu=${i} --format=csv,noheader > /dev/null 2>&1
            local smi_code=$?
            if [ "$smi_code" -eq 0 ]; then
                query_list+=("$i")
            else
                echo "当前 nvidia 驱动版本不支持 nvidia-smi --query-gpu=$i --format=csv 命令。" >> "$WARING_LOG"
            fi
        done
        QUERY_LIST=$(echo "${query_list[@]}" | sed 's/ /,/g')
        timeout $NVSMI_TIMEOUT nvidia-smi --query-gpu="$QUERY_LIST" --format=csv,noheader | sed 's/, /,/g' > "$NVSMI_INFO_FILE" 2>&1
    else
        echo -e "    ${RED_COLOR}nvidia-smi 执行失败 , $(timeout $NVSMI_TIMEOUT nvidia-smi)${RESET_COLOR}" >> "$ERROR_LOG"
        return 1
    fi
    
}

# 收集 GPU 信息并生成 JSON 输出
function collect_gpu_info() {
    # 获取 NVIDIA 和 CUDA 驱动版本
    #由于低版本cuda驱动没有version.json文件，local cuda_v=$(cat /usr/local/cuda/version.json 2> /dev/null | jq -r .cuda_nvcc.version)
    local cuda_v=$(/usr/local/cuda/bin/nvcc -V 2> /dev/null | grep -oP  "release \K[\d.]+")
    local nvidia_v=$(modinfo nvidia 2> /dev/null | awk '/^version/{print $NF}')

    # 获取 NVIDIA 信息
    while read -r bus_id_gpu;do
        local speed_cap=$(lspci -vvs "$bus_id_gpu"  | awk -F"," '/LnkCap:/{print $2}' | awk '{print $2}' | tr -d ',' | tr '[A-Z]' '[a-z]')   # 获取链路速度能力
        local speed_sta=$(lspci -vvs "$bus_id_gpu"  | awk -F"," '/LnkSta:/{print $1}' | awk '{print $3}' | tr -d ',' | tr '[A-Z]' '[a-z]')   # 获取链路速度状态
        local width_cap=$(lspci -vvs "$bus_id_gpu" | awk -F"," '/LnkCap:/{print $3}' | awk '{print $2}' | tr -d ',')                         # 获取链路宽度能力
        local width_sta=$(lspci -vvs "$bus_id_gpu" | awk -F"," '/LnkSta:/{print $2}' | awk '{print $2}' | tr -d ',')                         # 获取链路宽度状态
        local revision_status=$(lspci -s "$bus_id_gpu" | awk  '{print $NF}' | tr -d ' ' | tr -d ')')                                         # 获取修订状态

        if [[ -f "$NVSMI_INFO_FILE" && -s "$NVSMI_INFO_FILE" ]];then
            # 获取 nvidia-smi 查询项的索引
            local index_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "index") print i}}')
            local name_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "name") print i}}')
            local serial_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "serial") print i}}')
            local gpu_temp_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "temperature.gpu") print i}}')
            local gpu_tlimit_temp_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "temperature.gpu.tlimit") print i}}')
            local power_draw_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "power.draw") print i}}')
            local hw_slowdown_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "clocks_event_reasons.hw_slowdown") print i}}')
            local pcie_link_gen_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "pcie.link.gen.current") print i}}')
            local ecc_mode_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "ecc.mode.current") print i}}')
            local ecc_ce_total_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "ecc.errors.corrected.volatile.total") print i}}')
            local ecc_ue_total_index=$(echo "$QUERY_LIST" | awk -F, '{for(i=1;i<=NF;i++) {if ($i == "ecc.errors.uncorrected.volatile.total") print i}}')
            
            # 使用 nvidia-smi 获取 GPU 详细信息
			# 获取 GPU 索引
            local index=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F ',' "{print \$($index_index)}")
            # 获取产品名称
            local name=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$name_index}" 2> /dev/null)
            # 获取序列号
            local serial=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$serial_index}" 2> /dev/null)
            # 获取 GPU 当前温度
            local gpu_temp=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$gpu_temp_index}" 2> /dev/null)
            # 获取 GPU 温度上限
            local gpu_tlimit_temp=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$gpu_tlimit_temp_index}" 2> /dev/null)
            if [[ -z "$gpu_tlimit_temp"  || ! "$gpu_tlimit_temp" =~ ^-?[0-9]+([.][0-9]+)?$ ]];then
			    # 再次获取 GPU 温度上限
                local gpu_tlimit_temp=$(timeout $NVSMI_TIMEOUT nvidia-smi -q -i "$index" | awk -F: '/GPU T.Limit Temp/{print $(NF)}' | tr -d 'C' | tr -d ' ') 
            fi
            # 获取 GPU 减速温度
            local gpu_slowdown_temp=$(timeout "$NVSMI_TIMEOUT" nvidia-smi -q -i "$index" | awk '/GPU Slowdown Temp/{print $(NF-1)}')
            # 获取 GPU 功率读数
            local power_draw=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$power_draw_index}" 2> /dev/null)
            # 获取硬件减速原因
            local hw_slowdown=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$hw_slowdown_index}" 2> /dev/null)
            if [ -z "$hw_slowdown" ];then
			    # 再次获取硬件减速原因
                local hw_slowdown=$(timeout $NVSMI_TIMEOUT nvidia-smi -q -i "$index" | awk '/HW Slowdown/{print $(NF-1)" "$NF}')
            fi
            # 获取 PCIe 生成
            local pcie_link_gen=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$pcie_link_gen_index}" 2> /dev/null)
            # 获取 ECC 模式
            local ecc_mode=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$ecc_mode_index}" 2> /dev/null)
            # 获取 ECC 校正错误总数
            local ecc_ce_total=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$ecc_ce_total_index}" 2> /dev/null)
			# 获取 ECC 未校正错误总数
            local ecc_ue_total=$(grep -i "$bus_id_gpu" "$NVSMI_INFO_FILE" | awk -F, "{print \$$ecc_ue_total_index}" 2> /dev/null)

            # 获取 NVLink 信息
            local nvlink_counts=$(timeout $NVSMI_TIMEOUT nvidia-smi nvlink -s -i "$index" | grep "Link" | wc -l)
			# 获取 NVLink 带宽总和
            local nvlink_bandwidth=$(timeout $NVSMI_TIMEOUT nvidia-smi nvlink -s -i "$index" | awk -F": " '{print $2}' | tr -d 'GB/s' | awk '{s+=$1} END {print s}')

            # 获取 SRAM 和 Remapped Rows 信息
			# 获取 SRAM 阈值是否超过
            local sram_threshold_exceeded=$(timeout $NVSMI_TIMEOUT nvidia-smi -q -i $index | grep "SRAM Threshold Exceeded" | awk '{print $NF}')
            # 获取待映射行数
            local remapped_pending=$(timeout $NVSMI_TIMEOUT nvidia-smi -q -i $index | grep -A4 "Remapped Rows" | awk '/Pending/{print $NF}')
            # 获取映射发生次数
            local remapped_occurred=$(timeout $NVSMI_TIMEOUT nvidia-smi -q -i $index | grep -A4 "Remapped Rows" | awk '/Remapping/{print $NF}')
        fi

        echo "{
            \"index\":\"$index\",
            \"name\":\"$name\",
            \"serial\":\"$serial\",
            \"pci_bus_id\":\"$bus_id_gpu\",
            \"device_id\":\"$NVIDIA_DID\",
            \"revision_status\":\"$revision_status\",
            \"link_speed_cap\":\"$speed_cap\",
            \"link_speed_sta\":\"$speed_sta\",
            \"link_width_cap\":\"$width_cap\",
            \"link_width_sta\":\"$width_sta\",
            \"nvlink_counts\":\"$nvlink_counts\",
            \"nvlink_bandwidth\":\"$nvlink_bandwidth\",
            \"gpu_temp\":\"$gpu_temp\",
            \"gpu_tlimit_temp\":\"$gpu_tlimit_temp\",
            \"gpu_slowdown_temp\":\"$gpu_slowdown_temp\",
            \"power_draw\":\"$power_draw\",
            \"hw_slowdown\":\"$hw_slowdown\",
            \"pcie_link_gen\":\"$pcie_link_gen\",
            \"ecc_mode\":\"$ecc_mode\",
            \"ecc_ce_total\":\"$ecc_ce_total\",
            \"ecc_ue_total\":\"$ecc_ue_total\",
            \"sram_threshold_exceeded\":\"$sram_threshold_exceeded\",
            \"remapped_pending\":\"$remapped_pending\",
            \"remapped_occurred\":\"$remapped_occurred\"
        }" >> "${GPU_INFO_JSON}_gpu1"
    done <<< "$NVIDIA_DEVICES"

    # 将 GPU 信息合并成一个单独的 JSON 文件
    jq -sc . "${GPU_INFO_JSON}_gpu1" > "${GPU_INFO_JSON}_gpu2"

    # 遍历所有 NVIDIA 设备的根端口
    if [ -n "$NVIDIA_RPORT_BUS" ];then
        while read -r bus_id_rport;do
            local id=$(echo "$bus_id_rport" |awk -F':' '{print $1}')
            # 获取 Lnksta 和 Lnkcap 状态
            local speed_cap=$(lspci -vvs "$bus_id_rport"  | awk -F"," '/LnkCap:/{print $2}' | awk '{print $2}' | tr -d ',' | tr '[A-Z]' '[a-z]')
            local speed_sta=$(lspci -vvs "$bus_id_rport"  | awk -F"," '/LnkSta:/{print $1}' | awk '{print $3}' | tr -d ',' | tr '[A-Z]' '[a-z]')
            local width_cap=$(lspci -vvs "$bus_id_rport" | awk -F"," '/LnkCap:/{print $3}' | awk '{print $2}' | tr -d ',')
            local width_sta=$(lspci -vvs "$bus_id_rport" | awk -F"," '/LnkSta:/{print $2}' | awk '{print $2}' | tr -d ',')

            echo "{
                \"pci_bus_id\":\"$bus_id_rport\",
                \"link_speed_cap\":\"$speed_cap\",
                \"link_speed_sta\":\"$speed_sta\",
                \"link_width_cap\":\"$width_cap\",
                \"link_width_sta\":\"$width_sta\"
            }" >> "${GPU_INFO_JSON}_gpu_rport1"
        done <<< "$NVIDIA_RPORT_BUS"
        # 将 GPU 根端口信息合并成一个单独的 JSON 文件
        jq -sc . "${GPU_INFO_JSON}_gpu_rport1" > "${GPU_INFO_JSON}_gpu_rport2"
    else
        echo "null"> "${GPU_INFO_JSON}_gpu_rport2"
    fi

    # 遍历所有 NVIDIA Switch 的根端口
    if [ -n "$NVSWITCH_RPORT_BUS" ];then
        while read -r bus_id_rport;do
            local id=$(echo "$bus_id_rport" |awk -F':' '{print $1}')
            # 获取 Lnksta 和 Lnkcap 状态
            local speed_cap=$(lspci -vvs "$bus_id_rport"  | awk -F"," '/LnkCap:/{print $2}' | awk '{print $2}' | tr -d ',' | tr '[A-Z]' '[a-z]')
            local speed_sta=$(lspci -vvs "$bus_id_rport"  | awk -F"," '/LnkSta:/{print $1}' | awk '{print $3}' | tr -d ',' | tr '[A-Z]' '[a-z]')
            local width_cap=$(lspci -vvs "$bus_id_rport" | awk -F"," '/LnkCap:/{print $3}' | awk '{print $2}' | tr -d ',')
            local width_sta=$(lspci -vvs "$bus_id_rport" | awk -F"," '/LnkSta:/{print $2}' | awk '{print $2}' | tr -d ',')

            echo "{
                \"pci_bus_id\":\"$bus_id_rport\",
                \"link_speed_cap\":\"$speed_cap\",
                \"link_speed_sta\":\"$speed_sta\",
                \"link_width_cap\":\"$width_cap\",
                \"link_width_sta\":\"$width_sta\"
            }" >> "${GPU_INFO_JSON}_nvswitch_rport1"
        done <<< "$NVSWITCH_RPORT_BUS"
        # 将 NVSwitch 根端口信息合并成一个单独的 JSON 文件
        jq -sc . "${GPU_INFO_JSON}_nvswitch_rport1" > "${GPU_INFO_JSON}_nvswitch_rport2"
    else
        echo "null"> "${GPU_INFO_JSON}_nvswitch_rport2"
    fi

    # 获取 NVSwitch 信息
    while read -r bus_id_nvswitch;do
        local device_info=$(lspci -vvs "$bus_id_nvswitch")
        local speed_cap=$(echo "$device_info" | awk -F"," '/LnkCap:/{print $2}' | awk '{print $2}' | tr -d ',')   # 获取链路速度能力
        local speed_sta=$(echo "$device_info" | awk -F"," '/LnkSta:/{print $1}' | awk '{print $3}' | tr -d ',')   # 获取链路速度状态
        local width_cap=$(echo "$device_info" | awk -F"," '/LnkCap:/{print $3}' | awk '{print $2}' | tr -d ',')   # 获取链路宽度能力
        local width_sta=$(echo "$device_info" | awk -F"," '/LnkSta:/{print $2}' | awk '{print $2}' | tr -d ',')   # 获取链路宽度状态

        echo "{
            \"pci_bus_id\":\"$bus_id_nvswitch\",
            \"device_id\":\"$NVSWITCH_DID\",
            \"link_speed_cap\":\"$speed_cap\",
            \"link_speed_sta\":\"$speed_sta\",
            \"link_width_cap\":\"$width_cap\",
            \"link_width_sta\":\"$width_sta\"
        }" >> "${GPU_INFO_JSON}_nvswitch1"
    done <<< "$NVSWITCH_DEVICES"

    # 将 NVSwitch 信息合并成一个单独的 JSON 文件
    jq -sc . "${GPU_INFO_JSON}_nvswitch1" > "${GPU_INFO_JSON}_nvswitch2"

    # 生成最终的 JSON 输出
    echo "{
        \"cuda_version\":\"$cuda_v\",
        \"nvidia_version\":\"$nvidia_v\",
        \"nvidia_counts\":\"$NVIDIA_COUNTS\",
        \"nvswitch_counts\":\"$NVSWITCH_COUNTS\",
        \"gpu_rport\":$(cat "${GPU_INFO_JSON}_gpu_rport2"),
        \"nvswitch_rport\":$(cat "${GPU_INFO_JSON}_nvswitch_rport2"),
        \"gpus\":$(cat "${GPU_INFO_JSON}_gpu2"),
        \"nvswitchs\":$(cat "${GPU_INFO_JSON}_nvswitch2")
    }" > "$GPU_INFO_JSON"


}

# 显示 NVIDIA 和 CUDA 驱动版本
function show_gpu_driver(){
    local cuda_flag=0
    local nvidia_flag=0
    local cuda_v=$(jq -r .cuda_version "$GPU_INFO_JSON")
    local nvidia_v=$(jq -r .nvidia_version "$GPU_INFO_JSON")

    local iterm_nv="当前的 NVIDIA 驱动版本"
    local iterm_cuda="当前的 CUDA 驱动版本"
#    if [ -z "$cuda_v" ];then
#        echo -e "$iterm_cuda:\n    \033[1;31m未找到 CUDA 驱动! \033[0m" >> "$ERROR_LOG"
#        ((cuda_flag++))
#    fi
    if [ -z "$nvidia_v" ];then
        echo -e "$iterm_nv:\n    \033[1;31m未找到 NVIDIA 驱动! \033[0m" >> "$ERROR_LOG"
        ((nvidia_flag++))
    fi

#    if [ "$cuda_flag" -ne 0 ];then
#        printf "%-51s | ${RED_COLOR} %-4s ${RESET_COLOR} |\n" "$iterm_cuda" "FAIL"
#    fi
    if [ "$nvidia_flag" -ne 0 ];then
        printf "%-51s | ${RED_COLOR} %-4s ${RESET_COLOR} |\n" "$iterm_nv" "FAIL"
    fi

}

# 检查 NVIDIA 设备数量
function check_gpu_counts() {
    local check_flag=0
    # 获取 JSON 文件中 GPU 的索引数量
    local index_counts=$(jq -r .gpus[].index "$GPU_INFO_JSON" | wc -l)  
    # 检查 NVIDIA 设备数量是否符合预期
    if [ -z "$NVIDIA_COUNTS" ]; then
        echo -e "    \033[1;31m未找到 GPU 设备!\033[0m" >> "${GPU_ERROR_LOG}_nvidia"
        ((check_flag++))
    elif [[ "$NVIDIA_COUNTS" -ne "8" || "$index_counts" -ne "8" ]];then
        echo -e "    PCI_实际: ${NVIDIA_COUNTS},NV_实际: ${index_counts}, 预期: 8 .${RED_COLOR}PCI 物理链路中的 GPU 设备数量与 nvidia-smi 视图不一致。\033[0m" >> "${GPU_ERROR_LOG}_nvidia"
        ((check_flag++))
    fi

    local iterm="GPU 数量检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${GPU_ERROR_LOG}_nvidia")" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 设备总线修订状态
function check_gpu_bus_revision() {
    local check_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
	    # 获取第 i 个 GPU 的修订状态
        local revision=$(jq -r .gpus[$i].revision_status "$GPU_INFO_JSON")
		# 获取第 i 个 GPU 的 PCI 总线 ID
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")

        # 检查修订 ID
        if [[ "$revision" != "a1" ]]; then
            echo -e "    ${pci_bus_id} , ${RED_COLOR}GPU 的修订 ID 为 ff，表示设备可能离线。\033[0m" >> "${GPU_ERROR_LOG}_lspci_ff"
            ((check_flag++))
        fi
    done

    local iterm="GPU PCI BUS 修订状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_lspci_ff)" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 根端口链接状态
function check_gpu_rport_link_status() {
    local check_flag=0
    
    if [ -n "$NVIDIA_RPORT_BUS" ];then
        for ((i=0; i<"$NVIDIA_RPORT_COUNTS"; i++)); do
            local pci_bus_id=$(jq -r .gpu_rport[$i].pci_bus_id "$GPU_INFO_JSON")     # 获取第 i 个根端口的 PCI 总线 ID
            local width_sta=$(jq -r .gpu_rport[$i].link_width_sta "$GPU_INFO_JSON")  # 获取第 i 个根端口的实际链路宽度
            local width_cap=$(jq -r .gpu_rport[$i].link_width_cap "$GPU_INFO_JSON")  # 获取第 i 个根端口的链路宽度能力
            local speed_cap=$(jq -r .gpu_rport[$i].link_speed_cap "$GPU_INFO_JSON")  # 获取第 i 个根端口的链路速度能力
            local speed_sta=$(jq -r .gpu_rport[$i].link_speed_sta "$GPU_INFO_JSON")  # 获取第 i 个根端口的实际链路速度

            # 检查 PCIe 链路宽度是否符合预期
            if [[ -z "$width_sta" || "$width_sta" != "$width_cap" ]]; then
                echo -e "    ${pci_bus_id} 实际: ${width_sta}, 预期: ${width_cap}.${RED_COLOR}PCIE 链路速度不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_rport_link"
                ((check_flag++))
            fi

            # 检查 PCIe 链路速度是否符合预期
            if [[ -z "$speed_sta" || "$speed_sta" != "$speed_cap" ]]; then
                echo -e "    ${pci_bus_id} 实际: ${speed_sta}, 预期: ${speed_cap}.${RED_COLOR}PCIE 链路速度不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_rport_link"
                ((check_flag++))
            fi
        done
    else
        echo -e "    ${RED_COLOR}The $(lspci --version) 不支持 -PP 参数。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_rport_link"
        ((check_flag++))
    fi

    local iterm="GPU Root port 链接状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_rport_link)" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 设备链接状态
function check_gpu_link_status() {
    local check_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")    # 获取第 i 个 GPU 的 PCI 总线 ID
        local width_sta=$(jq -r .gpus[$i].link_width_sta "$GPU_INFO_JSON") # 获取第 i 个 GPU 的实际链路宽度
        local width_cap=$(jq -r .gpus[$i].link_width_cap "$GPU_INFO_JSON") # 获取第 i 个 GPU 的链路宽度能力
        local speed_cap=$(jq -r .gpus[$i].link_speed_cap "$GPU_INFO_JSON") # 获取第 i 个 GPU 的链路速度能力
        local speed_sta=$(jq -r .gpus[$i].link_speed_sta "$GPU_INFO_JSON") # 获取第 i 个 GPU 的实际链路速度

        # 检查 PCIe 链路宽度是否符合预期
        if [[ -z "$width_sta" || "$width_sta" != "$width_cap" ]]; then
            echo -e "    ${pci_bus_id} 实际: ${width_sta}, 预期: ${width_cap}.${RED_COLOR}PCIe 链路宽度不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_link"
            ((check_flag++))
        fi

        # 检查 PCIe 链路速度是否符合预期
        if [[ -z "$speed_sta" || "$speed_sta" != "$speed_cap" ]]; then
            echo -e "    ${pci_bus_id} 实际: ${speed_sta}, 预期: ${speed_cap}.${RED_COLOR}PCIe 链路速度不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_link"
            ((check_flag++))
        fi
    done

    local iterm="GPU PCIe 链接状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${GPU_ERROR_LOG}_gpu_link")" >> "$ERROR_LOG"
    fi
}

# 检查 NVSwitch 数量
function check_nvswitch_counts() {
    local check_flag=0

    # 检查 NVSwitch 数量是否符合预期
    local expected_nvswitch="${NVSWITCH_COUNTS_EXPECTATION[$NVSWITCH_DID]}"
    if [[ -n "$expected_nvswitch" && "$NVSWITCH_COUNTS" != "$expected_nvswitch" ]]; then
        echo -e "    实际: ${NVSWITCH_COUNTS}, 预期: ${expected_nvswitch}.${RED_COLOR}NVSwitch 数量不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_nvswitch"
        ((check_flag++))
    else
        # 确保 $expected_nvswitch 未定义或 NVSwitch 数量匹配
        if [[ -z "$expected_nvswitch" ]]; then
            echo -e "    ${RED_COLOR}未找到匹配的设备 ID! ${RESET_COLOR}" >> "${GPU_ERROR_LOG}_nvswitch"
            ((check_flag++))
        fi
    fi

    local iterm="GPU NVSwitche 数量检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${GPU_ERROR_LOG}_nvswitch")" >> "$ERROR_LOG"
    fi
}

# 检查 NVSwitch 链接状态
function check_nvswitch_link_status() {
    local check_flag=0

    for ((i=0; i<"$NVSWITCH_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .nvswitchs[$i].pci_bus_id "$GPU_INFO_JSON")
        local width_sta=$(jq -r .nvswitchs[$i].link_width_sta "$GPU_INFO_JSON")
        local speed_cap=$(jq -r .nvswitchs[$i].link_speed_cap "$GPU_INFO_JSON")
        local speed_sta=$(jq -r .nvswitchs[$i].link_speed_sta "$GPU_INFO_JSON")

        # 因为 A800 是特殊的，链带宽度值是 x1
        if [[ "$NVIDIA_DID" == "20f3" ]];then
            local width_cap="x1"
        else
            local width_cap=$(jq -r .nvswitchs[$i].link_width_cap "$GPU_INFO_JSON")
        fi

        # 检查 PCIe 链路宽度是否符合预期
        if [[ -z "$width_sta" || "$width_sta" != "$width_cap" ]]; then
            echo -e "    ${pci_bus_id} 实际: ${width_sta}, 预期: ${width_cap}.${RED_COLOR}PCIe 链路宽度不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_nvswitch_link"
            ((check_flag++))
        fi

        # 检查 PCIe 链路速度是否符合预期
        if [[ -z "$speed_sta" || "$speed_sta" != "$speed_cap" ]]; then
            echo -e "    ${pci_bus_id} 实际: ${speed_sta}, 预期: ${speed_cap}.${RED_COLOR}PCIe 链路速度不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_nvswitch_link"
            ((check_flag++))
        fi
    done

    local iterm="GPU NVSwitche PCIe 链接状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat "${GPU_ERROR_LOG}_nvswitch_link")" >> "$ERROR_LOG"
    fi
}

# 检查 NVSwitch 根端口链接状态
function check_nvswitch_rport_link_status() {
    local check_flag=0
    
    if [ -n "$NVSWITCH_RPORT_BUS" ];then
        for ((i=0; i<"$NVSWITCH_RPORT_COUNTS"; i++)); do
            local pci_bus_id=$(jq -r .nvswitch_rport[$i].pci_bus_id "$GPU_INFO_JSON")
            local width_sta=$(jq -r .nvswitch_rport[$i].link_width_sta "$GPU_INFO_JSON")
            local width_cap=$(jq -r .nvswitch_rport[$i].link_width_cap "$GPU_INFO_JSON")
            local speed_cap=$(jq -r .nvswitch_rport[$i].link_speed_cap "$GPU_INFO_JSON")
            local speed_sta=$(jq -r .nvswitch_rport[$i].link_speed_sta "$GPU_INFO_JSON")

            # 检查 PCIe 链路速度是否符合预期
            if [[ -z "$speed_sta" || "$speed_sta" != "$speed_cap" ]]; then
                echo -e "    ${pci_bus_id} 实际: ${speed_sta}, 预期: ${speed_cap}.${RED_COLOR}PCIe 链路速度不符合预期。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_nvswitch_rport_link"
                ((check_flag++))
            fi
        done
    else
        echo -e "    ${RED_COLOR}The $(lspci --version) 不支持 -PP 参数。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_nvswitch_rport_link"
        ((check_flag++))
    fi

    local iterm="GPU NVSwitche Root port 链接状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_nvswitch_rport_link)" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 设备温度状态
function check_gpu_temperature() {
    local check_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local index=$(jq -r .gpus[$i].index "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local gpu_temp=$(jq -r .gpus[$i].gpu_temp "$GPU_INFO_JSON")
        local gpu_tlimit_temp=$(jq -r .gpus[$i].gpu_tlimit_temp "$GPU_INFO_JSON")
        local gpu_slowdown_temp=$(jq -r .gpus[$i].gpu_slowdown_temp "$GPU_INFO_JSON")

        if ! [[ "$gpu_slowdown_temp" =~ ^[0-9]+$ ]];then
            if [[ "$gpu_temp" =~ ^[0-9]+$ && "$gpu_tlimit_temp" =~ ^[0-9]+$ ]];then
                local sum_gpu_temp=$(( $gpu_temp + $gpu_tlimit_temp ))
                # 检查 GPU 温度是否低于 80 度 Celsius
                if [[ "$sum_gpu_temp" -lt 80 ]]; then
                    echo -e "    $pci_bus_id , $serial , ${RED_COLOR}GPU 运行在低温度:  $sum_gpu_temp°C${RESET_COLOR}, 建议重启系统检查是否已恢复。" >> "${GPU_ERROR_LOG}_gpu_temperature"
                    ((check_flag++))
                fi
            else
                echo -e "    $pci_bus_id , $serial , ${RED_COLOR}获取 GPU 温度失败。${RESET_COLOR}, 建议重启系统检查是否已恢复。" >> "${GPU_ERROR_LOG}_gpu_temperature"
                ((check_flag++))
            fi
        else
            if [ "$gpu_slowdown_temp" -ne 89 ]; then
                echo -e "    $pci_bus_id , $serial , ${RED_COLOR}GPU 降频温度异常:  $gpu_slowdown_temp°C${RESET_COLOR}, 建议重启系统检查是否已恢复。" >> "${GPU_ERROR_LOG}_gpu_temperature"
                ((check_flag++))
            fi
        fi
    done

    local iterm="GPU 温度状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_temperature)" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 设备电源状态
function check_gpu_power() {
    local check_flag=0
    
    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local power_draw=$(jq -r .gpus[$i].power_draw "$GPU_INFO_JSON")

        # 检查 GPU 功耗是否为 "Unknown"
        if [[ "$power_draw" == "Unknown" ]]; then
            echo -e "    $pci_bus_id , $serial , ${RED_COLOR}GPU 功耗异常${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_powerstate"
            ((check_flag++))
        fi

    done

    local iterm="GPU 电源状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_powerstate)"  >> "$ERROR_LOG"
    fi 
}

# 检查 NVIDIA 设备硬件降频状态
function check_gpu_hw_slowdown() {
    local check_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local hw_slowdown=$(jq -r .gpus[$i].hw_slowdown "$GPU_INFO_JSON")
        
        if [ -n "$hw_slowdown" ];then
            # 检查 "HW Slowdown" 是否不是 "Not Active"
            if [[ "$hw_slowdown" != "Not Active" ]]; then
                echo -e "   ${RED_COLOR}$pci_bus_id , $serial , GPU 存在 'HW Slowdown' 情况。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_hwslowdown"
                ((check_flag++))
            fi
        else
            local hw_slowdown=$(nvidia-smi -q -i "$i" | awk '/HW Slowdown/{print $(NF-1)" "$NF}') # 使用 nvidia-smi 获取硬件降频状态
            # 检查 "HW Slowdown" 是否不是 "Not Active"
            if [[ "$hw_slowdown" != "Not Active" ]]; then
                echo -e "   ${RED_COLOR}$pci_bus_id , $serial , GPU 存在 'HW Slowdown' 情况。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_hwslowdown"
                ((check_flag++))
            fi
        fi

    done

    local iterm="GPU 硬件降频状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_hwslowdown)" >> "$ERROR_LOG"
    fi
}


# 检查 NVIDIA 设备 PCIe 链接代数
function check_gpu_pcie_link_gen() {
    local check_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local pcie_link_gen=$(jq -r .gpus[$i].pcie_link_gen "$GPU_INFO_JSON")
        local device_id=$(jq -r .gpus[$i].device_id "$GPU_INFO_JSON")

        case $device_id in
            #A Serial PCIe Gen 4.0
            *20b2*|*20f3*)
                if [[ "$pcie_link_gen" -lt  4 ]]; then
                    echo -e "   $pci_bus_id , $serial , ${RED_COLOR}GPU PCIe 链接代数检查失败。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_linkgen"
                    ((check_flag++))
                fi
                ;;
            # H Serial PCIe Gen 5.0
            *2330*|*2324*|*2329*|*2335*)
                if [[ "$pcie_link_gen" -lt  5 ]]; then
                    echo -e "   $pci_bus_id , $serial , ${RED_COLOR}GPU PCIe 链接代数检查失败。${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_linkgen"
                    ((check_flag++))
                fi
                ;;
            *)
                printf " 错误: 不支持的 GPU 模型 ${RED_COLOR} $gpu_model ${RESET_COLOR} , 脚本退出!\n
                A100    10de:20b2
                A800    10de:20f3
                H100    10de:2330
                H800    10de:2324
                H20    10de:2329
                H200    10de:2335\n" >> "${GPU_ERROR_LOG}_gpu_linkgen"
                ((check_flag++))
                ;;
        esac
    done

    local iterm="GPU PCIe 链接代数检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_linkgen | sort -u)" >> "$ERROR_LOG"
    fi 
}


# 检查 NVIDIA 设备 NVLink 数量
function check_gpu_nvlink_counts() {
    local check_flag=0

    local device_id="$NVIDIA_DID"
    local expected_nvlink_counts="${NVLINK_COUNTS_EXPECTATION[$device_id]}"
    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local nvlink_counts=$(jq -r .gpus[$i].nvlink_counts "$GPU_INFO_JSON")

        if [[ "$expected_nvlink_counts" != "$nvlink_counts" ]];then
            echo -e "    $pci_bus_id , $serial , ${RED_COLOR}GPU NVLink 数量异常: $nvlink_counts .${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_nvlink"
            ((check_flag++))
        fi
    done

    local iterm="GPU NVLink 数量检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_nvlink)" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 设备 NVLink 带宽
function check_gpu_nvlink_bandwidth() {
    local check_flag=0
    local device_id="$NVIDIA_DID"
    local expected_nvlink_bandwidth="${NVLINK_BANDWIDTH_EXPECTATION[$device_id]}"
    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local nvlink_bandwidth=$(jq -r .gpus[$i].nvlink_bandwidth "$GPU_INFO_JSON")

        if [[ "$expected_nvlink_bandwidth" != "$nvlink_bandwidth" ]];then
            echo -e "    $pci_bus_id , $serial , ${RED_COLOR}GPU NVLink 带宽异常: $nvlink_bandwidth .${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_nvlinkwidth"
            ((check_flag++))
        fi
    done

    local iterm="GPU NVLink 带宽检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_nvlinkwidth)" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 设备 ECC 模式
function check_gpu_ecc_mode() {
    local check_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local ecc_mode=$(jq -r .gpus[$i].ecc_mode "$GPU_INFO_JSON")

        # 检查 ECC 模式是否启用
        if [[ "$ecc_mode" != "Enabled" ]]; then
            echo -e "    $pci_bus_id , $serial , ${RED_COLOR}GPU ECC 模式异常:  $ecc_mode .${RESET_COLOR} 建议启用 ECC 模式"  >> "${GPU_ERROR_LOG}_gpu_eccmode"
            ((check_flag++))
        fi
    done

    local iterm="GPU ECC 模式检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_eccmode)" >> "$ERROR_LOG"
    fi
}

# 检查 NVIDIA 设备 ECC 数量
function check_gpu_ecc_counts() {
    local check_flag=0
    local reset_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local index=$(jq -r .gpus[$i].index "$GPU_INFO_JSON")
        local ecc_ce_total=$(jq -r .gpus[$i].ecc_ce_total "$GPU_INFO_JSON")
        local ecc_ue_total=$(jq -r .gpus[$i].ecc_ue_total "$GPU_INFO_JSON")
        local sram_threshold_exceeded=$(jq -r .gpus[$i].sram_threshold_exceeded "$GPU_INFO_JSON")

        if [[ "$ecc_ce_total" -gt 0 || "$ecc_ue_total" -gt 0 ]]; then
            local ecc_info="可纠正错误: $ecc_ce_total, 不可纠正错误: $ecc_ue_total"

            if [[ "$sram_threshold_exceeded" == "Yes" ]]; then
                printf "    $pci_bus_id , $serial , ${RED_COLOR}GPU$index SRAM 阈值是否超出: YES. $ecc_info。请报修。${RESET_COLOR}\n" >> "${GPU_ERROR_LOG}_gpu_ecccounts"
                ((check_flag++))
            elif [[ "$sram_threshold_exceeded" == "No" ]]; then
                printf "    $pci_bus_id , $serial , ${RED_COLOR}GPU$index 存在 ECC 错误，SRAM 阈值是否超出: No. $ecc_info。请重置此 GPU 或重启系统！${RESET_COLOR}\n" >> "${GPU_ERROR_LOG}_gpu_ecccounts"
                ((check_flag++))
                ((reset_flag++))
                if [[ ! " ${GPU_IDS[@]} " =~ " ${pci_bus_id} " ]]; then
                    GPU_IDS+=("$pci_bus_id")
                fi
            else
                printf "    $pci_bus_id , $serial , ${RED_COLOR}GPU$index 存在 ECC 错误。$ecc_info。请重置此 GPU 或重启系统！如无效请报修。${RESET_COLOR}\n" >> "${GPU_ERROR_LOG}_gpu_ecccounts"
                ((check_flag++))
                ((reset_flag++))
                if [[ ! " ${GPU_IDS[@]} " =~ " ${pci_bus_id} " ]]; then
                    GPU_IDS+=("$pci_bus_id")
                fi
            fi
        fi
    done

    local iterm="GPU ECC 数量检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_ecccounts)" >> "$ERROR_LOG"
    fi

    if [ "$reset_flag" -ne 0 ]; then
        PROJECTS_REQUIRING_RESET+=("GPU_ECC_Counts_Check")
    fi
}

# 检查 NVIDIA 设备行重映射状态
function check_gpu_row_remapping() {
    local check_flag=0

    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local serial=$(jq -r .gpus[$i].serial "$GPU_INFO_JSON")
        local index=$(jq -r .gpus[$i].index "$GPU_INFO_JSON")
        local remapped_pending=$(jq -r .gpus[$i].remapped_pending "$GPU_INFO_JSON")
        local remapped_occurred=$(jq -r .gpus[$i].remapped_occurred "$GPU_INFO_JSON")

        if [[ "$remapped_pending" != No || "$remapped_occurred" != No ]];then
            echo -e "    $pci_bus_id , $serial , ${RED_COLOR}GPU$index 待处理重映射： $remapped_pending,已发生重映射失败: $remapped_occurred ${RESET_COLOR},建议重启设备或重置 GPU 以确定重映射失败状态" >> "${GPU_ERROR_LOG}_remapping"
            ((check_flag++))
        fi
    done

    local iterm="GPU 行重映射检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_remapping)" >> "$ERROR_LOG"
    fi
}

# 记录 XID 事件日志
function log_xid_event() {
    local sn="$1"
    local xid="$2"
    local message="$3"
    local timestamp="$4"

    # 转换时间戳为可读格式
    local time_str=$(date -d "@$timestamp" "+%Y-%m-%d %H:%M:%S")

    echo -e "[$time_str] ${RED_COLOR}GPU$sn Xid $xid${RESET_COLOR} : $message" >> "${GPU_ERROR_LOG}_gpu_xid"
}

# 使用 nvidia-smi 查询 XID 事件
function check_gpu_xid_events() {
    if [ -f "/var/log/syslog" ]; then
        local logfile="/var/log/syslog"
    else
        local logfile="/var/log/messages"
    fi

    local check_flag=0
    local sn_info=$(jq -r ".gpus[] | \"\(.index),\(.pci_bus_id),\(.serial)\"" "$GPU_INFO_JSON")

    # 获取当前时间戳（用于时间筛选）
    local now=$(date +%s)
    local one_hour_ago=$((now - 3600))

    # 存储唯一XID事件和时间戳
    declare -A unique_xids

    # 从日志文件中读取 XID 事件
    while IFS= read -r line; do
        # 提取日志时间
        local logtime=$(echo "$line" | awk '{print $1" "$2" "$3}' | tr -d '[]')
        local timestamp=$(date -d "$logtime" +%s 2>/dev/null || date -d "${logtime%.*}" +%s 2>/dev/null || echo 0)

        # 只处理最近1小时内的日志
        [ $timestamp -lt $one_hour_ago ] && continue

        local busid=$(echo "$line" | awk -F'[()]' '{print substr($2, 5)}' | sed 's/0000://g')
        local xid=$(echo "$line" | awk -F': ' '{print $4}' | cut -d ',' -f 1)

        if [ -n "$busid" ] && [ -n "$xid" ]; then
            ((check_flag++))
            local unique_key="id_${busid}_${xid}"
            # 存储最早发生的时间戳
            if [[ -z "${unique_xids[$unique_key]}" ]] || [ $timestamp -lt ${unique_xids[$unique_key]} ]; then
                unique_xids[$unique_key]=$timestamp
            fi
        fi
    done <<< "$(grep "NVRM: Xid" $logfile)"

    # 只有当发现XID时才处理
    if [ $check_flag -gt 0 ]; then
        local check_flag=0
        > "${GPU_ERROR_LOG}_gpu_xid"

        for key in "${!unique_xids[@]}"; do
            local key_without_prefix="${key#id_}"
            local busid="${key_without_prefix%_*}"
            local xid="${key_without_prefix#*_}"
            local timestamp="${unique_xids[$key]}"

            local sn=$(echo "$sn_info" | tr ' ' '\n' | grep -i "$busid")

            if [ ! -z "$xid" ]; then
                ((check_flag++))
                case $xid in
                    13|31|43)
                        log_xid_event "$sn" "$xid" "请用户检查应用程序。" "$timestamp"
                        ;;
                    32)
                        log_xid_event "$sn" "$xid" "当 DMA 控制器报告与 GPU 之间的 PCI-E 总线通信流中的故障时会记录此事件。请报修" "$timestamp"
                        ;;
                    63|64)
                        log_xid_event "$sn" "$xid" "检测到 ECC 页面退役或行重映射记录事件。请报修" "$timestamp"
                        ;;
                    74)
                        log_xid_event "$sn" "$xid" "检测到 NVLink 错误。请报修" "$timestamp"
                        ;;
                    44|48|62|61|69|79)
                        log_xid_event "$sn" "$xid" "GPU 卡存在未知且无法纠正的错误。建议重置 GPU 卡或重启服务器。如无效请报修" "$timestamp"
                        ;;
                    92)
                        log_xid_event "$sn" "$xid" "检测到高单比特 ECC 错误率。请报修" "$timestamp"
                        ;;
                    94|95)
                        log_xid_event "$sn" "$xid" "检测到包含或不包含 ECC 错误。请报修" "$timestamp"
                        ;;
                    119)
                        log_xid_event "$sn" "$xid" "建议禁用 GSP。" "$timestamp"
                        ;;
                    *)
                        log_xid_event "$sn" "$xid" "未知 XID 事件: $xid" "$timestamp"
                        ;;
                esac
            fi
        done

        # 只有当有有效XID时才输出
        if [ $check_flag -ne 0 ]; then
            echo -e "最近1小时内GPU XID事件：\n$(cat ${GPU_ERROR_LOG}_gpu_xid)" >> "$ERROR_LOG"
        fi
    fi
}

# 检查 nvidia-fabricmanager 状态
function check_nvfabricmanager_status(){
    local check_flag=0

    if systemctl list-unit-files | grep -q "nvidia-fabricmanager.service"; then
        systemctl is-active --quiet nvidia-fabricmanager.service
        if [[ $? -eq 0 ]]; then
            local fm_version=$(nv-fabricmanager --version 2>/dev/null | grep -oP '\d+\.\d+\.\d+')
            local nvidia_v=$(jq -r .nvidia_version "$GPU_INFO_JSON")
            if [[ -z $fm_version ]]; then
                echo -e "    ${RED_COLOR}获取 nvidia-fabricmanager 版本失败。${RESET_COLOR}"  >> "${GPU_ERROR_LOG}_gpu_nvfab"
                ((check_flag++))
            else
                if [[ "$fm_version" != "$nvidia_v" ]]; then
                    echo -e "   ${RED_COLOR}版本不匹配: nvidia-fabricmanager ($fm_version) vs Nvidia-GPU 驱动 ($nvidia_v).${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_nvfab"
                    ((check_flag++))         
                fi
            fi
        else
            echo -e "   ${RED_COLOR}nvidia-fabricmanager.service 未运行.${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_nvfab"
            ((check_flag++))
        fi
    else
        echo -e "    ${RED_COLOR}nvidia-fabricmanager.service 不存在.${RESET_COLOR}" >> "${GPU_ERROR_LOG}_gpu_nvfab"
        ((check_flag++))
    fi

    local iterm="nvidia-fabricmanager 状态检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${GPU_ERROR_LOG}_gpu_nvfab)" >> "$ERROR_LOG"
    fi
}
# 检查 nvidia-smi -q 是否需要重置 GPU
function check_gpu_reset() {
    local check_flag=0

    # 提取 GPU 索引并检查是否需要重置
    for ((i=0; i<"$NVIDIA_COUNTS"; i++)); do
        local pci_bus_id=$(jq -r .gpus[$i].pci_bus_id "$GPU_INFO_JSON")
        local index=$(jq -r .gpus[$i].index "$GPU_INFO_JSON")
        local nvidia_q_info=$(timeout $NVSMI_TIMEOUT nvidia-smi -q -i "$index" | grep -q "GPU requires reset" && echo "Yes")
        
        if [[ "$nvidia_q_info" == "Yes" ]];then
            if [[ ! " ${GPU_IDS[@]} " =~ " ${pci_bus_id} " ]]; then
                GPU_IDS+=("$pci_bus_id")
            fi
            ((check_flag++))
        fi
    done

    local iterm="GPU 重置检查结果"
    if [ "$check_flag" -ne 0 ]; then
        PROJECTS_REQUIRING_RESET+=("GPU_reset_Check ")
    fi
}

# 检查 GPU 是否需要重置
function gpu_reset() {
    if [[ ${#PROJECTS_REQUIRING_RESET[@]} -gt 0 ]]; then
        printf "${GREEN_COLOR}以下项目需要重置 GPU:${RESET_COLOR}\n" >> "$ERROR_LOG"
        for project in "${PROJECTS_REQUIRING_RESET[@]}"; do
            echo -e "  - ${RED_COLOR} $project ${RESET_COLOR}" >> "$ERROR_LOG"
        done

        for gpu_busid in "${GPU_IDS[@]}"; do
            gpu_index=$(grep $gpu_busid $NVSMI_INFO_FILE | awk -F ',' '{print $1}')
            printf "    GPU $gpu_index 需要重置。请运行 ${RED_COLOR} nvidia-smi --gpu-reset -i $gpu_index ${RESET_COLOR} \n" >> "$ERROR_LOG"
        done

        get_gpu_pids >> "$ERROR_LOG"
    fi
}

# 获取与 GPU 相关的进程 ID
function get_gpu_pids() {
    pids=$(fuser /dev/nvidia* 2>/dev/null | tr -s ' ' '\n' | sort -u)
    printf "  ${RED_COLOR}GPU 相关进程:${RESET_COLOR} \n"
    echo "    PID    进程名"
    local services_to_stop=()
    for pid in $pids; do
        if [[ -d /proc/$pid ]]; then
            local pname=$(ps -p $pid -o comm= | tr -d '[:space:]')
            echo "    $pid    $pname"
            if [[ "$pname" == "elfin-epc" || "$pname" == "nvidia-persiste" || "$pname" == "nv-fabricmanage" ]]; then
                case "$pname" in
                    "elfin-epc")
                        service_name="elfin-epc.service"
                        ;;
                    "nvidia-persiste")
                        service_name="nvidia-persistenced.service"
                        ;;
                    "nv-fabricmanage")
                        service_name="nvidia-fabricmanager.service"
                        ;;
                esac
                if [[ ! " ${services_to_stop[@]} " =~ " ${service_name} " ]]; then
                    services_to_stop+=("$service_name")
                fi
            else
                echo "    请手动停止进程 $pname (PID $pid)."
            fi
        fi
    done
    if [[ ${#services_to_stop[@]} -gt 0 ]]; then
        printf "  ${RED_COLOR}请在重置 GPU 之前停止以下服务:${RESET_COLOR} \n"
        for service in "${services_to_stop[@]}"; do
            echo "    systemctl stop $service"
        done
        echo "    请在重置 GPU 之前停止所有使用 GPU 的进程。并使用 fuser 命令检查是否有进程仍在使用 GPU."
        printf "    ${GREEN_COLOR}fuser /dev/nvidia* ${RESET_COLOR} \n"
    fi
}

# 显示 GPU 和 IB 设备亲和性
function show_gpu_ib_port() {
    # 声明一个关联数组来按类别分组设备
    declare -A device_groups

    # 处理每个供应商 ID
    # Mellanox 供应商 ID 是 15b3
    # NVIDIA 供应商 ID 是 10de
    local netgpu_vendor_id="$NET_VENDOR_ID $GPU_VENDOR_ID"
    for vendor_id in $netgpu_vendor_id;do
        # 获取供应商详细信息
        vendor_info=$(lspci -nn -d "${vendor_id}:")

        # 列出设备并收集信息
        while read -r vendor_device;do
            device_busid=$(echo "$vendor_device" | awk '{print $1}')
            device_info=$(lspci -PP -s "$device_busid")

            # 使用 "/" 作为分隔符提取第一部分
            first_part=$(echo "$device_info" | awk -F'/' '{print $1}')

            # 按第一部分分组设备
            device_groups["$first_part"]+="$device_info"$'\n'
        done <<< "$vendor_info"
    done

    # 打印分类输出
    printf '%*s\n' 70 '' | tr ' ' -
    echo -e "${YELLOW_COLOR}显示分类设备:${RESET_COLOR}"
    for category in "${!device_groups[@]}"; do
        echo -e "类别: $category"
        echo -e "${device_groups[$category]}"
    done
}

# 合并系统信息和网络/GPU 信息到一个 JSON 文件中
function merge_data() {
    local sn=$(dmidecode -t1 | awk -F': ' '/Serial Number/{print $NF}')
    local manufacturer=$(dmidecode -t1 | awk -F': ' '/Manufacturer/{print $NF}')
    local product=$(dmidecode -t1 | awk -F': ' '/Product Name/{print $NF}')
    local net_info=$(cat "$NET_INFO_JSON")
    local gpu_info=$(cat "$GPU_INFO_JSON")

    echo "{
        \"SN\":\"$sn\",
        \"Manufacturer\":\"$manufacturer\",
        \"ProductName\":\"$product\",
        \"NET_INFO\":$net_info,
        \"GPU_INFO\":$gpu_info
    }" > "$MERGE_DATA_JSON"
    
    cat "$MERGE_DATA_JSON" |jq .
}

# 检查 nvidia_peermem 模块是否加载
function check_nvidia_peermem() {
    local check_flag=0

    # 检查模块是否加载
    if ! lsmod | grep -q nvidia_peermem; then
        echo -e "    ${RED_COLOR}nvidia_peermem 模块未加载${RESET_COLOR}" >> "${NET_ERROR_LOG}_peermem"
        ((check_flag++))
    else
        # 检查模块版本是否与NVIDIA驱动匹配
        local peermem_version=$(modinfo nvidia_peermem 2>/dev/null | awk '/^version:/{print $NF; exit}')
        local nvidia_version=$(jq -r .nvidia_version "$GPU_INFO_JSON")

        if [[ "$peermem_version" != "$nvidia_version" ]]; then
            echo -e "    ${RED_COLOR}nvidia_peermem 版本($peermem_version)与NVIDIA驱动($nvidia_version)不匹配${RESET_COLOR}" >> "${NET_ERROR_LOG}_peermem"
            ((check_flag++))
        fi
    fi

    local iterm="nvidia_peermem 模块检查结果"
    if [ "$check_flag" -ne 0 ]; then
        echo -e "$iterm:\n$(cat ${NET_ERROR_LOG}_peermem)" >> "$ERROR_LOG"
        echo -e "    如模块未加载建议执行: ${GREEN_COLOR}modprobe nvidia_peermem${RESET_COLOR} 加载模块，需保持模块版本一致" >> "$ERROR_LOG"
    fi
}

function check_acs_gdr(){
   LOCALNUM=$(lspci -vvv | grep -i acsctl|awk '{print $2}'|wc -l)
   TEMPNUM=0
   if [[ -z "$LOCALNUM" ]]; then
      echo "[异常]未找到ACS信息，需要弄下" >> "$ERROR_LOG"
   fi
   for i in $(lspci -vvv | grep -i acsctl|awk '{print $2}') 
   do 
      if [[ "$i" =~ SrcValid-$ ]]; then 
      ((TEMPNUM++))
      fi
   done
   if [ "$LOCALNUM" -ne "$TEMPNUM" ]; then 
      echo "[异常]acs 未完全关闭" >>"$ERROR_LOG"
   fi
   if ! lsmod | grep -i "peer" >/dev/null 2>&1; then
      echo "[异常]未开启 GDR,请执行 "echo 'nvidia_peermem' >> /etc/modules-load.d/modules.conf" " >>"$ERROR_LOG"
   elif  grep -i "peer"  /etc/modules-load.d/modules.conf  >/dev/null 2>&1; then
      echo "[异常] 执行下  modprobe nvidia_peermem " >>"$ERROR_LOG"
   fi
}




# 主函数，执行所有检查任务
function main() {
    if ! which jq >/dev/null 2>&1; then
        if which apt >/dev/null 2>&1; then
            sudo apt update > /dev/null 2>&1  # 更新包列表
            sudo apt -y install jq > /dev/null 2>&1  # 安装 jq
            if [ $? -ne 0 ]; then
                echo "Error: Install jq failed, please check /etc/apt/ !"  # 安装失败提示
                exit 1
            fi
        elif which yum >/dev/null 2>&1; then
            sudo yum -y install jq > /dev/null 2>&1  # 安装 jq
            if [ $? -ne 0 ]; then
                echo "Error: Install jq failed, please check /etc/yum.repos.d/ !"  # 安装失败提示
                exit 1
            fi
        else
            echo "Please install jq manually."  # 手动安装提示
            exit 1
        fi
    fi

    collect_net_info # 收集网络信息
#    check_net_fw # 检查网卡固件版本
    check_net_link_status # 检查网络链路状态
    check_net_rport_link_status # 检查远程端口链路状态
    check_net_log  # 检查网络日志
    
    check_acs_gdr #检查ACS是否关闭与GDR是否开启
    check_gpu_series # 检查 GPU 系列
    check_and_get_value # 检查并获取值
    local get_code=$?  # 获取返回码
    collect_gpu_info  # 收集 GPU 信息
    show_gpu_driver   # 显示 GPU 驱动信息
    check_gpu_counts  # 检查 GPU 数量
    check_gpu_bus_revision  # 检查 GPU 总线修订版
    check_gpu_rport_link_status  # 检查 GPU 远程端口链路状态
    check_gpu_link_status  # 检查 GPU 链路状态
    check_nvswitch_counts  # 检查 NVSwitch 数量
    check_nvswitch_link_status  # 检查 NVSwitch 链路状态
    check_nvidia_peermem #检查 nvidia_peermem 模块是否加载
    #check_nvswitch_rport_link_status  # 检查 NVSwitch 远程端口链路状态

    if [ "$get_code" -eq 0 ];then
#        check_gpu_temperature  # 检查 GPU 温度
        check_gpu_hw_slowdown  # 检查 GPU 硬件降速
        check_gpu_nvlink_counts   # 检查 NVLink 数量
        check_gpu_nvlink_bandwidth  # 检查 NVLink 带宽
        check_gpu_ecc_mode  # 检查 GPU ECC 模式
        #check_gpu_ecc_counts  # 检查 GPU ECC 数量
        #check_gpu_row_remapping  # 检查 GPU 行重映射
        #check_gpu_xid_events  # 检查 GPU XID 事件
        #check_gpu_reset   # 检查 GPU 是否需要重置
    fi
}

main  # 调用主函数
output_error  # 输出错误信息
#show_gpu_ib_port  # 显示 GPU 和 IB 设备亲和性

