#!/bin/bash

# ========== 基础设置 ==========
START_IP="10.4.1.1"
END_IP="10.4.1.254"
USERNAME="root"
IP_LIST_FILE="./hostfile"  # 可选，如果存在则用此文件中的 IP 列表
CHECK_SCRIPT_PATH="/public-nvme/baai/devops/nvidiaGpu_check_v3.sh"  # <<< 检查脚本路径

# 飞书信息
APP_ID="cli_a70cba7a4d12500c"
APP_SECRET="QXBY2TslFTlOGmFZFHDtoZ1hANHKucVr"
FEISHU_WEBHOOK="https://open.feishu.cn/open-apis/im/v1/messages?receive_id_type=chat_id"
# 测试组
#RECEIVE_ID="oc_5c3656a52b468bd97da973bd43374f65"
# 运维小组
#RECEIVE_ID="oc_922a582b6696cf4d3819cba45aec97ff"
# 突击队
#RECEIVE_ID="oc_6b6541560dc4df37e911fce6e596a1c0"
#巡检
RECEIVE_ID="oc_ceaf5328a52e7ffeb071ae2fbe947c05"

# ========== 输出文件 ==========
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
SCAN_RESULTS="./scan_Check_$TIMESTAMP.txt"
ERROR_LOG="./error_Check_$TIMESTAMP.txt"

rm -rf error_Check* scan_Check*

# ========== 并发设置 ==========
max_concurrent=100
active_jobs=0

# ========== 构造 IP 列表 ==========
ip_list=()
if [[ -f "$IP_LIST_FILE" ]]; then
  echo "使用文件中的 IP 列表：$IP_LIST_FILE"
  mapfile -t ip_list < "$IP_LIST_FILE"
else
  start=$(echo $START_IP | awk -F. '{print $4}')
  end=$(echo $END_IP | awk -F. '{print $4}')
  base_ip=$(echo $START_IP | awk -F. '{print $1"."$2"."$3}')
  for i in $(seq $start $end); do
    ip_list+=("${base_ip}.${i}")
  done
fi

# ========== 遍历 IP 并执行检查 ==========
for FULL_IP in "${ip_list[@]}"; do
  {
    echo "正在检查 $FULL_IP..."
    ssh_output=$(ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o BatchMode=yes \
      $USERNAME@$FULL_IP "bash $CHECK_SCRIPT_PATH 2>&1 || true" 2>&1)
    exit_status=$?

    if [ $exit_status -eq 0 ]; then
      if [ -n "$ssh_output" ]; then
        echo "在 $FULL_IP (NvidiaGPU_Check)中找到: $ssh_output" >> "$SCAN_RESULTS"
        # 添加分隔符，用于区分每台机器的输出
        echo "=========================================================" >> "$SCAN_RESULTS"
      else
        echo "$(date +%Y-%m-%d_%H:%M:%S) - 对于 $FULL_IP 没有匹配的输出。" >> "$ERROR_LOG"
        # 添加分隔符，用于区分每台机器的输出
        echo "=========================================================" >> "$ERROR_LOG"
      fi
    else
      echo "$(date +%Y-%m-%d_%H:%M:%S) - 连接失败或 $FULL_IP 出现错误: $ssh_output" >> "$ERROR_LOG"
      # 添加分隔符，用于区分每台机器的输出
      echo "=========================================================" >> "$ERROR_LOG"
    fi
    active_jobs=$((active_jobs - 1))
  } &

  active_jobs=$((active_jobs + 1))
  if [ $active_jobs -ge $max_concurrent ]; then
    wait -n
  fi
done

wait
echo "完成"

# ========== 汇总并发送飞书 ==========
#total_hosts=${#ip_list[@]}
#successful_checks=$(grep -c "NvidiaGPU_Check_v5" "$SCAN_RESULTS" 2>/dev/null || echo 0)
#failed_checks=$(grep -c "没有匹配" "$ERROR_LOG" 2>/dev/null || echo 0)
#ssh_failures=$(grep -c "连接失败" "$ERROR_LOG" 2>/dev/null || echo 0)
##
## 文本消息
#feishu_card=$(cat <<EOF
#{
#  "receive_id": "$RECEIVE_ID",
#  "msg_type": "text",
#  "content": "{\"text\":\"巡检任务执行完成\\\n总节点数: $total_hosts\\\n成功执行: $successful_checks\\\n未匹配输出: $failed_checks\\\nSSH 失败: $ssh_failures\"}",
#  "uuid": "$(uuidgen)"
#}
#EOF
#)
#
#echo "curl -s -X POST '$FEISHU_WEBHOOK' \
#        -H "Content-Type: application/json" \
#        -H "Authorization: Bearer $tenant_access_token" \
#        -d '$feishu_card'"
#
#curl -s -X POST "$FEISHU_WEBHOOK" \
#  -H "Content-Type: application/json" \
#  -H "Authorization: Bearer $tenant_access_token" \
#  -d "$feishu_card"
#
#
#
## 扫描结果文本内容（最多 100 行）
#SCAN_RESULTS_CONTENT=$(head -n 100 "$SCAN_RESULTS" \
#  | sed -r "s/\x1B\[[0-9;]*[mGK]//g" \
#  | sed ':a;N;$!ba;s/\n/\\n/g')
#ESCAPED_CONTENT=$(jq -Rs . <<< "{\"text\":\"$SCAN_RESULTS_CONTENT\"}")  # 做内容转义
#
#feishu_result=$(cat <<EOF
#{
#  "receive_id": "$RECEIVE_ID",
#  "msg_type": "text",
#  "content": $ESCAPED_CONTENT,
#  "uuid": "$(uuidgen)"
#}
#EOF
#)
#
#
#  echo "curl -s -X POST '$FEISHU_WEBHOOK' \
#          -H "Content-Type: application/json" \
#          -H "Authorization: Bearer $tenant_access_token" \
#          -d '$feishu_result'"
#
#curl -s -X POST "$FEISHU_WEBHOOK" \
#  -H "Content-Type: application/json" \
#  -H "Authorization: Bearer $tenant_access_token" \
#  -d "$feishu_result"

# ========== 获取 Feishu Token ==========
get_tenant_access_token() {
  token_response=$(timeout 10 curl -s -X POST 'https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal' \
    -H 'Content-Type: application/json' \
    -d "{
      \"app_id\": \"$APP_ID\",
      \"app_secret\": \"$APP_SECRET\"
    }")

  tenant_access_token=$(echo "$token_response" | jq -r '.tenant_access_token')
  if [[ "$tenant_access_token" == "null" ]]; then
    echo "获取飞书 tenant_access_token 失败。"
    exit 1
  fi
}
get_tenant_access_token

echo $tenant_access_token

# ========== 汇总并发送飞书 ==========
timestamp=$(date "+%Y-%m-%d %H:%M:%S")
total_hosts=${#ip_list[@]}

# 抽取异常 IP（执行成功但存在异常输出）
declare -A exception_map
while read -r ip; do [[ -n "$ip" ]] && exception_map["$ip"]=1; done < <(grep -oP '(?<=在 )([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)' "$SCAN_RESULTS")
exception_count=${#exception_map[@]}

# 抽取 SSH 失败的 IP
declare -A sshfail_map
while read -r ip; do
  [[ -n "$ip" ]] && sshfail_map["$ip"]=1
done < <(grep "连接失败" "$ERROR_LOG" | grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}' | sort -u)
ssh_failures=${#sshfail_map[@]}

# 计算成功数
successful_checks=$((total_hosts - ssh_failures - exception_count))

# 整理失败 IP 列表
ssh_failed_ips=$(printf "%s\n" "${!sshfail_map[@]}" | paste -sd "," -)
exception_ips=$(printf "%s\n" "${!exception_map[@]}" | paste -sd "," -)

#SCAN_RESULTS_CONTENT=$(head -n 10 "$SCAN_RESULTS" \
#  | sed -r "s/\x1B\[[0-9;]*[mGK]//g" \
#  | sed ':a;N;$!ba;s/\n/\\n/g' \
#  | sed 's/\\n/\\\\n/g')

SCAN_RESULTS_CONTENT=$(head -n 100 "$SCAN_RESULTS" \
                         | sed -r "s/\x1B\[[0-9;]*[mGK]//g" \
                         | sed ':a;N;$!ba;s/\n/\\n/g' \
                         | sed 's/\\n/\\\\n/g'\
                         | sed "s/[']/ /g")
#echo $SCAN_RESULTS_CONTENT


MSG_TEXT="GPU节点巡检完成\\\n\\\n巡检时间: $timestamp\\\n总节点数: $total_hosts\\\n成功执行: $successful_checks\\\n异常输出: $exception_count\\\nSSH 失败: $ssh_failures"

[[ -n "$exception_ips" ]] && MSG_TEXT="$MSG_TEXT\\\n异常节点: $exception_ips"
[[ -n "$ssh_failed_ips" ]] && MSG_TEXT="$MSG_TEXT\\\nSSH失败节点: $ssh_failed_ips"

MSG_TEXT="$MSG_TEXT\\\n\\\n需关注信息 (前 100 行):\\\n$SCAN_RESULTS_CONTENT"

feishu_message=$(cat <<EOF
{
  "receive_id": "$RECEIVE_ID",
  "msg_type": "text",
  "content": "{\"text\": \"$MSG_TEXT\"}",
  "uuid": "$(uuidgen)"
}
EOF
)

curl_cmd=$(cat <<EOF
timeout 10 curl -s -X POST "$FEISHU_WEBHOOK" \\
  -H "Content-Type: application/json" \\
  -H "Authorization: Bearer $tenant_access_token" \\
  -d '$feishu_message'
EOF
)

# 打印 curl 命令内容
echo ">>> 执行的飞书推送命令如下:"
echo "$curl_cmd"

# 实际执行
eval "$curl_cmd"


