3. What This Script Collects

• Total InfiniBand ports detected
• Number of Active ports
• Number of Down ports
• Number of Degraded ports (below expected rate)
• Lowest detected link rate
• Unreachable node count
• Timestamped log file

3.1. Requirements

• Linux-based login/management node
• Passwordless SSH access to compute nodes
• ibstat installed on compute nodes
• Bash shell environment

4. Script

#!/bin/bash

# =========================

# CONFIGURATION VARIABLES

# =========================

START=1                   # Starting node index

END=10                    # Ending node index

BASE_HOST="compute"       # Hostname prefix (example: compute01, compute02)

EXPECTED_RATE=400         # Expected IB rate (Gb/s)

LOGDIR="$HOME/gpu_logs/ib_health"

CMD="ibstat"

# =========================

mkdir -p "$LOGDIR"

LOGFILE="$LOGDIR/ib_health_$(date +%F_%H%M).log"

TMPFILE=$(mktemp)

{

echo "========================================================"

echo " InfiniBand Health Report"

echo " Date: $(date '+%F  %T')"

echo "========================================================"

echo

} > "$LOGFILE"

MISSING_NODES=0

TOTAL_NODES=0

DOWNLINK_NODES=0

echo ""

echo "Checking InfiniBand Health..."

echo "This may take a few minutes depending on node count."

echo ""

for i in $(seq $START $END); do

NODE_NAME=$(printf "%s%02d" "$BASE_HOST" "$i")

((TOTAL_NODES++))

echo "===================== Checking $NODE_NAME ====================="

OUTPUT=$(ssh -o ConnectTimeout=5 "$NODE_NAME" "$CMD" 2>&1)

SSH_STATUS=$?

if [ $SSH_STATUS -eq 255 ]; then

printf "%-15s | %-12s\n" "$NODE_NAME" "Unreachable" >> "$TMPFILE"

((MISSING_NODES++))

continue

fi

TOTAL_PORTS=$(echo "$OUTPUT" | grep -c "CA 'mlx")

ACTIVE_PORTS=$(echo "$OUTPUT" | grep -c "State: Active")

DOWN_PORTS=$(echo "$OUTPUT" | grep -c "State: Down")

DEGRADED_PORTS=$(echo "$OUTPUT" | grep -A1 "State: Active" | \

grep -o "Rate: [0-9]*" | \

awk -v rate=$EXPECTED_RATE '$2 < rate' | wc -l)

LOWEST_RATE=$(echo "$OUTPUT" | grep -o "Rate: [0-9]*" | \

awk '{print $2}' | sort -n | head -1)

[ -z "$LOWEST_RATE" ] && LOWEST_RATE="N/A"

if [ "$ACTIVE_PORTS" -eq "$TOTAL_PORTS" ] && \

[ "$DOWN_PORTS" -eq 0 ] && \

[ "$DEGRADED_PORTS" -eq 0 ]; then

echo " ✅ $NODE_NAME – all links healthy"

echo ""

continue

fi

if [ "$DOWN_PORTS" -gt 0 ] || [ "$DEGRADED_PORTS" -gt 0 ]; then

echo " ❌ Issue detected on $NODE_NAME – manual check required"

echo ""

((DOWNLINK_NODES++))

fi

printf "%-15s | %-11s | %-6s | %-4s | %-8s | %-6s\n" \

"$NODE_NAME" "$TOTAL_PORTS" "$ACTIVE_PORTS" \

"$DOWN_PORTS" "$DEGRADED_PORTS" "$LOWEST_RATE" >> "$TMPFILE"

done

{

echo "Node Name       | Total Ports | Active | Down | Degraded | Lowest Rate"

echo "-----------------------------------------------------------------------"

cat "$TMPFILE"

echo "-----------------------------------------------------------------------"

echo

echo "Nodes Checked: $TOTAL_NODES"

echo "Nodes Unreachable: $MISSING_NODES"

echo "Nodes with Down/Degraded Links: $DOWNLINK_NODES"

echo "========================================================"

echo "Log generated on: $(date)"

echo "========================================================"

} >> "$LOGFILE"

rm -f "$TMPFILE"

echo

echo "========================================================"

echo "Summary:"

echo "  IB Health Check Completed for $TOTAL_NODES Nodes"

echo "  Nodes with issues: $DOWNLINK_NODES"

echo "  Log saved at: $LOGFILE"

echo "========================================================"