3. What This Script Exactly Collects

• Total nodes checked
• GPU count per node (configurable)
• Nodes where:
• GPUs are missing
• NVIDIA-SMI fails
• SSH connection fails
• Date and time of execution
• Timestamped log file

• Linux-based login node
• Passwordless SSH access to compute nodes
• nvidia-smi installed on compute nodes
• Bash shell environment

~/gpu_logs/

4. Script

🔧 Modify the variables at the top to match your environment.

#!/bin/bash

# =========================

# CONFIGURATION VARIABLES

# =========================

START=1                  # Starting node index

END=10                   # Ending node index

BASE_HOST="compute"      # Base hostname prefix (example: compute01, compute02)

EXPECTED_GPUS=8          # Expected GPUs per node

LOGDIR="$HOME/gpu_logs"

# =========================

mkdir -p "$LOGDIR"

LOGFILE="$LOGDIR/gpu_detection_$(date +%F_%H%M).log"

TMPFILE=$(mktemp)

{

  echo "=========================================="

  echo " GPU Detection Report"

  echo " Date: $(date +%F)"

  echo " Time: $(date +%T)"

  echo "=========================================="

  echo

} > "$LOGFILE"

MISSING_NODES=0

TOTAL_NODES=0

CMD="nvidia-smi -L"

for i in $(seq $START $END); do

    NODE_NAME=$(printf "%s%02d" "$BASE_HOST" "$i")

    ((TOTAL_NODES++))

    echo "===================== $NODE_NAME ====================="

    OUTPUT=$(ssh -o ConnectTimeout=5 "$NODE_NAME" "$CMD" 2>&1)

    SSH_STATUS=$?

    if [ $SSH_STATUS -eq 255 ]; then

        echo "❌ Node Unreachable: $NODE_NAME"

        printf "%-15s | %-20s\n" "$NODE_NAME" "Unreachable" >> "$TMPFILE"

        ((MISSING_NODES++))

        echo

        continue

    fi

    if [[ -z "$OUTPUT" || "$OUTPUT" == *"NVIDIA-SMI has failed"* ]]; then

        echo "⚠️  NVIDIA-SMI failed on $NODE_NAME — please check"

        printf "%-15s | %-20s\n" "$NODE_NAME" "NVIDIA-SMI Failed" >> "$TMPFILE"

        ((MISSING_NODES++))

        echo

        continue

    fi

    DETECTED_INDICES=$(echo "$OUTPUT" | grep -oP 'GPU \K[0-9]+')

    DETECTED_COUNT=$(echo "$DETECTED_INDICES" | wc -l)

    if [ "$DETECTED_COUNT" -lt "$EXPECTED_GPUS" ]; then

        echo "⚠️  GPU(s) missing on $NODE_NAME — please check"

        printf "%-15s | %-20s\n" "$NODE_NAME" "GPU(s) Missing" >> "$TMPFILE"

        ((MISSING_NODES++))

    else

        echo "✅ All $EXPECTED_GPUS GPUs detected on $NODE_NAME"

    fi

    echo

done

{

  echo "Nodes Checked: $TOTAL_NODES"

  echo "Nodes with Issues: $MISSING_NODES"

  echo

  echo "---------------------------------------------"

  echo "Node Name       | Status"

  echo "---------------------------------------------"

  cat "$TMPFILE"

  echo "---------------------------------------------"

  echo

  if [ "$MISSING_NODES" -eq 0 ]; then

      echo "✅ All nodes have all $EXPECTED_GPUS GPUs detected."

  else

      echo "⚠️  Some nodes require attention."

  fi

  echo

  echo "================================================="

  echo "Log generated on: $(date)"

  echo "================================================="

} >> "$LOGFILE"

rm -f "$TMPFILE"

echo

echo "================================================="

echo "Summary:"

echo "  Nodes with issues: $MISSING_NODES"

echo "Log saved at: $LOGFILE"

echo "================================================="