ctdb/tools/ctdb_diagnostics

   1 #!/bin/sh
   2 # a script to test the basic setup of a CTDB/Samba install
   3 # tridge@samba.org September 2007
   4 # martin@meltin.net August 2010
   5
   6 usage ()
   7 {
   8     cat >&2 <<EOF
   9 Usage: ctdb_diagnostics [OPTION] ...
  10   options:
  11     -n <nodes>  Comma separated list of nodes to operate on
  12     -c          Ignore comment lines (starting with '#') in file comparisons
  13     -w          Ignore whitespace in file comparisons
  14     --no-ads    Do not use commands that assume an Active Directory Server
  15 EOF
  16     exit 1
  17
  18 }
  19
  20 nodes=$(ctdb listnodes -X | cut -d'|' -f2)
  21 bad_nodes=""
  22 diff_opts=
  23 no_ads=false
  24
  25 parse_options ()
  26 {
  27     temp=$(getopt -n "ctdb_diagnostics" -o "n:cwh" -l no-ads,help -- "$@")
  28
  29     [ $? != 0 ] && usage
  30
  31     eval set -- "$temp"
  32
  33     while true ; do
  34         case "$1" in
  35             -n) nodes=$(echo "$2" | sed -e 's@,@ @g') ; shift 2 ;;
  36             -c) diff_opts="${diff_opts} -I ^#.*" ; shift ;;
  37             -w) diff_opts="${diff_opts} -w" ; shift ;;
  38             --no-ads) no_ads=true ; shift ;;
  39             --) shift ; break ;;
  40             -h|--help|*) usage ;;
  41         esac
  42     done
  43
  44     [ $# -ne 0 ] && usage
  45 }
  46
  47 parse_options "$@"
  48
  49 # Use 5s ssh timeout if EXTRA_SSH_OPTS doesn't set a timeout.
  50 case "$EXTRA_SSH_OPTS" in
  51     *ConnectTimeout=*) : ;;
  52     *)
  53         export EXTRA_SSH_OPTS="${EXTRA_SSH_OPTS} -o ConnectTimeout=5"
  54 esac
  55
  56 # Filter nodes.  Remove any nodes we can't contact from $node and add
  57 # them to $bad_nodes.
  58 _nodes=""
  59 for _i in $nodes ; do
  60     if onnode "$_i" true >/dev/null 2>&1 ; then
  61         _nodes="${_nodes}${_nodes:+ }${_i}"
  62     else
  63         bad_nodes="${bad_nodes}${bad_nodes:+,}${_i}"
  64     fi
  65 done
  66 nodes="$_nodes"
  67
  68 nodes_comma=$(echo "$nodes" | sed -e 's@[[:space:]]@,@g')
  69
  70 PATH="$PATH:/sbin:/usr/sbin:/usr/lpp/mmfs/bin"
  71
  72 # list of config files that must exist and that we check are the same
  73 # on the nodes
  74 if [ -d /etc/sysconfig ] ; then
  75     CONFIG_FILES_MUST="/etc/krb5.conf /etc/hosts /usr/local/etc/ctdb/nodes /etc/sysconfig/ctdb /etc/resolv.conf /etc/nsswitch.conf /etc/sysctl.conf /etc/samba/smb.conf /etc/fstab /etc/multipath.conf /etc/pam.d/system-auth /etc/sysconfig/nfs /etc/exports /etc/vsftpd/vsftpd.conf"
  76 else
  77     CONFIG_FILES_MUST="/etc/krb5.conf /etc/hosts /usr/local/etc/ctdb/nodes /etc/default/ctdb /etc/resolv.conf /etc/nsswitch.conf /etc/sysctl.conf /etc/samba/smb.conf /etc/fstab /etc/multipath.conf /etc/pam.d/system-auth /etc/default/nfs /etc/exports /etc/vsftpd/vsftpd.conf"
  78 fi
  79
  80 # list of config files that may exist and should be checked that they
  81 # are the same on the nodes
  82 CONFIG_FILES_MAY="/usr/local/etc/ctdb/public_addresses /usr/local/etc/ctdb/static-routes"
  83
  84 2>&1
  85
  86 cat <<EOF
  87 --------------------------------------------------------------------
  88 ctdb_diagnostics starting. This script will gather information about
  89 your ctdb cluster. You should send the output of this script along
  90 with any ctdb or clustered Samba bug reports.
  91 --------------------------------------------------------------------
  92 EOF
  93
  94 date
  95
  96 error() {
  97     msg="$1"
  98     echo "ERROR: $msg"
  99     NUM_ERRORS=$((NUM_ERRORS + 1))
 100     echo " ERROR[$NUM_ERRORS]: $msg" >> "$ERRORS"
 101 }
 102
 103 show_file() {
 104     fname="$1"
 105     _fdetails=$(ls -l "$fname" 2>&1)
 106     echo "  ================================"
 107     echo "  File: $fname"
 108     echo "  $_fdetails"
 109     sed 's/^/  /' "$fname" 2>&1
 110     echo "  ================================"
 111 }
 112
 113 show_all() {
 114     echo "running $1 on nodes $nodes_comma"
 115     onnode "$nodes_comma" "hostname; date; $1 2>&1 | sed 's/^/  /'" 2>&1
 116 }
 117
 118 show_and_compare_files () {
 119
 120     fmt="$1" ; shift
 121
 122     for f ; do
 123         _bf=$(basename "$f")
 124         first=true
 125
 126         for n in $nodes ; do
 127
 128             if $first ; then
 129                 onnode "$n" [ -r "$f" ] || {
 130                     # This function takes a format string
 131                     # shellcheck disable=SC2059
 132                     msg=$(printf "$fmt" "$f" "$n")
 133                     error "$msg"
 134                     continue 2;
 135                 }
 136
 137                 fstf="${tmpdir}/${_bf}.node${n}"
 138                 onnode "$n" cat "$f" >"$fstf" 2>&1
 139
 140                 _fdetails=$(onnode "$n" ls -l "$f" 2>&1)
 141                 echo "  ================================"
 142                 echo "  File (on node $n): $f"
 143                 echo "  $_fdetails"
 144                 sed 's/^/  /' "$fstf"
 145                 echo "  ================================"
 146                 first=false
 147             else
 148                 echo "Testing for same config file $f on node $n"
 149                 tmpf="${tmpdir}/${_bf}.node${n}"
 150                 onnode "$n" cat "$f" >"$tmpf" 2>&1
 151                 # Intentional multi-word splitting on diff_opts
 152                 # shellcheck disable=SC2086
 153                 diff $diff_opts "$fstf" "$tmpf" >/dev/null 2>&1 || {
 154                     error "File $f is different on node $n"
 155                     diff -u $diff_opts "$fstf" "$tmpf"
 156                 }
 157                 rm -f "$tmpf"
 158             fi
 159         done
 160
 161         rm -f "$fstf"
 162     done
 163 }
 164
 165 if ! tmpdir=$(mktemp -d) ; then
 166     echo "Unable to create a temporary directory"
 167     exit 1
 168 fi
 169 ERRORS="${tmpdir}/diag_err"
 170 NUM_ERRORS=0
 171
 172 cat <<EOF
 173 Diagnosis started on these nodes:
 174 $nodes_comma
 175 EOF
 176
 177 if [ -n "$bad_nodes" ] ; then
 178     cat <<EOF
 179
 180 NOT RUNNING DIAGNOSTICS on these uncontactable nodes:
 181 $bad_nodes
 182 EOF
 183
 184 fi
 185
 186 cat <<EOF
 187
 188 For reference, here is the nodes file on the current node...
 189 EOF
 190
 191 show_file /usr/local/etc/ctdb/nodes
 192
 193 cat <<EOF
 194 --------------------------------------------------------------------
 195 Comping critical config files on nodes $nodes_comma
 196 EOF
 197
 198 # Intentional multi-word splitting on CONFIG_FILES_MUST
 199 # shellcheck disable=SC2086
 200 show_and_compare_files \
 201     "%s is missing on node %d" \
 202     $CONFIG_FILES_MUST
 203
 204 # Intentional multi-word splitting on CONFIG_FILES_MAY
 205 # shellcheck disable=SC2086
 206 show_and_compare_files \
 207     "Optional file %s is not present on node %d" \
 208     $CONFIG_FILES_MAY
 209
 210 cat <<EOF
 211 --------------------------------------------------------------------
 212 Checking for clock drift
 213 EOF
 214 t=$(date +%s)
 215 for i in $nodes; do
 216     t2=$(onnode "$i" date +%s)
 217     d=$((t2 - t))
 218     if [ "$d" -gt 30 -o "$d" -lt -30 ]; then
 219         error "time on node $i differs by $d seconds"
 220     fi
 221 done
 222
 223 cat <<EOF
 224 --------------------------------------------------------------------
 225 Showing software versions
 226 EOF
 227 show_all "uname -a"
 228 [ -x /bin/rpm ] && {
 229     show_all "rpm -qa | egrep 'samba|ctdb|gpfs'"
 230 }
 231 [ -x /usr/bin/dpkg-query ] && {
 232     show_all "/usr/bin/dpkg-query --show 'ctdb'"
 233     show_all "/usr/bin/dpkg-query --show 'samba'"
 234     #show_all "/usr/bin/dpkg-query --show 'gpfs'"
 235 }
 236
 237
 238 cat <<EOF
 239 --------------------------------------------------------------------
 240 Showing ctdb status and recent log entries
 241 EOF
 242 show_all "ctdb status; ctdb ip"
 243 show_all "ctdb statistics"
 244 show_all "ctdb uptime"
 245 show_all "ctdb listvars"
 246 show_all "ctdb getdbmap"
 247 show_all "ctdb -X getdbmap | awk -F'|' 'NR > 1 {print \$3}' | sort | xargs -n 1 ctdb dbstatistics"
 248
 249 echo "Showing log.ctdb"
 250 show_all "test -f /usr/local/var/log/log.ctdb && tail -100 /usr/local/var/log/log.ctdb"
 251
 252 echo "Showing log.ctdb"
 253 show_all "test -f /usr/local/var/log/log.ctdb && tail -100 /usr/local/var/log/log.ctdb"
 254
 255 show_all "tail -200 /var/log/messages"
 256 show_all "ls -lRs /usr/local/var/lib/ctdb"
 257 show_all "ls -lRs /usr/local/etc/ctdb"
 258
 259
 260 cat <<EOF
 261 --------------------------------------------------------------------
 262 Showing system and process status
 263 EOF
 264 show_all "df"
 265 show_all "df -i"
 266 show_all "mount"
 267 show_all "w"
 268 show_all "ps axfwu"
 269 show_all "dmesg"
 270 show_all "/sbin/lspci"
 271 show_all "dmidecode"
 272 show_all "cat /proc/partitions"
 273 show_all "cat /proc/cpuinfo"
 274 show_all "cat /proc/scsi/scsi"
 275 show_all "/sbin/ifconfig -a"
 276 show_all "/sbin/ifconfig -a"
 277 show_all "/sbin/ip addr list"
 278 show_all "/sbin/route -n"
 279 show_all "netstat -s"
 280 show_all "free"
 281 show_all "crontab -l"
 282 show_all "sysctl -a"
 283 show_all "iptables -L -n"
 284 show_all "iptables -L -n -t nat"
 285 show_all "/usr/sbin/rpcinfo -p"
 286 show_all "/usr/sbin/showmount -a"
 287 show_all "/usr/sbin/showmount -e"
 288 show_all "/usr/sbin/nfsstat -v"
 289 [ -x /sbin/multipath ] && {
 290     show_all "/sbin/multipath -ll"
 291 }
 292 [ -x /sbin/chkconfig ] && {
 293     show_all "/sbin/chkconfig --list"
 294 }
 295 [ -x /usr/sbin/getenforce ] && {
 296     show_all "/usr/sbin/getenforce"
 297 }
 298 [ -d /proc/net/bonding ] && {
 299     for f in /proc/net/bonding/*; do
 300         show_all "cat $f"
 301     done
 302 }
 303
 304 cat <<EOF
 305 --------------------------------------------------------------------
 306 Showing Samba status
 307 EOF
 308 show_all "smbstatus -n -B"
 309 if $no_ads ; then
 310     echo
 311     echo "Skipping \"net ads testjoin\" as requested"
 312     echo
 313 else
 314     show_all "net ads testjoin"
 315 fi
 316 show_all "net conf list"
 317 show_all "lsof -n | grep smbd"
 318 show_all "lsof -n | grep ctdbd"
 319 show_all "netstat -tan"
 320 if $no_ads ; then
 321     echo
 322     echo "Skipping \"net ads info\" as requested"
 323     echo
 324 else
 325     show_all "net ads info"
 326 fi
 327 show_all "date"
 328 show_all "smbclient -U% -L 127.0.0.1"
 329 WORKGROUP=$(testparm -s --parameter-name=WORKGROUP 2> /dev/null)
 330 show_all id "$WORKGROUP/Administrator"
 331 show_all "wbinfo -p"
 332 show_all "wbinfo --online-status"
 333 show_all "smbd -b"
 334
 335 date
 336 echo "Diagnostics finished with $NUM_ERRORS errors"
 337
 338 [ -r "$ERRORS" ] && {
 339     cat "$ERRORS"
 340     rm -f "$ERRORS"
 341 }
 342
 343 rm -rf "$tmpdir"
 344
 345 exit $NUM_ERRORS
 346