ctdb/tools/onnode

   1 #!/bin/bash
   2
   3 # Run commands on CTDB nodes.
   4
   5 # See http://ctdb.samba.org/ for more information about CTDB.
   6
   7 # Copyright (C) Martin Schwenke  2008
   8
   9 # Based on an earlier script by Andrew Tridgell and Ronnie Sahlberg.
  10
  11 # Copyright (C) Andrew Tridgell  2007
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 3 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, see <http://www.gnu.org/licenses/>.
  25
  26 prog=$(basename $0)
  27
  28 usage ()
  29 {
  30     cat >&2 <<EOF
  31 Usage: onnode [OPTION] ... <NODES> <COMMAND> ...
  32   options:
  33     -c          Run in current working directory on specified nodes.
  34     -f          Specify nodes file, overrides CTDB_NODES_FILE.
  35     -i          Keep standard input open - the default is to close it.
  36     -n          Allow nodes to be specified by name.
  37     -o <prefix> Save standard output from each node to file <prefix>.<ip>
  38     -p          Run command in parallel on specified nodes.
  39     -P          Push given files to nodes instead of running commands.
  40     -q          Do not print node addresses (overrides -v).
  41     -v          Print node address even for a single node.
  42   <NODES>       "all", "any", "ok" (or "healthy"), "con" (or "connected"),
  43                 "rm" (or "recmaster"), "lvs" (or "lvsmaster"),
  44                 "natgw" (or "natgwlist"); or
  45                 a node number (0 base); or
  46                 a hostname (if -n is specified); or
  47                 list (comma separated) of <NODES>; or
  48                 range (hyphen separated) of node numbers.
  49 EOF
  50     exit 1
  51
  52 }
  53
  54 invalid_nodespec ()
  55 {
  56     echo "Invalid <nodespec>" >&2 ; echo >&2
  57     usage
  58 }
  59
  60 # Defaults.
  61 current=false
  62 parallel=false
  63 verbose=false
  64 quiet=false
  65 prefix=""
  66 names_ok=false
  67 push=false
  68 stdin=false
  69
  70 ctdb_base="${CTDB_BASE:-/etc/ctdb}"
  71
  72 parse_options ()
  73 {
  74     # $POSIXLY_CORRECT means that the command passed to onnode can
  75     # take options and getopt won't reorder things to make them
  76     # options ot onnode.
  77     local temp
  78     # Not on the previous line - local returns 0!
  79     temp=$(POSIXLY_CORRECT=1 getopt -n "$prog" -o "cf:hno:pqvPi" -l help -- "$@")
  80
  81     [ $? != 0 ] && usage
  82
  83     eval set -- "$temp"
  84
  85     while true ; do
  86         case "$1" in
  87             -c) current=true ; shift ;;
  88             -f) CTDB_NODES_FILE="$2" ; shift 2 ;;
  89             -n) names_ok=true ; shift ;;
  90             -o) prefix="$2" ; shift 2 ;;
  91             -p) parallel=true ; shift ;;
  92             -q) quiet=true ; shift ;;
  93             -v) verbose=true ; shift ;;
  94             -P) push=true ; shift ;;
  95             -i) stdin=true ; shift ;;
  96             --) shift ; break ;;
  97             -h|--help|*) usage ;; # Shouldn't happen, so this is reasonable.
  98         esac
  99     done
 100
 101     [ $# -lt 2 ] && usage
 102
 103     nodespec="$1" ; shift
 104     command="$@"
 105 }
 106
 107 echo_nth ()
 108 {
 109     local n="$1" ; shift
 110
 111     shift $n
 112     local node="$1"
 113
 114     if [ -n "$node" -a "$node" != "#DEAD" ] ; then
 115         echo $node
 116     else
 117         echo "${prog}: \"node ${n}\" does not exist" >&2
 118         exit 1
 119     fi
 120 }
 121
 122 parse_nodespec ()
 123 {
 124     # Subshell avoids hacks to restore $IFS.
 125     (
 126         IFS=","
 127         for i in $1 ; do
 128             case "$i" in
 129                 *-*) seq "${i%-*}" "${i#*-}" 2>/dev/null || invalid_nodespec ;;
 130                 # Separate lines for readability.
 131                 all|any|ok|healthy|con|connected) echo "$i" ;;
 132                 rm|recmaster|lvs|lvsmaster|natgw|natgwlist) echo "$i" ;;
 133                 *)
 134                     [ $i -gt -1 ] 2>/dev/null || $names_ok || invalid_nodespec
 135                     echo $i
 136             esac
 137         done
 138     )
 139 }
 140
 141 ctdb_status_output="" # cache
 142 get_nodes_with_status ()
 143 {
 144     local all_nodes="$1"
 145     local status="$2"
 146
 147     if [ -z "$ctdb_status_output" ] ; then
 148         ctdb_status_output=$(ctdb -X status 2>&1)
 149         if [ $? -ne 0 ] ; then
 150             echo "${prog}: unable to get status of CTDB nodes" >&2
 151             echo "$ctdb_status_output" >&2
 152             exit 1
 153         fi
 154         local nl="
 155 "
 156         ctdb_status_output="${ctdb_status_output#*${nl}}"
 157     fi
 158
 159     (
 160         local i
 161         IFS="${IFS}|"
 162         while IFS="" read i ; do
 163
 164             set -- $i # split line on colons
 165             shift     # line starts with : so 1st field is empty
 166             local pnn="$1" ; shift
 167             local ip="$1" ; shift
 168
 169             case "$status" in
 170                 healthy)
 171                     # If any bit is 1, don't match this address.
 172                     local s
 173                     for s ; do
 174                         [ "$s" != "1" ] || continue 2
 175                     done
 176                     ;;
 177                 connected)
 178                     # If disconnected bit is not 0, don't match this address.
 179                     [ "$1" = "0" ] || continue
 180                     ;;
 181                 *)
 182                     invalid_nodespec
 183             esac
 184
 185             echo_nth "$pnn" $all_nodes
 186         done <<<"$ctdb_status_output"
 187     )
 188 }
 189
 190 ctdb_props="" # cache
 191 get_node_with_property ()
 192 {
 193     local all_nodes="$1"
 194     local prop="$2"
 195
 196     local prop_node=""
 197     if [ "${ctdb_props##:${prop}:}" = "$ctdb_props" ] ; then
 198         # Not in cache.
 199         prop_node=$(ctdb "$prop" -X 2>/dev/null)
 200         if [ $? -eq 0 ] ; then
 201             if [ "$prop" = "natgwlist" ] ; then
 202                 prop_node="${prop_node%% *}" # 1st word
 203                 if [ "$prop_node" = "-1" ] ; then
 204                     # This works around natgwlist returning 0 even
 205                     # when there's no natgw.
 206                     prop_node=""
 207                 fi
 208             else
 209                 # We only want the first line.
 210                 local nl="
 211 "
 212                 prop_node="${prop_node%%${nl}*}"
 213             fi
 214         else
 215             prop_node=""
 216         fi
 217
 218         if [ -n "$prop_node" ] ; then
 219             # Add to cache.
 220             ctdb_props="${ctdb_props}${ctdb_props:+ }:${prop}:${prop_node}"
 221         fi
 222     else
 223         # Get from cache.
 224         prop_node="${ctdb_props##:${prop}:}"
 225         prop_node="${prop_node%% *}"
 226     fi
 227
 228     if [ -n "$prop_node" ] ; then
 229         echo_nth "$prop_node" $all_nodes
 230     else
 231         echo "${prog}: No ${prop} available" >&2
 232         exit 1
 233     fi
 234 }
 235
 236 get_any_available_node ()
 237 {
 238     local all_nodes="$1"
 239
 240     # We do a recursive onnode to find which nodes are up and running.
 241     local out=$($0 -pq all ctdb pnn 2>&1)
 242     local line
 243     while read line ; do
 244         local pnn="${line#PNN:}"
 245         if [ "$pnn" != "$line" ] ; then
 246             echo_nth "$pnn" $all_nodes
 247             return 0
 248         fi
 249         # Else must be an error message from a down node.
 250     done <<<"$out"
 251     return 1
 252 }
 253
 254 get_nodes ()
 255 {
 256     local all_nodes
 257
 258     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
 259         all_nodes="$CTDB_NODES_SOCKETS"
 260     else
 261         local f="${ctdb_base}/nodes"
 262         if [ -n "$CTDB_NODES_FILE" ] ; then
 263             f="$CTDB_NODES_FILE"
 264             if [ ! -e "$f" -a "${f#/}" = "$f" ] ; then
 265                 # $f is relative, try in $ctdb_base
 266                 f="${ctdb_base}/${f}"
 267             fi
 268         fi
 269
 270         if [ ! -r "$f" ] ; then
 271             echo "${prog}: unable to open nodes file  \"${f}\"" >&2
 272             exit 1
 273         fi
 274
 275         all_nodes=$(sed -e 's@#.*@@g' -e 's@ *@@g' -e 's@^$@#DEAD@' "$f")
 276     fi
 277
 278     local nodes=""
 279     local n
 280     for n in $(parse_nodespec "$1") ; do
 281         [ $? != 0 ] && exit 1  # Required to catch exit in above subshell.
 282         case "$n" in
 283             all)
 284                 echo "${all_nodes//#DEAD/}"
 285                 ;;
 286             any)
 287                 get_any_available_node "$all_nodes" || exit 1
 288                 ;;
 289             ok|healthy)
 290                 get_nodes_with_status "$all_nodes" "healthy" || exit 1
 291                 ;;
 292             con|connected)
 293                 get_nodes_with_status "$all_nodes" "connected" || exit 1
 294                 ;;
 295             rm|recmaster)
 296                 get_node_with_property "$all_nodes" "recmaster" || exit 1
 297                 ;;
 298             lvs|lvsmaster)
 299                 get_node_with_property "$all_nodes" "lvsmaster" || exit 1
 300                 ;;
 301             natgw|natgwlist)
 302                 get_node_with_property "$all_nodes" "natgwlist" || exit 1
 303                 ;;
 304             [0-9]|[0-9][0-9]|[0-9][0-9][0-9])
 305                 echo_nth $n $all_nodes
 306                 ;;
 307             *)
 308                 $names_ok || invalid_nodespec
 309                 echo $n
 310         esac
 311     done
 312 }
 313
 314 push()
 315 {
 316     local host="$1"
 317     local files="$2"
 318
 319     local f
 320     for f in $files ; do
 321         $verbose && echo "Pushing $f"
 322         case "$f" in
 323             /*) rsync "$f" "[${host}]:${f}" ;;
 324             *)  rsync "${PWD}/${f}" "[${host}]:${PWD}/${f}" ;;
 325         esac
 326     done
 327 }
 328
 329 fakessh ()
 330 {
 331     CTDB_SOCKET="$1" sh -c "$2" 3>/dev/null
 332 }
 333
 334 stdout_filter ()
 335 {
 336     if [ -n "$prefix" ] ; then
 337         cat >"${prefix}.${n//\//_}"
 338     elif $verbose && $parallel ; then
 339         sed -e "s@^@[$n] @"
 340     else
 341         cat
 342     fi
 343 }
 344
 345 stderr_filter ()
 346 {
 347     if $verbose && $parallel ; then
 348         sed -e "s@^@[$n] @"
 349     else
 350         cat
 351     fi
 352 }
 353
 354 ######################################################################
 355
 356 parse_options "$@"
 357
 358 ssh_opts=
 359 if $push ; then
 360     SSH=push
 361     EXTRA_SSH_OPTS=""
 362 else
 363     $current && command="cd $PWD && $command"
 364
 365     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
 366         SSH=fakessh
 367         EXTRA_SSH_OPTS=""
 368     else
 369         # Could "2>/dev/null || true" but want to see errors from typos in file.
 370         [ -r "${ctdb_base}/onnode.conf" ] && . "${ctdb_base}/onnode.conf"
 371         [ -n "$SSH" ] || SSH=ssh
 372         if [ "$SSH" = "ssh" ] ; then
 373             if $parallel || ! $stdin ; then
 374                 ssh_opts="-n"
 375             fi
 376         else
 377             : # rsh? All bets are off!
 378         fi
 379     fi
 380 fi
 381
 382 ######################################################################
 383
 384 nodes=$(get_nodes "$nodespec")
 385 [ $? != 0 ] && exit 1   # Required to catch exit in above subshell.
 386
 387 if $quiet ; then
 388     verbose=false
 389 else
 390     # If $nodes contains a space or a newline then assume multiple nodes.
 391     nl="
 392 "
 393     [ "$nodes" != "${nodes%[ ${nl}]*}" ] && verbose=true
 394 fi
 395
 396 pids=""
 397 trap 'kill -TERM $pids 2>/dev/null' INT TERM
 398 # There's a small race here where the kill can fail if no processes
 399 # have been added to $pids and the script is interrupted.  However,
 400 # the part of the window where it matter is very small.
 401 retcode=0
 402 for n in $nodes ; do
 403     set -o pipefail 2>/dev/null
 404     if $parallel ; then
 405         { exec 3>&1 ; { $SSH $ssh_opts $EXTRA_SSH_OPTS $n "$command" | stdout_filter >&3 ; } 2>&1 | stderr_filter ; } &
 406         pids="${pids} $!"
 407     else
 408         if $verbose ; then
 409             echo >&2 ; echo ">> NODE: $n <<" >&2
 410         fi
 411
 412         { exec 3>&1 ; { $SSH $ssh_opts $EXTRA_SSH_OPTS $n "$command" | stdout_filter >&3 ; } 2>&1 | stderr_filter ; }
 413         [ $? = 0 ] || retcode=$?
 414     fi
 415 done
 416
 417 $parallel && {
 418     for p in $pids; do
 419         wait $p
 420         [ $? = 0 ] || retcode=$?
 421     done
 422 }
 423
 424 exit $retcode