ctdb/config/events.d/10.interface

   1 #!/bin/sh
   2
   3 #################################
   4 # interface event script for ctdb
   5 # this adds/removes IPs from your
   6 # public interface
   7
   8 [ -n "$CTDB_BASE" ] || \
   9     export CTDB_BASE=$(cd -P $(dirname "$0") ; dirname "$PWD")
  10
  11 . $CTDB_BASE/functions
  12 loadconfig
  13
  14 [ -z "$CTDB_PUBLIC_ADDRESSES" ] && {
  15         CTDB_PUBLIC_ADDRESSES=$CTDB_BASE/public_addresses
  16 }
  17
  18 [ ! -f "$CTDB_PUBLIC_ADDRESSES" ] && {
  19         if [ "$1" = "init" ]; then
  20                 echo "No public addresses file found. Nothing to do for 10.interfaces"
  21         fi
  22         exit 0
  23 }
  24
  25 mark_up ()
  26 {
  27     up_interfaces_found=true
  28     ctdb setifacelink $1 up >/dev/null 2>&1
  29 }
  30
  31 mark_down ()
  32 {
  33     fail=true
  34     ctdb setifacelink $1 down >/dev/null 2>&1
  35 }
  36
  37 # This sets $all_interfaces as a side-effect.
  38 get_all_interfaces ()
  39 {
  40     # Get all the interfaces listed in the public_addresses file
  41     all_interfaces=$(sed -e "s/^[^\t ]*[\t ]*//" -e "s/,/ /g" -e "s/[\t ]*$//" $CTDB_PUBLIC_ADDRESSES)
  42
  43     # Add some special interfaces if they're defined
  44     [ "$CTDB_PUBLIC_INTERFACE" ] && all_interfaces="$CTDB_PUBLIC_INTERFACE $all_interfaces"
  45     [ "$CTDB_NATGW_PUBLIC_IFACE" ] && all_interfaces="$CTDB_NATGW_PUBLIC_IFACE $all_interfaces"
  46
  47     # Get the interfaces for which CTDB has public IPs configured.
  48     # That is, for all but the 1st line, get the 1st field.
  49     ctdb_ifaces=$(ctdb -X ifaces | sed -e '1d' -e 's@^|@@' -e 's@|.*@@')
  50
  51     # Add $ctdb_interfaces and uniquify
  52     all_interfaces=$(echo $all_interfaces $ctdb_ifaces | tr ' ' '\n' | sort -u)
  53 }
  54
  55 monitor_interfaces()
  56 {
  57         get_all_interfaces
  58
  59         fail=false
  60         up_interfaces_found=false
  61
  62         # Note that this loop must not exit early.  It must process
  63         # all interfaces so that the correct state for each interface
  64         # is set in CTDB using mark_up/mark_down.  If there is a
  65         # problem with an interface then set fail=true and continue.
  66         for iface in $all_interfaces ; do
  67
  68             ip link show $iface 2>/dev/null >/dev/null || {
  69                 echo "ERROR: Interface $iface does not exist but it is used by public addresses."
  70                 mark_down $iface
  71                 continue
  72             }
  73
  74             # These interfaces are sometimes bond devices
  75             # When we use VLANs for bond interfaces, there will only
  76             # be an entry in /proc for the underlying real interface
  77             realiface=`echo $iface |sed -e 's/\..*$//'`
  78             bi=$(get_proc "net/bonding/$realiface" 2>/dev/null) && {
  79                 echo "$bi" | grep -q 'Currently Active Slave: None' && {
  80                         echo "ERROR: No active slaves for bond device $realiface"
  81                         mark_down $iface
  82                         continue
  83                 }
  84                 echo "$bi" | grep -q '^MII Status: up' || {
  85                         echo "ERROR: public network interface $realiface is down"
  86                         mark_down $iface
  87                         continue
  88                 }
  89                 echo "$bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
  90                         # This works around a bug in the driver where the
  91                         # overall bond status can be up but none of the actual
  92                         # physical interfaces have a link.
  93                         echo "$bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
  94                                 echo "ERROR: No active slaves for 802.ad bond device $realiface"
  95                                 mark_down $iface
  96                                 continue
  97                         }
  98                 }
  99                 mark_up $iface
 100                 continue
 101             }
 102
 103             case $iface in
 104             lo*)
 105                 # loopback is always working
 106                 mark_up $iface
 107                 ;;
 108             ib*)
 109                 # we dont know how to test ib links
 110                 mark_up $iface
 111                 ;;
 112             *)
 113                 ethtool $iface | grep -q 'Link detected: yes' || {
 114                     # On some systems, this is not successful when a
 115                     # cable is plugged but the interface has not been
 116                     # brought up previously. Bring the interface up
 117                     # and try again...
 118                     ip link set $iface up
 119                     ethtool $iface | grep -q 'Link detected: yes' || {
 120                         echo "ERROR: No link on the public network interface $iface"
 121                         mark_down $iface
 122                         continue
 123                     }
 124                 }
 125                 mark_up $iface
 126                 ;;
 127             esac
 128
 129         done
 130
 131         $fail || return 0
 132
 133         $up_interfaces_found && \
 134             [ "$CTDB_PARTIALLY_ONLINE_INTERFACES" = "yes" ] && \
 135             return 0
 136
 137         return 1
 138 }
 139
 140 # Sets: iface, ip, maskbits, family
 141 get_iface_ip_maskbits_family ()
 142 {
 143     _iface_in="$1"
 144     ip="$2"
 145     _maskbits_in="$3"
 146
 147     set -- $(ip_maskbits_iface "$ip")
 148     if [ -n "$1" ] ; then
 149         maskbits="$1"
 150         iface="$2"
 151         family="$3"
 152
 153         if [ "$iface" != "$_iface_in" ] ; then
 154             printf \
 155                 'WARNING: Public IP %s hosted on interface %s but VNN says %s\n' \
 156                 "$ip" "$iface" "$_iface_in"
 157         fi
 158         if [ "$maskbits" != "$_maskbits_in" ] ; then
 159             printf \
 160                 'WARNING: Public IP %s has %s bit netmask but VNN says %s\n' \
 161                     "$ip" "$maskbits" "$_maskbits_in"
 162         fi
 163     else
 164         die "ERROR: Unable to determine interface for IP ${ip}"
 165     fi
 166 }
 167
 168 ctdb_check_args "$@"
 169
 170 case "$1" in
 171      #############################
 172      # called when ctdbd starts up
 173      init)
 174         # make sure that we only respond to ARP messages from the NIC where
 175         # a particular ip address is associated.
 176         get_proc sys/net/ipv4/conf/all/arp_filter >/dev/null 2>&1 && {
 177             set_proc sys/net/ipv4/conf/all/arp_filter 1
 178         }
 179
 180         _promote="sys/net/ipv4/conf/all/promote_secondaries"
 181         get_proc "$_promote" >/dev/null 2>&1 || \
 182             die "Public IPs only supported if promote_secondaries is available"
 183
 184         # make sure we drop any ips that might still be held if
 185         # previous instance of ctdb got killed with -9 or similar
 186         drop_all_public_ips
 187         ;;
 188
 189      #############################
 190      # called after ctdbd has done its initial recovery
 191      # and we start the services to become healthy
 192      startup)
 193         monitor_interfaces
 194         ;;
 195
 196
 197      ################################################
 198      # called when ctdbd wants to claim an IP address
 199      takeip)
 200         iface=$2
 201         ip=$3
 202         maskbits=$4
 203
 204         add_ip_to_iface $iface $ip $maskbits || {
 205                 exit 1;
 206         }
 207
 208         # cope with the script being killed while we have the interface blocked
 209         case "$ip" in
 210             *:*) family="inet6" ;;
 211             *)   family="inet"  ;;
 212         esac
 213         iptables_wrapper $family -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
 214
 215         flush_route_cache
 216         ;;
 217
 218
 219      ##################################################
 220      # called when ctdbd wants to release an IP address
 221      releaseip)
 222         # releasing an IP is a bit more complex than it seems. Once the IP
 223         # is released, any open tcp connections to that IP on this host will end
 224         # up being stuck. Some of them (such as NFS connections) will be unkillable
 225         # so we need to use the killtcp ctdb function to kill them off. We also
 226         # need to make sure that no new connections get established while we are
 227         # doing this! So what we do is this:
 228         # 1) firewall this IP, so no new external packets arrive for it
 229         # 2) use netstat -tn to find existing connections, and kill them
 230         # 3) remove the IP from the interface
 231         # 4) remove the firewall rule
 232         shift
 233         get_iface_ip_maskbits_family "$@"
 234
 235         # we do an extra delete to cope with the script being killed
 236         iptables_wrapper $family -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
 237         iptables_wrapper $family -I INPUT -i $iface -d $ip -j DROP
 238         kill_tcp_connections $ip
 239
 240         delete_ip_from_iface $iface $ip $maskbits || {
 241             iptables_wrapper $family \
 242                              -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
 243                 exit 1
 244         }
 245
 246         iptables_wrapper $family -D INPUT -i $iface -d $ip -j DROP 2> /dev/null
 247
 248         flush_route_cache
 249         ;;
 250
 251      ##################################################
 252      # called when ctdbd wants to update an IP address
 253      updateip)
 254         # moving an IP is a bit more complex than it seems.
 255         # First we drop all traffic on the old interface.
 256         # Then we try to add the ip to the new interface and before
 257         # we finally remove it from the old interface.
 258         #
 259         # 1) firewall this IP, so no new external packets arrive for it
 260         # 2) remove the IP from the old interface (and new interface, to be sure)
 261         # 3) add the IP to the new interface
 262         # 4) remove the firewall rule
 263         # 5) use ctdb gratiousarp to propagate the new mac address
 264         # 6) use netstat -tn to find existing connections, and tickle them
 265         _oiface=$2
 266         niface=$3
 267         _ip=$4
 268         _maskbits=$5
 269
 270         get_iface_ip_maskbits_family "$_oiface" "$ip" "$maskbits"
 271         oiface="$iface"
 272
 273         # we do an extra delete to cope with the script being killed
 274         iptables_wrapper $family -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null
 275         iptables_wrapper $family -I INPUT -i $oiface -d $ip -j DROP
 276
 277         delete_ip_from_iface $oiface $ip $maskbits 2>/dev/null
 278         delete_ip_from_iface $niface $ip $maskbits 2>/dev/null
 279
 280         add_ip_to_iface $niface $ip $maskbits || {
 281             iptables_wrapper $family \
 282                              -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null
 283             exit 1
 284         }
 285
 286         # cope with the script being killed while we have the interface blocked
 287         iptables_wrapper $family -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null
 288
 289         flush_route_cache
 290
 291         # propagate the new mac address
 292         ctdb gratiousarp $ip $niface
 293
 294         # tickle all existing connections, so that dropped packets
 295         # are retransmited and the tcp streams work
 296
 297         tickle_tcp_connections $ip
 298
 299         ;;
 300
 301      monitor)
 302         monitor_interfaces || exit 1
 303         ;;
 304     *)
 305         ctdb_standard_event_handler "$@"
 306         ;;
 307 esac
 308
 309 exit 0
 310