ctdb/config/events.d/13.per_ip_routing

   1 #!/bin/sh
   2
   3 [ -n "$CTDB_BASE" ] || \
   4     export CTDB_BASE=$(cd -P $(dirname "$0") ; dirname "$PWD")
   5
   6 . $CTDB_BASE/functions
   7 loadconfig
   8
   9 service_name=per_ip_routing
  10
  11 # Do nothing if unconfigured
  12 [ -n "$CTDB_PER_IP_ROUTING_CONF" ] || exit 0
  13
  14 table_id_prefix="ctdb."
  15
  16 [ -n "$CTDB_PER_IP_ROUTING_RULE_PREF" ] || \
  17     die "error: CTDB_PER_IP_ROUTING_RULE_PREF not configured"
  18
  19 [ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -lt "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] 2>/dev/null || \
  20     die "error: CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW] and/or CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] improperly configured"
  21
  22 if [ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -le 253 -a \
  23     255 -le "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] ; then
  24     die "error: range CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW]..CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] must not include 253-255"
  25 fi
  26
  27 have_link_local_config ()
  28 {
  29     [ "$CTDB_PER_IP_ROUTING_CONF" = "__auto_link_local__" ]
  30 }
  31
  32 if ! have_link_local_config && [ ! -r "$CTDB_PER_IP_ROUTING_CONF" ] ; then
  33     die "error: CTDB_PER_IP_ROUTING_CONF=$CTDB_PER_IP_ROUTING_CONF file not found"
  34 fi
  35
  36 ######################################################################
  37
  38 ipv4_is_valid_addr()
  39 {
  40     _ip="$1"
  41
  42     _count=0
  43     # Get the shell to break up the address into 1 word per octet
  44     for _o in $(export IFS="." ; echo $_ip) ; do
  45         # The 2>/dev/null stops output from failures where an "octet"
  46         # is not numeric.  The test will still fail.
  47         if ! [ 0 -le $_o -a $_o -le 255 ] 2>/dev/null ; then
  48             return 1
  49         fi
  50         _count=$(($_count + 1))
  51     done
  52
  53     # A valid IPv4 address has 4 octets
  54     [ $_count -eq 4 ]
  55 }
  56
  57 ensure_ipv4_is_valid_addr ()
  58 {
  59     _event="$1"
  60     _ip="$2"
  61
  62     ipv4_is_valid_addr "$_ip" || {
  63         echo "$0: $_event not an ipv4 address skipping IP:$_ip"
  64         exit 0
  65     }
  66 }
  67
  68 ipv4_host_addr_to_net ()
  69 {
  70     _host="$1"
  71     _maskbits="$2"
  72
  73     # Convert the host address to an unsigned long by splitting out
  74     # the octets and doing the math.
  75     _host_ul=0
  76     for _o in $(export IFS="." ; echo $_host) ; do
  77         _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
  78     done
  79
  80     # Calculate the mask and apply it.
  81     _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
  82     _net_ul=$(( $_host_ul & $_mask_ul ))
  83
  84     # Now convert to a network address one byte at a time.
  85     _net=""
  86     for _o in $(seq 1 4) ; do
  87         _net="$(($_net_ul & 255))${_net:+.}${_net}"
  88         _net_ul=$(($_net_ul >> 8))
  89     done
  90
  91     echo "${_net}/${_maskbits}"
  92 }
  93
  94 ######################################################################
  95
  96 # Setup a table id to use for the given IP.  We don't need to know it,
  97 # it just needs to exist in /etc/iproute2/rt_tables.  Fail if no free
  98 # table id could be found in the configured range.
  99 ensure_table_id_for_ip ()
 100 {
 101     _ip=$1
 102
 103     _f="$CTDB_ETCDIR/iproute2/rt_tables"
 104     # This file should always exist, but...
 105     if [ ! -f "$_f" ] ; then
 106         mkdir -p $(dirname "$_f")
 107         touch "$_f"
 108     fi
 109
 110     # Maintain a table id for each IP address we've ever seen in
 111     # rt_tables.  We use a "ctdb." prefix on the label.
 112     _label="${table_id_prefix}${_ip}"
 113
 114     # This finds either the table id corresponding to the label or a
 115     # new unused one (that is greater than all the used ones in the
 116     # range).
 117     (
 118         # Note that die() just gets us out of the subshell...
 119         flock --timeout 30 0 || \
 120             die "ensure_table_id_for_ip: failed to lock file $_f"
 121
 122         _new=$CTDB_PER_IP_ROUTING_TABLE_ID_LOW
 123         while read _t _l ; do
 124             # Skip comments
 125             case "$_t" in
 126                 \#*) continue ;;
 127             esac
 128             # Found existing: done
 129             if [ "$_l" = "$_label" ] ; then
 130                 return 0
 131             fi
 132             # Potentially update the new table id to be used.  The
 133             # redirect stops error spam for a non-numeric value.
 134             if [ $_new -le $_t -a \
 135                 $_t -le $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH ] 2>/dev/null ; then
 136                 _new=$(($_t + 1))
 137             fi
 138         done
 139
 140         # If the new table id is legal then add it to the file and
 141         # print it.
 142         if [ $_new -le $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH ] ; then
 143             printf "%d\t%s\n" "$_new" "$_label" >>"$_f"
 144             return 0
 145         else
 146             return 1
 147         fi
 148     ) <"$_f"
 149 }
 150
 151 # Clean up all the table ids that we might own.
 152 clean_up_table_ids ()
 153 {
 154     _f="$CTDB_ETCDIR/iproute2/rt_tables"
 155     # Even if this didn't exist on the system, adding a route will
 156     # have created it.  What if we startup and immediately shutdown?
 157     if [ ! -f "$_f" ] ; then
 158         mkdir -p $(dirname "$_f")
 159         touch "$_f"
 160     fi
 161
 162     (
 163         # Note that die() just gets us out of the subshell...
 164         flock --timeout 30 0 || \
 165             die "clean_up_table_ids: failed to lock file $_f"
 166
 167         # Delete any items from the file that have a table id in our
 168         # range or a label matching our label.  Preserve comments.
 169         _tmp="${_f}.$$.ctdb"
 170         awk -v min="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" \
 171             -v max="$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" \
 172             -v pre="$table_id_prefix" \
 173             '/^#/ || \
 174              !(min <= $1 && $1 <= max) && \
 175              !(index($2, pre) == 1) \
 176              { print $0 }' "$_f" >"$_tmp"
 177
 178         mv "$_tmp" "$_f"
 179         # The lock is gone - don't do anything else here
 180     ) <"$_f"
 181 }
 182
 183 ######################################################################
 184
 185 # This prints the config for an IP, which is either relevant entries
 186 # from the config file or, if set to the magic link local value, some
 187 # link local routing config for the IP.
 188 get_config_for_ip ()
 189 {
 190     _ip="$1"
 191
 192     if have_link_local_config ; then
 193         # When parsing public_addresses also split on '/'.  This means
 194         # that we get the maskbits as item #2 without further parsing.
 195         while IFS="/$IFS" read _i _maskbits _x ; do
 196             if [ "$_ip" = "$_i" ] ; then
 197                 echo -n "$_ip "; ipv4_host_addr_to_net "$_ip" "$_maskbits"
 198             fi
 199         done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 200     else
 201         while read _i _rest ; do
 202             if [ "$_ip" = "$_i" ] ; then
 203                 printf "%s\t%s\n" "$_ip" "$_rest"
 204             fi
 205         done <"$CTDB_PER_IP_ROUTING_CONF"
 206     fi
 207 }
 208
 209 ip_has_configuration ()
 210 {
 211     _ip="$1"
 212
 213     [ -n "$(get_config_for_ip $_ip)" ]
 214 }
 215
 216 add_routing_for_ip ()
 217 {
 218     _iface="$1"
 219     _ip="$2"
 220
 221     # Do nothing if no config for this IP.
 222     ip_has_configuration "$_ip" || return 0
 223
 224     ensure_table_id_for_ip "$_ip" || \
 225         die "add_routing_for_ip: out of table ids in range $CTDB_PER_IP_ROUTING_TABLE_ID_LOW - $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH"
 226
 227     _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
 228     _table_id="${table_id_prefix}${_ip}"
 229
 230     del_routing_for_ip "$_ip" >/dev/null 2>&1
 231
 232     ip rule add from "$_ip" pref "$_pref" table "$_table_id" || \
 233         die "add_routing_for_ip: failed to add rule for $_ip"
 234
 235     # Add routes to table for any lines matching the IP.
 236     get_config_for_ip "$_ip" |
 237     while read _i _dest _gw ; do
 238         _r="$_dest ${_gw:+via} $_gw dev $_iface table $_table_id"
 239         ip route add $_r || \
 240             die "add_routing_for_ip: failed to add route: $_r"
 241     done
 242 }
 243
 244 del_routing_for_ip ()
 245 {
 246     _ip="$1"
 247
 248     _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
 249     _table_id="${table_id_prefix}${_ip}"
 250
 251     # Do this unconditionally since we own any matching table ids.
 252     # However, print a meaningful message if something goes wrong.
 253     _cmd="ip rule del from $_ip pref $_pref table $_table_id"
 254     _out=$($_cmd 2>&1) || \
 255         cat <<EOF
 256 WARNING: Failed to delete policy routing rule
 257   Command "$_cmd" failed:
 258   $_out
 259 EOF
 260     # This should never usually fail, so don't redirect output.
 261     # However, it can fail when deleting a rogue IP, since there will
 262     # be no routes for that IP.  In this case it should only fail when
 263     # the rule deletion above has already failed because the table id
 264     # is invalid.  Therefore, go to a little bit of trouble to indent
 265     # the failure message so that it is associated with the above
 266     # warning message and doesn't look too nasty.
 267     ip route flush table $_table_id 2>&1 | sed -e 's@^.@  &@'
 268 }
 269
 270 ######################################################################
 271
 272 flush_rules_and_routes ()
 273 {
 274         ip rule show |
 275         while read _p _x _i _x _t ; do
 276             # Remove trailing colon after priority/preference.
 277             _p="${_p%:}"
 278             # Only remove rules that match our priority/preference.
 279             [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
 280
 281             echo "Removing ip rule for public address $_i for routing table $_t"
 282             ip rule del from "$_i" table "$_t" pref "$_p"
 283             ip route flush table "$_t" 2>/dev/null
 284         done
 285 }
 286
 287 # Add any missing routes.  Some might have gone missing if, for
 288 # example, all IPs on the network were removed (possibly if the
 289 # primary was removed).  If $1 is "force" then (re-)add all the
 290 # routes.
 291 add_missing_routes ()
 292 {
 293     ctdb ip -v -X | {
 294         read _x # skip header line
 295
 296         # Read the rest of the lines.  We're only interested in the
 297         # "IP" and "ActiveInterface" columns.  The latter is only set
 298         # for addresses local to this node, making it easy to skip
 299         # non-local addresses.  For each IP local address we check if
 300         # the relevant routing table is populated and populate it if
 301         # not.
 302         while IFS="|" read _x _ip _x _iface _x ; do
 303             [ -n "$_iface" ] || continue
 304
 305             _table_id="${table_id_prefix}${_ip}"
 306             if [ -z "$(ip route show table $_table_id 2>/dev/null)" -o \
 307                 "$1" = "force" ]  ; then
 308                 add_routing_for_ip "$_iface" "$_ip"
 309             fi
 310         done
 311     } || exit $?
 312 }
 313
 314 # Remove rules/routes for addresses that we're not hosting.  If a
 315 # releaseip event failed in an earlier script then we might not have
 316 # had a chance to remove the corresponding rules/routes.
 317 remove_bogus_routes ()
 318 {
 319     # Get a IPs current hosted by this node, each anchored with '@'.
 320     _ips=$(ctdb ip -v -X | awk -F'|' 'NR > 1 && $4 != "" {printf "@%s@\n", $2}')
 321
 322     ip rule show |
 323     while read _p _x _i _x _t ; do
 324         # Remove trailing colon after priority/preference.
 325         _p="${_p%:}"
 326         # Only remove rules that match our priority/preference.
 327         [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
 328         # Only remove rules for which we don't have an IP.  This could
 329         # be done with grep, but let's do it with shell prefix removal
 330         # to avoid unnecessary processes.  This falls through if
 331         # "@${_i}@" isn't present in $_ips.
 332         [ "$_ips" = "${_ips#*@${_i}@}" ] || continue
 333
 334         echo "Removing ip rule/routes for unhosted public address $_i"
 335         del_routing_for_ip "$_i"
 336     done
 337 }
 338
 339 ######################################################################
 340
 341 service_reconfigure ()
 342 {
 343     add_missing_routes "force"
 344     remove_bogus_routes
 345
 346     # flush our route cache
 347     set_proc sys/net/ipv4/route/flush 1
 348 }
 349
 350 ######################################################################
 351
 352 ctdb_check_args "$@"
 353
 354 ctdb_service_check_reconfigure
 355
 356 case "$1" in
 357     startup)
 358         flush_rules_and_routes
 359
 360         # make sure that we only respond to ARP messages from the NIC
 361         # where a particular ip address is associated.
 362         get_proc sys/net/ipv4/conf/all/arp_filter >/dev/null 2>&1 && {
 363             set_proc sys/net/ipv4/conf/all/arp_filter 1
 364         }
 365         ;;
 366
 367     shutdown)
 368         flush_rules_and_routes
 369         clean_up_table_ids
 370         ;;
 371
 372     takeip)
 373         iface=$2
 374         ip=$3
 375         maskbits=$4
 376
 377         ensure_ipv4_is_valid_addr "$1" "$ip"
 378         add_routing_for_ip "$iface" "$ip"
 379
 380         # flush our route cache
 381         set_proc sys/net/ipv4/route/flush 1
 382
 383         ctdb gratiousarp "$ip" "$iface"
 384         ;;
 385
 386     updateip)
 387         oiface=$2
 388         niface=$3
 389         ip=$4
 390         maskbits=$5
 391
 392         ensure_ipv4_is_valid_addr "$1" "$ip"
 393         add_routing_for_ip "$niface" "$ip"
 394
 395         # flush our route cache
 396         set_proc sys/net/ipv4/route/flush 1
 397
 398         ctdb gratiousarp "$ip" "$niface"
 399         tickle_tcp_connections "$ip"
 400         ;;
 401
 402     releaseip)
 403         iface=$2
 404         ip=$3
 405         maskbits=$4
 406
 407         ensure_ipv4_is_valid_addr "$1" "$ip"
 408         del_routing_for_ip "$ip"
 409         ;;
 410
 411     ipreallocated)
 412         add_missing_routes
 413         remove_bogus_routes
 414         ;;
 415
 416     *)
 417         ctdb_standard_event_handler "$@"
 418         ;;
 419 esac
 420
 421 exit 0