ctdb/config/events.d/13.per_ip_routing

   1 #!/bin/sh
   2
   3 [ -n "$CTDB_BASE" ] || \
   4     export CTDB_BASE=$(cd -P $(dirname "$0") ; dirname "$PWD")
   5
   6 . $CTDB_BASE/functions
   7 loadconfig
   8
   9 service_name=per_ip_routing
  10
  11 # Do nothing if unconfigured
  12 [ -n "$CTDB_PER_IP_ROUTING_CONF" ] || exit 0
  13
  14 table_id_prefix="ctdb."
  15
  16 [ -n "$CTDB_PER_IP_ROUTING_RULE_PREF" ] || \
  17     die "error: CTDB_PER_IP_ROUTING_RULE_PREF not configured"
  18
  19 [ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -lt "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] 2>/dev/null || \
  20     die "error: CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW] and/or CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] improperly configured"
  21
  22 have_link_local_config ()
  23 {
  24     [ "$CTDB_PER_IP_ROUTING_CONF" = "__auto_link_local__" ]
  25 }
  26
  27 if ! have_link_local_config && [ ! -r "$CTDB_PER_IP_ROUTING_CONF" ] ; then
  28     die "error: CTDB_PER_IP_ROUTING_CONF=$CTDB_PER_IP_ROUTING_CONF file not found"
  29 fi
  30
  31 ######################################################################
  32
  33 ipv4_is_valid_addr()
  34 {
  35     _ip="$1"
  36
  37     _count=0
  38     # Get the shell to break up the address into 1 word per octet
  39     for _o in $(export IFS="." ; echo $_ip) ; do
  40         # The 2>/dev/null stops output from failures where an "octet"
  41         # is not numeric.  The test will still fail.
  42         if ! [ 0 -le $_o -a $_o -le 255 ] 2>/dev/null ; then
  43             return 1
  44         fi
  45         _count=$(($_count + 1))
  46     done
  47
  48     # A valid IPv4 address has 4 octets
  49     [ $_count -eq 4 ]
  50 }
  51
  52 ensure_ipv4_is_valid_addr ()
  53 {
  54     _event="$1"
  55     _ip="$2"
  56
  57     ipv4_is_valid_addr "$_ip" || {
  58         echo "$0: $_event not an ipv4 address skipping IP:$_ip"
  59         exit 0
  60     }
  61 }
  62
  63 ipv4_host_addr_to_net ()
  64 {
  65     _host="$1"
  66     _maskbits="$2"
  67
  68     # Convert the host address to an unsigned long by splitting out
  69     # the octets and doing the math.
  70     _host_ul=0
  71     for _o in $(export IFS="." ; echo $_host) ; do
  72         _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
  73     done
  74
  75     # Calculate the mask and apply it.
  76     _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
  77     _net_ul=$(( $_host_ul & $_mask_ul ))
  78
  79     # Now convert to a network address one byte at a time.
  80     _net=""
  81     for _o in $(seq 1 4) ; do
  82         _net="$(($_net_ul & 255))${_net:+.}${_net}"
  83         _net_ul=$(($_net_ul >> 8))
  84     done
  85
  86     echo "${_net}/${_maskbits}"
  87 }
  88
  89 ######################################################################
  90
  91 # Setup a table id to use for the given IP.  We don't need to know it,
  92 # it just needs to exist in /etc/iproute2/rt_tables.  Fail if no free
  93 # table id could be found in the configured range.
  94 ensure_table_id_for_ip ()
  95 {
  96     _ip=$1
  97
  98     _f="$CTDB_ETCDIR/iproute2/rt_tables"
  99     # This file should always exist, but...
 100     if [ ! -f "$_f" ] ; then
 101         mkdir -p $(dirname "$_f")
 102         touch "$_f"
 103     fi
 104
 105     # Maintain a table id for each IP address we've ever seen in
 106     # rt_tables.  We use a "ctdb." prefix on the label.
 107     _label="${table_id_prefix}${_ip}"
 108
 109     # This finds either the table id corresponding to the label or a
 110     # new unused one (that is greater than all the used ones in the
 111     # range).
 112     (
 113         # Note that die() just gets us out of the subshell...
 114         flock --timeout 30 0 || \
 115             die "ensure_table_id_for_ip: failed to lock file $_f"
 116
 117         _new=$CTDB_PER_IP_ROUTING_TABLE_ID_LOW
 118         while read _t _l ; do
 119             # Skip comments
 120             case "$_t" in
 121                 \#*) continue ;;
 122             esac
 123             # Found existing: done
 124             if [ "$_l" = "$_label" ] ; then
 125                 return 0
 126             fi
 127             # Potentially update the new table id to be used.  The
 128             # redirect stops error spam for a non-numeric value.
 129             if [ $_new -le $_t -a \
 130                 $_t -le $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH ] 2>/dev/null ; then
 131                 _new=$(($_t + 1))
 132             fi
 133         done
 134
 135         # If the new table id is legal then add it to the file and
 136         # print it.
 137         if [ $_new -le $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH ] ; then
 138             printf "%d\t%s\n" "$_new" "$_label" >>"$_f"
 139             return 0
 140         else
 141             return 1
 142         fi
 143     ) <"$_f"
 144 }
 145
 146 # Clean up all the table ids that we might own.
 147 clean_up_table_ids ()
 148 {
 149     _f="$CTDB_ETCDIR/iproute2/rt_tables"
 150     # Even if this didn't exist on the system, adding a route will
 151     # have created it.  What if we startup and immediately shutdown?
 152     if [ ! -f "$_f" ] ; then
 153         mkdir -p $(dirname "$_f")
 154         touch "$_f"
 155     fi
 156
 157     (
 158         # Note that die() just gets us out of the subshell...
 159         flock --timeout 30 0 || \
 160             die "clean_up_table_ids: failed to lock file $_f"
 161
 162         # Delete any items from the file that have a table id in our
 163         # range or a label matching our label.  Preserve comments.
 164         _tmp="${_f}.$$.ctdb"
 165         awk -v min="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" \
 166             -v max="$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" \
 167             -v pre="$table_id_prefix" \
 168             '/^#/ || \
 169              !(min <= $1 && $1 <= max) && \
 170              !(index($2, pre) == 1) \
 171              { print $0 }' "$_f" >"$_tmp"
 172
 173         mv "$_tmp" "$_f"
 174         # The lock is gone - don't do anything else here
 175     ) <"$_f"
 176 }
 177
 178 ######################################################################
 179
 180 # This prints the config for an IP, which is either relevant entries
 181 # from the config file or, if set to the magic link local value, some
 182 # link local routing config for the IP.
 183 get_config_for_ip ()
 184 {
 185     _ip="$1"
 186
 187     if have_link_local_config ; then
 188         # When parsing public_addresses also split on '/'.  This means
 189         # that we get the maskbits as item #2 without further parsing.
 190         while IFS="/$IFS" read _i _maskbits _x ; do
 191             if [ "$_ip" = "$_i" ] ; then
 192                 echo -n "$_ip "; ipv4_host_addr_to_net "$_ip" "$_maskbits"
 193             fi
 194         done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 195     else
 196         while read _i _rest ; do
 197             if [ "$_ip" = "$_i" ] ; then
 198                 printf "%s\t%s\n" "$_ip" "$_rest"
 199             fi
 200         done <"$CTDB_PER_IP_ROUTING_CONF"
 201     fi
 202 }
 203
 204 ip_has_configuration ()
 205 {
 206     _ip="$1"
 207
 208     [ -n "$(get_config_for_ip $_ip)" ]
 209 }
 210
 211 add_routing_for_ip ()
 212 {
 213     _iface="$1"
 214     _ip="$2"
 215
 216     # Do nothing if no config for this IP.
 217     ip_has_configuration "$_ip" || return 0
 218
 219     ensure_table_id_for_ip "$_ip" || \
 220         die "add_routing_for_ip: out of table ids in range $CTDB_PER_IP_ROUTING_TABLE_ID_LOW - $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH"
 221
 222     _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
 223     _table_id="${table_id_prefix}${_ip}"
 224
 225     del_routing_for_ip "$_ip" >/dev/null 2>&1
 226
 227     ip rule add from "$_ip" pref "$_pref" table "$_table_id" || \
 228         die "add_routing_for_ip: failed to add rule for $_ip"
 229
 230     # Add routes to table for any lines matching the IP.
 231     get_config_for_ip "$_ip" |
 232     while read _i _dest _gw ; do
 233         _r="$_dest ${_gw:+via} $_gw dev $_iface table $_table_id"
 234         ip route add $_r || \
 235             die "add_routing_for_ip: failed to add route: $_r"
 236     done
 237 }
 238
 239 del_routing_for_ip ()
 240 {
 241     _ip="$1"
 242
 243     _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
 244     _table_id="${table_id_prefix}${_ip}"
 245
 246     # Do this unconditionally since we own any matching table ids.
 247     # However, print a meaningful message if something goes wrong.
 248     _cmd="ip rule del from $_ip pref $_pref table $_table_id"
 249     _out=$($_cmd 2>&1) || \
 250         cat <<EOF
 251 WARNING: Failed to delete policy routing rule
 252   Command "$_cmd" failed:
 253   $_out
 254 EOF
 255     # This should never usually fail, so don't redirect output.
 256     # However, it can fail when deleting a rogue IP, since there will
 257     # be no routes for that IP.  In this case it should only fail when
 258     # the rule deletion above has already failed because the table id
 259     # is invalid.  Therefore, go to a little bit of trouble to indent
 260     # the failure message so that it is associated with the above
 261     # warning message and doesn't look too nasty.
 262     ip route flush table $_table_id 2>&1 | sed -e 's@^.@  &@'
 263 }
 264
 265 ######################################################################
 266
 267 flush_rules_and_routes ()
 268 {
 269         ip rule show |
 270         while read _p _x _i _x _t ; do
 271             # Remove trailing colon after priority/preference.
 272             _p="${_p%:}"
 273             # Only remove rules that match our priority/preference.
 274             [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
 275
 276             echo "Removing ip rule for public address $_i for routing table $_t"
 277             ip rule del from "$_i" table "$_t" pref "$_p"
 278             ip route flush table "$_t" 2>/dev/null
 279         done
 280 }
 281
 282 # Add any missing routes.  Some might have gone missing if, for
 283 # example, all IPs on the network were removed (possibly if the
 284 # primary was removed).  If $1 is "force" then (re-)add all the
 285 # routes.
 286 add_missing_routes ()
 287 {
 288     ctdb ip -v -Y | {
 289         read _x # skip header line
 290
 291         # Read the rest of the lines.  We're only interested in the
 292         # "IP" and "ActiveInterface" columns.  The latter is only set
 293         # for addresses local to this node, making it easy to skip
 294         # non-local addresses.  For each IP local address we check if
 295         # the relevant routing table is populated and populate it if
 296         # not.
 297         while IFS=":" read _x _ip _x _iface _x ; do
 298             [ -n "$_iface" ] || continue
 299
 300             _table_id="${table_id_prefix}${_ip}"
 301             if [ -z "$(ip route show table $_table_id 2>/dev/null)" -o \
 302                 "$1" = "force" ]  ; then
 303                 add_routing_for_ip "$_iface" "$_ip"
 304             fi
 305         done
 306     } || exit $?
 307 }
 308
 309 # Remove rules/routes for addresses that we're not hosting.  If a
 310 # releaseip event failed in an earlier script then we might not have
 311 # had a chance to remove the corresponding rules/routes.
 312 remove_bogus_routes ()
 313 {
 314     # Get a IPs current hosted by this node, each anchored with '@'.
 315     _ips=$(ctdb ip -v -Y | awk -F: 'NR > 1 && $4 != "" {printf "@%s@\n", $2}')
 316
 317     ip rule show |
 318     while read _p _x _i _x _t ; do
 319         # Remove trailing colon after priority/preference.
 320         _p="${_p%:}"
 321         # Only remove rules that match our priority/preference.
 322         [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
 323         # Only remove rules for which we don't have an IP.  This could
 324         # be done with grep, but let's do it with shell prefix removal
 325         # to avoid unnecessary processes.  This falls through if
 326         # "@${_i}@" isn't present in $_ips.
 327         [ "$_ips" = "${_ips#*@${_i}@}" ] || continue
 328
 329         echo "Removing ip rule/routes for unhosted public address $_i"
 330         del_routing_for_ip "$_i"
 331     done
 332 }
 333
 334 ######################################################################
 335
 336 service_reconfigure ()
 337 {
 338     add_missing_routes "force"
 339     remove_bogus_routes
 340
 341     # flush our route cache
 342     set_proc sys/net/ipv4/route/flush 1
 343 }
 344
 345 ######################################################################
 346
 347 ctdb_check_args "$@"
 348
 349 ctdb_service_check_reconfigure
 350
 351 case "$1" in
 352     startup)
 353         flush_rules_and_routes
 354
 355         # make sure that we only respond to ARP messages from the NIC
 356         # where a particular ip address is associated.
 357         get_proc sys/net/ipv4/conf/all/arp_filter >/dev/null 2>&1 && {
 358             set_proc sys/net/ipv4/conf/all/arp_filter 1
 359         }
 360         ;;
 361
 362     shutdown)
 363         flush_rules_and_routes
 364         clean_up_table_ids
 365         ;;
 366
 367     takeip)
 368         iface=$2
 369         ip=$3
 370         maskbits=$4
 371
 372         ensure_ipv4_is_valid_addr "$1" "$ip"
 373         add_routing_for_ip "$iface" "$ip"
 374
 375         # flush our route cache
 376         set_proc sys/net/ipv4/route/flush 1
 377
 378         ctdb gratiousarp "$ip" "$iface"
 379         ;;
 380
 381     updateip)
 382         oiface=$2
 383         niface=$3
 384         ip=$4
 385         maskbits=$5
 386
 387         ensure_ipv4_is_valid_addr "$1" "$ip"
 388         add_routing_for_ip "$niface" "$ip"
 389
 390         # flush our route cache
 391         set_proc sys/net/ipv4/route/flush 1
 392
 393         ctdb gratiousarp "$ip" "$niface"
 394         tickle_tcp_connections "$ip"
 395         ;;
 396
 397     releaseip)
 398         iface=$2
 399         ip=$3
 400         maskbits=$4
 401
 402         ensure_ipv4_is_valid_addr "$1" "$ip"
 403         del_routing_for_ip "$ip"
 404         ;;
 405
 406     ipreallocated)
 407         add_missing_routes
 408         remove_bogus_routes
 409         ;;
 410
 411     *)
 412         ctdb_standard_event_handler "$@"
 413         ;;
 414 esac
 415
 416 exit 0