ctdb/config/events.d/13.per_ip_routing

   1 #!/bin/sh
   2
   3 [ -n "$CTDB_BASE" ] || \
   4     CTDB_BASE=$(d=$(dirname "$0") ; cd -P "$d" ; dirname "$PWD")
   5
   6 . "${CTDB_BASE}/functions"
   7
   8 loadconfig
   9
  10 # service_name is used by various functions
  11 # shellcheck disable=SC2034
  12 service_name=per_ip_routing
  13
  14 # Do nothing if unconfigured
  15 [ -n "$CTDB_PER_IP_ROUTING_CONF" ] || exit 0
  16
  17 table_id_prefix="ctdb."
  18
  19 [ -n "$CTDB_PER_IP_ROUTING_RULE_PREF" ] || \
  20     die "error: CTDB_PER_IP_ROUTING_RULE_PREF not configured"
  21
  22 [ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -lt "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] 2>/dev/null || \
  23     die "error: CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW] and/or CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] improperly configured"
  24
  25 if [ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -le 253 -a \
  26     255 -le "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] ; then
  27     die "error: range CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW]..CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] must not include 253-255"
  28 fi
  29
  30 have_link_local_config ()
  31 {
  32     [ "$CTDB_PER_IP_ROUTING_CONF" = "__auto_link_local__" ]
  33 }
  34
  35 if ! have_link_local_config && [ ! -r "$CTDB_PER_IP_ROUTING_CONF" ] ; then
  36     die "error: CTDB_PER_IP_ROUTING_CONF=$CTDB_PER_IP_ROUTING_CONF file not found"
  37 fi
  38
  39 service_state_dir=$(ctdb_setup_service_state_dir) || exit $?
  40
  41 ######################################################################
  42
  43 ipv4_is_valid_addr()
  44 {
  45     _ip="$1"
  46
  47     _count=0
  48     # Get the shell to break up the address into 1 word per octet
  49     # Intentional word splitting here
  50     # shellcheck disable=SC2086
  51     for _o in $(export IFS="." ; echo $_ip) ; do
  52         # The 2>/dev/null stops output from failures where an "octet"
  53         # is not numeric.  The test will still fail.
  54         if ! [ 0 -le $_o -a $_o -le 255 ] 2>/dev/null ; then
  55             return 1
  56         fi
  57         _count=$((_count + 1))
  58     done
  59
  60     # A valid IPv4 address has 4 octets
  61     [ $_count -eq 4 ]
  62 }
  63
  64 ensure_ipv4_is_valid_addr ()
  65 {
  66     _event="$1"
  67     _ip="$2"
  68
  69     ipv4_is_valid_addr "$_ip" || {
  70         echo "$0: $_event not an ipv4 address skipping IP:$_ip"
  71         exit 0
  72     }
  73 }
  74
  75 ipv4_host_addr_to_net ()
  76 {
  77     _host="$1"
  78     _maskbits="$2"
  79
  80     # Convert the host address to an unsigned long by splitting out
  81     # the octets and doing the math.
  82     _host_ul=0
  83     # Intentional word splitting here
  84     # shellcheck disable=SC2086
  85     for _o in $(export IFS="." ; echo $_host) ; do
  86         _host_ul=$(( (_host_ul << 8) + _o)) # work around Emacs color bug
  87     done
  88
  89     # Calculate the mask and apply it.
  90     _mask_ul=$(( 0xffffffff << (32 - _maskbits) ))
  91     _net_ul=$(( _host_ul & _mask_ul ))
  92
  93     # Now convert to a network address one byte at a time.
  94     _net=""
  95     for _o in $(seq 1 4) ; do
  96         _net="$((_net_ul & 255))${_net:+.}${_net}"
  97         _net_ul=$((_net_ul >> 8))
  98     done
  99
 100     echo "${_net}/${_maskbits}"
 101 }
 102
 103 ######################################################################
 104
 105 ensure_rt_tables ()
 106 {
 107     rt_tables="$CTDB_SYS_ETCDIR/iproute2/rt_tables"
 108     rt_tables_lock="${service_state_dir}/rt_tables_lock"
 109
 110     # This file should always exist.  Even if this didn't exist on the
 111     # system, adding a route will have created it.  What if we startup
 112     # and immediately shutdown?  Let's be sure.
 113     if [ ! -f "$rt_tables" ] ; then
 114         mkdir -p "${rt_tables%/*}" # dirname
 115         touch "$rt_tables"
 116     fi
 117 }
 118
 119 # Setup a table id to use for the given IP.  We don't need to know it,
 120 # it just needs to exist in /etc/iproute2/rt_tables.  Fail if no free
 121 # table id could be found in the configured range.
 122 ensure_table_id_for_ip ()
 123 {
 124     _ip=$1
 125
 126     ensure_rt_tables
 127
 128     # Maintain a table id for each IP address we've ever seen in
 129     # rt_tables.  We use a "ctdb." prefix on the label.
 130     _label="${table_id_prefix}${_ip}"
 131
 132     # This finds either the table id corresponding to the label or a
 133     # new unused one (that is greater than all the used ones in the
 134     # range).
 135     (
 136         # Note that die() just gets us out of the subshell...
 137         flock --timeout 30 9 || \
 138             die "ensure_table_id_for_ip: failed to lock file $rt_tables"
 139
 140         _new="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW"
 141         while read _t _l ; do
 142             # Skip comments
 143             case "$_t" in
 144                 \#*) continue ;;
 145             esac
 146             # Found existing: done
 147             if [ "$_l" = "$_label" ] ; then
 148                 return 0
 149             fi
 150             # Potentially update the new table id to be used.  The
 151             # redirect stops error spam for a non-numeric value.
 152             if [ "$_new" -le "$_t" -a \
 153                 "$_t" -le "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] 2>/dev/null ; then
 154                 _new=$((_t + 1))
 155             fi
 156         done <"$rt_tables"
 157
 158         # If the new table id is legal then add it to the file and
 159         # print it.
 160         if [ "$_new" -le "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] ; then
 161             printf "%d\t%s\n" "$_new" "$_label" >>"$rt_tables"
 162             return 0
 163         else
 164             return 1
 165         fi
 166     ) 9>"$rt_tables_lock"
 167 }
 168
 169 # Clean up all the table ids that we might own.
 170 clean_up_table_ids ()
 171 {
 172     ensure_rt_tables
 173
 174     (
 175         # Note that die() just gets us out of the subshell...
 176         flock --timeout 30 9 || \
 177             die "clean_up_table_ids: failed to lock file $rt_tables"
 178
 179         # Delete any items from the file that have a table id in our
 180         # range or a label matching our label.  Preserve comments.
 181         _tmp="${rt_tables}.$$.ctdb"
 182         awk -v min="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" \
 183             -v max="$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" \
 184             -v pre="$table_id_prefix" \
 185             '/^#/ ||
 186              !(min <= $1 && $1 <= max) &&
 187              !(index($2, pre) == 1) {
 188                 print $0 }' "$rt_tables" >"$_tmp"
 189
 190         mv "$_tmp" "$rt_tables"
 191     ) 9>"$rt_tables_lock"
 192 }
 193
 194 ######################################################################
 195
 196 # This prints the config for an IP, which is either relevant entries
 197 # from the config file or, if set to the magic link local value, some
 198 # link local routing config for the IP.
 199 get_config_for_ip ()
 200 {
 201     _ip="$1"
 202
 203     if have_link_local_config ; then
 204         # When parsing public_addresses also split on '/'.  This means
 205         # that we get the maskbits as item #2 without further parsing.
 206         while IFS="/$IFS" read _i _maskbits _x ; do
 207             if [ "$_ip" = "$_i" ] ; then
 208                 printf "%s" "$_ip "; ipv4_host_addr_to_net "$_ip" "$_maskbits"
 209             fi
 210         done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 211     else
 212         while read _i _rest ; do
 213             if [ "$_ip" = "$_i" ] ; then
 214                 printf "%s\t%s\n" "$_ip" "$_rest"
 215             fi
 216         done <"$CTDB_PER_IP_ROUTING_CONF"
 217     fi
 218 }
 219
 220 ip_has_configuration ()
 221 {
 222     _ip="$1"
 223
 224     _conf=$(get_config_for_ip "$_ip")
 225     [ -n "$_conf" ]
 226 }
 227
 228 add_routing_for_ip ()
 229 {
 230     _iface="$1"
 231     _ip="$2"
 232
 233     # Do nothing if no config for this IP.
 234     ip_has_configuration "$_ip" || return 0
 235
 236     ensure_table_id_for_ip "$_ip" || \
 237         die "add_routing_for_ip: out of table ids in range $CTDB_PER_IP_ROUTING_TABLE_ID_LOW - $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH"
 238
 239     _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
 240     _table_id="${table_id_prefix}${_ip}"
 241
 242     del_routing_for_ip "$_ip" >/dev/null 2>&1
 243
 244     ip rule add from "$_ip" pref "$_pref" table "$_table_id" || \
 245         die "add_routing_for_ip: failed to add rule for $_ip"
 246
 247     # Add routes to table for any lines matching the IP.
 248     get_config_for_ip "$_ip" |
 249     while read _i _dest _gw ; do
 250         _r="$_dest ${_gw:+via} $_gw dev $_iface table $_table_id"
 251         # Intentionally unquoted multi-word value here
 252         # shellcheck disable=SC2086
 253         ip route add $_r || \
 254             die "add_routing_for_ip: failed to add route: $_r"
 255     done
 256 }
 257
 258 del_routing_for_ip ()
 259 {
 260     _ip="$1"
 261
 262     _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
 263     _table_id="${table_id_prefix}${_ip}"
 264
 265     # Do this unconditionally since we own any matching table ids.
 266     # However, print a meaningful message if something goes wrong.
 267     _cmd="ip rule del from $_ip pref $_pref table $_table_id"
 268     _out=$($_cmd 2>&1) || \
 269         cat <<EOF
 270 WARNING: Failed to delete policy routing rule
 271   Command "$_cmd" failed:
 272   $_out
 273 EOF
 274     # This should never usually fail, so don't redirect output.
 275     # However, it can fail when deleting a rogue IP, since there will
 276     # be no routes for that IP.  In this case it should only fail when
 277     # the rule deletion above has already failed because the table id
 278     # is invalid.  Therefore, go to a little bit of trouble to indent
 279     # the failure message so that it is associated with the above
 280     # warning message and doesn't look too nasty.
 281     ip route flush table "$_table_id" 2>&1 | sed -e 's@^.@  &@'
 282 }
 283
 284 ######################################################################
 285
 286 flush_rules_and_routes ()
 287 {
 288         ip rule show |
 289         while read _p _x _i _x _t ; do
 290             # Remove trailing colon after priority/preference.
 291             _p="${_p%:}"
 292             # Only remove rules that match our priority/preference.
 293             [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
 294
 295             echo "Removing ip rule for public address $_i for routing table $_t"
 296             ip rule del from "$_i" table "$_t" pref "$_p"
 297             ip route flush table "$_t" 2>/dev/null
 298         done
 299 }
 300
 301 # Add any missing routes.  Some might have gone missing if, for
 302 # example, all IPs on the network were removed (possibly if the
 303 # primary was removed).  If $1 is "force" then (re-)add all the
 304 # routes.
 305 add_missing_routes ()
 306 {
 307     $CTDB ip -v -X | {
 308         read _x # skip header line
 309
 310         # Read the rest of the lines.  We're only interested in the
 311         # "IP" and "ActiveInterface" columns.  The latter is only set
 312         # for addresses local to this node, making it easy to skip
 313         # non-local addresses.  For each IP local address we check if
 314         # the relevant routing table is populated and populate it if
 315         # not.
 316         while IFS="|" read _x _ip _x _iface _x ; do
 317             [ -n "$_iface" ] || continue
 318
 319             _table_id="${table_id_prefix}${_ip}"
 320             if [ -z "$(ip route show table "$_table_id" 2>/dev/null)" -o \
 321                 "$1" = "force" ]  ; then
 322                 add_routing_for_ip "$_iface" "$_ip"
 323             fi
 324         done
 325     } || exit $?
 326 }
 327
 328 # Remove rules/routes for addresses that we're not hosting.  If a
 329 # releaseip event failed in an earlier script then we might not have
 330 # had a chance to remove the corresponding rules/routes.
 331 remove_bogus_routes ()
 332 {
 333     # Get a IPs current hosted by this node, each anchored with '@'.
 334     _ips=$($CTDB ip -v -X | awk -F'|' 'NR > 1 && $4 != "" {printf "@%s@\n", $2}')
 335
 336     # x is intentionally ignored
 337     # shellcheck disable=SC2034
 338     ip rule show |
 339     while read _p _x _i _x _t ; do
 340         # Remove trailing colon after priority/preference.
 341         _p="${_p%:}"
 342         # Only remove rules that match our priority/preference.
 343         [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
 344         # Only remove rules for which we don't have an IP.  This could
 345         # be done with grep, but let's do it with shell prefix removal
 346         # to avoid unnecessary processes.  This falls through if
 347         # "@${_i}@" isn't present in $_ips.
 348         [ "$_ips" = "${_ips#*@${_i}@}" ] || continue
 349
 350         echo "Removing ip rule/routes for unhosted public address $_i"
 351         del_routing_for_ip "$_i"
 352     done
 353 }
 354
 355 ######################################################################
 356
 357 service_reconfigure ()
 358 {
 359     add_missing_routes "force"
 360     remove_bogus_routes
 361
 362     # flush our route cache
 363     set_proc sys/net/ipv4/route/flush 1
 364 }
 365
 366 ######################################################################
 367
 368 ctdb_check_args "$@"
 369
 370 case "$1" in
 371 startup)
 372         flush_rules_and_routes
 373
 374         # make sure that we only respond to ARP messages from the NIC
 375         # where a particular ip address is associated.
 376         get_proc sys/net/ipv4/conf/all/arp_filter >/dev/null 2>&1 && {
 377             set_proc sys/net/ipv4/conf/all/arp_filter 1
 378         }
 379         ;;
 380
 381 shutdown)
 382         flush_rules_and_routes
 383         clean_up_table_ids
 384         ;;
 385
 386 takeip)
 387         iface=$2
 388         ip=$3
 389         # maskbits included here so argument order is obvious
 390         # shellcheck disable=SC2034
 391         maskbits=$4
 392
 393         ensure_ipv4_is_valid_addr "$1" "$ip"
 394         add_routing_for_ip "$iface" "$ip"
 395
 396         # flush our route cache
 397         set_proc sys/net/ipv4/route/flush 1
 398
 399         $CTDB gratarp "$ip" "$iface"
 400         ;;
 401
 402 updateip)
 403         # oiface, maskbits included here so argument order is obvious
 404         # shellcheck disable=SC2034
 405         oiface=$2
 406         niface=$3
 407         ip=$4
 408         # shellcheck disable=SC2034
 409         maskbits=$5
 410
 411         ensure_ipv4_is_valid_addr "$1" "$ip"
 412         add_routing_for_ip "$niface" "$ip"
 413
 414         # flush our route cache
 415         set_proc sys/net/ipv4/route/flush 1
 416
 417         $CTDB gratarp "$ip" "$niface"
 418         tickle_tcp_connections "$ip"
 419         ;;
 420
 421 releaseip)
 422         iface=$2
 423         ip=$3
 424         # maskbits included here so argument order is obvious
 425         # shellcheck disable=SC2034
 426         maskbits=$4
 427
 428         ensure_ipv4_is_valid_addr "$1" "$ip"
 429         del_routing_for_ip "$ip"
 430         ;;
 431
 432 ipreallocated)
 433         add_missing_routes
 434         remove_bogus_routes
 435         ;;
 436
 437 reconfigure)
 438         ctdb_service_reconfigure
 439         ;;
 440 esac
 441
 442 exit 0