ctdb-daemon: Schedule running of callback if there are no event scripts

[Samba.git] / ctdb / config / functions

# Hey Emacs, this is a -*- shell-script -*- !!!

# utility functions for ctdb event scripts

if [ -z "$CTDB_BASE" ] ; then
    echo 'CTDB_BASE unset in CTDB functions file'
    exit 1
fi
export CTDB_BASE

# CTDB_VARDIR is used elsewhere
# shellcheck disable=SC2034
CTDB_VARDIR="/usr/local/var/lib/ctdb"
ctdb_rundir="/usr/local/var/run/ctdb"

CTDB="${CTDB:-/usr/local/bin/ctdb}"

# Only (and always) override these variables in test code

if [ -z "$CTDB_SCRIPT_VARDIR" ] ; then
    CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/state"
fi

if [ -z "$CTDB_SYS_ETCDIR" ] ; then
    CTDB_SYS_ETCDIR="/etc"
fi

if [ -z "$CTDB_HELPER_BINDIR" ] ; then
    CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
fi

#######################################
# pull in a system config file, if any

rewrite_ctdb_options ()
{
    case "$CTDB_DBDIR" in
        tmpfs|tmpfs:*)
            _opts_defaults="mode=700"
            # Get any extra options specified after colon
            if [ "$CTDB_DBDIR" = "tmpfs" ] ; then
                _opts=""
            else
                _opts="${CTDB_DBDIR#tmpfs:}"
            fi
            # It is OK to repeat mount options - last value wins.
            # CTDB_DBDIR_TMPFS_OPTIONS is used by ctdbd_wrapper
            # shellcheck disable=SC2034
            CTDB_DBDIR_TMPFS_OPTIONS="${_opts_defaults}${_opts:+,}${_opts}"

            CTDB_DBDIR="${ctdb_rundir}/CTDB_DBDIR"
            ;;
        *)
            # shellcheck disable=SC2034
            CTDB_DBDIR_TMPFS_OPTIONS=""
    esac
}

_loadconfig() {

    if [ -z "$1" ] ; then
        foo="${service_config:-${service_name}}"
        if [ -n "$foo" ] ; then
            loadconfig "$foo"
            return
        fi
    fi

    if [ "$1" != "ctdb" ] ; then
        loadconfig "ctdb"
    fi

    if [ -z "$1" ] ; then
        return
    fi

    if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/$1" ]; then
        . "${CTDB_SYS_ETCDIR}/sysconfig/$1"
    elif [ -f "${CTDB_SYS_ETCDIR}/default/$1" ]; then
        . "${CTDB_SYS_ETCDIR}/default/$1"
    elif [ -f "${CTDB_BASE}/sysconfig/$1" ]; then
        . "${CTDB_BASE}/sysconfig/$1"
    fi

    if [ "$1" = "ctdb" ] ; then
        _config="${CTDBD_CONF:-${CTDB_BASE}/ctdbd.conf}"
        if [ -r "$_config" ] ; then
            . "$_config"
        fi
        rewrite_ctdb_options
    fi
}

loadconfig () {
    _loadconfig "$@"
}

##############################################################

# CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
# configuration file.
debug ()
{
    if [ "${CTDB_SCRIPT_DEBUGLEVEL:-2}" -ge 4 ] ; then
        # If there are arguments then echo them.  Otherwise expect to
        # use stdin, which allows us to pass lots of debug using a
        # here document.
        if [ -n "$1" ] ; then
            echo "DEBUG: $*"
        else
            sed -e 's@^@DEBUG: @'
        fi
    else
        if [ -z "$1" ] ; then
            cat >/dev/null
        fi
    fi
}

die ()
{
    _msg="$1"
    _rc="${2:-1}"

    echo "$_msg" >&2
    exit "$_rc"
}

# Log given message or stdin to either syslog or a CTDB log file
# $1 is the tag passed to logger if syslog is in use.
script_log ()
{
    _tag="$1" ; shift

    case "$CTDB_LOGGING" in
        file:*|"")
            if [ -n "$CTDB_LOGGING" ] ; then
                _file="${CTDB_LOGGING#file:}"
            else
                _file="/usr/local/var/log/log.ctdb"
            fi
            {
                if [ -n "$*" ] ; then
                    echo "$*"
                else
                    cat
                fi
            } >>"$_file"
            ;;
        *)
            # Handle all syslog:* variants here too.  There's no tool to do
            # the lossy things, so just use logger.
            logger -t "ctdbd: ${_tag}" "$*"
            ;;
    esac
}

# When things are run in the background in an eventscript then logging
# output might get lost.  This is the "solution".  :-)
background_with_logging ()
{
    (
        "$@" 2>&1 </dev/null |
        script_log "${script_name}&"
    )&

    return 0
}

##############################################################
# check number of args for different events
ctdb_check_args ()
{
    case "$1" in
        takeip|releaseip)
            if [ $# != 4 ]; then
                echo "ERROR: must supply interface, IP and maskbits"
                exit 1
            fi
            ;;
        updateip)
            if [ $# != 5 ]; then
                echo "ERROR: must supply old interface, new interface, IP and maskbits"
                exit 1
            fi
            ;;
    esac
}

##############################################################
# determine on what type of system (init style) we are running
detect_init_style()
{
    # only do detection if not already set:
    [ -z "$CTDB_INIT_STYLE" ] || return

    if [ -x /sbin/startproc ]; then
        CTDB_INIT_STYLE="suse"
    elif [ -x /sbin/start-stop-daemon ]; then
        CTDB_INIT_STYLE="debian"
    else
        CTDB_INIT_STYLE="redhat"
    fi
}

######################################################
# simulate /sbin/service on platforms that don't have it
# _service() makes it easier to hook the service() function for
# testing.
_service ()
{
  _service_name="$1"
  _op="$2"

  # do nothing, when no service was specified
  [ -z "$_service_name" ] && return

  if [ -x /sbin/service ]; then
      $_nice /sbin/service "$_service_name" "$_op"
  elif [ -x /usr/sbin/service ]; then
      $_nice /usr/sbin/service "$_service_name" "$_op"
  elif [ -x /bin/systemctl ]; then
      $_nice /bin/systemctl "$_op" "$_service_name"
  elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
      $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
  elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
      $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
  fi
}

service()
{
    _nice=""
    _service "$@"
}

######################################################
# simulate /sbin/service (niced) on platforms that don't have it
nice_service()
{
    _nice="nice"
    _service "$@"
}

######################################################
# Cached retrieval of PNN from local node.  This never changes so why
# open a client connection to the server each time this is needed?
ctdb_get_pnn ()
{
    _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
    if [ ! -f "$_pnn_file" ] ; then
        $CTDB pnn >"$_pnn_file"
    fi

    cat "$_pnn_file"
}

# Cached retrieval of private IP address from local node.  This never
# changes.
ctdb_get_ip_address ()
{
    _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
    if [ ! -f "$_ip_addr_file" ] ; then
        $CTDB -X nodestatus |
            awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
    fi

    # ip_address is used by caller
    # shellcheck disable=SC2034
    cat "$_ip_addr_file"
}

######################################################
# wrapper around /proc/ settings to allow them to be hooked
# for testing
# 1st arg is relative path under /proc/, 2nd arg is value to set
set_proc ()
{
    echo "$2" >"/proc/$1"
}

set_proc_maybe ()
{
    if [ -w "/proc/$1" ] ; then
        set_proc "$1" "$2"
    fi
}

######################################################
# wrapper around getting file contents from /proc/ to allow
# this to be hooked for testing
# 1st arg is relative path under /proc/
get_proc ()
{
    cat "/proc/$1"
}

######################################################
# Print up to $_max kernel stack traces for processes named $_program
program_stack_traces ()
{
    _prog="$1"
    _max="${2:-1}"

    _count=1
    for _pid in $(pidof "$_prog") ; do
        [ "$_count" -le "$_max" ] || break

        # Do this first to avoid racing with process exit
        _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
        if [ -n "$_stack" ] ; then
            echo "Stack trace for ${_prog}[${_pid}]:"
            echo "$_stack"
            _count=$((_count + 1))
        fi
    done
}

######################################################
# Ensure $service_name is set
assert_service_name ()
{
    [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
}

######################################################
# check a set of directories is available
# return 1 on a missing directory
# directories are read from stdin
######################################################
ctdb_check_directories_probe()
{
    while IFS="" read d ; do
        case "$d" in
            *%*)
                continue
                ;;
            *)
                [ -d "${d}/." ] || return 1
        esac
    done
}

######################################################
# check a set of directories is available
# directories are read from stdin
######################################################
ctdb_check_directories()
{
    ctdb_check_directories_probe || {
        echo "ERROR: $service_name directory \"$d\" not available"
        exit 1
    }
}

######################################################
# check a set of tcp ports
# usage: ctdb_check_tcp_ports <ports...>
######################################################

# This flag file is created when a service is initially started.  It
# is deleted the first time TCP port checks for that service succeed.
# Until then ctdb_check_tcp_ports() prints a more subtle "error"
# message if a port check fails.
_ctdb_check_tcp_common ()
{
    assert_service_name
    _d="${CTDB_SCRIPT_VARDIR}/failcount"
    _ctdb_service_started_file="${_d}/${service_name}.started"
}

ctdb_check_tcp_init ()
{
    _ctdb_check_tcp_common
    mkdir -p "${_ctdb_service_started_file%/*}" # dirname
    touch "$_ctdb_service_started_file"
}

# Check whether something is listening on all of the given TCP ports
# using the "ctdb checktcpport" command.
ctdb_check_tcp_ports()
{
    if [ -z "$1" ] ; then
        echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
        exit 1
    fi

    for _p ; do  # process each function argument (port)
        _cmd="$CTDB checktcpport $_p"
        _out=$($_cmd 2>&1)
        _ret=$?
        case "$_ret" in
            0)
                _ctdb_check_tcp_common
                if [ ! -f "$_ctdb_service_started_file" ] ; then
                    echo "ERROR: $service_name tcp port $_p is not responding"
                    debug "\"ctdb checktcpport $_p\" was able to bind to port"
                else
                    echo "INFO: $service_name tcp port $_p is not responding"
                fi

                return 1
                ;;
            98)
                # Couldn't bind, something already listening, next port...
                continue
                ;;
            *)
                echo "ERROR: unexpected error running \"ctdb checktcpport\""
                debug <<EOF
$CTDB checktcpport (exited with $_ret) with output:
$_out"
EOF
                return $_ret
        esac
    done

    # All ports listening
    _ctdb_check_tcp_common
    rm -f "$_ctdb_service_started_file"
    return 0
}

######################################################
# check a unix socket
# usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
######################################################
ctdb_check_unix_socket() {
    socket_path="$1"
    [ -z "$socket_path" ] && return

    if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
        echo "ERROR: $service_name socket $socket_path not found"
        return 1
    fi
}

######################################################
# check a command returns zero status
# usage: ctdb_check_command <command>
######################################################
ctdb_check_command ()
{
    _out=$("$@" 2>&1) || {
        echo "ERROR: $* returned error"
        echo "$_out" | debug
        exit 1
    }
}

################################################
# kill off any TCP connections with the given IP
################################################
kill_tcp_connections ()
{
    _iface="$1"
    _ip="$2"

    _oneway=false
    if [ "$3" = "oneway" ] ; then
        _oneway=true
    fi

    get_tcp_connections_for_ip "$_ip" | {
        _killcount=0
        _connections=""
        _nl="
"
        while read _dst _src; do
            _destport="${_dst##*:}"
            __oneway=$_oneway
            case $_destport in
                # we only do one-way killtcp for CIFS
                139|445) __oneway=true ;;
            esac

            echo "Killing TCP connection $_src $_dst"
            _connections="${_connections}${_nl}${_src} ${_dst}"
            if ! $__oneway ; then
                _connections="${_connections}${_nl}${_dst} ${_src}"
            fi

            _killcount=$((_killcount + 1))
        done

        if [ $_killcount -eq 0 ] ; then
            return
        fi

        echo "$_connections" | \
                "${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
                echo "Failed to kill TCP connections"
                return
        }

        _remaining=$(get_tcp_connections_for_ip "$_ip" | wc -l)

        if [ "$_remaining" -eq 0 ] ; then
                echo "Killed $_killcount TCP connections to released IP $_ip"
                return
        fi

        _t="${_remaining}/${_killcount}"
        echo "Failed to kill TCP connections for IP $_ip (${_t} remaining)"
    }
}

##################################################################
# kill off the local end for any TCP connections with the given IP
##################################################################
kill_tcp_connections_local_only ()
{
    kill_tcp_connections "$@" "oneway"
}

##################################################################
# tickle any TCP connections with the given IP
##################################################################
tickle_tcp_connections ()
{
    _ip="$1"

    get_tcp_connections_for_ip "$_ip" |
    {
        _failed=false

        while read dest src; do
            echo "Tickle TCP connection $src $dest"
            $CTDB tickle "$src" "$dest" >/dev/null 2>&1 || _failed=true
            echo "Tickle TCP connection $dest $src"
            $CTDB tickle "$dest" "$src" >/dev/null 2>&1 || _failed=true
        done

        if $_failed ; then
            echo "Failed to send tickle control"
        fi
    }
}

get_tcp_connections_for_ip ()
{
    _ip="$1"

    ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
}

########################################################

add_ip_to_iface ()
{
    _iface=$1
    _ip=$2
    _maskbits=$3

    # Ensure interface is up
    ip link set "$_iface" up || \
        die "Failed to bringup interface $_iface"

    # Only need to define broadcast for IPv4
    case "$_ip" in
        *:*) _bcast=""      ;;
        *)   _bcast="brd +" ;;
    esac

    # Intentionally unquoted multi-word value here
    # shellcheck disable=SC2086
    ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
        echo "Failed to add $_ip/$_maskbits on dev $_iface"
        return 1
    }

    # Wait 5 seconds for IPv6 addresses to stop being tentative...
    if [ -z "$_bcast" ] ; then
        for _x in $(seq 1 10) ; do
            ip addr show to "${_ip}/128" | grep -q "tentative" || break
            sleep 0.5
        done

        # If the address was a duplicate then it won't be on the
        # interface so flag an error.
        _t=$(ip addr show to "${_ip}/128")
        case "$_t" in
            "")
                echo "Failed to add $_ip/$_maskbits on dev $_iface"
                return 1
                ;;
            *tentative*|*dadfailed*)
                echo "Failed to add $_ip/$_maskbits on dev $_iface"
                ip addr del "$_ip/$_maskbits" dev "$_iface"
                return 1
                ;;
        esac
    fi
}

delete_ip_from_iface()
{
    _iface=$1
    _ip=$2
    _maskbits=$3

    # This could be set globally for all interfaces but it is probably
    # better to avoid surprises, so limit it the interfaces where CTDB
    # has public IP addresses.  There isn't anywhere else convenient
    # to do this so just set it each time.  This is much cheaper than
    # remembering and re-adding secondaries.
    set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1

    ip addr del "$_ip/$_maskbits" dev "$_iface" || {
        echo "Failed to del $_ip on dev $_iface"
        return 1
    }
}

# If the given IP is hosted then print 2 items: maskbits and iface
ip_maskbits_iface ()
{
    _addr="$1"

    case "$_addr" in
        *:*) _bits=128 ;;
        *)   _bits=32  ;;
    esac
    ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
        awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
                       sub("@.*", "", iface) }
             $1 ~ /inet/ { mask = $2; sub(".*/", "", mask);
                           print mask, iface }'
}

drop_ip ()
{
    _addr="${1%/*}"  # Remove optional maskbits

    # Intentional word splitting here
    # shellcheck disable=SC2046
    set -- $(ip_maskbits_iface "$_addr")
    if [ -n "$1" ] ; then
        _maskbits="$1"
        _iface="$2"
        echo "Removing public address $_addr/$_maskbits from device $_iface"
        delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
    fi
}

drop_all_public_ips ()
{
        # _x is intentionally ignored
        # shellcheck disable=SC2034
        while read _ip _x ; do
                drop_ip "$_ip"
        done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
}

flush_route_cache ()
{
    set_proc_maybe sys/net/ipv4/route/flush 1
    set_proc_maybe sys/net/ipv6/route/flush 1
}

########################################################
# Interface monitoring

# If the interface is a virtual one (e.g. VLAN) then get the
# underlying interface
interface_get_real ()
{
    # Output of "ip link show <iface>"
    _iface_info="$1"

    # Extract the full interface description to see if it is a VLAN
    _t=$(echo "$_iface_info" |
                awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
                               print iface }')
    case "$_t" in
        *@*)
            # VLAN: use the underlying interface, after the '@'
            echo "${_t##*@}"
            ;;
        *)
            # Not a regular VLAN.  For backward compatibility, assume
            # there is some other sort of VLAN that doesn't have the
            # '@' in the output and only use what is before a '.'.  If
            # there is no '.' then this will be the whole interface
            # name.
            echo "${_t%%.*}"
    esac
}

# Check whether an interface is operational
interface_monitor ()
{
    _iface="$1"

    _iface_info=$(ip link show "$_iface" 2>&1) || {
        echo "ERROR: Monitored interface ${_iface} does not exist"
        return 1
    }


    # If the interface is a virtual one (e.g. VLAN) then get the
    # underlying interface.
    _realiface=$(interface_get_real "$_iface_info")

    if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null) ; then
        # This is a bond: various monitoring strategies
        echo "$_bi" | grep -q 'Currently Active Slave: None' && {
            echo "ERROR: No active slaves for bond device ${_realiface}"
            return 1
        }
        echo "$_bi" | grep -q '^MII Status: up' || {
            echo "ERROR: public network interface ${_realiface} is down"
            return 1
        }
        echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
            # This works around a bug in the driver where the
            # overall bond status can be up but none of the actual
            # physical interfaces have a link.
            echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
                echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
                return 1
            }
        }

        return 0
    else
        # Not a bond
        case "$_iface" in
            lo*)
                # loopback is always working
                return 0
                ;;
            ib*)
                # we don't know how to test ib links
                return 0
                ;;
            *)
                ethtool "$_iface" | grep -q 'Link detected: yes' || {
                    # On some systems, this is not successful when a
                    # cable is plugged but the interface has not been
                    # brought up previously. Bring the interface up
                    # and try again...
                    ip link set "$_iface" up
                    ethtool "$_iface" | grep -q 'Link detected: yes' || {
                        echo "ERROR: No link on the public network interface ${_iface}"
                        return 1
                    }
                }
                return 0
                ;;
        esac
    fi
}

########################################################
# Simple counters
_ctdb_counter_common () {
    _service_name="${1:-${service_name:-${script_name}}}"
    _counter_file="${CTDB_SCRIPT_VARDIR}/failcount/${_service_name}"
    mkdir -p "${_counter_file%/*}" # dirname
}
# Some code passes an argument
# shellcheck disable=SC2120
ctdb_counter_init () {
    _ctdb_counter_common "$1"

    >"$_counter_file"
}
ctdb_counter_incr () {
    _ctdb_counter_common "$1"

    # unary counting using newlines!
    echo >>"$_counter_file"
}
ctdb_counter_get () {
    _ctdb_counter_common "$1"
    # unary counting!
    stat -c "%s" "$_counter_file" 2>/dev/null || echo 0
}

########################################################

ctdb_setup_service_state_dir ()
{
        _s="${1:-${service_name}}"

        _service_state_dir="${CTDB_SCRIPT_VARDIR}/service_state/${_s}"
        mkdir -p "$_service_state_dir" ||
                die "Error creating state dir \"${_service_state_dir}\""

        echo "$_service_state_dir"
}

########################################################
# Managed status history, for auto-start/stop

_ctdb_managed_common ()
{
    _ctdb_managed_file="${CTDB_SCRIPT_VARDIR}/managed_history/${service_name}"
}

ctdb_service_managed ()
{
    _ctdb_managed_common
    mkdir -p "${_ctdb_managed_file%/*}" # dirname
    touch "$_ctdb_managed_file"
}

ctdb_service_unmanaged ()
{
    _ctdb_managed_common
    rm -f "$_ctdb_managed_file"
}

is_ctdb_previously_managed_service ()
{
    _ctdb_managed_common
    [ -f "$_ctdb_managed_file" ]
}

##################################################################
# Reconfigure a service on demand

_ctdb_service_reconfigure_common ()
{
    _d="${CTDB_SCRIPT_VARDIR}/service_status/${service_name}"
    mkdir -p "$_d"
    _ctdb_service_reconfigure_flag="$_d/reconfigure"
}

ctdb_service_needs_reconfigure ()
{
    _ctdb_service_reconfigure_common
    [ -e "$_ctdb_service_reconfigure_flag" ]
}

ctdb_service_set_reconfigure ()
{
    _ctdb_service_reconfigure_common
    >"$_ctdb_service_reconfigure_flag"
}

ctdb_service_unset_reconfigure ()
{
    _ctdb_service_reconfigure_common
    rm -f "$_ctdb_service_reconfigure_flag"
}

ctdb_service_reconfigure ()
{
    echo "Reconfiguring service \"${service_name}\"..."
    ctdb_service_unset_reconfigure
    service_reconfigure || return $?
    # Intentionally have this use $service_name as default
    # shellcheck disable=SC2119
    ctdb_counter_init
}

# Default service_reconfigure() function does nothing.
service_reconfigure ()
{
    :
}

ctdb_reconfigure_take_lock ()
{
        _ctdb_service_reconfigure_common
        _lock="${_d}/reconfigure_lock"
        mkdir -p "${_lock%/*}" # dirname
        touch "$_lock"

        (
                flock 9
                # This is overkill but will work if we need to extend
                # this to allow certain events to run multiple times
                # in parallel (e.g. takeip) and write multiple PIDs to
                # the file.
                {
                        read _locker_event
                        if [ -n "$_locker_event" ] ; then
                                while read _pid ; do
                                        if [ -n "$_pid" -a "$_pid" != $$ ] && \
                                           kill -0 "$_pid" 2>/dev/null ; then
                                                exit 1
                                        fi
                                done
                        fi
                } <"$_lock"

                printf "%s\n%s\n" "$event_name" $$ >"$_lock"
                exit 0
    ) 9>"${_lock}.flock"
}

ctdb_reconfigure_release_lock ()
{
    _ctdb_service_reconfigure_common
    _lock="${_d}/reconfigure_lock"

    rm -f "$_lock"
}

ctdb_replay_monitor_status ()
{
    echo "Replaying previous status for this script due to reconfigure..."
    # Leading separator ('|') is missing in some versions...
    _out=$($CTDB scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
    # Output looks like this:
    # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
    # This is the cheapest way of getting fields in the middle.
    # Intentional word splitting here
    # shellcheck disable=SC2046,2086
    set -- $(IFS="|" ; echo $_out)
    _code="$3"
    _status="$4"
    # The error output field can include colons so we'll try to
    # preserve them.  The weak checking at the beginning tries to make
    # this work for both broken (no leading '|') and fixed output.
    _out="${_out%|}"
    _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
    case "$_status" in
        OK) : ;;  # Do nothing special.
        TIMEDOUT)
            # Recast this as an error, since we can't exit with the
            # correct negative number.
            _code=1
            _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
            ;;
        DISABLED)
            # Recast this as an OK, since we can't exit with the
            # correct negative number.
            _code=0
            _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
            ;;
        *) : ;;  # Must be ERROR, do nothing special.
    esac
    if [ -n "$_err_out" ] ; then
        echo "$_err_out"
    fi
    exit $_code
}

ctdb_service_check_reconfigure ()
{
    assert_service_name

    # We only care about some events in this function.  For others we
    # return now.
    case "$event_name" in
        monitor|ipreallocated|reconfigure) : ;;
        *) return 0 ;;
    esac

    if ctdb_reconfigure_take_lock ; then
        # No events covered by this function are running, so proceed
        # with gay abandon.
        case "$event_name" in
            reconfigure)
                (ctdb_service_reconfigure)
                exit $?
                ;;
            ipreallocated)
                if ctdb_service_needs_reconfigure ; then
                    ctdb_service_reconfigure
                fi
                ;;
        esac

        ctdb_reconfigure_release_lock
    else
        # Somebody else is running an event we don't want to collide
        # with.  We proceed with caution.
        case "$event_name" in
            reconfigure)
                # Tell whoever called us to retry.
                exit 2
                ;;
            ipreallocated)
                # Defer any scheduled reconfigure and just run the
                # rest of the ipreallocated event, as per the
                # eventscript.  There's an assumption here that the
                # event doesn't depend on any scheduled reconfigure.
                # This is true in the current code.
                return 0
                ;;
            monitor)
                # There is most likely a reconfigure in progress so
                # the service is possibly unstable.  As above, we
                # defer any scheduled reconfigured.  We also replay
                # the previous monitor status since that's the best
                # information we have.
                ctdb_replay_monitor_status
                ;;
        esac
    fi
}

##################################################################
# Does CTDB manage this service? - and associated auto-start/stop

ctdb_compat_managed_service ()
{
    if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
        CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
    fi
}

is_ctdb_managed_service ()
{
    assert_service_name

    # $t is used just for readability and to allow better accurate
    # matching via leading/trailing spaces
    t=" $CTDB_MANAGED_SERVICES "

    # Return 0 if "<space>$service_name<space>" appears in $t
    if [ "${t#* ${service_name} }" != "${t}" ] ; then
        return 0
    fi

    # If above didn't match then update $CTDB_MANAGED_SERVICES for
    # backward compatibility and try again.
    ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
    ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
    ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
    ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
    ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
    ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
    ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
    ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"

    t=" $CTDB_MANAGED_SERVICES "

    # Return 0 if "<space>$service_name<space>" appears in $t
    [ "${t#* ${service_name} }" != "${t}" ]
}

ctdb_start_stop_service ()
{
    assert_service_name

    # Allow service-start/service-stop pseudo-events to start/stop
    # services when we're not auto-starting/stopping and we're not
    # monitoring.
    case "$event_name" in
        service-start)
            if is_ctdb_managed_service ; then
                die 'service-start event not permitted when service is managed'
            fi
            if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
                die 'service-start event not permitted with CTDB_SERVICE_AUTOSTARTSTOP=yes'
            fi
            ctdb_service_start
            exit $?
            ;;
        service-stop)
            if is_ctdb_managed_service ; then
                die 'service-stop event not permitted when service is managed'
            fi
            if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
                die 'service-stop event not permitted with CTDB_SERVICE_AUTOSTARTSTOP=yes'
            fi
            ctdb_service_stop
            exit $?
            ;;
    esac

    # Do nothing unless configured to...
    [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0

    [ "$event_name" = "monitor" ] || return 0

    if is_ctdb_managed_service ; then
        if ! is_ctdb_previously_managed_service ; then
            echo "Starting service \"$service_name\" - now managed"
            background_with_logging ctdb_service_start
            exit $?
        fi
    else
        if is_ctdb_previously_managed_service ; then
            echo "Stopping service \"$service_name\" - no longer managed"
            background_with_logging ctdb_service_stop
            exit $?
        fi
    fi
}

ctdb_service_start ()
{
    # The service is marked managed if we've ever tried to start it.
    ctdb_service_managed

    service_start || return $?

    # Intentionally have this use $service_name as default
    # shellcheck disable=SC2119
    ctdb_counter_init
    ctdb_check_tcp_init
}

ctdb_service_stop ()
{
    ctdb_service_unmanaged
    service_stop
}

# Default service_start() and service_stop() functions.
 
# These may be overridden in an eventscript.
service_start ()
{
    service "$service_name" start
}

service_stop ()
{
    service "$service_name" stop
}

##################################################################

# This exists only for backward compatibility with 3rd party scripts
# that call it
ctdb_standard_event_handler ()
{
    :
}

iptables_wrapper ()
{
    _family="$1" ; shift
    if [ "$_family" = "inet6" ] ; then
        _iptables_cmd="ip6tables"
    else
        _iptables_cmd="iptables"
    fi

    # iptables doesn't like being re-entered, so flock-wrap it.
    flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
}

# AIX (and perhaps others?) doesn't have mktemp
# type is commonly supported and more portable than which(1)
# shellcheck disable=SC2039
if ! type mktemp >/dev/null 2>&1 ; then
    mktemp ()
    {
        _dir=false
        if [ "$1" = "-d" ] ; then
            _dir=true
            shift
        fi
        _d="${TMPDIR:-/tmp}"
        _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
            md5sum | \
            sed -e 's@\(..........\).*@\1@')
        _t="${_d}/tmp.${_hex10}"
        (
            umask 077
            if $_dir ; then
                mkdir "$_t"
            else
                >"$_t"
            fi
        )
        echo "$_t"
    }
fi

######################################################################
# NFS callout handling

nfs_callout_init ()
{
        _state_dir="$1"

        if [ -z "$CTDB_NFS_CALLOUT" ] ; then
                CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
        fi
        # Always export, for statd callout
        export CTDB_NFS_CALLOUT

        # If the callout wants to use this then it must create it
        export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"

        # Export, if set, for use by clustered NFS callouts
        if [ -n "$CTDB_NFS_STATE_FS_TYPE" ] ; then
                export CTDB_NFS_STATE_FS_TYPE
        fi
        if [ -n "$CTDB_NFS_STATE_MNT" ] ; then
                export CTDB_NFS_STATE_MNT
        fi

        nfs_callout_cache="${_state_dir}/nfs_callout_cache"
        nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
        nfs_callout_cache_ops="${nfs_callout_cache}/ops"
}

nfs_callout_register ()
{
    mkdir -p "$nfs_callout_cache_ops"
    rm -f "$nfs_callout_cache_ops"/*

    echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"

    _t=$(eval "$CTDB_NFS_CALLOUT" "register")
    if [ -n "$_t" ] ; then
        echo "$_t" |
            while IFS="" read _op ; do
                touch "${nfs_callout_cache_ops}/${_op}"
            done
    else
        touch "${nfs_callout_cache_ops}/ALL"
    fi
}

nfs_callout ()
{
    # Re-run registration if $CTDB_NFS_CALLOUT has changed
    _prev=""
    if [ -r "$nfs_callout_cache_callout" ] ; then
        read _prev <"$nfs_callout_cache_callout"
    fi
    if [ "$CTDB_NFS_CALLOUT" != "$_prev" ] ; then
        nfs_callout_register
    fi

    # Run the operation if it is registered...
    if [ -e "${nfs_callout_cache_ops}/${1}" ] || \
           [ -e "${nfs_callout_cache_ops}/ALL" ]; then
        eval "$CTDB_NFS_CALLOUT" "$@"
    fi
}

########################################################
# tickle handling
########################################################

update_tickles ()
{
        _port="$1"

        tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
        mkdir -p "$tickledir"

        # What public IPs do I hold?
        _pnn=$(ctdb_get_pnn)
        _ips=$($CTDB -X ip | awk -F'|' -v pnn="$_pnn" '$3 == pnn {print $2}')

        # IPs and port as ss filters
        _ip_filter=""
        for _ip in $_ips ; do
            _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
        done
        _port_filter="sport == :${_port}"

        # Record connections to our public IPs in a temporary file.
        # This temporary file is in CTDB's private state directory and
        # $$ is used to avoid a very rare race involving CTDB's script
        # debugging.  No security issue, nothing to see here...
        _my_connections="${tickledir}/${_port}.connections.$$"
        # Parentheses are needed around the filters for precedence but
        # the parentheses can't be empty!
        ss -tn state established \
           "${_ip_filter:+( ${_ip_filter} )}" \
           "${_port_filter:+( ${_port_filter} )}" |
        awk 'NR > 1 {print $4, $3}' |
        sort >"$_my_connections"

        # Record our current tickles in a temporary file
        _my_tickles="${tickledir}/${_port}.tickles.$$"
        for _i in $_ips ; do
                $CTDB -X gettickles "$_i" "$_port" |
                awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
        done |
        sort >"$_my_tickles"

        # Add tickles for connections that we haven't already got tickles for
        comm -23 "$_my_connections" "$_my_tickles" |
        while read _src _dst ; do
                $CTDB addtickle "$_src" "$_dst"
        done

        # Remove tickles for connections that are no longer there
        comm -13 "$_my_connections" "$_my_tickles" |
        while read _src _dst ; do
                $CTDB deltickle "$_src" "$_dst"
        done

        rm -f "$_my_connections" "$_my_tickles"

        # Remove stale files from killed scripts
        # Files can't have spaces in name, more portable than -print0/-0
        # shellcheck disable=SC2038
        (cd "$tickledir" && find . -type f -mmin +10 | xargs -r rm)
}

########################################################
# load a site local config file
########################################################

[ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
        . "$CTDB_RC_LOCAL"
}

[ -x "${CTDB_BASE}/rc.local" ] && {
        . "${CTDB_BASE}/rc.local"
}

[ -d "${CTDB_BASE}/rc.local.d" ] && {
        for i in "${CTDB_BASE}/rc.local.d"/* ; do
                [ -x "$i" ] && . "$i"
        done
}

script_name="${0##*/}"       # basename
event_name="$1"