qa/169

   1 #! /bin/sh
   2 # PCP QA Test No. 169
   3 #
   4 # Test pmcd timeouts with the pmcd PMDA's pmcd.control.timeout
   5 #
   6 # Test setting the app's timeouts < pmcd's timeout and vice versa
   7 #
   8 # Copyright (c) 1995-2002 Silicon Graphics, Inc.  All Rights Reserved.
   9 #
  10
  11 seq=`basename $0`
  12 echo "QA output created by $seq"
  13
  14 # get standard environment, filters and checks
  15 . ./common.product
  16 . ./common.check
  17 . ./common.filter
  18
  19 echo ""
  20 echo "Test pmcd timeouts with the pmcd PMDA's pmcd.control.timeout"
  21 echo ""
  22
  23 LOCALHOST=`hostname`
  24 signal=$PCP_BINADM_DIR/pmsignal
  25
  26 _filter_dots()
  27 {
  28     sed -e 's/\.\.\.\.\.*/[dots]/'
  29 }
  30
  31 _cleanup()
  32 {
  33     $sudo $PCP_RC_DIR/pcp stop >$tmp.stop
  34     # primary pmlogger exiting during the test is racey depending on timing
  35     # of fetches and stopping/starting pmcd (this is also not under test).
  36     _filter_pcp_stop <$tmp.stop | _filter_dots | sed -e '/logger not running/d'
  37     _restore_config $PCP_PMCDCONF_PATH
  38     $sudo $PCP_RC_DIR/pcp start >$tmp.start
  39     _filter_pcp_start <$tmp.start | _filter_dots
  40     _wait_for_pmcd
  41     _wait_for_pmlogger
  42
  43     rm -f $tmp.*
  44 }
  45
  46 status=1        # failure is the default!
  47 trap "_cleanup; exit \$status" 0 1 2 3 15
  48 rm -f $seq.full
  49
  50 # real QA test starts here
  51
  52 # This value is passed to processes this script creates on the command line.
  53 # It's unlikely to occur anywhere in a ps output and is used as a tag to
  54 # terminate or check for the existence of the processes later.
  55 #
  56 TAG=000000660066
  57
  58
  59 _check_pmcd_procs()
  60 {
  61     n_pmcd=`ps $PCP_PS_ALL_FLAGS \
  62             | tee $tmp.pmcd \
  63             | fgrep "$PCP_PMCD_PATH" \
  64             | grep -v grep \
  65             | wc -l \
  66             | tr -d ' '`
  67     if [ "X$n_pmcd" != "X$1" ]
  68     then
  69         echo "Expected $1 pmcd processes but found $n_pmcd:"
  70         cat $tmp.pmcd
  71         cat $PCP_PMCDLOG_PATH
  72         file $PCP_LOG_DIR/core* $PCP_LOG_DIR/pmcd/core*
  73         return 1
  74     else
  75         return 0
  76     fi
  77 }
  78
  79 _set_timeouts()
  80 {
  81     pmcd_timeout=$1
  82     app_timeout=$2
  83
  84     echo "app timeout = $app_timeout"
  85     echo "pmcd timeout = $pmcd_timeout"
  86     PMCD_REQUEST_TIMEOUT=$app_timeout
  87     export PMCD_REQUEST_TIMEOUT
  88
  89     pmstore pmcd.control.timeout $pmcd_timeout
  90
  91 }
  92
  93 _echo()
  94 {
  95     echo "$*"
  96     echo "$*" >>$seq.full
  97 }
  98
  99 rm -rf $tmp.*
 100 pmns=$tmp.pmns
 101
 102 # use a dummy pmns for pminfo
 103 dummy_domain=160 # choose non-existent domain
 104 cat >$pmns <<EOF
 105 root {
 106     dummyproc   $dummy_domain:0:0
 107 }
 108 EOF
 109
 110 # pmcd will run a dumb_pmda, masquerading as the IRIX agent.  This will
 111 # not respond to PDUs from PMCD, which should then time it out.
 112 # The pmcd PMDA is required so that pmcd.control.timeout can be modified.
 113 # We don't need any other PMDAs and in particular all pmlogger and pmie
 114 # processes are stopped before pmcd is reconfigured.
 115 # Save the current pmcd.conf so it can be restored afterwards.
 116
 117 _save_config $PCP_PMCDCONF_PATH
 118 cat >$tmp.tmp <<End-Of-File
 119 # QA test 169 dummy pmcd.conf (blame Jonathan if you find this!)
 120 #
 121 dummyproc       $dummy_domain   pipe    binary  $here/src/dumb_pmda -d $dummy_domain $TAG
 122 End-Of-File
 123
 124 $PCP_AWK_PROG <$PCP_PMCDCONF_PATH >>$tmp.tmp '$1 == "pmcd" { print }'
 125
 126 pmcdlog=$PCP_PMCDLOG_PATH
 127
 128 $sudo $signal -a -s TERM pmie
 129 $sudo $signal -a -s TERM pmlogger
 130
 131 $sudo cp $tmp.tmp $PCP_PMCDCONF_PATH
 132 $sudo $PCP_RC_DIR/pmcd start >$tmp.start
 133 _filter_pcp_start <$tmp.start | sed -e 's/\.\.\.\.\.*/[dots]/'
 134 _wait_for_pmcd
 135 if _check_pmcd_procs 1
 136 then
 137     :
 138 else
 139     _echo "pmcd didn't start!"
 140     _echo ""
 141     cat $pmcdlog >>$seq.full
 142     exit 1
 143 fi
 144
 145 ps $PCP_PS_ALL_FLAGS >$tmp.ps
 146 egrep '[P]ID|[p]m' $tmp.ps >>$seq.full
 147 if grep "[d]umb_pmda" $tmp.ps >/dev/null
 148 then
 149     :
 150 else
 151     _echo "pmcd didn't create the dumb_pmda agent"
 152     _echo ""
 153     cat $pmcdlog >>$seq.full
 154     exit 1
 155 fi
 156
 157 # force to a known state
 158 #
 159 pmstore pmcd.control.timeout 5 >/dev/null
 160
 161 # for debugging
 162 #
 163 #debug# pmstore pmcd.control.tracenobuf 1 >/dev/null
 164
 165 #
 166 # pmcd agent timeout < application time out
 167 #
 168 _echo ""
 169 _echo "*** pmcd agent timeout < app timeout ***"
 170 _echo "*** pmcd will killoff agent ***"
 171 _echo ""
 172 _set_timeouts 5 10
 173
 174 # Fetch from dummy domain.  This should cause the dumb_pmda agent to be
 175 # timed-out by pmcd, resulting in its termination.
 176 #
 177 _echo ""
 178 _echo "Expect dummyproc to fail (IPC protocol failure):"
 179 pminfo -n $pmns -d dummyproc
 180 _echo ""
 181 # make sure pmcd has had a chance to wait() on the pmda process
 182 #
 183 sleep 2
 184 pminfo -n $pmns -d dummyproc >/dev/null 2>&1
 185
 186 ps $PCP_PS_ALL_FLAGS >$tmp.ps
 187 egrep '[P]ID|[p]m' $tmp.ps >>$seq.full
 188 if grep "[d]umb_pmda" <$tmp.ps >/dev/null
 189 then
 190     _echo "Error: pmcd didn't terminate the dumb_pmda agent"
 191     cat $pmcdlog >>$seq.full
 192     exit 1
 193 else
 194     _echo "pmcd terminated dummy the dumb_pmda agent as desired"
 195 fi
 196
 197 echo >>$seq.full
 198 cat $PCP_LOG_DIR/pmcd/pmcd.log >>$seq.full
 199 echo >>$seq.full
 200 cat $PCP_LOG_DIR/pmcd/dumb_pmda.log >>$seq.full
 201
 202 # Restart pmcd and its agents
 203 _echo ""
 204 $sudo $signal -a -s HUP pmcd
 205 sleep 1
 206 _wait_for_pmcd
 207 if _check_pmcd_procs 1
 208 then
 209     :
 210 else
 211     _echo "pmcd was sent SIGHUP and died"
 212     _echo ""
 213     cat $pmcdlog >>$seq.full
 214     exit 1
 215 fi
 216
 217 ps $PCP_PS_ALL_FLAGS >$tmp.ps
 218 egrep '[P]ID|[p]m' $tmp.ps >>$seq.full
 219 if grep "[d]umb_pmda" <$tmp.ps >/dev/null
 220 then
 221     psline=`ps $PCP_PS_ALL_FLAGS | grep "[d]umb_pmda"`
 222     old_dummy_pid=`echo $psline | cut -d ' ' -f 2`
 223 else
 224     _echo "pmcd SIGHUP didn't restart the dumb_pmda agent"
 225     _echo ""
 226     cat $pmcdlog >>$seq.full
 227     exit 1
 228 fi
 229
 230 #
 231 # pmcd agent timeout > application time out
 232 #
 233 _echo ""
 234 _echo "*** pmcd agent timeout > app timeout ***"
 235 _echo "*** pmcd will not killoff agent until later ***"
 236 _echo ""
 237 _set_timeouts 10 5
 238
 239 # Fetch from dummy domain.  This should cause the dumb_pmda agent to be
 240 # timed-out by pmcd, resulting in its termination.
 241 #
 242 _echo ""
 243 _echo "Expect dummyproc to fail (timeout for pmcd failure):"
 244 #DEBUG# date
 245 pminfo -n $pmns -d dummyproc
 246 #DEBUG# date
 247 _echo ""
 248
 249 ps $PCP_PS_ALL_FLAGS >$tmp.ps
 250 egrep '[P]ID|[p]m' $tmp.ps >>$seq.full
 251 if grep "[d]umb_pmda" <$tmp.ps >/dev/null
 252 then
 253     _echo "pmcd did NOT terminate the dumb_pmda agent as desired"
 254
 255     # make sure it is really the same process - check pid
 256     psline=`ps $PCP_PS_ALL_FLAGS | grep "[d]umb_pmda"`
 257     new_dummy_pid=`echo $psline | cut -d ' ' -f 2`
 258     if [ $old_dummy_pid -ne $new_dummy_pid ]
 259     then
 260         _echo "Error: dumb_pmda proc has been recreated"
 261         _echo "pid mismatch: $old_dummy_pid versus $new_dummy_pid"
 262         exit 1
 263     fi
 264 else
 265     _echo "Error: pmcd terminated the dumb_pmda agent"
 266     cat $pmcdlog >>$seq.full
 267     exit 1
 268 fi
 269
 270 _echo ""
 271 _echo "Now wait for pmcd to timeout..."
 272 _echo ""
 273 sleep 8
 274
 275 ps $PCP_PS_ALL_FLAGS >$tmp.ps
 276 egrep '[P]ID|[p]m' $tmp.ps >>$seq.full
 277 if grep "[d]umb_pmda" <$tmp.ps >/dev/null
 278 then
 279     _echo "Error: pmcd didn't terminate the dumb_pmda agent"
 280     cat $pmcdlog >>$seq.full
 281     exit 1
 282 else
 283     _echo "pmcd terminated the dumb_pmda agent as desired"
 284 fi
 285
 286 echo >>$seq.full
 287 cat $PCP_LOG_DIR/pmcd/pmcd.log >>$seq.full
 288 echo >>$seq.full
 289 cat $PCP_LOG_DIR/pmcd/dumb_pmda.log >>$seq.full
 290
 291 _echo ""
 292 status=0