qa/308

   1 #! /bin/sh
   2 # PCP QA Test No. 308
   3 # pmlogger and pmlc for proc metrics ... indom problems
   4 # pv #452417
   5 #
   6 # Copyright (c) 1995-2002 Silicon Graphics, Inc.  All Rights Reserved.
   7 #
   8
   9 seq=`basename $0`
  10 echo "QA output created by $seq"
  11
  12 # get standard filters
  13 . ./common.product
  14 . ./common.filter
  15 . ./common.check
  16
  17 [ -d $PCP_PMDAS_DIR/proc ] || _notrun "proc PMDA directory is not installed"
  18
  19 _filter()
  20 {
  21     # some of this is because different platforms produce different
  22     # PMIDs for the proc metrics, and the name of the init process is
  23     # different
  24     # And on some Linux systems (e.g. Fedora 18), pid 1 is not init, but
  25     # /usr/lib/systemd/systemd --switched-root --system --deserialize ...
  26     #
  27     _filter_pmdumplog \
  28     | grep proc.psinfo.ppid \
  29     | sed -e 's/000*/zeroes.../' \
  30           -e 's/ 60\.8\.3 / PMID /' \
  31           -e 's/ 3\.1\.8 / PMID /' \
  32           -e 's/ 3\.8\.3 / PMID /' \
  33           -e 's/ init \[[0-9]*]/ init/' \
  34           -e 's/ \/etc\/init\([ "]\)/ init\1/' \
  35           -e 's/ \/sbin\/init\([ "]\)/ init\1/' \
  36           -e 's/ \/usr\/lib\/systemd\/systemd\([ "]\)/ init\1/' \
  37           -e 's/init .*"/init"/'
  38 }
  39
  40 signal=$PCP_BINADM_DIR/pmsignal
  41 status=1        # failure is the default!
  42 trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
  43
  44 if [ $PCP_PLATFORM = darwin -o $PCP_PLATFORM = solaris ]
  45 then
  46     _notrun "No proc metrics for $PCP_PLATFORM"
  47 fi
  48
  49 eval `./getpmcdhosts -L -n 2 -m 'proc.psinfo.ppid>=0' | sed -e 's/ / host2=/' -e 's/^/host1=/'`
  50 if [ -z "$host1" ]
  51 then
  52     _notrun "Cannot get first remote pmcd host with proc.psinfo.ppid metrics"
  53 fi
  54 if [ -z "$host2" ]
  55 then
  56     _notrun "Cannot get second remote pmcd host with proc.psinfo.ppid metrics"
  57 fi
  58
  59 rm -f $seq.full
  60 echo "host1=$host1" >>$seq.full
  61 echo "host2=$host2" >>$seq.full
  62 done_clean=false
  63
  64 install_on_cleanup=false
  65 pminfo proc >/dev/null 2>&1 && install_on_cleanup=true
  66
  67 _cleanup()
  68 {
  69     if $done_clean
  70     then
  71         :
  72     else
  73         if $install_on_cleanup
  74         then
  75             ( cd $PCP_PMDAS_DIR/proc; $sudo ./Install </dev/null >/dev/null 2>&1 )
  76         else
  77             ( cd $PCP_PMDAS_DIR/proc; $sudo ./Remove </dev/null >/dev/null 2>&1 )
  78         fi
  79         done_clean=true
  80     fi
  81     $sudo rm -f $tmp.*
  82     exit $status
  83 }
  84 trap "_cleanup; exit \$status" 0 1 2 3 15
  85 ( cd $PCP_PMDAS_DIR/proc; $sudo ./Install </dev/null >/dev/null 2>&1 )
  86
  87 # real QA test starts here
  88
  89 echo | tee -a $seq.full
  90 echo "pmlogger directly ..." | tee -a $seq.full
  91 inst=`pminfo -F proc.psinfo.ppid \
  92       | sed -n -e '/inst \[1 /{
  93 s/^[^"]*"/"/
  94 s/] value .*//
  95 p
  96 }'`
  97
  98 if [ -z "$inst" ]
  99 then
 100     echo "Arrgh ... cannot find inst for local pid 1" | tee -a $seq.full
 101     pminfo -F proc.psinfo.ppid >>$seq.full
 102     exit
 103 fi
 104
 105 echo "inst=$inst" >>$seq.full
 106 ps $PCP_PS_ALL_FLAGS | egrep "PID|$inst" >>$seq.full
 107
 108 cat <<end-of-file >$tmp.conf
 109 log advisory on once { proc.psinfo.ppid[$inst] }
 110 end-of-file
 111
 112 echo "=== pmlogger config ===" >>$seq.full
 113 cat $tmp.conf >>$seq.full
 114
 115 rm -f $tmp.0 $tmp.index $tmp.meta
 116 pmlogger -s 1 -c $tmp.conf -l $tmp.log $tmp
 117 status=$?
 118 if [ $status -ne 0 ]
 119 then
 120     echo "Warning: pmlogger exit status=$status"
 121     cat $tmp.log
 122 fi
 123 x=`pmdumplog $tmp | _filter`
 124 if [ -z "$x" ]
 125 then
 126     echo "Error: pmdumplog | filter produces nothing"
 127     pmdumplog -a $tmp
 128 else
 129     echo "$x"
 130 fi
 131
 132 echo | tee -a $seq.full
 133 echo "pmlc for local pmlogger ..." | tee -a $seq.full
 134 rm -f $tmp.0 $tmp.index $tmp.meta $tmp.log
 135 pmlogger -L -s 1 -c /dev/null -l $tmp.log $tmp &
 136 pid=$!
 137 _wait_for_pmlogger $pid $tmp.log
 138 ( echo "connect $!"; cat $tmp.conf ) | pmlc
 139 sleep 2
 140 x=`pmdumplog $tmp | _filter`
 141 if [ -z "$x" ]
 142 then
 143     echo "Error: pmdumplog | filter produces nothing"
 144     $signal -s TERM $pid
 145     pmdumplog -a $tmp
 146 else
 147     echo "$x"
 148 fi
 149
 150 echo | tee -a $seq.full
 151 echo "pmlc local, pmcd and pmlogger remote ..." | tee -a $seq.full
 152
 153 inst=`pminfo -F -h $host1 proc.psinfo.ppid \
 154       | sed -n -e '/inst \[1 /{
 155 s/^[^"]*"/"/
 156 s/] value .*//
 157 p
 158 }'`
 159 if [ -z "$inst" ]
 160 then
 161     echo "Arrgh ... cannot find inst for host $host1 pid 1" | tee -a $seq.full
 162     pminfo -F -h $host1 proc.psinfo.ppid >>$seq.full
 163     exit
 164 fi
 165 cat <<end-of-file >$tmp.conf
 166 log advisory on once { proc.psinfo.ppid[$inst] }
 167 end-of-file
 168
 169 echo "inst=$inst" >>$seq.full
 170 ssh -q pcpqa@$host1 ps -e | egrep "PID|$inst" >>$seq.full
 171 echo "=== pmlogger config ===" >>$seq.full
 172 cat $tmp.conf >>$seq.full
 173
 174 cat <<end-of-file >$tmp.cmd
 175 . \$PCP_DIR/etc/pcp.env
 176 rm -f /tmp/\$\$.*
 177 ( \$PCP_BINADM_DIR/pmlogger -L -s 1 -c /dev/null -l /tmp/\$\$.log /tmp/\$\$ </dev/null >/dev/null 2>&1 ) &
 178 echo rem_pid=\$! rem_tmp=\$\$
 179 end-of-file
 180
 181 scp -q $tmp.cmd pcpqa@$host1:$tmp.cmd
 182 eval `ssh -q pcpqa@$host1 /bin/sh $tmp.cmd`
 183
 184 if [ -z "$rem_pid" ]
 185 then
 186     echo "Arrgh ... failed to get remote pmlogger pid"
 187     exit
 188 fi
 189
 190 if [ -z "$rem_tmp" ]
 191 then
 192     echo "Arrgh ... failed to get remote archive basename"
 193     exit
 194 fi
 195
 196 # if this fails, /hosts/$host1 may not help to diagnose the failure
 197 # if $host1 is not exporting filesystems via NFS and/or the local
 198 # host is not mounting them via NFS ... only matters if the remote
 199 # pmlogger does not start ...
 200 #
 201 _wait_for_pmlogger "-h $host1 $rem_pid" /hosts/$host1/tmp/$rem_tmp.log
 202
 203 ( echo "connect $rem_pid@$host1"; cat $tmp.conf ) | pmlc
 204 sleep 2
 205 rm -f $tmp.0 $tmp.meta $tmp.index
 206 scp -q pcpqa@$host1:/tmp/$rem_tmp.index $tmp.index
 207 scp -q pcpqa@$host1:/tmp/$rem_tmp.meta $tmp.meta
 208 scp -q pcpqa@$host1:/tmp/$rem_tmp.0 $tmp.0
 209 x=`pmdumplog $tmp | _filter`
 210 if [ -z "$x" ]
 211 then
 212     echo "Error: pmdumplog | filter produces nothing"
 213     $signal -s TERM $pid
 214     pmdumplog -a $tmp
 215 else
 216     echo "$x"
 217 fi
 218 ssh -q pcpqa@$host1 "rm -f /tmp/$rem_tmp.*"
 219
 220 echo | tee -a $seq.full
 221 echo "pmlc local, pmcd remote on one host, pmlogger remote on a different host ..." | tee -a $seq.full
 222 # need this in the format of the pmcd that pmlogger is connected to
 223 #
 224 inst=`pminfo -F -h $host2 proc.psinfo.ppid \
 225       | sed -n -e '/inst \[1 /{
 226 s/^[^"]*"/"/
 227 s/] value .*//
 228 p
 229 }'`
 230 if [ -z "$inst" ]
 231 then
 232     echo "Arrgh ... cannot find inst for host $host2 pid 1" | tee -a $seq.full
 233     pminfo -F -h $host2 proc.psinfo.ppid >>$seq.full
 234     exit
 235 fi
 236 cat <<end-of-file >$tmp.conf
 237 log advisory on once { proc.psinfo.ppid[$inst] }
 238 end-of-file
 239
 240 echo "inst=$inst" >>$seq.full
 241 ssh -q pcpqa@$host2 ps -e | egrep "PID|$inst" >>$seq.full
 242 echo "=== pmlogger config ===" >>$seq.full
 243 cat $tmp.conf >>$seq.full
 244
 245 cat <<end-of-file >$tmp.cmd
 246 . \$PCP_DIR/etc/pcp.env
 247 rm -f /tmp/\$\$.*
 248 ( \$PCP_BINADM_DIR/pmlogger -h $host2 -L -s 1 -c /dev/null -l /tmp/\$\$.log /tmp/\$\$ </dev/null >/dev/null 2>&1 ) &
 249 echo rem_pid=\$! rem_tmp=\$\$
 250 end-of-file
 251
 252 scp $tmp.cmd pcpqa@$host1:$tmp.cmd
 253 eval `ssh -q pcpqa@$host1 /bin/sh $tmp.cmd`
 254
 255 if [ -z "$rem_pid" ]
 256 then
 257     echo "Arrgh ... failed to get remote pmlogger pid"
 258     exit
 259 fi
 260
 261 if [ -z "$rem_tmp" ]
 262 then
 263     echo "Arrgh ... failed to get remote archive basename"
 264     exit
 265 fi
 266
 267 # see note above about /hosts/$host1
 268 #
 269 _wait_for_pmlogger "-h $host1 $rem_pid" /hosts/$host1/tmp/$rem_tmp.log
 270
 271 ( echo "connect $rem_pid@$host1"; cat $tmp.conf ) | pmlc
 272 sleep 2
 273 rm -f $tmp.0 $tmp.meta $tmp.index
 274 scp pcpqa@$host1:/tmp/$rem_tmp.index $tmp.index
 275 scp pcpqa@$host1:/tmp/$rem_tmp.meta $tmp.meta
 276 scp pcpqa@$host1:/tmp/$rem_tmp.0 $tmp.0
 277 x=`pmdumplog $tmp | _filter`
 278 if [ -z "$x" ]
 279 then
 280     echo "Error: pmdumplog | filter produces nothing"
 281     $signal -s TERM $pid
 282     pmdumplog -a $tmp
 283 else
 284     echo "$x"
 285 fi
 286 ssh -q pcpqa@$host1 "rm -f /tmp/$rem_tmp.*"
 287
 288 # success, all done
 289 status=0
 290 exit