scripts/dbginfo.sh

   1 #!/bin/bash
   2
   3 ###############################################################################
   4 # collect some configuration, trace, and debug
   5 # information about the S390 Linux system
   6 #
   7 # Copyright IBM Corp. 2002, 2006.
   8 ###############################################################################
   9
  10
  11 # variables
  12 #
  13
  14 SCRIPTNAME="dbginfo.sh"
  15 WORKDIR=DBGINFO-`date +%Y-%m-%d-%H-%M-%S`-`hostname`
  16 rc_check_zvm=0
  17 # write output to following directory
  18 WORKPFX=/tmp
  19 WORKPATH=$WORKPFX/$WORKDIR
  20 # write output to following files
  21 CMDOUTPUT=runtime.out
  22 VM_CMDOUTPUT=zvm_runtime.out
  23 SYSFSFILELIST=sysfsfiles.out
  24 FCPCONFOUTPUT=scsi
  25 MOUNT_POINT_DEBUGFS="/sys/kernel/debug"
  26 kernel_version_tmp=0
  27
  28 LOGFILE=dbginfo.log
  29
  30 # procfs entries to be collected (except s390dbf and scsi)
  31 PROCFILES=" \
  32     /proc/sysinfo \
  33     /proc/version \
  34     /proc/cpuinfo \
  35     /proc/meminfo \
  36     /proc/buddyinfo \
  37     /proc/slabinfo \
  38     /proc/modules \
  39     /proc/mounts \
  40     /proc/partitions \
  41     /proc/stat \
  42     /proc/devices \
  43     /proc/misc \
  44     /proc/qeth \
  45     /proc/qeth_perf \
  46     /proc/qeth_ipa_takeover \
  47     /proc/cmdline \
  48     /proc/crypto \
  49     /proc/diskstats \
  50     /proc/dasd/devices \
  51     /proc/dasd/statistics \
  52     /proc/sys/kernel/spin_retry \
  53     /proc/sys/vm/cmm_pages \
  54     /proc/sys/vm/cmm_timed_pages \
  55     /proc/sys/vm/cmm_timeout \
  56     /proc/sys/vm/swappiness \
  57     "
  58 PROCFILES_24=" \
  59     /proc/subchannels \
  60     /proc/chpids \
  61     /proc/chandev \
  62     /proc/ksyms \
  63     /proc/lvm/global \
  64     "
  65
  66 # log files to be collected
  67 LOGFILES=" \
  68     /var/log/messages* \
  69     /var/log/boot.log* \
  70     /var/log/IBMtape.trace \
  71     /var/log/IBMtape.errorlog \
  72     /var/log/lin_tape.trace \
  73     /var/log/lin_tape.errorlog \
  74     /var/log/sa/* \
  75     "
  76 # config files to be collected;
  77 # additional all files "modules.dep" are collected
  78 CONFIGFILES=" \
  79     /etc/ccwgroup.conf \
  80     /etc/chandev.conf \
  81     /etc/modules.conf \
  82     /etc/modprobe.conf* \
  83     /etc/fstab \
  84     /etc/syslog.conf \
  85     /etc/sysconfig \
  86     /etc/crontab \
  87     /etc/cron.* \
  88     /etc/exports \
  89     /etc/sysctl.conf \
  90     /etc/zipl.conf \
  91     /etc/lvm \
  92     /etc/IBMtaped.conf \
  93     /etc/lin_taped.conf \
  94     /etc/multipath.conf \
  95     /etc/rc.d \
  96     /etc/zfcp.conf \
  97     "
  98
  99 #sysfs files NOT to be collected
 100 SYSFSFILEEXCLUDES="\
 101         trace_pipe\
 102         "
 103
 104 # collect output of following commands;
 105 # commands are separated by ':'
 106 CMDS="uname -a\
 107     :uptime\
 108     :iptables -L\
 109     :ulimit -a\
 110     :ps -eo pid,tid,nlwp,policy,user,tname,ni,pri,psr,sgi_p,stat,wchan,start_time,time,pcpu,pmem,vsize,size,rss,share,command\
 111     :ps axX\
 112     :dmesg -s 1048576\
 113     :ifconfig -a\
 114     :route -n\
 115     :ip route list\
 116     :ip rule list\
 117     :ip neigh list\
 118     :ip link show\
 119     :ipcs -a\
 120     :netstat -pantu\
 121     :netstat -s\
 122     :dmsetup ls\
 123     :dmsetup ls --tree\
 124     :dmsetup table\
 125     :dmsetup table --target multipath\
 126     :dmsetup status\
 127     :multipath -v6 -ll\
 128     :multipath -d\
 129     :lsqeth\
 130     :lscss\
 131     :lsdasd\
 132     :ziorep_config -ADM\
 133     :lsmod\
 134     :lsdev\
 135     :lsscsi\
 136     :lstape\
 137     :lszfcp\
 138     :lszfcp -D\
 139     :lszfcp -V\
 140     :SPident\
 141     :rpm -qa\
 142     :sysctl -a\
 143     :lsof\
 144     :mount\
 145     :df -h\
 146     :pvpath -qa
 147     :ls -la /boot\
 148     :java -version\
 149     :cat /root/.bash_history\
 150     "
 151
 152 #z/VM commands
 153
 154 VM_CMDS="q userid\
 155         :q users\
 156         :q v osa\
 157         :q v dasd\
 158         :q v crypto\
 159         :q v fcp\
 160         :q v pav\
 161         :q v st\
 162         :q st\
 163         :q xstore\
 164         :q xstore user system\
 165         :q sxspages\
 166         :q v sw\
 167         :q vmlan\
 168         :q vswitch\
 169         :q vswitch details\
 170         :q vswitch access\
 171         :q set\
 172         :q comm\
 173         :q controller all\
 174         :q cplevel\
 175         :q cpus\
 176         :q fcp\
 177         :q frames\
 178         :q lan\
 179         :q lan all details\
 180         :q lan all access\
 181         :q cache\
 182         :q nic\
 183         :q pav\
 184         :q privclass\
 185         :q proc\
 186         :q qioass\
 187         :q spaces\
 188         :q swch all\
 189         :q timezone\
 190         :q trace\
 191         :q vtod\
 192         :q srm\
 193         :q mdcache\
 194         :q alloc page\
 195         :q alloc spool\
 196         :q dump\
 197         :q reorder \*\
 198         :ind load\
 199         :ind sp\
 200         :ind user\
 201         "
 202 #
 203 # function definitions
 204 #
 205
 206 # print version info
 207 printversion()
 208 {
 209     cat <<EOF
 210 $SCRIPTNAME: Debug information script version %S390_TOOLS_VERSION%
 211 Copyright IBM Corp. 2002, 2006
 212 EOF
 213 }
 214
 215 # print usage and help
 216 printhelp()
 217 {
 218     cat <<EOF
 219 Usage: $SCRIPTNAME [OPTIONS]
 220
 221 This script collects some runtime, configuration and
 222 trace information about your Linux for zSeries system
 223 for debugging purposes. It also traces information
 224 about z/VM if the Linux runs under z/VM.
 225 This information is written to a file
 226         /tmp/DBGINFO-[date]-[time]-[hostname].tgz
 227 where [date] and [time] are the date and time when debug
 228 data is collected. [hostname] indicates the hostname of the
 229 system the data was collected from.
 230
 231 Options:
 232
 233         -h, --help       print this help
 234         -v, --version    print version information
 235
 236 Please report bugs to: linux390@de.ibm.com
 237 EOF
 238 }
 239
 240 # copy file $1 to $WORKPATH
 241 collect_file_contents()
 242 {
 243     echo "  $1" >>  $LOGFILE
 244     if [ ! -e $1 ]
 245     then
 246         echo "  WARNING: No such file: \"$1\"" >> $LOGFILE
 247         return 1
 248     elif [ ! -r $1 ]
 249     then
 250         echo "  WARNING: Permission denied: \"$1\"" >> $LOGFILE
 251         return 1
 252     else
 253         if [ ! -e $WORKPATH`dirname $1` ]
 254         then
 255             mkdir --parents $WORKPATH`dirname $1`
 256         fi
 257         cp -r -d -L --parents $1 $WORKPATH 2>> $LOGFILE
 258 #       head -c 10m $1 >> $2
 259         if [ $? -ne 0 ]
 260         then
 261             echo "  WARNING: cp failed for file: \"$1\"" >> $LOGFILE
 262             return 1
 263         else
 264             return 0
 265         fi
 266     fi
 267 }
 268
 269 # append output of command $1 to file $2
 270 collect_cmd_output()
 271 {
 272     local raw_cmd=`echo $1 | awk '{ print $1 }'`;
 273
 274     echo "#######################################################">>$2
 275
 276     # check if command exists
 277     which $raw_cmd >/dev/null 2>&1;
 278     if [ $? -ne 0 ]
 279     then
 280         # check if command is a builtin
 281         command -v $raw_cmd >/dev/null;
 282         [ $? -ne 0 ] && echo "command '$raw_cmd' not available">>$2 && return 1;
 283     fi
 284
 285     echo "$USER@$HOST> $1">>$2
 286     $1 1>>$2 2>&1
 287     echo "" >>$2
 288     if [ $? -ne 0 ]
 289     then
 290         echo "  WARNING: Command not successfully completed: \"$1\"">>  $2
 291         return 1
 292     else
 293         return 0
 294     fi
 295
 296 }
 297
 298 # check cmd line arguments
 299 check_cmdline()
 300 {
 301     # currently no options available
 302     if [ $# -eq 1 ]
 303     then
 304         if [ $1 = '-h' ] ||  [ $1 = '--help' ]
 305         then
 306             printhelp
 307         elif [ $1 = '-v' ] ||  [ $1 = '--version' ]
 308         then
 309             printversion
 310         else
 311             echo
 312             echo "$SCRIPTNAME: invalid option $1"
 313             echo "Try '$SCRIPTNAME --help' for more information"
 314             echo
 315         fi
 316         exit 0
 317     elif [[ $# -ge 1 ]]
 318     then
 319         echo "ERROR: Invalid number of arguments"
 320         echo
 321         printhelp
 322         exit 1
 323     fi
 324 }
 325
 326 # change into temporary directory; if necessary create the directory
 327 prepare_workdir()
 328 {
 329     if [ ! -e $WORKPFX ]
 330     then
 331         mkdir $WORKPFX
 332     fi
 333
 334     if [ -e $WORKPATH ]
 335     then
 336         # remove old stuff
 337         echo "Clean up target directory $WORKPATH"
 338         rm -rf $WORKPATH/*
 339     else
 340         echo "Create target directory $WORKPATH"
 341         mkdir $WORKPATH
 342     fi
 343     echo "Change to target directory $WORKPATH"
 344     cd $WORKPATH
 345 }
 346
 347 # collect single proc fs entries
 348 # (PRCFILES should not contain /proc/scsi and /proc/s390dbf)
 349 collect_procfs()
 350 {
 351     echo "Get procfs entries" | tee -a $LOGFILE
 352     for i in $*
 353     do
 354         collect_file_contents $i
 355     done
 356 }
 357
 358 # collect procfs entries of /proc/s390dbf
 359 collect_s390dbf()
 360 {
 361     echo "Get entries of /proc/s390dbf" | tee -a $LOGFILE
 362     if [ -e /proc/s390dbf ]
 363     then
 364         for i in `find /proc/s390dbf -type f \
 365                   -not -path "*/raw" -not -path "*/flush"`
 366         do
 367             collect_file_contents $i
 368         done
 369     else
 370         echo "  WARNING: /proc/s390dbf not found" | tee -a $LOGFILE
 371     fi
 372 }
 373
 374 # collect procfs entries of /proc/scsi
 375 collect_procfs_scsi()
 376 {
 377     echo "Get entries of /proc/scsi" | tee -a $LOGFILE
 378     if [ -e /proc/scsi ]
 379     then
 380         for i in `find /proc/scsi -type f \
 381                   -perm +0444`
 382         do
 383             collect_file_contents $i
 384         done
 385     else
 386         echo "  WARNING: /proc/scsi not found" >> $LOGFILE
 387     fi
 388 }
 389
 390 #check for excluded files in sysfs
 391 check_for_excludes()
 392 {
 393         for filename in ${SYSFSFILEEXCLUDES[@]}
 394         do
 395                 if [ `basename $1` = $filename ]
 396                 then
 397                         return 1
 398                 fi
 399         done
 400         return 0
 401 }
 402
 403 # collect sysfs entries
 404 collect_sysfs()
 405 {
 406     local rc_mount=""
 407     # check if debugfs is mounted
 408     mount | grep -q $MOUNT_POINT_DEBUGFS
 409     rc_mount=$?
 410     if [ $rc_mount -eq 1 ] && [ $kernel_version_tmp -ge 13 ]; then
 411         mount -t debugfs debugfs $MOUNT_POINT_DEBUGFS
 412     fi
 413
 414     echo "Get file list of /sys" | tee -a $LOGFILE
 415         collect_cmd_output "ls -Rl /sys" $SYSFSFILELIST
 416
 417     echo "Get entries of /sys" | tee -a $LOGFILE
 418     for i in `find /sys -noleaf -type f -perm +444`
 419         do
 420                 if [ -e $i ]
 421                 then
 422                         if check_for_excludes $i
 423                         then
 424                                 collect_file_contents $i
 425                         fi
 426                 else
 427                         echo "  WARNING: $i not found" | tee -a $LOGFILE
 428                 fi
 429         done
 430     #unmount debugfs if not mounted at the beginning
 431     if [ $rc_mount -eq 1 ] && [ $kernel_version_tmp -ge 13 ]; then
 432         umount $MOUNT_POINT_DEBUGFS
 433     fi
 434
 435 }
 436
 437 # collect output of commands
 438 collect_cmdsout()
 439 {
 440     local commands=$1
 441     local outputfile=$2
 442     if [ $rc_check_zvm -eq 1 ]; then
 443         echo "Saving z/VM runtime information into $outputfile" | tee -a $LOGFILE
 444     else
 445         echo "Saving runtime information into $outputfile" | tee -a $LOGFILE
 446     fi
 447     _IFS_ORIG=$IFS
 448     IFS=:
 449     for i in $commands
 450     do
 451         IFS=$_IFS_ORIG
 452         if [ $rc_check_zvm -eq 1 ]; then
 453                 i="$cp_tool $i"
 454         fi
 455         collect_cmd_output "$i" "$outputfile"
 456         IFS=:
 457     done
 458     IFS=$_IFS_ORIG
 459 }
 460
 461 # config files and module dependencies
 462 collect_config()
 463 {
 464     echo "Copy config files" | tee -a $LOGFILE
 465     for i in $CONFIGFILES
 466     do
 467         collect_file_contents $i
 468     done
 469     for i in `find /lib/modules -name modules.dep`
 470     do
 471         collect_file_contents $i
 472     done
 473 }
 474
 475 # Check if we run under z/VM and which Linux cp tool is installed
 476 check_zvm()
 477 {
 478     cp_tool=''
 479     echo "Check if we run under z/VM" | tee -a $LOGFILE
 480     # Are we running under z/VM
 481     cat /proc/sysinfo | grep -q "z/VM"
 482     if [ $? -eq 1 ]
 483     then
 484         echo " Running in LPAR" | tee -a $LOGFILE
 485         return 0
 486     else
 487         echo " Running under z/VM" | tee -a $LOGFILE
 488         # Is vmcp installed
 489         which vmcp > /dev/null 2>&1
 490         if [ $? -eq 1 ]
 491         then
 492                 # is hcp installed
 493                 which hcp > /dev/null 2>&1
 494                 if [ $? -eq 1 ]
 495                 then
 496                         echo " No cp tool installed" | tee -a $LOGFILE
 497                         return 0
 498                 else
 499                         cp_tool=hcp
 500                         echo " Installed CP tool: $cp_tool" | tee -a $LOGFILE
 501                         return 1
 502                 fi
 503         else
 504                 cp_tool=vmcp
 505                 echo " Installed CP tool: $cp_tool" | tee -a $LOGFILE
 506                 return 1
 507         fi
 508      fi
 509 }
 510
 511 # Capture z/VM data
 512 get_zvm_data()
 513 {
 514         lsmod | grep -q $cp_tool
 515         rc_lsmod=$?
 516         if [ $rc_lsmod -eq 1 ]; then
 517                 if [ $cp_tool = "hcp" ]; then
 518                         modprobe cpint
 519                         sleep 2
 520                 else
 521                         modprobe $cp_tool
 522                         sleep 2
 523                 fi
 524         fi
 525         collect_cmdsout "$VM_CMDS" "$VM_CMDOUTPUT"
 526         if [ $rc_lsmod -eq 1 ]; then
 527                 if [ $cp_tool = "hcp" ]; then
 528                         rmmod cpint
 529                 else
 530                         rmmod $cp_tool
 531                 fi
 532         fi
 533 }
 534
 535 # Caputure FCP configuration data if possible
 536 get_fcp_data()
 537 {
 538     which ziomon_fcpconf >/dev/null 2>&1;
 539     [ $? -eq 0 ] && ziomon_fcpconf -o $FCPCONFOUTPUT;
 540 }
 541
 542 # log files
 543 collect_log()
 544 {
 545     echo "Copy log files" | tee -a $LOGFILE
 546     for i in $LOGFILES
 547     do
 548         collect_file_contents $i
 549     done
 550 }
 551
 552 # create gzip-ped tar file
 553 create_package()
 554 {
 555     cd $WORKPATH/..
 556     tar -czf $WORKDIR.tgz $WORKDIR 2>/dev/null
 557     cleanup;
 558     echo
 559     echo "Collected data was saved to:"
 560     echo "  $WORKPFX/$WORKDIR.tgz"
 561 }
 562
 563 cleanup()
 564 {
 565     [ -d $WORKPATH ] && rm -rf $WORKPATH;
 566 }
 567
 568 emergency_exit()
 569 {
 570     cleanup;
 571     exit;
 572 }
 573
 574 #
 575 # start of script
 576 #
 577
 578 trap emergency_exit SIGHUP SIGTERM SIGINT
 579
 580 kernel_version=`uname -r`
 581 kernel_version_tmp=`echo ${kernel_version:4:2} | sed s/[^0-9]//g`
 582 check_cmdline $*
 583 prepare_workdir
 584 printversion >$LOGFILE
 585 if [ $kernel_version_tmp -lt 13 ]
 586         then
 587                 collect_s390dbf
 588         fi
 589 collect_procfs $PROCFILES
 590 collect_cmdsout "$CMDS" "$CMDOUTPUT"
 591 if [ "${kernel_version:2:1}" \> 4 ]
 592     then
 593         collect_sysfs
 594     else
 595         collect_procfs $PROCFILES_24
 596 fi
 597 collect_procfs_scsi
 598 check_zvm
 599 rc_check_zvm=$?
 600 if [ $rc_check_zvm -eq 1 ]; then
 601         get_zvm_data
 602 fi
 603 get_fcp_data
 604 collect_config
 605 collect_log
 606 create_package
 607
 608 #
 609 # end of script
 610 #
 611
 612 #EOF