usr/src/cmd/ztest/zloop.bash

   1 #!/bin/bash
   2
   3 #
   4 # CDDL HEADER START
   5 #
   6 # This file and its contents are supplied under the terms of the
   7 # Common Development and Distribution License ("CDDL"), version 1.0.
   8 # You may only use this file in accordance with the terms of version
   9 # 1.0 of the CDDL.
  10 #
  11 # A full copy of the text of the CDDL should have accompanied this
  12 # source.  A copy of the CDDL is also available via the Internet at
  13 # http://www.illumos.org/license/CDDL.
  14 #
  15 # CDDL HEADER END
  16 #
  17
  18 #
  19 # Copyright (c) 2015, 2017 by Delphix. All rights reserved.
  20 #
  21
  22 set -x
  23 export BITS=64
  24 export UMEM_DEBUG=default,verbose
  25 export UMEM_LOGGING=transaction,contents
  26 set +x
  27
  28 sparc_32=sparc
  29 sparc_64=sparcv9
  30 i386_32=i386
  31 i386_64=amd64
  32 ARCH=`uname -p`
  33 eval 'ARCHBITS=${'"${ARCH}_${BITS}"'}'
  34 BIN=$ROOT/usr/bin/${ARCHBITS}
  35 SBIN=$ROOT/usr/sbin/${ARCHBITS}
  36 DEFAULTWORKDIR=/var/tmp
  37 DEFAULTCOREDIR=/var/tmp/zloop
  38
  39 function usage
  40 {
  41         echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \
  42             "[ -- [extra ztest parameters]]\n" \
  43             "\n" \
  44             "  This script runs ztest repeatedly with randomized arguments.\n" \
  45             "  If a crash is encountered, the ztest logs, any associated\n" \
  46             "  vdev files, and core file (if one exists) are moved to the\n" \
  47             "  output directory ($DEFAULTCOREDIR by default). Any options\n" \
  48             "  after the -- end-of-options marker will be passed to ztest.\n" \
  49             "\n" \
  50             "  Options:\n" \
  51             "    -t  Total time to loop for, in seconds. If not provided,\n" \
  52             "        zloop runs forever.\n" \
  53             "    -f  Specify working directory for ztest vdev files.\n" \
  54             "    -c  Specify a core dump directory to use.\n" \
  55             "    -h  Print this help message.\n" \
  56             "" >&2
  57 }
  58
  59 function or_die
  60 {
  61         $@
  62         if [[ $? -ne 0 ]]; then
  63                 echo "Command failed: $@"
  64                 exit 1
  65         fi
  66 }
  67
  68 function store_core
  69 {
  70         if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then
  71                 coreid=$(/bin/date "+zloop-%y%m%d-%H%M%S")
  72                 foundcrashes=$(($foundcrashes + 1))
  73
  74                 dest=$coredir/$coreid
  75                 or_die /bin/mkdir $dest
  76                 or_die /bin/mkdir $dest/vdev
  77
  78                 echo "*** ztest crash found - moving logs to $coredir/$coreid"
  79
  80                 or_die /bin/mv ztest.history $dest/
  81                 or_die /bin/mv ztest.out $dest/
  82                 or_die /bin/mv $workdir/ztest* $dest/vdev/
  83                 or_die /bin/mv $workdir/zpool.cache $dest/vdev/
  84
  85                 # check for core
  86                 if [[ -f core ]]; then
  87                         corestatus=$(mdb -e "::status" core)
  88                         corestack=$(mdb -e "::stack" core)
  89
  90                         # Dump core + logs to stored directory
  91                         echo "$corestatus" >>$dest/status
  92                         echo "$corestack" >>$dest/status
  93                         or_die /bin/mv core $dest/
  94
  95                         # Record info in cores logfile
  96                         echo "*** core @ $coredir/$coreid/core:" | /bin/tee -a ztest.cores
  97                         echo "$corestatus" | /bin/tee -a ztest.cores
  98                         echo "$corestack" | /bin/tee -a ztest.cores
  99                         echo "" | /bin/tee -a ztest.cores
 100                 fi
 101                 echo "continuing..."
 102         fi
 103 }
 104
 105 set -x
 106 export PATH=${BIN}:${SBIN}
 107 export LD_LIBRARY_PATH=$ROOT/lib/$BITS:$ROOT/usr/lib/$BITS
 108 set +x
 109
 110 # parse arguments
 111 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
 112 coredir=$DEFAULTCOREDIR
 113 workdir=$DEFAULTWORKDIR
 114 timeout=0
 115 while getopts ":ht:c:f:" opt; do
 116         case $opt in
 117                 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
 118                 c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
 119                 f ) [[ $OPTARG ]] && workdir=$(/usr/bin/readlink -f $OPTARG) ;;
 120                 h ) usage
 121                     exit 2
 122                     ;;
 123                 * ) echo "Invalid argument: -$OPTARG";
 124                     usage
 125                     exit 1
 126         esac
 127 done
 128 # pass remaining arguments on to ztest
 129 shift $((OPTIND - 1))
 130
 131 if [[ -f core ]]; then
 132         echo "There's a core dump here you might want to look at first."
 133         exit 1
 134 fi
 135
 136 if [[ ! -d $coredir ]]; then
 137         echo "core dump directory ($coredir) does not exist, creating it."
 138         or_die /bin/mkdir -p $coredir
 139 fi
 140
 141 if [[ ! -w $coredir ]]; then
 142         echo "core dump directory ($coredir) is not writable."
 143         exit 1
 144 fi
 145
 146 or_die /bin/rm -f ztest.history
 147 or_die /bin/rm -f ztest.cores
 148
 149 # Allow core files to be written to cwd if that's currently disabled.
 150 sudo coreadm -e process
 151
 152 ztrc=0          # ztest return value
 153 foundcrashes=0  # number of crashes found so far
 154 starttime=$(/bin/date +%s)
 155 curtime=$starttime
 156
 157 # if no timeout was specified, loop forever.
 158 while [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do
 159         zopt="-VVVVV"
 160
 161         # switch between common arrangements & fully randomized
 162         if [[ $((RANDOM % 2)) -eq 0 ]]; then
 163                 mirrors=2
 164                 raidz=0
 165                 parity=1
 166                 vdevs=2
 167         else
 168                 mirrors=$(((RANDOM % 3) * 1))
 169                 parity=$(((RANDOM % 3) + 1))
 170                 raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
 171                 vdevs=$(((RANDOM % 3) + 3))
 172         fi
 173         align=$(((RANDOM % 2) * 3 + 9))
 174         runtime=$((RANDOM % 100))
 175         passtime=$((RANDOM % (runtime / 3 + 1) + 10))
 176         size=128m
 177
 178         zopt="$zopt -m $mirrors"
 179         zopt="$zopt -r $raidz"
 180         zopt="$zopt -R $parity"
 181         zopt="$zopt -v $vdevs"
 182         zopt="$zopt -a $align"
 183         zopt="$zopt -T $runtime"
 184         zopt="$zopt -P $passtime"
 185         zopt="$zopt -s $size"
 186         zopt="$zopt -f $workdir"
 187
 188         cmd="ztest $zopt $@"
 189         desc="$(/bin/date '+%m/%d %T') $cmd"
 190         echo "$desc" | /bin/tee -a ztest.history
 191         echo "$desc" >>ztest.out
 192         $BIN/$cmd >>ztest.out 2>&1
 193         ztrc=$?
 194         /bin/egrep '===|WARNING' ztest.out >>ztest.history
 195         $SBIN/zdb -U $workdir/zpool.cache -DD ztest >>ztest.ddt 2>&1
 196
 197         store_core
 198
 199         curtime=$(/bin/date +%s)
 200 done
 201
 202 echo "zloop finished, $foundcrashes crashes found"
 203
 204 /bin/uptime >>ztest.out
 205
 206 if [[ $foundcrashes -gt 0 ]]; then
 207         exit 1
 208 fi