tests/kdc/check-iprop.in

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2006 - 2007 Kungliga Tekniska Högskolan
   4 # (Royal Institute of Technology, Stockholm, Sweden).
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # 1. Redistributions of source code must retain the above copyright
  12 #    notice, this list of conditions and the following disclaimer.
  13 #
  14 # 2. Redistributions in binary form must reproduce the above copyright
  15 #    notice, this list of conditions and the following disclaimer in the
  16 #    documentation and/or other materials provided with the distribution.
  17 #
  18 # 3. Neither the name of the Institute nor the names of its contributors
  19 #    may be used to endorse or promote products derived from this software
  20 #    without specific prior written permission.
  21 #
  22 # THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
  23 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25 # ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
  26 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32 # SUCH DAMAGE.
  33
  34 top_builddir="@top_builddir@"
  35 env_setup="@env_setup@"
  36 objdir="@objdir@"
  37
  38 db_type=@db_type@
  39
  40 . ${env_setup}
  41
  42 # If there is no useful db support compiled in, disable test
  43 ${have_db} || exit 77
  44
  45
  46 # Don't run this test in AFS, since it lacks support for AF_UNIX
  47 expr "X`/bin/pwd || pwd`" : "X/afs/.*" > /dev/null 2>/dev/null && exit 77
  48
  49 R=TEST.H5L.SE
  50
  51 port=@port@
  52 ipropport=@ipropport@
  53 ipropport2=@ipropport2@
  54
  55 cache="FILE:${objdir}/cache.krb5"
  56 keytabfile=${objdir}/iprop.keytab
  57 keytab="FILE:${keytabfile}"
  58
  59 kdc="${kdc} --addresses=localhost -P $port"
  60 kadmin="${kadmin} -r $R"
  61 kinit="${kinit} -c $cache ${afs_no_afslog}"
  62
  63 # We'll test iprop, and in particular, hierarchical iprop.  This means we'll
  64 # have a setup like:
  65 #
  66 #  ipropd-master -> ipropd-slave -> 2nd ipropd-master -> 2nd ipropd-slave
  67
  68 # Waiting for incremental propagation is inherently difficult because we don't
  69 # have a way for ipropd-slave to signal this script that it has received
  70 # updates.  Well, it does have a way to signal a possible ipropd-master for
  71 # hierarchical iprop, but we don't have a way to get that signal here.
  72 #
  73 # FIXME: Add a private interface for async waiting for iprop.
  74 #
  75 # What we do is we have a set of utility functions:
  76 #
  77 #  - get_iprop_ver   [N] -> checks that N (default to 1) ops have made it over
  78 #  - get_iprop_ver2  [N] -> same, but for second ipropd-slave instance
  79 #
  80 #  - wait_for            -> repeat a command until it succeeds or too many tries
  81 #  - wait_for_slave  [N] -> wait for N ops to make it over (calls get_iprop_ver)
  82 #  - wait_for_slave2 [N] -> same, but for second ipropd-slave instance
  83 #
  84 # In particular the wait_for* functions busy-wait for a max amount of time,
  85 # with sleeps in between.
  86 #
  87 # NOTE: get_iprop_ver and get_iprop_ver2 keep hidden state.
  88 #
  89 # When first called, get_iprop_ver / get_iprop_ver2 save the current version
  90 # numbers.  Thereafter they check that N ops have been received.
  91 #
  92 # It is critical to account for every incremental op via get_iprop_ver /
  93 # get_iprop_ver2, or wait_for_slave / wait_for_slave2, otherwise this test will
  94 # be racy and will have spurious failures!
  95 #
  96 # The pattern should be something like this:
  97 #
  98 #   echo "Add host"
  99 #   ${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1
 100 #   wait_for_slave
 101 #   ^^^^^^^^^^^^^^
 102 #      waits for 1 operation
 103 #
 104 # or
 105 #
 106 #   echo "Rollover host keys"
 107 #   ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
 108 #   ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
 109 #   ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
 110 #   wait_for_slave 3
 111 #   ^^^^^^^^^^^^^^^^
 112 #      waits for the three operations
 113 #
 114 # So though all operations must be accounted for, they need not be accounted
 115 # one by one.
 116
 117 slave_ver_from_master_old=
 118 slave_ver_from_master_new=
 119 slave_ver_old=
 120 slave_ver_new=
 121 get_iprop_ver () {
 122     min_change=${1:-1}
 123     slave_ver_from_master_new=`grep '^iprop/' iprop-stats | head -1 | awk '{print $3}'`
 124     slave_ver_new=`grep 'up-to-date with version:' iprop-slave-status | awk '{print $4}'`
 125     if [ -z "$slave_ver_from_master_new" -o -z "$slave_ver_new" ]; then
 126         return 1
 127     fi
 128     if [ x"$slave_ver_from_master_new" != x"$slave_ver_new" ]; then
 129         return 1
 130     fi
 131     if [ x"$slave_ver_from_master_old" != x ]; then
 132         change=`expr "$slave_ver_from_master_new" - "$slave_ver_from_master_old"`
 133         if [ "$change" -lt "$min_change" ]; then
 134             return 1
 135         fi
 136     fi
 137     slave_ver_from_master_old=$slave_ver_from_master_new
 138     slave_ver_old=$slave_ver_new
 139     return 0
 140 }
 141
 142 slave_ver_from_master_old2=
 143 slave_ver_from_master_new2=
 144 slave_ver_old2=
 145 slave_ver_new2=
 146 get_iprop_ver2 () {
 147     min_change=${1:-1}
 148     slave_ver_from_master_new2=`grep '^iprop/' iprop-stats2 | head -1 | awk '{print $3}'`
 149     slave_ver_new2=`grep 'up-to-date with version:' iprop-slave-status2 | awk '{print $4}'`
 150     if [ -z "$slave_ver_from_master_new2" -o -z "$slave_ver_new2" ]; then
 151         return 1
 152     fi
 153     if [ x"$slave_ver_from_master_new2" != x"$slave_ver_new2" ]; then
 154         return 1
 155     fi
 156     if [ x"$slave_ver_from_master_old2" != x ]; then
 157         change=`expr "$slave_ver_from_master_new2" - "$slave_ver_from_master_old2"`
 158         if [ "$change" -lt "$min_change" ]; then
 159             return 1
 160         fi
 161     fi
 162     slave_ver_from_master_old2=$slave_ver_from_master_new2
 163     slave_ver_old2=$slave_ver_new2
 164     return 0
 165 }
 166
 167 waitsec=65
 168 sleeptime=2
 169 wait_for () {
 170     msg=$1
 171     shift
 172     t=0
 173     while ! "$@"; do
 174         sleep $sleeptime;
 175         t=`expr $t + $sleeptime`
 176         if [ $t -gt $waitsec ]; then
 177             echo "Waited too long for $msg"
 178             exit 1
 179         fi
 180     done
 181     return 0
 182 }
 183
 184 check_pidfile_is_dead () {
 185     if test ! -f lt-${1}.pid -a ! -f ${1}.pid; then
 186         return 0
 187     fi
 188     _pid=`cat lt-${1}.pid ${1}.pid 2>/dev/null`
 189     if [ -z "$_pid" ]; then
 190         return 0
 191     fi
 192     if kill -0 $_pid 2>/dev/null; then
 193         return 1
 194     fi
 195     return 0
 196 }
 197
 198 wait_for_slave () {
 199     wait_for "iprop versions to change and/or slave to catch up" get_iprop_ver "$@"
 200 }
 201
 202 wait_for_slave2 () {
 203     wait_for "iprop versions to change and/or second slave to catch up" get_iprop_ver2 "$@"
 204 }
 205
 206 wait_for_master_down () {
 207     wait_for "master to exit" check_pidfile_is_dead ipropd-master
 208 }
 209
 210 wait_for_slave_down () {
 211     wait_for "slave to exit" check_pidfile_is_dead ipropd-slave
 212 }
 213
 214 KRB5_CONFIG="${objdir}/krb5.conf"
 215 export KRB5_CONFIG
 216
 217 rm -f ${keytabfile}
 218 rm -f current-db*
 219 rm -f current*.log
 220 rm -f out-*
 221 rm -f mkey.file*
 222 rm -f messages.log messages.log
 223
 224 > messages.log
 225 > messages.log2
 226
 227 echo Creating database
 228 ${kadmin} -l <<EOF || exit 1
 229 init --realm-max-ticket-life=1day --realm-max-renewable-life=1month ${R}
 230 add -p foo --use-defaults user@${R}
 231 add --random-key --use-defaults iprop/localhost@${R}
 232 ext -k ${keytab} iprop/localhost@${R}
 233 add --random-key --use-defaults iprop/slave.test.h5l.se@${R}
 234 ext -k ${keytab} iprop/slave.test.h5l.se@${R}
 235 EOF
 236
 237 echo foo > ${objdir}/foopassword
 238
 239 echo "Test log recovery"
 240 ${kadmin} -l add --random-key --use-defaults recovtest@${R} || exit 1
 241 # Test theory: save the log, make a change and save the record it
 242 # produced, restore the log, append to it the saved record, then add dummy
 243 # record.
 244
 245 # Save the log
 246 cp current.log current.log.tmp
 247 ls -l current.log.tmp | awk '{print $5}' > tmp
 248 read sz < tmp
 249 # Make a change
 250 ${kadmin} -l mod -a requires-pre-auth recovtest@${R} || exit 1
 251 ${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1
 252 # Save the resulting log record
 253 ls -l current.log | awk '{print $5}' > tmp
 254 read nsz < tmp
 255 rm tmp
 256 dd bs=1 if=current.log skip=$sz of=current.log.tmp.saved-record count=`expr $nsz - $sz` 2>/dev/null
 257 # Undo the change
 258 ${kadmin} -l mod -a -requires-pre-auth recovtest@${R} || exit 1
 259 ${kadmin} -l get recovtest@${R} | grep 'Attributes:.$' > /dev/null || exit 1
 260 # Restore the log
 261 cp current.log current.log.save
 262 mv current.log.tmp current.log
 263 # Append the saved record
 264 cat current.log.tmp.saved-record >> current.log
 265 rm current.log.tmp.saved-record
 266 # Check that we still see the principal as modified after another write forcing
 267 # log recovery.
 268 ${kadmin} -l add --random-key --use-defaults dummy@${R} || exit 1
 269 ${kadmin} -l del dummy@${R} || exit 1
 270 ${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1
 271
 272 # -- foo
 273 ipds=
 274 ipdm=
 275 kdcpid=
 276
 277 > iprop-stats
 278 > iprop-stats2
 279 rm -f iprop-slave-status iprop-slave-status2
 280
 281 ipropd_slave2=$ipropd_slave
 282 ipropd_master2=$ipropd_master
 283 ipropd_slave="${ipropd_slave} --status-file=iprop-slave-status --port=$ipropport"
 284 ipropd_slave="${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab}"
 285 ipropd_slave="${ipropd_slave} --detach localhost"
 286 ipropd_master="${ipropd_master} --hostname=localhost -k ${keytab}"
 287 ipropd_master="${ipropd_master} --port=$ipropport"
 288 ipropd_master="${ipropd_master} --database=${objdir}/current-db --detach"
 289
 290 ipropd_slave2="${ipropd_slave2} --status-file=iprop-slave-status2 --port=$ipropport2"
 291 ipropd_slave2="${ipropd_slave2} --hostname=slave.test.h5l.se -k ${keytab}"
 292 ipropd_slave2="${ipropd_slave2} --pidfile-basename=ipropd-slave2"
 293 ipropd_slave2="${ipropd_slave2} --detach localhost"
 294 ipropd_master2="${ipropd_master2} --hostname=localhost -k ${keytab}"
 295 ipropd_master2="${ipropd_master2} --port=$ipropport2"
 296 ipropd_master2="${ipropd_master2} --pidfile-basename=ipropd-master2"
 297 ipropd_master2="${ipropd_master2} --database=${objdir}/current-db.slave --detach"
 298
 299 cleanup() {
 300     echo 'killing ipropd s + m + kdc'
 301     test -n "$ipdm" && kill -9 $ipdm        >/dev/null 2>/dev/null
 302     test -n "$ipdm2" && kill -9 $ipdm2      >/dev/null 2>/dev/null
 303     test -n "$ipds" && kill -9 $ipds        >/dev/null 2>/dev/null
 304     test -n "$ipds2" && kill -9 $ipds2      >/dev/null 2>/dev/null
 305     test -n "$kdcpid" && kill -9 $kdcpid    >/dev/null 2>/dev/null
 306     trap '' EXIT INT TERM
 307     tail messages.log
 308     tail iprop-stats
 309     exit 1
 310 }
 311 trap cleanup EXIT INT TERM
 312
 313 echo Starting kdc ; > messages.log
 314 ${kdc} --detach --testing || { echo "kdc failed to start"; cat messages.log; exit 1; }
 315 kdcpid=`getpid kdc`
 316
 317 echo "starting master" ; > messages.log
 318 env ${HEIM_MALLOC_DEBUG} \
 319 ${ipropd_master} || { echo "ipropd-master failed to start"; exit 1; }
 320 ipdm=`getpid ipropd-master`
 321
 322 echo "starting slave" ; > messages.log
 323 env ${HEIM_MALLOC_DEBUG} \
 324 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 325 ${ipropd_slave} || { echo "ipropd-slave failed to start"; exit 1; }
 326 ipds=`getpid ipropd-slave`
 327 sh ${wait_kdc} ipropd-slave messages.log 'slave status change: up-to-date' || exit 1
 328 get_iprop_ver || exit 1
 329
 330 echo "checking slave is up"
 331 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null || exit 1
 332 ${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; }
 333
 334 # Also setup a second master on the slave, then a second slave to pull from the
 335 # second master.
 336 echo "starting master2" ; > messages.log
 337 env ${HEIM_MALLOC_DEBUG} \
 338 KRB5_CONFIG="${objdir}/krb5-master2.conf" \
 339 ${ipropd_master2} || { echo "second ipropd-master failed to start"; exit 1; }
 340 ipdm2=`getpid ipropd-master2`
 341
 342 echo "starting slave2" ; > messages.log
 343 env ${HEIM_MALLOC_DEBUG} \
 344 KRB5_CONFIG="${objdir}/krb5-slave2.conf" \
 345 ${ipropd_slave2} || { echo "ipropd-slave failed to start"; exit 1; }
 346 ipds2=`getpid ipropd-slave2`
 347 sh ${wait_kdc} ipropd-slave messages2.log 'slave status change: up-to-date' || exit 1
 348 wait_for "Slave sees new host" get_iprop_ver2 0 || exit 1
 349
 350 # ----------------- checking: pushing lives changes
 351
 352 slave_get() { KRB5_CONFIG="${objdir}/krb5-slave.conf" ${kadmin} -l get "$@"; }
 353 slave_check_exists() {
 354     # Creation with a random key is not atomic, there are at present
 355     # 3 log entries to create a random key principal, the entry is
 356     # "invalid" for the first two of these.  We wait for the entry to
 357     # exist and not be invalid
 358     #
 359     attrs=`slave_get -o attributes "$@" 2>/dev/null` || return 1
 360     echo $attrs | egrep 'Attributes:' | egrep -v invalid >/dev/null || return 1
 361     get_iprop_ver 0
 362 }
 363
 364 echo "Add host"
 365 ${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1
 366 wait_for_slave
 367 wait_for "Slave sees new host" slave_check_exists "host/foo@${R}"
 368
 369 echo "Rollover host keys"
 370 ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
 371 ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
 372 ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
 373 wait_for_slave 3
 374 slave_get host/foo@${R} | \
 375     ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' '
 376 ' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d '
 377 ' | ${EGREP} 1234 > /dev/null || exit 1
 378
 379 wait_for_slave2 4
 380
 381 echo "Delete 3DES keys"
 382 ${kadmin} -l del_enctype host/foo@${R} des3-cbc-sha1
 383 wait_for_slave
 384 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 385 ${kadmin} -l get host/foo@${R} | \
 386     ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' '
 387 ' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d '
 388 ' | ${EGREP} 1234 > /dev/null || exit 1
 389 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 390 ${kadmin} -l get host/foo@${R} | \
 391     ${EGREP} 'Keytypes:.*des3-cbc-sha1' > /dev/null && exit 1
 392
 393 echo "Change policy host"
 394 ${kadmin} -l modify --policy=default host/foo@${R} || exit 1
 395 wait_for_slave
 396 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 397 ${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null || exit 1
 398
 399 echo "Rename host"
 400 ${kadmin} -l rename host/foo@${R} host/bar@${R} || exit 1
 401 wait_for_slave
 402 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 403 ${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null && exit 1
 404 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 405 ${kadmin} -l get host/bar@${R} > /dev/null || exit 1
 406
 407 wait_for_slave2 3
 408
 409 echo "Delete host"
 410 ${kadmin} -l delete host/bar@${R} || exit 1
 411 wait_for_slave
 412 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 413 ${kadmin} -l get host/bar@${R} > /dev/null 2>/dev/null && exit 1
 414
 415 # See note below in LMDB sanity checking
 416 echo "Re-add host"
 417 ${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1
 418 ${kadmin} -l add --random-key --use-defaults host/bar@${R} || exit 1
 419 wait_for_slave 2
 420 wait_for "Slave sees re-added host" slave_check_exists "host/bar@${R}"
 421
 422 wait_for_slave2 3
 423
 424 echo "kill slave and remove log and database"
 425 > iprop-stats
 426 sh ${leaks_kill} ipropd-slave $ipds || exit 1
 427 rm -f iprop-slave-status
 428
 429 wait_for_slave_down
 430 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Down' iprop-stats >/dev/null || exit 1
 431
 432 # ----------------- checking: slave is missing changes while down
 433
 434 rm current.slave.log current-db.slave* || exit 1
 435
 436 echo "doing changes while slave is down"
 437 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 438 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 439
 440 echo "Making a copy of the master log file"
 441 cp ${objdir}/current.log ${objdir}/current.log.tmp
 442
 443 # ----------------- checking: checking that master and slaves resyncs
 444
 445 echo "starting slave again" ; > messages.log
 446 > iprop-stats
 447 env ${HEIM_MALLOC_DEBUG} \
 448 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 449 ${ipropd_slave} || { echo "ipropd-slave failed to start"; exit 1; }
 450 ipds=`getpid ipropd-slave`
 451
 452 echo "checking slave is up again"
 453 wait_for "slave to start and connect to master" \
 454     ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null
 455 wait_for_slave 2
 456 wait_for_slave2 2
 457 ${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; }
 458 echo "checking for replay problems"
 459 ${EGREP} 'Entry already exists in database' messages.log && exit 1
 460
 461 echo "compare versions on master and slave logs (no lock)"
 462 KRB5_CONFIG=${objdir}/krb5-slave.conf \
 463 ${iprop_log} last-version -n > slave-last.tmp
 464 ${iprop_log} last-version -n > master-last.tmp
 465 cmp master-last.tmp slave-last.tmp || exit 1
 466
 467 echo "kill slave and remove log and database"
 468 sh ${leaks_kill} ipropd-slave $ipds || exit 1
 469 wait_for_slave_down
 470
 471 rm current.slave.log current-db.slave* || exit 1
 472 > iprop-stats
 473 rm -f iprop-slave-status
 474 echo "starting slave" ; > messages.log
 475 env ${HEIM_MALLOC_DEBUG} \
 476 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 477 ${ipropd_slave} || { echo "ipropd-slave failed to start"; exit 1; }
 478 ipds=`getpid ipropd-slave`
 479 wait_for_slave 0
 480
 481 echo "checking slave is up again"
 482 wait_for "slave to start and connect to master" \
 483     ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null
 484 ${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; }
 485 echo "checking for replay problems"
 486 ${EGREP} 'Entry already exists in database' messages.log && exit 1
 487
 488 # ----------------- checking: checking live truncation of master log
 489
 490 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 491 wait_for_slave
 492 wait_for_slave2
 493
 494 echo "live truncate on master log"
 495 ${iprop_log} truncate -K 5 || exit 1
 496 wait_for_slave 0
 497
 498 echo "Killing master and slave"
 499 sh ${leaks_kill} ipropd-master $ipdm || exit 1
 500 sh ${leaks_kill} ipropd-slave $ipds || exit 1
 501
 502 rm -f iprop-slave-status
 503
 504 wait_for_slave_down
 505 wait_for_master_down
 506
 507 echo "compare versions on master and slave logs"
 508 KRB5_CONFIG=${objdir}/krb5-slave.conf \
 509 ${iprop_log} last-version > slave-last.tmp
 510 ${iprop_log} last-version > master-last.tmp
 511 cmp master-last.tmp slave-last.tmp || exit 1
 512
 513 # ----------------- checking: master going backward
 514 > iprop-stats
 515 > messages.log
 516
 517 echo "Going back to old version of the master log file"
 518 cp ${objdir}/current.log.tmp ${objdir}/current.log
 519
 520 echo "starting master"  ; > messages.log
 521 env ${HEIM_MALLOC_DEBUG} \
 522 ${ipropd_master} || { echo "ipropd-master failed to start"; exit 1; }
 523 ipdm=`getpid ipropd-master`
 524
 525 echo "starting slave" ; > messages.log
 526 env ${HEIM_MALLOC_DEBUG} \
 527 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 528 ${ipropd_slave} || { echo "ipropd-slave failed to start"; exit 1; }
 529 ipds=`getpid ipropd-slave`
 530 wait_for_slave -1
 531
 532 echo "checking slave is up again"
 533 wait_for "slave to start and connect to master" \
 534     ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null
 535 ${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; }
 536 echo "checking for replay problems"
 537 ${EGREP} 'Entry already exists in database' messages.log && exit 1
 538
 539 echo "pushing one change"
 540 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 541 wait_for_slave
 542 wait_for_slave2 0
 543
 544 echo "Killing master"
 545 sh ${leaks_kill} ipropd-master $ipdm || exit 1
 546
 547 wait_for_master_down
 548
 549 wait_for "slave to disconnect" \
 550   ${EGREP} 'disconnected' iprop-slave-status >/dev/null
 551
 552 if ! tail -30 messages.log | grep 'disconnected for server' > /dev/null; then
 553     echo "client didnt disconnect"
 554     exit 1
 555 fi
 556
 557 echo "probing for slave pid"
 558 kill -0 ${ipds}  || { echo "slave no longer there"; exit 1; }
 559
 560 > messages.log
 561
 562 echo "Staring master again" ; > messages.log
 563 env ${HEIM_MALLOC_DEBUG} \
 564 ${ipropd_master} || { echo "ipropd-master failed to start"; exit 1; }
 565 ipdm=`getpid ipropd-master`
 566
 567 echo "probing for slave pid"
 568 kill -0 ${ipds}  || { echo "slave no longer there"; exit 1; }
 569
 570
 571 echo "pushing one change"
 572 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 573 wait_for_slave
 574 wait_for_slave2
 575
 576 echo "shutting down all services"
 577
 578 leaked=false
 579 sh ${leaks_kill} kdc $kdcpid || leaked=true
 580 sh ${leaks_kill} ipropd-master $ipdm || leaked=true
 581 sh ${leaks_kill} ipropd-slave $ipds || leaked=true
 582 sh ${leaks_kill} ipropd-master $ipdm2 || leaked=true
 583 sh ${leaks_kill} ipropd-slave $ipds2 || leaked=true
 584 rm -f iprop-slave-status
 585 trap "" EXIT
 586 $leaked && exit 1
 587
 588 echo "compare versions on master and slave logs"
 589 KRB5_CONFIG=${objdir}/krb5-slave.conf \
 590 ${iprop_log} last-version > slave-last.tmp
 591 ${iprop_log} last-version > master-last.tmp
 592 cmp master-last.tmp slave-last.tmp || exit 1
 593
 594 if [ "$db_type" = lmdb ] && type mdb_stat > /dev/null 2>&1; then
 595     # Sanity check that we have the same number of principals at the HDB
 596     # and LMDB levels.
 597     #
 598     # We should also do this for the sqlite backend, but that would
 599     # require a sqlite3(1) shell that is capable of opening our HDB
 600     # files.
 601     echo "checking that principals in DB == entries in LMDB"
 602     # Add one to match lmdb overhead
 603     princs=`(echo; ${kadmin} -l list '*') | wc -l`
 604     entries=`mdb_stat -n current-db.mdb | grep 'Entries:' | awk '{print $2}'`
 605     [ "$princs" -eq "$entries" ] || exit 1
 606 fi
 607
 608 exit 0