ctdb/tests/simple/70_recoverpdbbyseqnum.sh

   1 #!/bin/bash
   2
   3 test_info()
   4 {
   5     cat <<EOF
   6 The tunable RecoverPDBBySeqNum controls how we perform recovery
   7 on persistent databases.
   8 The default is that persistent databases are recovered exactly the same
   9 way as normal databases. That is that we recover record by record.
  10
  11 If RecoverPDBBySeqNum is set to 1 AND if a record with the key
  12 "__db_sequence_number__" can be found in the database, then instead we will
  13 perform the recovery by picking the copy of the database from the node
  14 that has the highest sequence number and ignore the content on all other
  15 nodes.
  16
  17
  18 Prerequisites:
  19
  20 * An active CTDB cluster with at least 2 active nodes.
  21
  22 Steps:
  23
  24 1. Verify that the status on all of the ctdb nodes is 'OK'.
  25 2. create a persistent test database
  26 3. test that RecoveryPDBBySeqNum==0 and no seqnum record blends the database
  27    during recovery
  28 4. test that RecoveryPDBBySeqNum==0 and seqnum record blends the database
  29    during recovery
  30 5. test that RecoveryPDBBySeqNum==1 and no seqnum record blends the database
  31    during recovery
  32 6. test that RecoveryPDBBySeqNum==1 and seqnum record does not blend the database
  33    during recovery
  34
  35 Expected results:
  36
  37 * that 3,4,5 will blend the databases and that 6 will recovery the highest seqnum
  38   database
  39
  40 EOF
  41 }
  42
  43 . "${TEST_SCRIPTS_DIR}/integration.bash"
  44
  45 ctdb_test_init "$@"
  46
  47 set -e
  48
  49 cluster_is_healthy
  50
  51 # Reset configuration
  52 ctdb_restart_when_done
  53
  54 try_command_on_node 0 "$CTDB listnodes"
  55 num_nodes=$(echo "$out" | wc -l)
  56
  57 add_record_per_node ()
  58 {
  59     _i=0
  60     while [ $_i -lt $num_nodes ] ; do
  61         _k="KEY${_i}"
  62         _d="DATA${_i}"
  63         echo "Store key(${_k}) data(${_d}) on node ${_i}"
  64         db_ctdb_tstore $_i "$test_db" "$_k" "$_d"
  65         _i=$(($_i + 1))
  66     done
  67 }
  68
  69 test_db="persistent_test.tdb"
  70 echo "Create persistent test database \"$test_db\""
  71 try_command_on_node 0 $CTDB attach "$test_db" persistent
  72
  73
  74 echo "Setting RecoverPDBBySeqNum=0"
  75 try_command_on_node all $CTDB setvar "RecoverPDBBySeqNum" 0
  76
  77 # 3.
  78 # If RecoverPDBBySeqNum==0  and no __db_sequence_number__
  79 # recover record by record
  80 #
  81 # wipe database
  82 echo
  83 echo "Test that RecoverPDBBySeqNum=0 and no __db_sequence_number__ blends the database during recovery"
  84
  85 echo "Wipe test database"
  86 try_command_on_node 0 $CTDB wipedb "$test_db"
  87
  88 add_record_per_node
  89
  90 # force a recovery
  91 echo "Force a recovery"
  92 try_command_on_node 0 $CTDB recover
  93
  94 # check that we now have both records on node 0
  95 num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
  96 if [ $num_records = "$num_nodes" ] ; then
  97     echo "OK: databases were blended"
  98 else
  99     echo "BAD: we did not end up with the expected $num_nodes records after the recovery"
 100     exit 1
 101 fi
 102
 103 # 4.
 104 # If RecoverPDBBySeqNum==0  and __db_sequence_number__
 105 # recover record by record
 106 #
 107 # wipe database
 108 echo
 109 echo "Test that RecoverPDBBySeqNum=0 and __db_sequence_number__ blends the database during recovery"
 110
 111 echo "Wipe the test database"
 112 try_command_on_node 0 $CTDB wipedb persistent_test.tdb
 113
 114 add_record_per_node
 115
 116 echo "Add __db_sequence_number__==5 record to all nodes"
 117 pnn=0
 118 while [ $pnn -lt $num_nodes ] ; do
 119     db_ctdb_tstore_dbseqnum $pnn "$test_db" 5
 120     pnn=$(($pnn + 1))
 121 done
 122
 123 echo "Set __db_sequence_number__ to 7 on node 0"
 124 db_ctdb_tstore_dbseqnum 0 "$test_db" 7
 125
 126 echo "Set __db_sequence_number__ to 8 on node 1"
 127 db_ctdb_tstore_dbseqnum 1 "$test_db" 8
 128
 129 # force a recovery
 130 echo "Force a recovery"
 131 try_command_on_node 0 $CTDB recover
 132
 133 # check that we now have both records on node 0
 134 num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
 135 if [ $num_records = "$num_nodes" ] ; then
 136     echo "OK: databases were blended"
 137 else
 138     echo "BAD: we did not end up with the expected $num_nodes records after the recovery"
 139     try_command_on_node -v 0 $CTDB cattdb "$test_db"
 140     exit 1
 141 fi
 142
 143
 144 # set RecoverPDBBySeqNum=1
 145 echo
 146 echo "Setting RecoverPDBBySeqNum to 1"
 147 try_command_on_node all $CTDB setvar "RecoverPDBBySeqNum" 1
 148
 149
 150 # 5,
 151 # If RecoverPDBBySeqNum==1  and no __db_sequence_number__
 152 # recover whole database
 153 #
 154 # wipe database
 155 echo
 156 echo "Test that RecoverPDBBySeqNum=1 and no __db_sequence_number__ does not blend the database during recovery"
 157 echo "Wipe the test database"
 158 try_command_on_node 0 $CTDB wipedb "$test_db"
 159
 160 add_record_per_node
 161
 162 # force a recovery
 163 echo force a recovery
 164 try_command_on_node 0 $CTDB recover
 165
 166 # Check that we now have 1 record on node 0
 167 num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
 168 if [ $num_records = "1" ] ; then
 169     echo "OK: databases were not blended"
 170 else
 171     echo "BAD: we did not end up with the expected single record after the recovery"
 172     exit 1
 173 fi
 174
 175
 176
 177 # 6,
 178 # If RecoverPDBBySeqNum==1  and __db_sequence_number__
 179 # recover whole database
 180 #
 181 # wipe database
 182 echo
 183 echo test that RecoverPDBBySeqNum==1 and __db_sequence_number__ does not blend the database during recovery
 184 echo wipe the test database
 185 try_command_on_node 0 $CTDB wipedb persistent_test.tdb
 186
 187 add_record_per_node
 188
 189 echo "Add __db_sequence_number__==5 record to all nodes"
 190 pnn=0
 191 while [ $pnn -lt $num_nodes ] ; do
 192     db_ctdb_tstore_dbseqnum $pnn "$test_db" 5
 193     pnn=$(($pnn + 1))
 194 done
 195
 196 echo "Set __db_sequence_number__ to 7 on node 0"
 197 db_ctdb_tstore_dbseqnum 0 "$test_db" 7
 198
 199 echo "Set __db_sequence_number__ to 8 on node 1"
 200 db_ctdb_tstore_dbseqnum 1 "$test_db" 8
 201
 202
 203 # force a recovery
 204 echo force a recovery
 205 try_command_on_node 0 $CTDB recover
 206
 207 # check that we now have both records on node 0
 208 num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
 209 if [ $num_records = "1" ] ; then
 210     echo "OK: databases were not blended"
 211 else
 212     echo "BAD: we did not end up with the expected single record after the recovery"
 213     exit 1
 214 fi