ctdb/config/debug_locks.sh

   1 #!/bin/sh
   2
   3 # This script parses /proc/locks and finds the processes that are holding
   4 # locks on CTDB databases.  For all those processes the script dumps a
   5 # stack trace using gstack.
   6 #
   7 # This script can be used only if Samba is configured to use fcntl locks
   8 # rather than mutex locks.
   9
  10 [ -n "$CTDB_BASE" ] || \
  11     export CTDB_BASE=$(cd -P $(dirname "$0") ; echo "$PWD")
  12
  13 . "$CTDB_BASE/functions"
  14
  15 loadconfig ctdb
  16
  17 # Create sed expression to convert inodes to names
  18 sed_cmd=$( ls -li "$CTDB_DBDIR"/*.tdb.* "$CTDB_DBDIR_PERSISTENT"/*.tdb.* |
  19            sed -e "s#${CTDB_DBDIR}/\(.*\)#\1#" \
  20                -e "s#${CTDB_DBDIR_PERSISTENT}/\(.*\)#\1#" |
  21            awk '{printf "s#[0-9]*:[0-9]*:%s #%s #\n", $1, $10}' )
  22
  23 # Parse /proc/locks and extract following information
  24 #    pid process_name tdb_name offsets [W]
  25 out=$( cat /proc/locks |
  26     grep -F "POSIX  ADVISORY  WRITE" |
  27     awk '{ if($2 == "->") { print $6, $7, $8, $9, "W" } else { print $5, $6, $7, $8 } }' |
  28     while read pid rest ; do
  29         pname=$(readlink /proc/$pid/exe)
  30         echo $pid $pname $rest
  31     done | sed -e "$sed_cmd" | grep "\.tdb" )
  32
  33 if [ -n "$out" ]; then
  34     # Log information about locks
  35     echo "$out" | logger -t "ctdbd-lock"
  36
  37     # Find processes that are waiting for locks
  38     dbs=$(echo "$out" | grep "W$" | awk '{print $3}')
  39     all_pids=""
  40     for db in $dbs ; do
  41         pids=$(echo "$out" | grep -v "W$" | grep "$db" | grep -v ctdbd | awk '{print $1}')
  42         all_pids="$all_pids $pids"
  43     done
  44     pids=$(echo $all_pids | sort -u)
  45
  46     # For each process waiting, log stack trace
  47     for pid in $pids ; do
  48         gstack $pid | logger -t "ctdbd-lock $pid"
  49 #       gcore -o /var/log/core-deadlock-ctdb $pid
  50     done
  51 fi
  52
  53 exit 0