3 # This script parses /proc/locks and finds the processes that are holding
4 # locks on CTDB databases. For all those processes the script dumps a
7 # This script can be used only if Samba is configured to use fcntl locks
8 # rather than mutex locks.
10 [ -n "$CTDB_BASE" ] || \
11 CTDB_BASE=$(d=$(dirname "$0") ; cd -P "$d" ; echo "$PWD")
13 . "${CTDB_BASE}/functions"
15 # Load/cache database options from configuration file
21 echo "===== Start of debug locks PID=$$ ====="
23 # Create sed expression to convert inodes to names.
24 # Filenames don't contain dashes and we want basenames
25 # shellcheck disable=SC2035
26 sed_cmd=$(cd "$CTDB_DBDIR" &&
27 stat -c "s#[0-9a-f]*:[0-9a-f]*:%i #%n #" *.tdb.* 2>/dev/null ;
28 cd "$CTDB_DBDIR_PERSISTENT" &&
29 stat -c "s#[0-9a-f]*:[0-9a-f]*:%i #%n #" *.tdb.* 2>/dev/null)
31 # Parse /proc/locks and extract following information
32 # pid process_name tdb_name offsets [W]
33 out=$( grep -F "POSIX ADVISORY WRITE" /proc/locks |
34 awk '{ if($2 == "->") { print $6, $7, $8, $9, "W" } else { print $5, $6, $7, $8 } }' |
35 while read pid rest ; do
36 pname=$(readlink "/proc/${pid}/exe")
37 echo "$pid $pname $rest"
38 done | sed -e "$sed_cmd" | grep '\.tdb' )
40 if [ -n "$out" ]; then
41 # Log information about locks
44 # Find processes that are waiting for locks
45 dbs=$(echo "$out" | grep "W$" | awk '{print $3}')
48 pids=$(echo "$out" | grep -v "W$" | grep "$db" | grep -v ctdbd | awk '{print $1}')
49 all_pids="$all_pids $pids"
51 # Use word splitting to squash whitespace
52 # shellcheck disable=SC2086
53 pids=$(echo $all_pids | tr ' ' '\n' | sort -u)
55 # For each process waiting, log stack trace
57 echo "----- Stack trace for PID=$pid -----"
58 # x is intentionally ignored
59 # shellcheck disable=SC2034
60 read x x state x <"/proc/${pid}/stat"
61 if [ "$state" = "D" ] ; then
62 # Don't run gstack on a process in D state since
63 # gstack will hang until the process exits D state.
64 # Although it is possible for a process to transition
65 # to D state after this check, it is unlikely because
66 # if a process is stuck in D state then it is probably
67 # the reason why this script was called. Note that a
68 # kernel stack almost certainly won't help diagnose a
69 # deadlock... but it will probably give us someone to
71 echo "----- Process in D state, printing kernel stack only"
72 cat "/proc/${pid}/stack"
75 # gcore -o /var/log/core-deadlock-ctdb $pid
80 echo "===== End of debug locks PID=$$ ====="
81 )9>"${CTDB_SCRIPT_VARDIR}/debug_locks.lock" | script_log "ctdbd-lock"