scripts: Add helper script to log locking information using /proc/locks

This finds any processes locking tdb databases used by CTDB and logs
stack trace for each process.

Signed-off-by: Amitay Isaacs <amitay@gmail.com>

diff --git a/Makefile.in b/Makefile.in
index e3d6d5bfab37c4eb1f749ccbda17d6a89e9d4e81..136ae308d54957fa944dbc6e7bf5bf1722a24b17 100755 (executable)
--- a/Makefile.in
+++ b/Makefile.in
@@ -264,6 +264,7 @@ install: all
        ${INSTALLCMD} -m 644 config/functions $(DESTDIR)$(etcdir)/ctdb
        ${INSTALLCMD} -m 755 config/statd-callout $(DESTDIR)$(etcdir)/ctdb
        ${INSTALLCMD} -m 755 config/interface_modify.sh $(DESTDIR)$(etcdir)/ctdb
+       ${INSTALLCMD} -m 755 config/debug_locks.sh $(DESTDIR)$(etcdir)/ctdb
        ${INSTALLCMD} -m 644 config/events.d/README $(DESTDIR)$(docdir)/ctdb/README.eventscripts
        ${INSTALLCMD} -m 644 doc/recovery-process.txt $(DESTDIR)$(docdir)/ctdb/recovery-process.txt
        ${INSTALLCMD} -m 755 config/events.d/00.ctdb $(DESTDIR)$(etcdir)/ctdb/events.d

diff --git a/config/debug_locks.sh b/config/debug_locks.sh
new file mode 100755 (executable)
index 0000000..5b9efed
--- /dev/null
+++ b/config/debug_locks.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+# Create sed expression to convert inodes to names
+sed_cmd=$( ls -li /var/ctdb/*.tdb.* /var/ctdb/persistent/*.tdb.* |
+          sed -e "s#/var/ctdb[/persistent]*/\(.*\)#\1#" |
+          awk '{printf "s#[0-9]*:[0-9]*:%s #%s #\n", $1, $10}' )
+
+# Parse /proc/locks and extract following information
+#    pid process_name tdb_name offsets [W]
+out=$( cat /proc/locks |
+    grep -F "POSIX  ADVISORY  WRITE" |
+    awk '{ if($2 == "->") { print $6, $7, $8, $9, "W" } else { print $5, $6, $7, $8 } }' |
+    while read pid rest ; do
+       pname=$(readlink /proc/$pid/exe)
+       echo $pid $pname $rest
+    done | sed -e "$sed_fu" | grep "\.tdb" )
+
+if [ -n "$out" ]; then
+    # Log information about locks
+    echo "$out" | logger -t "debug-lock"
+
+    # Find processes that are waiting for locks
+    dbs=$(echo "$out" | grep "W$" | awk '{print $3}')
+    all_pids=""
+    for db in $dbs ; do
+       pids=$(echo "$out" | grep -v "W$" | grep "$db" | grep -v ctdbd | awk '{print $1}')
+       all_pids="$all_pids $pids"
+    done
+    pids=$(echo $all_pids | sort -u)
+
+    # For each process waiting, log stack trace
+    for pid in $pids ; do
+       gstack $pid | logger -t "debug-lock $pid"
+#      gcore -o /var/log/core-deadlock-ctdb $pid
+    done
+fi
+
+exit 0

diff --git a/packaging/RPM/ctdb.spec.in b/packaging/RPM/ctdb.spec.in
index de8193dee1fed41b80621b7e15d7f935619008b4..39a5df9db4ca672d13a594ecf9677198d66d8db8 100644 (file)
--- a/packaging/RPM/ctdb.spec.in
+++ b/packaging/RPM/ctdb.spec.in
@@ -113,6 +113,7 @@ rm -rf $RPM_BUILD_ROOT
 %{_sysconfdir}/ctdb/events.d/91.lvs
 %{_sysconfdir}/ctdb/statd-callout
 %{_sysconfdir}/ctdb/interface_modify.sh
+%{_sysconfdir}/ctdb/debug_locks.sh
 %{_sbindir}/ctdbd
 %{_bindir}/ctdb
 %{_bindir}/smnotify