linux命令之pstack

很多时候我们想知道在Linux下后台程序到底运行到哪里了，卡住了吗，出错了吗，最简单的我们会使用

# ps auxf | grep <process_name>

来查看后台程序的状态，可是如果想知道的更多，那就可以用到pstack这个命令了。

首先举一个简单的例子(test.c)来引出这个命令 

#include <stdio.h>#include <unistd.h>#include <pthread.h>void *thread_proc(void *data){    printf("a new thread creaded\n");    while (1)    {        sleep(10);    }    return (void *)0;}int main(){    pthread_t tid;    pthread_create(&tid, NULL, thread_proc, NULL);    pthread_join(tid, NULL);    return 0;}

# gcc -g -Wall -Werror test.c -o test -pthread

# ./test  &

# ps auxf | grep test           // 得到进程号(pid)

# pstack $pid                    

结果如下

// ****begin**************************

Thread 2 (Thread 0x7f7c78fc2710 (LWP 3198)):
#0  0x0000003c0f2a6a8d in nanosleep () from /lib64/libc.so.6
#1  0x0000003c0f2a6900 in sleep () from /lib64/libc.so.6
#2  0x0000000000400624 in thread_proc ()
#3  0x0000003c0f6077e1 in start_thread () from /lib64/libpthread.so.0
#4  0x0000003c0f2e153d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f7c78fc4700 (LWP 3197)):
#0  0x0000003c0f60803d in pthread_join () from /lib64/libpthread.so.0
#1  0x000000000040065a in main ()

// ****end**************************

简单的分析一下， Thread 1为主线程， 首先执行的是main函数，接着停在pthread_join一直等待另一个线程的结束；Thread 2为主线程create出来的线程，通过结果我们可以看到，首先执行的是clone, 执行

# man clone                 // 可以看到clone的详细介绍，这里可以理解为在main函数中创建多线程

然后调用sleep， 而最终调用的是nanosleep

所以通过pstack 就可以很容易的后台的进程现在在干什么，而且也可以用来分析卡住的进程卡在哪里了。

再来看看pstack究竟是什么

# which pstack

/usr/bin/pstack                    // 显示命令所在位置

# ll /usr/bin/pastck

lrwxrwxrwx. 1 root root 6 Aug  1 21:10 /usr/bin/pstack -> gstack                 // 原来pstack是一个链接文件

# vi /usr/bin/gstack            // 分析一下gstack, 原来pstack就是由gdb执行的shell脚本，gdb原来这么强大啊，下一篇文章就来分析一下gdb

 17 backtrace="bt" 18 if test -d /proc/$1/task ; then 19     # Newer kernel; has a task/ directory. 20     if test `/bin/ls /proc/$1/task | /usr/bin/wc -l` -gt 1 2>/dev/null ; then 21         backtrace="thread apply all bt" 22     fi 23 elif test -f /proc/$1/maps ; then 24     # Older kernel; go by it loading libpthread. 25     if /bin/grep -e libpthread /proc/$1/maps > /dev/null 2>&1 ; then 26         backtrace="thread apply all bt" 27     fi 28 fi

 38 # Run GDB, strip out unwanted noise. 39 $GDB --quiet $readnever -nx /proc/$1/exe $1 <<EOF 2>&1 |  40 set width 0 41 set height 0 42 set pagination no 43 $backtrace 44 EOF 45 /bin/sed -n \ 46     -e 's/^\((gdb) \)*//' \ 47     -e '/^#/p' \ 48     -e '/^Thread/p'

核心代码如上，这里先不做分析，到下一章gdb中一起分析。

文章出处    http://www.cnblogs.com/mumuxinfei/p/4366708.html

pstack检测死锁

既然pstack可以打印出该进程的所有线程的情况，那它自然就可以用来检测死锁了。首先下一个死锁的例子

# vi dead_lock.c

#include <stdio.h>#include <pthread.h>#include <unistd.h>pthread_mutex_t mutex_1 = PTHREAD_MUTEX_INITIALIZER;pthread_mutex_t mutex_2 = PTHREAD_MUTEX_INITIALIZER;void *thread1_proc(void *data){    pthread_mutex_lock(&mutex_1);    sleep(1);    pthread_mutex_lock(&mutex_2);    pthread_mutex_unlock(&mutex_2);    pthread_mutex_unlock(&mutex_1);    return (void *)0;}void *thread2_proc(void *data){    pthread_mutex_lock(&mutex_2);    sleep(1);    pthread_mutex_lock(&mutex_1);    pthread_mutex_unlock(&mutex_1);    pthread_mutex_unlock(&mutex_2);    return (void *)0;}int main(){    pthread_t tid1, tid2;    pthread_create(&tid1, NULL, thread1_proc, NULL);    pthread_create(&tid2, NULL, thread2_proc, NULL);    pthread_join(tid1, NULL);    pthread_join(tid2, NULL);    return 0;}

# gcc -g -Wall -Werror dead_lock.c -pthread -o test

# ./test            // 则进程死锁一直卡住了

# pstack $pid 

Thread 3 (Thread 0x7f0489d8e710 (LWP 3616)):
#0  0x0000003c0f60e034 in __lll_lock_wait () from /lib64/libpthread.so.0
#1  0x0000003c0f609345 in _L_lock_870 () from /lib64/libpthread.so.0
#2  0x0000003c0f609217 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3  0x000000000040068e in thread1_proc ()
#4  0x0000003c0f6077e1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003c0f2e153d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f048938d710 (LWP 3617)):
#0  0x0000003c0f60e034 in __lll_lock_wait () from /lib64/libpthread.so.0
#1  0x0000003c0f609345 in _L_lock_870 () from /lib64/libpthread.so.0
#2  0x0000003c0f609217 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3  0x00000000004006d3 in thread2_proc ()
#4  0x0000003c0f6077e1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003c0f2e153d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f0489d90700 (LWP 3615)):
#0  0x0000003c0f60803d in pthread_join () from /lib64/libpthread.so.0
#1  0x000000000040073d in main ()

可以看到Thread 2和Thread 3都在等待锁，就是等待别人释放自己想要锁的那把锁， 但是并不能看出来是否是死锁，继续使用gdb分析

# gdb -p $pid

# info thread        // 打印所有的线程信息

  3 Thread 0x7f645eb23710 (LWP 3687)  0x0000003c0f60e034 in __lll_lock_wait () from /lib64/libpthread.so.0
  2 Thread 0x7f645e122710 (LWP 3688)  0x0000003c0f60e034 in __lll_lock_wait () from /lib64/libpthread.so.0
* 1 Thread 0x7f645eb25700 (LWP 3686)  0x0000003c0f60803d in pthread_join () from /lib64/libpthread.so.0

*表示gdb锁定的线程，切换到第二个线程去查看

# thread 2          // 切换到第2个线程, 可以看到线程id 为 0x7f645e122710, 而LWP指定的值是gdb用来唯一标示该进程中线程的，便于调试的时候追踪

[Switching to thread 2 (Thread 0x7f645e122710 (LWP 3688))]#0  0x0000003c0f60e034 in __lll_lock_wait ()  

# bt                   // bt 可以打印函数堆栈，却无法看到函数参数，

#0  0x0000003c0f60e034 in __lll_lock_wait () from /lib64/libpthread.so.0
#1  0x0000003c0f609345 in _L_lock_870 () from /lib64/libpthread.so.0
#2  0x0000003c0f609217 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3  0x00000000004006d3 in thread2_proc (data=0x0) at dead_lock.c:23
#4  0x0000003c0f6077e1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003c0f2e153d in clone () from /lib64/libc.so.6

# frame 3       // 打印第三帧信息(#2).每次函数调用都会有压栈的过程，而frame则记录栈中的帧信息，

#3  0x00000000004006d3 in thread2_proc (data=0x0) at dead_lock.c:23
23     pthread_mutex_lock(&mutex_1);

# p mutext_1  // 打印mutex_1的值 ,  __owner表示gdb中标示线程的值，即LWP

$1 = {__data = {__lock = 2, __count = 0, __owner = 3687, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0,
      __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000g\016\000\000\001", '\000' <repeats 26 times>, __align = 2}
# thread 3

# frame 3

# p mutex_2

$2 = {__data = {__lock = 2, __count = 0, __owner = 3688, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0,
      __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000h\016\000\000\001", '\000' <repeats 26 times>, __align = 2}

分析可以知道死锁了， 因为LWP(3688)在等待LWP(3687)所拥有的mutex_1, 而同时LWP(3688)又在等待LWP(3688)所拥有的mutex_2, 死锁。

如果每个线程在去取锁的时候都打印一条日志记录自己取到了哪个锁，或者正打算去取哪个锁，这样如果程序卡住的话可以通过查询日志看到是否有死锁，添加代码如下

#define pthread_mutex_lock(arg1, arg2) \do { \   printf("the thread %s try to  get the lock %s\n", arg1, #arg2);  \   pthread_mutex_lock(arg2);   \   printf("the thread %s get the lock %s\n", arg1, #arg2);  \}while(0);

修改部分如下

pthread_mutex_lock("thread1", &mutex_1);   // thread2部分相同sleep(1);pthread_mutex_lock("thread1", &mutex_2);

...

重新编译后执行

# ./test

the thread thread2 try to  get the lock &mutex_2
the thread thread2 get the lock &mutex_2
the thread thread1 try to  get the lock &mutex_1
the thread thread1 get the lock &mutex_1
the thread thread2 try to  get the lock &mutex_1
the thread thread1 try to  get the lock &mutex_2

然后就可以分析得出死锁的结论。

参考博文:   http://www.cnblogs.com/mumuxinfei/p/4365697.html