1 什么是死鎖
死鎖 (DeadLock): 是指兩個(gè)或兩個(gè)以上的進(jìn)程(線程)在執(zhí)行過程中,因爭奪資源而造成的一種互相等待的現(xiàn)象,若無外力作用,它們都將無法推進(jìn)下去。此時(shí)稱系統(tǒng)處于死鎖狀態(tài)或系統(tǒng)產(chǎn)生了死鎖,這些永遠(yuǎn)在互相等待的進(jìn)程(線程)稱為死鎖進(jìn)程(線程)。 由于資源占用是互斥的,當(dāng)某個(gè)進(jìn)程提出申請資源后,使得有關(guān)進(jìn)程(線程)在無外力協(xié)助下,永遠(yuǎn)分配不到必需的資源而無法繼續(xù)運(yùn)行,這就產(chǎn)生了一種特殊現(xiàn)象死鎖。
一種交叉持鎖死鎖的情形,此時(shí)執(zhí)行程序中兩個(gè)或多個(gè)線程發(fā)生永久堵塞(等待),每個(gè)線程都在等待被其它線程占用并堵塞了的資源。例如,如果線程 1 鎖住了記錄 A 并等待記錄 B,而線程 2 鎖住了記錄 B 并等待記錄 A,這樣兩個(gè)線程就發(fā)生了死鎖現(xiàn)象。在計(jì)算機(jī)系統(tǒng)中 , 如果系統(tǒng)的資源分配策略不當(dāng),更常見的可能是程序員寫的程序有錯(cuò)誤等,則會導(dǎo)致進(jìn)程因競爭資源不當(dāng)而產(chǎn)生死鎖的現(xiàn)象。
2 產(chǎn)生死鎖的四個(gè)必要條件
(1)互斥使用(資源獨(dú)占)
一個(gè)資源每次只能給一個(gè)進(jìn)程使用(比如寫操作)
(2)占有且等待
進(jìn)程在申請新的資源的同時(shí),保持對原有資源的占有
(3)不可搶占
資源申請者不能強(qiáng)行從資源占有者手中奪取資源,資源只能由占有者自愿釋放
(4)循環(huán)等待
P1等待P2占有的資源,P2等待P3的資源,...Pn等待P1的資源,形成一個(gè)進(jìn)程等待回路
3 一個(gè)例子及圖示
3.1 圖示
進(jìn)程在執(zhí)行一些代碼之后,子線程 1 獲得了鎖 A,正試圖獲得鎖 B,子線程 2 此時(shí)獲得了鎖 B,正試圖獲得鎖 A,這樣子線程 1 和子線程 2 將沒有辦法得到鎖 A 和鎖 B,因?yàn)樗鼈兏髯员粚Ψ秸加校肋h(yuǎn)不會釋放,從而發(fā)生了死鎖的現(xiàn)象。
3.2 代碼
#include <unistd.h>
#include <pthread.h>
#include <string.h>
pthread_mutex_t mutexA = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t mutexB = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t mutexC = PTHREAD_MUTEX_INITIALIZER;
static int counterA = 0;
static int counterB = 0;
int func1()
{
pthread_mutex_lock(&mutexA);
++counterA;
sleep(1);
pthread_mutex_lock(&mutexB);
++counterB;
pthread_mutex_unlock(&mutexB);
pthread_mutex_unlock(&mutexA);
return counterA;
}
int func2()
{
pthread_mutex_lock(&mutexB);
++counterB;
sleep(1);
pthread_mutex_lock(&mutexA);
++counterA;
pthread_mutex_unlock(&mutexA);
pthread_mutex_unlock(&mutexB);
return counterB;
}
void* start_routine1(void* arg)
{
while (1)
{
int iRetValue = func1();
if (iRetValue == 100000)
{
pthread_exit(NULL);
}
}
}
void* start_routine2(void* arg)
{
while (1)
{
int iRetValue = func2();
if (iRetValue == 100000)
{
pthread_exit(NULL);
}
}
}
void* start_routine(void* arg)
{
while (1)
{
sleep(1);
char szBuf[128];
memset(szBuf, 0, sizeof(szBuf));
strcpy(szBuf, (char*)arg);
}
}
int main()
{
pthread_t tid[4];
if (pthread_create(&tid[0], NULL, &start_routine1, NULL) != 0)
{
_exit(1);
}
if (pthread_create(&tid[1], NULL, &start_routine2, NULL) != 0)
{
_exit(1);
}
if (pthread_create(&tid[2], NULL, &start_routine, "thread3") != 0)
{
_exit(1);
}
if (pthread_create(&tid[3], NULL, &start_routine, "thread3") != 0)
{
_exit(1);
}
sleep(5);
//pthread_cancel(tid[0]);
pthread_join(tid[0], NULL);
pthread_join(tid[1], NULL);
pthread_join(tid[2], NULL);
pthread_join(tid[3], NULL);
pthread_mutex_destroy(&mutexA);
pthread_mutex_destroy(&mutexB);
pthread_mutex_destroy(&mutexC);
return 0;
}
3.3編譯程序并執(zhí)行
[hadoop@spark ~]$ gcc -g deadlock.c -o deadlock -lpthread
[hadoop@spark ~]$ ./deadlock
[hadoop@spark ~]$ ps -ef |grep -i deadlock
hadoop 103176 101848 0 15:42 pts/0 00:00:00 ./deadlock
4 使用 pstack 和 gdb 工具對死鎖程序進(jìn)行分析
4.1 pstack
第一次:
[hadoop@spark ~]$ pstack 103176
Thread 5 (Thread 0x7f3af34d2700 (LWP 103177)):
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3af389fdcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f3af389fc98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000004008ee in func1 () at deadlock.c:18
#4 0x000000000040098b in start_routine1 (arg=0x0) at deadlock.c:43
#5 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f3af2cd1700 (LWP 103178)):
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3af389fdcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f3af389fc98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000040094a in func2 () at deadlock.c:31
#4 0x00000000004009b9 in start_routine2 (arg=0x0) at deadlock.c:56
#5 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f3af24d0700 (LWP 103179)):
#0 0x00007f3af35921ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f3af3592044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f3af1ccf700 (LWP 103180)):
#0 0x00007f3af35921ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f3af3592044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f3af3cbd740 (LWP 103176)):
#0 0x00007f3af389ef57 in pthread_join () from /lib64/libpthread.so.0
#1 0x0000000000400aee in main () at deadlock.c:99
[hadoop@spark ~]$
第二次:
[hadoop@spark ~]$
[hadoop@spark ~]$ pstack 103176
Thread 5 (Thread 0x7f3af34d2700 (LWP 103177)):
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3af389fdcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f3af389fc98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000004008ee in func1 () at deadlock.c:18
#4 0x000000000040098b in start_routine1 (arg=0x0) at deadlock.c:43
#5 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f3af2cd1700 (LWP 103178)):
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3af389fdcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f3af389fc98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000040094a in func2 () at deadlock.c:31
#4 0x00000000004009b9 in start_routine2 (arg=0x0) at deadlock.c:56
#5 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f3af24d0700 (LWP 103179)):
#0 0x00007f3af35921ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f3af3592044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f3af1ccf700 (LWP 103180)):
#0 0x00007f3af35921ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f3af3592044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f3af3cbd740 (LWP 103176)):
#0 0x00007f3af389ef57 in pthread_join () from /lib64/libpthread.so.0
#1 0x0000000000400aee in main () at deadlock.c:99
[hadoop@spark ~]$
連續(xù)多次查看這個(gè)進(jìn)程的函數(shù)調(diào)用關(guān)系堆棧, 死鎖線程將一直處于等鎖的狀態(tài),對比多次的函數(shù)調(diào)用堆棧輸出結(jié)果,確定哪兩個(gè)線程(或者幾個(gè)線程)一直沒有變化且一直處于等鎖的狀態(tài)。
分析:
根據(jù)上面的輸出對比,線程 1 和線程 2 由第一次 pstack 輸出的處在 sleep 函數(shù)變化為第二次 pstack 輸出的處在 memset 函數(shù)。但是線程 4 和線程 5 一直處在等鎖狀態(tài)(pthread_mutex_lock),在連續(xù)兩次的 pstack 信息輸出中沒有變化,所以我們可以推測線程 4 和線程 5 發(fā)生了死鎖。
然后通過 gdb attach 到死鎖進(jìn)程
[hadoop@spark ~]$ gdb attach 103176
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-100.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
attach: No such file or directory.
Attaching to process 103176
Reading symbols from /home/hadoop/deadlock...done.
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[New LWP 103180]
[New LWP 103179]
[New LWP 103178]
[New LWP 103177]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
0x00007f3af389ef57 in pthread_join () from /lib64/libpthread.so.0
Missing separate debuginfos, use: debuginfo-install glibc-2.17-196.el7_4.2.x86_64
(gdb) info thread
Id Target Id Frame
5 Thread 0x7f3af34d2700 (LWP 103177) "deadlock" 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
4 Thread 0x7f3af2cd1700 (LWP 103178) "deadlock" 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
3 Thread 0x7f3af24d0700 (LWP 103179) "deadlock" 0x00007f3af35921ad in nanosleep () from /lib64/libc.so.6
2 Thread 0x7f3af1ccf700 (LWP 103180) "deadlock" 0x00007f3af35921ad in nanosleep () from /lib64/libc.so.6
* 1 Thread 0x7f3af3cbd740 (LWP 103176) "deadlock" 0x00007f3af389ef57 in pthread_join () from /lib64/libpthread.so.0
(gdb)
查看線程 4 和線程 5 的輸出
(gdb) thread 5
[Switching to thread 5 (Thread 0x7f3af34d2700 (LWP 103177))]
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
(gdb) where
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3af389fdcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f3af389fc98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000004008ee in func1 () at deadlock.c:18
#4 0x000000000040098b in start_routine1 (arg=0x0) at deadlock.c:43
#5 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
(gdb) f 3
#3 0x00000000004008ee in func1 () at deadlock.c:18
18 pthread_mutex_lock(&mutexB);
(gdb) thread 4
[Switching to thread 4 (Thread 0x7f3af2cd1700 (LWP 103178))]
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
(gdb) where
#0 0x00007f3af38a442d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f3af389fdcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f3af389fc98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000040094a in func2 () at deadlock.c:31
#4 0x00000000004009b9 in start_routine2 (arg=0x0) at deadlock.c:56
#5 0x00007f3af389de25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f3af35cb34d in clone () from /lib64/libc.so.6
(gdb) f 3
#3 0x000000000040094a in func2 () at deadlock.c:31
31 pthread_mutex_lock(&mutexA);
(gdb) p mutexA
$1 = {__data = {__lock = 2, __count = 0, __owner = 103177, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
__size = "\002\000\000\000\000\000\000\000\t\223\001\000\001", '\000' <repeats 26 times>, __align = 2}
(gdb) p mutexB
$2 = {__data = {__lock = 2, __count = 0, __owner = 103178, __nusers = 1, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
__size = "\002\000\000\000\000\000\000\000\n\223\001\000\001", '\000' <repeats 26 times>, __align = 2}
(gdb) p mutexC
$3 = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}
(gdb)
從上面可以發(fā)現(xiàn),線程 4 正試圖獲得鎖 mutexA,但是鎖 mutexA 已經(jīng)被 LWP 為 103177 的線程得到(__owner = 103177),線程 5 正試圖獲得鎖 mutexB,但是鎖 mutexB 已經(jīng)被 LWP 為 103178 的 得到(__owner = 103178),從 pstack 的輸出可以發(fā)現(xiàn),LWP 103177 與線程 5 是對應(yīng)的,LWP 103178 與線程 4 是對應(yīng)的。所以我們可以得出, 線程 4 和線程 5 發(fā)生了交叉持鎖的死鎖現(xiàn)象。查看線程的源代碼發(fā)現(xiàn),線程 4 和線程 5 同時(shí)使用 mutexA 和 mutexB,且申請順序不合理。
5 利用core文件分析
運(yùn)行./deadlock(編譯的時(shí)候加調(diào)試選項(xiàng)-g) 死鎖阻塞
[hadoop@spark ~]$ ulimit -c unlimited
[hadoop@spark ~]$ ./deadlock
[hadoop@spark ~]$ ps -ef |grep deadlock
hadoop 33397 32933 0 08:49 pts/0 00:00:00 ./deadlock
hadoop 33472 102129 0 08:50 pts/2 00:00:00 grep --color=auto deadlock
[hadoop@spark ~]$ pstack 33397
Thread 5 (Thread 0x7f71f6f11700 (LWP 33398)):
#0 0x00007f71f72e342d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f71f72dedcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f71f72dec98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000004008ee in func1 () at deadlock.c:18
#4 0x000000000040098b in start_routine1 (arg=0x0) at deadlock.c:43
#5 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f71f6710700 (LWP 33399)):
#0 0x00007f71f72e342d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f71f72dedcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f71f72dec98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000040094a in func2 () at deadlock.c:31
#4 0x00000000004009b9 in start_routine2 (arg=0x0) at deadlock.c:56
#5 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f71f5f0f700 (LWP 33400)):
#0 0x00007f71f6fd11ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f71f6fd1044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f71f570e700 (LWP 33401)):
#0 0x00007f71f6fd11ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f71f6fd1044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f71f76fc740 (LWP 33397)):
#0 0x00007f71f72ddf57 in pthread_join () from /lib64/libpthread.so.0
#1 0x0000000000400aee in main () at deadlock.c:99
Ctrl+\ 產(chǎn)生core dump
[hadoop@spark ~]$ ./deadlock
^\Quit (core dumped)
[hadoop@spark ~]$ ls core.*
core.33397
[hadoop@spark ~]$ gdb deadlock core.33397
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-100.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /home/hadoop/deadlock...done.
[New LWP 33397]
[New LWP 33398]
[New LWP 33399]
[New LWP 33400]
[New LWP 33401]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `./deadlock'.
Program terminated with signal 3, Quit.
#0 0x00007f71f72ddf57 in pthread_join () from /lib64/libpthread.so.0
Missing separate debuginfos, use: debuginfo-install glibc-2.17-196.el7_4.2.x86_64
(gdb) thread apply all bt
Thread 5 (Thread 0x7f71f570e700 (LWP 33401)):
#0 0x00007f71f6fd11ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f71f6fd1044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f71f5f0f700 (LWP 33400)):
#0 0x00007f71f6fd11ad in nanosleep () from /lib64/libc.so.6
#1 0x00007f71f6fd1044 in sleep () from /lib64/libc.so.6
#2 0x00000000004009ed in start_routine (arg=0x400be0) at deadlock.c:69
#3 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f71f6710700 (LWP 33399)):
#0 0x00007f71f72e342d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f71f72dedcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f71f72dec98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000040094a in func2 () at deadlock.c:31
#4 0x00000000004009b9 in start_routine2 (arg=0x0) at deadlock.c:56
#5 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f71f6f11700 (LWP 33398)):
#0 0x00007f71f72e342d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f71f72dedcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f71f72dec98 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000004008ee in func1 () at deadlock.c:18
#4 0x000000000040098b in start_routine1 (arg=0x0) at deadlock.c:43
#5 0x00007f71f72dce25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f71f700a34d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f71f76fc740 (LWP 33397)):
#0 0x00007f71f72ddf57 in pthread_join () from /lib64/libpthread.so.0
#1 0x0000000000400aee in main () at deadlock.c:99