openssl RAND_poll导致死锁

服务器死锁了，但并不是我们的锁导致的，是openssl导致的，以下是死锁具体内容

-----------------------------------------
DebugInfo = 0x002a1710
*** WARNING: Unable to verify checksum for ESServer.exe
Critical section = 0x005e4278 (ESServer!gDBabc+0x0)
LOCKED
LockCount = 0x2
WaiterWoken = No
OwningThread = 0x00000638
RecursionCount = 0x1
LockSemaphore = 0x444
SpinCount = 0x00000000
-----------------------------------------

-----------------------------------------
DebugInfo = 0x778f0820
Critical section = 0x00250130 (+0x250130)
LOCKED
LockCount = 0xC
WaiterWoken = No
OwningThread = 0x00000efc
RecursionCount = 0x1
LockSemaphore = 0x568
SpinCount = 0x00000fa0
-----------------------------------------

-----------------------------------------
DebugInfo = 0x002969f8
Critical section = 0x01410130 (+0x1410130)
LOCKED
LockCount = 0x2
WaiterWoken = No
OwningThread = 0x00001238
RecursionCount = 0x1
LockSemaphore = 0x578
SpinCount = 0x00000fa0
-----------------------------------------

  35 Id: 1390.638 Suspend: 1 Teb: 7ef60000 Unfrozen
ChildEBP RetAddr Args to Child
08b7f810 77848dcd 00000568 00000000 00000000 ntdll!ZwWaitForSingleObject+0x15
08b7f874 77848d98 00000000 00000000 00250000 ntdll!RtlpWaitOnCriticalSection+0x155
08b7f89c 778722ee 00250130 30b5d835 00250000 ntdll!RtlEnterCriticalSection+0x152
08b7f980 7783a5d7 000000d0 000000d8 00250254 ntdll!RtlpAllocateHeap+0x15c
08b7f9f8 76a617fd 00250000 00000000 000000d0 ntdll!RtlAllocateHeap+0x1e3
08b7fa0c 75ad4a98 76b2f6f8 000000d0 08b7fa5c ole32!CRetailMalloc_Alloc+0x16 [d:\longhorn\com\ole32\com\class\memapi.cxx @ 641]
08b7fa2c 75ad4b30 08b7fa6c 00000038 08b7fa50 oleaut32!InitAppData+0x43
08b7fa3c 75ad45e5 08b7fa5c 00000039 00000080 oleaut32!GetAppData+0x23
08b7fa50 75ad46c7 08b7fa6c 00000000 08b7fae8 oleaut32!SysAllocStringLen+0x2d
08b7fa60 00549dfc 08b7fa6c 00650073 0065006c oleaut32!SysAllocString+0x2c
08b7fae8 0042a92a 005dce44 00000000 00000000 ESServer!_com_util::ConvertStringToBSTR+0x6c
08b7fb58 00531b1d 005dce44 01b0a450 00000000 ESServer!CDataBase::RecordRead+0x5a [F:\code\10.9\epol\EPolServer\DataBase.cpp @ 291]
08b7fde8 00531479 01b0a450 00000000 08b7ff94 ESServer!CSuperiorReport::GetRmUserid+0x4d [F:\code\10.9\epol\EPolServer\SuperiorReport.cpp @ 2395]
08b7ff1c 0052696d 00000000 00000000 00000000 ESServer!CSuperiorReport::OnTimmer+0x109 [F:\code\10.9\epol\EPolServer\SuperiorReport.cpp @ 2309]
08b7ff88 75c3eccb 01b0a450 08b7ffd4 7788d24d ESServer!CSuperiorReport::MainThread+0xcd [F:\code\10.9\epol\EPolServer\SuperiorReport.cpp @ 800]
08b7ff94 7788d24d 01b0a450 30b5de61 00000000 kernel32!BaseThreadInitThunk+0xe
08b7ffd4 7788d45f 005268a0 01b0a450 ffffffff ntdll!__RtlUserThreadStart+0x23
08b7ffec 00000000 005268a0 01b0a450 00000000 ntdll!_RtlUserThreadStart+0x1b

  39 Id: 1390.efc Suspend: 1 Teb: 7ef33000 Unfrozen
ChildEBP RetAddr Args to Child
0a54f2bc 77848dcd 00000578 00000000 00000000 ntdll!ZwWaitForSingleObject+0x15
0a54f320 77848d98 00000000 00000000 0a54f4d0 ntdll!RtlpWaitOnCriticalSection+0x155
0a54f348 77849532 01410130 00000000 00000000 ntdll!RtlEnterCriticalSection+0x152
0a54f37c 778a5ad1 01410000 3256d5d1 0a54f510 ntdll!RtlLockHeap+0x3d
0a54f464 778a5f8c 0a54f4d0 77881d27 00000000 ntdll!RtlpQueryExtendedHeapInformation+0xbd
0a54f4a4 7788230e 00000000 00000002 0a54f4d0 ntdll!RtlQueryHeapInformation+0x4a
0a54f548 7786a424 0a780000 00400000 0a54f67c ntdll!RtlQueryProcessHeapInformation+0x285
0a54f5c4 75c3f650 00001390 00000014 0a780000 ntdll!RtlQueryProcessDebugInformation+0x2ab
*** ERROR: Symbol file could not be found. Defaulted to export symbols for libeay32.dll -
0a54f5f4 009bb475 0a780000 00000001 00000020 kernel32!Heap32Next+0x51
WARNING: Stack unwind information not available. Following frames may be wrong.
0a54f65c 009d50cc 00ded14c 018ae177 00000021 libeay32!RAND_poll+0x495
00000000 00000000 00000000 00000000 00000000 libeay32!ASN1_item_ex_d2i+0x10fc

  16 Id: 1390.1238 Suspend: 1 Teb: 7ef78000 Unfrozen
ChildEBP RetAddr Args to Child
0837f5e0 77848dcd 00000568 00000000 00000000 ntdll!ZwWaitForSingleObject+0x15
0837f644 77848d98 00000000 00000000 00250000 ntdll!RtlpWaitOnCriticalSection+0x155
0837f66c 778722ee 00250130 3035d6e5 00250000 ntdll!RtlEnterCriticalSection+0x152
0837f750 7783a5d7 000003f8 00000400 00000000 ntdll!RtlpAllocateHeap+0x15c
0837f7c8 7783defb 00250000 00800000 000003f8 ntdll!RtlAllocateHeap+0x1e3
0837f810 7783dd7e 3035d93d 00250000 00000000 ntdll!RtlpAllocateUserBlock+0xa2
0837f888 7783acbd 0a670048 0a670048 00000000 ntdll!RtlpLowFragHeapAllocFromContext+0x785
0837f8f4 77880341 00250000 00000000 00000020 ntdll!RtlAllocateHeap+0x17c
0837f904 7788086b 01410000 0a670048 00000000 ntdll!RtlpAllocateDebugInfo+0x24
0837f940 7783d7d4 0a670048 00000000 00000000 ntdll!RtlInitializeCriticalSectionEx+0x93
0837f954 77854993 0a670048 0a670048 01410000 ntdll!RtlInitializeCriticalSection+0x12
0837f968 77854b52 00000000 00000800 01410000 ntdll!RtlpInitializeLowFragHeap+0x28
0837f978 77854ac8 3035d805 014100cc 01410000 ntdll!RtlpCreateLowFragHeap+0x28
0837f9b0 77899033 01410000 0141019c 0837faa4 ntdll!RtlpActivateLowFragmentationHeap+0xc9
0837f9c0 7783abb7 01410000 3035db11 01410000 ntdll!RtlpPerformHeapMaintenance+0x2a
0837faa4 7783a5d7 00000018 00000020 0141019c ntdll!RtlpAllocateHeap+0x175
0837fb1c 77263146 01410000 00000000 00000018 ntdll!RtlAllocateHeap+0x1e3
0837fb30 77263a82 00000018 00000000 00000000 ws2_32!operator new+0x16
0837fb44 7546795f 01410850 0000057c 0837fd0c ws2_32!WPUModifyIFSHandle+0x40
0837fd64 7727bbdb 000003a4 0837fddc 0837fdd8 mswsock!WSPAccept+0xacc
0837fd98 7727be0d 000003a4 0837fddc 0837fdd8 ws2_32!WSAAccept+0x85
0837fdb4 0040dabe 000003a4 0837fddc 0837fdd8 ws2_32!accept+0x17
0837ff88 75c3eccb 005e5408 0837ffd4 7788d24d ESServer!CCS::ListenForDownload+0x19e [F:\code\10.9\epol\EPolServer\CS.cpp @ 1483]
0837ff94 7788d24d 005e5408 3035de61 00000000 kernel32!BaseThreadInitThunk+0xe
0837ffd4 7788d45f 0040d920 005e5408 ffffffff ntdll!__RtlUserThreadStart+0x23
0837ffec 00000000 0040d920 005e5408 00000000 ntdll!_RtlUserThreadStart+0x1b

 

参考以下网址
http://social.msdn.microsoft.com/Forums/zh-SG/windbg/thread/1f5765fd-c77d-44c4-95d3-a9e9d233f553

有人也遇到过类似问题

Hello.

I'm seeing a deadlock in the Windows heap that I'd like to get some expert eyes on before declaring it a Windows bug. The deadlock is between one thread calling MSVCR100!realloc and another thread calling kernel32!Heap32Next.

I initially thought this was caused by heap corruption, but I've turned on all the heap verification flags and nothing shows up. Further, there are no async exceptions being thrown or caught in the process before the deadlock occurs.

Even further, if I recompile OpenSSL without the code that calls Heap32Next (and friends), the deadlock disappears. This leads me to conclude that this is a Windows bug, as the heap snapshot functions don't come with a "no multi-threading" warning label.

Minidump: http://www.plexapp.com/mfeingol/bugs/plex-laika-47/minidump.zip
/MA dump: http://www.plexapp.com/mfeingol/bugs/plex-laika-47/dump_ma.zip
Relevant symbols and binaries: http://www.plexapp.com/mfeingol/bugs/plex-laika-47/Symbols.zip

This is on Windows 7 SP1 x64, running a 32-bit process with app-verifier enabled. The deadlock occurs when app-verified is disabled, but is much harder to reproduce.

In this particular instance, threads 0 and 7 are deadlocked:

0:007> ~
   0 Id: 2d44.2740 Suspend: 1 Teb: fffdd000 Unfrozen
. 7 Id: 2d44.2c14 Suspend: 1 Teb: fffa3000 Unfrozen

Thread 0 is waiting on critical section 05d00138, which is held by Thread 7:

0:007> !cs 05d00138
-----------------------------------------
Critical section = 0x05d00138 (+0x5D00138)
DebugInfo = 0x775b4980
LOCKED
LockCount = 0x1
WaiterWoken = No
OwningThread = 0x00002c14
RecursionCount = 0x1
LockSemaphore = 0x4B0
SpinCount = 0x00000fa0

Thread 7 is waiting on critical section 08150138, which is held by Thread 0:

0:007> !cs 08150138
-----------------------------------------
Critical section = 0x08150138 (+0x8150138)
DebugInfo = 0x05d07248
LOCKED
LockCount = 0x1
WaiterWoken = No
OwningThread = 0x00002740
RecursionCount = 0x1
LockSemaphore = 0x504
SpinCount = 0x00000fa0

This is a classic deadlock...

Stack traces for both threads:

0:000> kb
ChildEBP RetAddr Args to Child
0103e7c4 774e8dd4 000004b0 00000000 00000000 ntdll!ZwWaitForSingleObject+0x15
0103e828 774e8cb8 00000000 00000000 00000001 ntdll!RtlpWaitOnCriticalSection+0x13e
0103e850 77580d03 05d00138 787c127d 00000000 ntdll!RtlEnterCriticalSection+0x150
0103e894 7753ae1e 05d00000 11000102 00000048 ntdll!RtlDebugAllocateHeap+0x9d
0103e978 774e3cce 00000048 00000000 00000000 ntdll!RtlpAllocateHeap+0xc4
0103e9fc 650ca682 05d00000 01000002 00000048 ntdll!RtlAllocateHeap+0x23a
0103ea20 650c8f6e 05a61000 05d00000 01000002 verifier!AVrfpDphNormalHeapAllocate+0xb2
0103ea54 77580c96 05a60000 01000002 00000020 verifier!AVrfDebugPageHeapAllocate+0x30e
0103eaa0 7753ae1e 05a60000 01000002 00000020 ntdll!RtlDebugAllocateHeap+0x30
0103eb84 774e3cce 00000020 00000000 00000000 ntdll!RtlpAllocateHeap+0xc4
0103ec08 774e7ea9 05a60000 01000002 00000020 ntdll!RtlAllocateHeap+0x23a
0103ec18 774e4829 08150000 0a5c7370 00000000 ntdll!RtlpAllocateDebugInfo+0x28
0103ec54 774e2c54 0a5c7370 00000000 00000000 ntdll!RtlInitializeCriticalSectionEx+0x93
0103ec68 774efd6c 0a5c7370 0a5c7370 08150000 ntdll!RtlInitializeCriticalSection+0x12
0103ec7c 774efd3a 00000000 00000800 08150000 ntdll!RtlpInitializeLowFragHeap+0x28
0103ec8c 774efae4 787c162d 081500cc 08150000 ntdll!RtlpCreateLowFragHeap+0x28
0103ecc4 774efb8c 08150000 081501ec 0103edb0 ntdll!RtlpActivateLowFragmentationHeap+0xc9
0103ecd4 774efb58 08150000 787c1759 09ed0000 ntdll!RtlpPerformHeapMaintenance+0x2a
0103edb0 774e3cce 0000005f 00000068 081501ec ntdll!RtlpAllocateHeap+0x172
0103ee34 650ca682 08150000 00001002 0000005f ntdll!RtlAllocateHeap+0x23a
0103ee58 650c8f6e 09ed1000 08150000 00001002 verifier!AVrfpDphNormalHeapAllocate+0xb2
0103ee8c 650c92b2 09ed0000 00001002 00000037 verifier!AVrfDebugPageHeapAllocate+0x30e
0103eecc 77580fc3 09ed0002 00001002 0a599438 verifier!AVrfDebugPageHeapReAllocate+0x1a2
0103ef1c 7753e171 09ed0000 00000000 0a599438 ntdll!RtlDebugReAllocateHeap+0x33
0103ef98 650dcd9f 09ed0000 00000000 0a599438 ntdll!RtlReAllocateHeap+0x54
0103f034 698d4707 09ed0000 00000000 0a599438 verifier!AVrfpRtlReAllocateHeap+0xf3
0103f054 5b198359 0a599438 00000037 065180c1 MSVCR100!realloc+0x45
0103f070 68f48fad 0815faf8 06519c58 06518bc8 cpluff!end_element_handler+0x229

0:007> kb
ChildEBP RetAddr Args to Child
0a2bed7c 774e8dd4 00000504 00000000 00000000 ntdll!ZwWaitForSingleObject+0x15
0a2bede0 774e8cb8 00000000 00000000 0a2bef90 ntdll!RtlpWaitOnCriticalSection+0x13e
0a2bee08 774e8185 08150138 00000000 09ed0000 ntdll!RtlEnterCriticalSection+0x150
0a2bee3c 7756f6c5 08150000 735415cd 0a2befd0 ntdll!RtlLockHeap+0x3d
0a2bef24 7753db37 0a2bef90 77553a65 00000000 ntdll!RtlpQueryExtendedHeapInformation+0xbd
0a2bef64 775540ff 00000000 00000002 0a2bef90 ntdll!RtlQueryHeapInformation+0x4a
0a2bf008 7753302c 0ab70000 75420000 0a2bf11c ntdll!RtlQueryProcessHeapInformation+0x288
0a2bf084 754b599b 00002d44 00000014 0ab70000 ntdll!RtlQueryProcessDebugInformation+0x28a
0a2bf0b4 0a31135e 0ab70000 0a2bf84c 00000003 kernel32!Heap32Next+0x4d
0a2bf5c4 0a3116c5 6b258c7a 6b2b2000 6b2b2290 libeay32!RAND_poll+0x58e

Benhaha
There is some mention of this on Raymond Chen's blog:

* http://blogs.msdn.com/b/oldnewthing/archive/2012/03/23/10286665.aspx#comments

Short answer: the Heap32* functions are only for diagnostic purposes and shouldn't be used otherwise. Certainly they shouldn't be used to generate entropy for cryptographic purposes. That's what CryptGenRandom is for. (If you don't like that consider using the performance monitor data returned under HKEY_PERFORMANCE_DATA as an entropy source instead.)

mfeingol Plex

Benhaha:

I agree with your observation that Heap32* shouldn't be used. Someone should talk to the OpenSSL authors about that.

However, if an API is offered to the public, then it shouldn't deadlock itself. It's really that simple.

It's okay if an API is slow. It's okay if an API is mostly useless. But calling it shouldn't deadlock your program. If it does, the API isn't even safe for diagnostic purposes; it's simply unsafe.

 

两篇文章，与这个相关
http://social.msdn.microsoft.com/Forums/zh-SG/windbg/thread/1f5765fd-c77d-44c4-95d3-a9e9d233f553

http://blogs.msdn.com/b/oldnewthing/archive/2012/03/23/10286665.aspx#comments

 

可能需要更换openssl版本到最新的
参考
http://blog.csdn.net/piao_polar/article/details/6860110 

 

openssl的dll中用到诸如heap32next这样函数

文中提到
the Heap32* functions are only for diagnostic purposes and shouldn't be used otherwise

不然就有可能会导致性能问题

 

openssl报的BUG

http://rt.openssl.org/Ticket/Display.html?id=2100&user=guest&pass=guest