Debugging Android core dumps

Debugging Android core dumps

When a process crashes, a core dump (called Tombstone in Android) is printed out in the logcat and stored under/data/tombstones/tombstone_nn, where nn is just a counter.

To get some useful info out of it we need to dereference it with the source. The libraries on the target are stripped of symbols, so you’ll need the unstripped ones from /out/product/name/symbols/system/libwhatever.so.

Then you can use the addr2line located in ${android-src}/prebuilt/linux-x86/toolchain/arm-eabi-4.4.0/bin/arm-eabi-addr2line to dereference it.

Getting hands on, the crash I am looking at looks like:

*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***Build fingerprint: 'unknown'pid: 1384, tid: 1384  >>> /system/bin/mediaserver <<<signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr e08f3007 r0 69708128  r1 00000000  r2 00000002  r3 00000000 r4 00000000  r5 69708128  r6 69709214  r7 0000adf8 r8 00000000  r9 00000000  10 00000000  fp 00000000 ip 697108f8  sp 7e8fca08  lr e08f3003  pc 4013eb04  cpsr 80000010 d0  968f6dec00000000  d1  6f20656c69709214 d2  6e6f635f726f6c62  d3  2e6e6f697372652f d4  0000000000000000  d5  0000000000000000 d6  3f80000000000000  d7  00000000ced2e124 d8  0000000000000000  d9  0000000000000000 d10 0000000000000000  d11 0000000000000000 d12 0000000000000000  d13 0000000000000000 d14 0000000000000000  d15 0000000000000000 d16 0000000000000000  d17 0000000000000000 d18 0000000000000000  d19 0000000000000000 d20 0000000000000000  d21 0000000000000000 d22 0000000000000000  d23 0000000000000000 d24 0000000000000000  d25 0000000000000000 d26 0000000000000000  d27 0000000000000000 d28 0000000000000000  d29 0000000000000000 d30 0000000000000000  d31 0000000000000000 scr 00000010         #00  pc 0003eb04  /system/lib/libasound.so         #01  pc 0003edc8  /system/lib/libasound.so         #02  pc 0000a9bc  /system/lib/libaudio.so         #03  pc 0000c2d8  /system/lib/libaudio.so         #04  pc 000234f8  /system/lib/libaudioflinger.so         #05  pc 00029a06  /system/lib/libaudioflinger.so         #06  pc 000088f2  /system/bin/mediaserver         #07  pc 000089e6  /system/bin/mediaserver         #08  pc 00014b52  /system/lib/libc.socode around pc:4013eae4 e1a0e00f e59cf000 e8bd8010 e92d40104013eaf4 e590e010 e1a03001 e3a02002 e3a010004013eb04 e59ec004 e1a0e00f e59cf000 e8bd80104013eb14 e590c010 e1a03002 e92d4010 e59c20084013eb24 e3120008 03e00015 08bd8010 e1a02001code around lr:e08f2fe0 ffffffff ffffffff ffffffff ffffffffe08f2ff0 ffffffff ffffffff ffffffff ffffffffe08f3000 ffffffff ffffffff ffffffff ffffffffe08f3010 ffffffff ffffffff ffffffff ffffffffe08f3020 ffffffff ffffffff ffffffff ffffffffstack:    7e8fc9c8  0000aed0    7e8fc9cc  697107b8    7e8fc9d0  0000afa0    7e8fc9d4  6970c17f  /system/lib/libaudio.so    7e8fc9d8  00000000    7e8fc9dc  0000aa98    7e8fc9e0  0000aaf8    7e8fc9e4  0000aac8    7e8fc9e8  68223204    7e8fc9ec  0000b854    7e8fc9f0  0000aaf8    7e8fc9f4  00000000    7e8fc9f8  0000001b    7e8fc9fc  0000aeb8    7e8fca00  df002777    7e8fca04  e3a070ad#00 7e8fca08  00000000    7e8fca0c  4013edcc  /system/lib/libasound.so#01 7e8fca10  0000adc8    7e8fca14  3f800000    7e8fca18  00000000    7e8fca1c  6970a9c0  /system/lib/libaudio.so#02 7e8fca20  0000adc8    7e8fca24  6970c2db  /system/lib/libaudio.so#03 7e8fca28  0000adc8    7e8fca2c  68d234fb  /system/lib/libaudioflinger.so#04 7e8fca30  0000adc8    7e8fca34  00000000    7e8fca38  00000000    7e8fca3c  3f800000    7e8fca40  00000000    7e8fca44  68d29a0b  /system/lib/libaudioflinger.so#05 7e8fca48  00000000    7e8fca4c  00000000    7e8fca50  00000000    7e8fca54  00000000    7e8fca58  00000000    7e8fca5c  00000000    7e8fca60  00000000    7e8fca64  00000000    7e8fca68  7e8fcab4    7e8fca6c  00000000    7e8fca70  00000000    7e8fca74  00000000    7e8fca78  00000000    7e8fca7c  0000adc8    7e8fca80  6821764d  /system/lib/libbinder.so    7e8fca84  0000ad50    7e8fca88  7e8fcaec    7e8fca8c  000088f5  /system/bin/mediaserver#06 7e8fca90  00000019    7e8fca94  7e8fc660    7e8fca98  0000ad98    7e8fca9c  0000ad50    7e8fcaa0  7e8fcab4    7e8fcaa4  7e8fcae4    7e8fcaa8  00000001    7e8fcaac  000089eb  /system/bin/mediaserver#07 7e8fcab0  00000001    7e8fcab4  7e8fcae4    7e8fcab8  0000ad50    7e8fcabc  0000b848    7e8fcac0  00008894  /system/bin/mediaserver    7e8fcac4  6fd14b55  /system/lib/libc.so#08 7e8fcac8  00000000    7e8fcacc  00000000    7e8fcad0  00000000    7e8fcad4  00000000    7e8fcad8  00000000    7e8fcadc  ffffffe4    7e8fcae0  00000001    7e8fcae4  7e8fcbf0    7e8fcae8  00000000    7e8fcaec  7e8fcc08    7e8fcaf0  7e8fcc45    7e8fcaf4  7e8fcc6d    7e8fcaf8  7e8fcc80    7e8fcafc  7e8fcc95    7e8fcb00  7e8fccb0    7e8fcb04  7e8fccc3    7e8fcb08  7e8fcce0    7e8fcb0c  7e8fcd00

Which can be dereferenced as follows:

#00  pc 0003eb04  /system/lib/libasound.so${android-src}/prebuilt/linux-x86/toolchain/arm-eabi-4.4.0/bin/arm-eabi-addr2line -f -e out/target/product/imx51_ccwmx51js/symbols/system/lib/libasound.so 0003eb04snd_mixer_selem_has_playback_channel/home/alex/Projects/android/external/alsa-lib/src/mixer/simple.c:270

I’ll omit the addr2line call from now on and just show the results.

#01  pc 0003edc8  /system/lib/libasound.sosnd_mixer_selem_set_playback_volume_all/home/alex/Projects/android/external/alsa-lib/src/mixer/simple.c:478#02  pc 0000a9bc  /system/lib/libaudio.so_ZN7android9ALSAMixer15snd_mixer_selem_set_playback_volume_all/home/alex/Projects/android/hardware/alsa_sound/ALSAMixer.cpp:237#03  pc 0000c2d8  /system/lib/libaudio.so_ZN7android18A2dpAudioInterface15setMasterVolumeEf/home/alex/Projects/android/frameworks/base/services/audioflinger/A2dpAudioInterface.cpp:210#04  pc 000234f8  /system/lib/libaudioflinger.so_ZN7android12AudioFlinger15setMasterVolumeEf/home/alex/Projects/android/frameworks/base/services/audioflinger/AudioFlinger.cpp:445#05  pc 00029a06  /system/lib/libaudioflinger.soAudioFlinger/home/alex/Projects/android/frameworks/base/services/audioflinger/AudioFlinger.cpp:144#06  pc 000088f2  /system/bin/mediaserver_ZN7android13BinderServiceINS_12AudioFlingerEE7publishEv/home/alex/Projects/android/frameworks/base/include/binder/BinderService.h:39#07  pc 000089e6  /system/bin/mediaserver_ZN7android13BinderServiceINS_12AudioFlingerEE11instantiateEv/home/alex/Projects/android/frameworks/base/include/binder/BinderService.h:50#08  pc 00014b52  /system/lib/libc.so__libc_init/home/alex/Projects/android/bionic/libc/bionic/libc_init_dynamic.c:114

Update: If you want a semiautomatic way, try the following script:

#!/bin/bash# $1 is the name of the file including the dump# $2 is the name of the library# parse_stack.py needs to be in your path# CustomizeANDROID_BASE=PRODUCT=# Do not change after this lineSOLIB_NAME=$2PATH_TO_SOLIB=${ANDROID_BASE}/out/target/product/${PRODUCT}/symbols/system/lib/${SOLIB_NAME}${ANDROID_BASE}/prebuilt/linux-x86/toolchain/arm-eabi-4.4.0/bin/arm-eabi-objdump -S ${PATH_TO_SOLIB} > ${SOLIB_NAME}.armparse_stack.py ${SOLIB_NAME}.arm $1rm ${SOLIB_NAME}.arm

With parse_stack.py in your path:

#!/usr/bin/python# -*- coding: utf-8 -*-import sysimport reimport ossohead = re.compile('(.+\.so):')funchead = re.compile('([0-9a-f]{8}) <(.+)>:')funcline = re.compile('^[ ]+([0-9a-f]+):.+')def parsestack( lines, libname ):    crashline = re.compile('.+pc.([0-9a-f]{8}).+%s' % libname )    ret = []    for l in lines:        m = crashline.match(l)        if m:            addr =  m.groups()[0]            ret.append(int(addr,16))    return retdef parseasm( lines ):    ret = []    current = None    restartcode = False;    for l in lines:        m = funchead.match(l)        if m:            if current:                ret.append(current)            startaddr, funcname =  m.groups()            current = [ funcname, int(startaddr,16), int(startaddr,16), int(startaddr,16), [] ]            continue        m = funcline.match(l)        if m:            restartcode = True;            addr =  m.groups()[0]            if current != None:                current[3] = int(addr,16)            continue        m = sohead.match(l)        if m:            so =  m.groups()[0]            so = os.path.split(so)[1]            continue        #Assume anything else is a code line        if restartcode:#            print 'XXX',l            restartcode = False;            ret.append(current)            current = [ current[0], current[1], current[3], current[3], [] ]        if current != None:            current[4].append(l);    return so, retif __name__=="__main__":    asm, stack = sys.argv[1],sys.argv[2]    libname, asm = parseasm( file(asm).read().split('\n') )    stack = parsestack( file(stack).read().split('\n'), libname )    for addr in stack:        for func, funcstart, segstart, segend, code in asm:            if addr >= segstart and addr <= segend:                print "0x%08x:%32s + 0x%04x %s" % ( addr, func, addr-funcstart, "".join(["\n"+x for x in code]))

To use it you will need a file containing a dump. You can get it from logcat with:

user@computer:$ adb logcat -d > logcat.txt

Using the one above as reference, we would call the script with:

user@computer:$ android_stack_dump.sh my.dump libasound.so
android_stack_dump.sh my.dump libaudio.so
android_stack_dump.sh my.dump libaudioflinger.so
android_stack_dump.sh my.dump libc.so