Doku Wiki

An example of debugging a kernel panic by disassembling an ARM Linux kernel.

[  128.459816] Unable to handle kernel NULL pointer dereference at virtual address 00000000
[  128.468018] Unable to handle kernel NULL pointer dereference at virtual address 00000002
[  128.468026] pgd = d8f38000
[  128.468035] [00000002] *pgd=1a660831, *pte=00000000, *ppte=00000000
[  128.468041] Internal error: Oops: 817 [#1] PREEMPT SMP ARM
[  128.468046] Modules linked in:
[  128.468062] task: d8f29300 ti: d8f70000 task.ti: d8f70000
[  128.468074] PC is at futex_wait_setup+0x90/0x14c
[  128.468080] LR is at get_parent_ip+0x10/0x2c
[  128.468086] pc : [<c00704d8>]    lr : [<c0040f30>]    psr: 600d0013
[  128.468086] sp : d8f71df8  ip : f66b1657  fp : 00000000
[  128.468090] r10: 00000002  r9 : 00000000  r8 : d8f71e88
[  128.468094] r7 : d8f70000  r6 : b56b1654  r5 : 00000002  r4 : d8f71ea4
[  128.468099] r3 : d96c4240  r2 : 000002ce  r1 : b56b1654  r0 : d8f71e04
[  128.468104] Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment user
[  128.468109] Control: 10c5387d  Table: 18f3806a  DAC: 00000015

As you can see:

• kernel panic occurred in futex_wait_setup, offset 0x90
• register r5 contains the value 00000002

From kernel/futex.c:

/* The key must be already stored in q->key. */
static inline struct futex_hash_bucket *queue_lock(struct futex_q *q)
        __acquires(&hb->lock)
{
        struct futex_hash_bucket *hb;

        hb = hash_futex(&q->key);

        /*
         * Increment the counter before taking the lock so that
         * a potential waker won't miss a to-be-slept task that is
         * waiting for the spinlock. This is safe as all queue_lock()
         * users end up calling queue_me(). Similarly, for housekeeping,
         * decrement the counter at queue_unlock() when some error has
         * occurred and we don't end up adding the task to the list.
         */
        hb_waiters_inc(hb);

        q->lock_ptr = &hb->lock;

        spin_lock(&hb->lock); /* implies MB (A) */
        return hb;
}

/**
 * futex_wait_setup() - Prepare to wait on a futex
 * @uaddr:      the futex userspace address
 * @val:        the expected value 
 * @flags:      futex flags (FLAGS_SHARED, etc.)
 * @q:          the associated futex_q
 * @hb:         storage for hash_bucket pointer to be returned to caller
 *
 * Setup the futex_q and locate the hash_bucket.  Get the futex value and
 * compare it with the expected value.  Handle atomic faults internally.
 * Return with the hb lock held and a q.key reference on success, and unlocked
 * with no q.key reference on failure.
 *
 * Return:
 *  0 - uaddr contains val and hb has been locked;
 * <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
 */
static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
                           struct futex_q *q, struct futex_hash_bucket **hb)
{
        u32 uval;
        int ret;

        /*
         * Access the page AFTER the hash-bucket is locked.
         * Order is important:
         *
         *   Userspace waiter: val = var; if (cond(val)) futex_wait(&var, val);
         *   Userspace waker:  if (cond(var)) { var = new; futex_wake(&var); }
         *
         * The basic logical guarantee of a futex is that it blocks ONLY
         * if cond(var) is known to be true at the time of blocking, for
         * any cond.  If we locked the hash-bucket after testing *uaddr, that
         * would open a race condition where we could block indefinitely with
         * cond(var) false, which would violate the guarantee.
         *      
         * On the other hand, we insert q and release the hash-bucket only
         * after testing *uaddr.  This guarantees that futex_wait() will NOT
         * absorb a wakeup if *uaddr does not match the desired values
         * while the syscall executes.
         */
retry:
        ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, VERIFY_READ);
        if (unlikely(ret != 0))
                return ret;
 
retry_private:
        *hb = queue_lock(q);

        ret = get_futex_value_locked(&uval, uaddr);

        if (ret) {
                queue_unlock(*hb);

                ret = get_user(uval, uaddr);
                if (ret)
                        goto out;

                if (!(flags & FLAGS_SHARED))
                        goto retry_private;

                put_futex_key(&q->key);
                goto retry;
        }

        if (uval != val) {
                queue_unlock(*hb);
                ret = -EWOULDBLOCK;
        }

out:
        if (ret)
                put_futex_key(&q->key);
        return ret;
}

Using:

$ arm-wrs-linux-gnueabi-objdump -D -m arm vmlinux

c0070448 <futex_wait_setup>:
c0070448:       e92d4ff0        push    {r4, r5, r6, r7, r8, r9, sl, fp, lr}
c007044c:       e24dd014        sub     sp, sp, #20
c0070450:       e1a08003        mov     r8, r3
c0070454:       e1a06000        mov     r6, r0
c0070458:       e1a0300d        mov     r3, sp
c007045c:       e59d5038        ldr     r5, [sp, #56]   ; 0x38
c0070460:       e3c37d7f        bic     r7, r3, #8128   ; 0x1fc0
c0070464:       e1a0a001        mov     sl, r1
c0070468:       e3c7703f        bic     r7, r7, #63     ; 0x3f
c007046c:       e1a0b002        mov     fp, r2
c0070470:       e288401c        add     r4, r8, #28
c0070474:       e20b9001        and     r9, fp, #1
c0070478:       e3a03000        mov     r3, #0
c007047c:       e1a00006        mov     r0, r6
c0070480:       e1a02004        mov     r2, r4
c0070484:       e1a01009        mov     r1, r9
c0070488:       ebfffd3f        bl      c006f98c <get_futex_key>
c007048c:       e3500000        cmp     r0, #0
c0070490:       1a00003d        bne     c007058c <futex_wait_setup+0x144>
c0070494:       e1a00004        mov     r0, r4
c0070498:       ebfffc7d        bl      c006f694 <hash_futex>
c007049c:       e1a03000        mov     r3, r0
c00704a0:       f590f000        pldw    [r0]
c00704a4:       e1902f9f        ldrex   r2, [r0]
c00704a8:       e2822001        add     r2, r2, #1
c00704ac:       e1801f92        strex   r1, r2, [r0]
c00704b0:       e3310000        teq     r1, #0
c00704b4:       1afffffa        bne     c00704a4 <futex_wait_setup+0x5c>
c00704b8:       f57ff05b        dmb     ish
c00704bc:       e2800004        add     r0, r0, #4
c00704c0:       e5880018        str     r0, [r8, #24]
c00704c4:       e58d3004        str     r3, [sp, #4]
c00704c8:       eb08c1b2        bl      c02a0b98 <_raw_spin_lock>
c00704cc:       e1a01006        mov     r1, r6
c00704d0:       e59d3004        ldr     r3, [sp, #4]
c00704d4:       e28d000c        add     r0, sp, #12
c00704d8:       e5853000        str     r3, [r5]
c00704dc:       ebfffdea        bl      c006fc8c <get_futex_value_locked>
c00704e0:       e3500000        cmp     r0, #0
c00704e4:       0a000016        beq     c0070544 <futex_wait_setup+0xfc>
c00704e8:       e5953000        ldr     r3, [r5]
c00704ec:       e2830004        add     r0, r3, #4
c00704f0:       e58d3004        str     r3, [sp, #4]
c00704f4:       eb08c243        bl      c02a0e08 <_raw_spin_unlock>
c00704f8:       e59d3004        ldr     r3, [sp, #4]
c00704fc:       f593f000        pldw    [r3]
c0070500:       e1932f9f        ldrex   r2, [r3]
c0070504:       e2422001        sub     r2, r2, #1
c0070508:       e1831f92        strex   r1, r2, [r3]
c007050c:       e3310000        teq     r1, #0
c0070510:       1afffffa        bne     c0070500 <futex_wait_setup+0xb8>
c0070514:       e5971008        ldr     r1, [r7, #8]
c0070518:       e1a00006        mov     r0, r6
c007051c:       e2411001        sub     r1, r1, #1
c0070520:       eb0447df        bl      c01824a4 <__get_user_4>
c0070524:       e2503000        subs    r3, r0, #0
c0070528:       e58d200c        str     r2, [sp, #12]
c007052c:       1a000011        bne     c0070578 <futex_wait_setup+0x130>
c0070530:       e3590000        cmp     r9, #0
c0070534:       0affffd6        beq     c0070494 <futex_wait_setup+0x4c>
c0070538:       e1a00004        mov     r0, r4
c007053c:       ebffff52        bl      c007028c <drop_futex_key_refs>
c0070540:       eaffffca        b       c0070470 <futex_wait_setup+0x28>
c0070544:       e59d300c        ldr     r3, [sp, #12]
c0070548:       e153000a        cmp     r3, sl
c007054c:       0a00000e        beq     c007058c <futex_wait_setup+0x144>
c0070550:       e5955000        ldr     r5, [r5]
c0070554:       e2850004        add     r0, r5, #4
c0070558:       eb08c22a        bl      c02a0e08 <_raw_spin_unlock>
c007055c:       f595f000        pldw    [r5]
c0070560:       e1953f9f        ldrex   r3, [r5]
c0070564:       e2433001        sub     r3, r3, #1
c0070568:       e1852f93        strex   r2, r3, [r5]
c007056c:       e3320000        teq     r2, #0
c0070570:       1afffffa        bne     c0070560 <futex_wait_setup+0x118>
c0070574:       e3e0300a        mvn     r3, #10
c0070578:       e1a00004        mov     r0, r4
c007057c:       e58d3004        str     r3, [sp, #4]
c0070580:       ebffff41        bl      c007028c <drop_futex_key_refs>
c0070584:       e59d3004        ldr     r3, [sp, #4]
c0070588:       e1a00003        mov     r0, r3
c007058c:       e28dd014        add     sp, sp, #20
c0070590:       e8bd8ff0        pop     {r4, r5, r6, r7, r8, r9, sl, fp, pc}