linux/arch/arm/mm/fault.c
<<
>>
Prefs
   1/*
   2 *  linux/arch/arm/mm/fault.c
   3 *
   4 *  Copyright (C) 1995  Linus Torvalds
   5 *  Modifications for ARM processor (c) 1995-2004 Russell King
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License version 2 as
   9 * published by the Free Software Foundation.
  10 */
  11#include <linux/module.h>
  12#include <linux/signal.h>
  13#include <linux/mm.h>
  14#include <linux/hardirq.h>
  15#include <linux/init.h>
  16#include <linux/kprobes.h>
  17#include <linux/uaccess.h>
  18#include <linux/page-flags.h>
  19#include <linux/sched.h>
  20#include <linux/highmem.h>
  21#include <linux/perf_event.h>
  22
  23#include <asm/exception.h>
  24#include <asm/pgtable.h>
  25#include <asm/system_misc.h>
  26#include <asm/system_info.h>
  27#include <asm/tlbflush.h>
  28
  29#include "fault.h"
  30
  31#ifdef CONFIG_MMU
  32
  33#ifdef CONFIG_KPROBES
  34static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr)
  35{
  36        int ret = 0;
  37
  38        if (!user_mode(regs)) {
  39                /* kprobe_running() needs smp_processor_id() */
  40                preempt_disable();
  41                if (kprobe_running() && kprobe_fault_handler(regs, fsr))
  42                        ret = 1;
  43                preempt_enable();
  44        }
  45
  46        return ret;
  47}
  48#else
  49static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr)
  50{
  51        return 0;
  52}
  53#endif
  54
  55/*
  56 * This is useful to dump out the page tables associated with
  57 * 'addr' in mm 'mm'.
  58 */
  59void show_pte(struct mm_struct *mm, unsigned long addr)
  60{
  61        pgd_t *pgd;
  62
  63        if (!mm)
  64                mm = &init_mm;
  65
  66        pr_alert("pgd = %p\n", mm->pgd);
  67        pgd = pgd_offset(mm, addr);
  68        pr_alert("[%08lx] *pgd=%08llx",
  69                        addr, (long long)pgd_val(*pgd));
  70
  71        do {
  72                pud_t *pud;
  73                pmd_t *pmd;
  74                pte_t *pte;
  75
  76                if (pgd_none(*pgd))
  77                        break;
  78
  79                if (pgd_bad(*pgd)) {
  80                        pr_cont("(bad)");
  81                        break;
  82                }
  83
  84                pud = pud_offset(pgd, addr);
  85                if (PTRS_PER_PUD != 1)
  86                        pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
  87
  88                if (pud_none(*pud))
  89                        break;
  90
  91                if (pud_bad(*pud)) {
  92                        pr_cont("(bad)");
  93                        break;
  94                }
  95
  96                pmd = pmd_offset(pud, addr);
  97                if (PTRS_PER_PMD != 1)
  98                        pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd));
  99
 100                if (pmd_none(*pmd))
 101                        break;
 102
 103                if (pmd_bad(*pmd)) {
 104                        pr_cont("(bad)");
 105                        break;
 106                }
 107
 108                /* We must not map this if we have highmem enabled */
 109                if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
 110                        break;
 111
 112                pte = pte_offset_map(pmd, addr);
 113                pr_cont(", *pte=%08llx", (long long)pte_val(*pte));
 114#ifndef CONFIG_ARM_LPAE
 115                pr_cont(", *ppte=%08llx",
 116                       (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
 117#endif
 118                pte_unmap(pte);
 119        } while(0);
 120
 121        pr_cont("\n");
 122}
 123#else                                   /* CONFIG_MMU */
 124void show_pte(struct mm_struct *mm, unsigned long addr)
 125{ }
 126#endif                                  /* CONFIG_MMU */
 127
 128/*
 129 * Oops.  The kernel tried to access some page that wasn't present.
 130 */
 131static void
 132__do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
 133                  struct pt_regs *regs)
 134{
 135        /*
 136         * Are we prepared to handle this kernel fault?
 137         */
 138        if (fixup_exception(regs))
 139                return;
 140
 141        /*
 142         * No handler, we'll have to terminate things with extreme prejudice.
 143         */
 144        bust_spinlocks(1);
 145        pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
 146                 (addr < PAGE_SIZE) ? "NULL pointer dereference" :
 147                 "paging request", addr);
 148
 149        show_pte(mm, addr);
 150        die("Oops", regs, fsr);
 151        bust_spinlocks(0);
 152        do_exit(SIGKILL);
 153}
 154
 155/*
 156 * Something tried to access memory that isn't in our memory map..
 157 * User mode accesses just cause a SIGSEGV
 158 */
 159static void
 160__do_user_fault(struct task_struct *tsk, unsigned long addr,
 161                unsigned int fsr, unsigned int sig, int code,
 162                struct pt_regs *regs)
 163{
 164        struct siginfo si;
 165
 166#ifdef CONFIG_DEBUG_USER
 167        if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
 168            ((user_debug & UDBG_BUS)  && (sig == SIGBUS))) {
 169                printk(KERN_DEBUG "%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
 170                       tsk->comm, sig, addr, fsr);
 171                show_pte(tsk->mm, addr);
 172                show_regs(regs);
 173        }
 174#endif
 175
 176        tsk->thread.address = addr;
 177        tsk->thread.error_code = fsr;
 178        tsk->thread.trap_no = 14;
 179        si.si_signo = sig;
 180        si.si_errno = 0;
 181        si.si_code = code;
 182        si.si_addr = (void __user *)addr;
 183        force_sig_info(sig, &si, tsk);
 184}
 185
 186void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 187{
 188        struct task_struct *tsk = current;
 189        struct mm_struct *mm = tsk->active_mm;
 190
 191        /*
 192         * If we are in kernel mode at this point, we
 193         * have no context to handle this fault with.
 194         */
 195        if (user_mode(regs))
 196                __do_user_fault(tsk, addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
 197        else
 198                __do_kernel_fault(mm, addr, fsr, regs);
 199}
 200
 201#ifdef CONFIG_MMU
 202#define VM_FAULT_BADMAP         0x010000
 203#define VM_FAULT_BADACCESS      0x020000
 204
 205/*
 206 * Check that the permissions on the VMA allow for the fault which occurred.
 207 * If we encountered a write fault, we must have write permission, otherwise
 208 * we allow any permission.
 209 */
 210static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
 211{
 212        unsigned int mask = VM_READ | VM_WRITE | VM_EXEC;
 213
 214        if (fsr & FSR_WRITE)
 215                mask = VM_WRITE;
 216        if (fsr & FSR_LNX_PF)
 217                mask = VM_EXEC;
 218
 219        return vma->vm_flags & mask ? false : true;
 220}
 221
 222static int __kprobes
 223__do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
 224                unsigned int flags, struct task_struct *tsk)
 225{
 226        struct vm_area_struct *vma;
 227        int fault;
 228
 229        vma = find_vma(mm, addr);
 230        fault = VM_FAULT_BADMAP;
 231        if (unlikely(!vma))
 232                goto out;
 233        if (unlikely(vma->vm_start > addr))
 234                goto check_stack;
 235
 236        /*
 237         * Ok, we have a good vm_area for this
 238         * memory access, so we can handle it.
 239         */
 240good_area:
 241        if (access_error(fsr, vma)) {
 242                fault = VM_FAULT_BADACCESS;
 243                goto out;
 244        }
 245
 246        return handle_mm_fault(mm, vma, addr & PAGE_MASK, flags);
 247
 248check_stack:
 249        /* Don't allow expansion below FIRST_USER_ADDRESS */
 250        if (vma->vm_flags & VM_GROWSDOWN &&
 251            addr >= FIRST_USER_ADDRESS && !expand_stack(vma, addr))
 252                goto good_area;
 253out:
 254        return fault;
 255}
 256
 257static int __kprobes
 258do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 259{
 260        struct task_struct *tsk;
 261        struct mm_struct *mm;
 262        int fault, sig, code;
 263        unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 264
 265        if (notify_page_fault(regs, fsr))
 266                return 0;
 267
 268        tsk = current;
 269        mm  = tsk->mm;
 270
 271        /* Enable interrupts if they were enabled in the parent context. */
 272        if (interrupts_enabled(regs))
 273                local_irq_enable();
 274
 275        /*
 276         * If we're in an interrupt or have no user
 277         * context, we must not take the fault..
 278         */
 279        if (in_atomic() || !mm)
 280                goto no_context;
 281
 282        if (user_mode(regs))
 283                flags |= FAULT_FLAG_USER;
 284        if (fsr & FSR_WRITE)
 285                flags |= FAULT_FLAG_WRITE;
 286
 287        /*
 288         * As per x86, we may deadlock here.  However, since the kernel only
 289         * validly references user space from well defined areas of the code,
 290         * we can bug out early if this is from code which shouldn't.
 291         */
 292        if (!down_read_trylock(&mm->mmap_sem)) {
 293                if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc))
 294                        goto no_context;
 295retry:
 296                down_read(&mm->mmap_sem);
 297        } else {
 298                /*
 299                 * The above down_read_trylock() might have succeeded in
 300                 * which case, we'll have missed the might_sleep() from
 301                 * down_read()
 302                 */
 303                might_sleep();
 304#ifdef CONFIG_DEBUG_VM
 305                if (!user_mode(regs) &&
 306                    !search_exception_tables(regs->ARM_pc))
 307                        goto no_context;
 308#endif
 309        }
 310
 311        fault = __do_page_fault(mm, addr, fsr, flags, tsk);
 312
 313        /* If we need to retry but a fatal signal is pending, handle the
 314         * signal first. We do not need to release the mmap_sem because
 315         * it would already be released in __lock_page_or_retry in
 316         * mm/filemap.c. */
 317        if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
 318                return 0;
 319
 320        /*
 321         * Major/minor page fault accounting is only done on the
 322         * initial attempt. If we go through a retry, it is extremely
 323         * likely that the page will be found in page cache at that point.
 324         */
 325
 326        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
 327        if (!(fault & VM_FAULT_ERROR) && flags & FAULT_FLAG_ALLOW_RETRY) {
 328                if (fault & VM_FAULT_MAJOR) {
 329                        tsk->maj_flt++;
 330                        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
 331                                        regs, addr);
 332                } else {
 333                        tsk->min_flt++;
 334                        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
 335                                        regs, addr);
 336                }
 337                if (fault & VM_FAULT_RETRY) {
 338                        /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
 339                        * of starvation. */
 340                        flags &= ~FAULT_FLAG_ALLOW_RETRY;
 341                        flags |= FAULT_FLAG_TRIED;
 342                        goto retry;
 343                }
 344        }
 345
 346        up_read(&mm->mmap_sem);
 347
 348        /*
 349         * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
 350         */
 351        if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS))))
 352                return 0;
 353
 354        /*
 355         * If we are in kernel mode at this point, we
 356         * have no context to handle this fault with.
 357         */
 358        if (!user_mode(regs))
 359                goto no_context;
 360
 361        if (fault & VM_FAULT_OOM) {
 362                /*
 363                 * We ran out of memory, call the OOM killer, and return to
 364                 * userspace (which will retry the fault, or kill us if we
 365                 * got oom-killed)
 366                 */
 367                pagefault_out_of_memory();
 368                return 0;
 369        }
 370
 371        if (fault & VM_FAULT_SIGBUS) {
 372                /*
 373                 * We had some memory, but were unable to
 374                 * successfully fix up this page fault.
 375                 */
 376                sig = SIGBUS;
 377                code = BUS_ADRERR;
 378        } else {
 379                /*
 380                 * Something tried to access memory that
 381                 * isn't in our memory map..
 382                 */
 383                sig = SIGSEGV;
 384                code = fault == VM_FAULT_BADACCESS ?
 385                        SEGV_ACCERR : SEGV_MAPERR;
 386        }
 387
 388        __do_user_fault(tsk, addr, fsr, sig, code, regs);
 389        return 0;
 390
 391no_context:
 392        __do_kernel_fault(mm, addr, fsr, regs);
 393        return 0;
 394}
 395#else                                   /* CONFIG_MMU */
 396static int
 397do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 398{
 399        return 0;
 400}
 401#endif                                  /* CONFIG_MMU */
 402
 403/*
 404 * First Level Translation Fault Handler
 405 *
 406 * We enter here because the first level page table doesn't contain
 407 * a valid entry for the address.
 408 *
 409 * If the address is in kernel space (>= TASK_SIZE), then we are
 410 * probably faulting in the vmalloc() area.
 411 *
 412 * If the init_task's first level page tables contains the relevant
 413 * entry, we copy the it to this task.  If not, we send the process
 414 * a signal, fixup the exception, or oops the kernel.
 415 *
 416 * NOTE! We MUST NOT take any locks for this case. We may be in an
 417 * interrupt or a critical region, and should only copy the information
 418 * from the master page table, nothing more.
 419 */
 420#ifdef CONFIG_MMU
 421static int __kprobes
 422do_translation_fault(unsigned long addr, unsigned int fsr,
 423                     struct pt_regs *regs)
 424{
 425        unsigned int index;
 426        pgd_t *pgd, *pgd_k;
 427        pud_t *pud, *pud_k;
 428        pmd_t *pmd, *pmd_k;
 429
 430        if (addr < TASK_SIZE)
 431                return do_page_fault(addr, fsr, regs);
 432
 433        if (user_mode(regs))
 434                goto bad_area;
 435
 436        index = pgd_index(addr);
 437
 438        pgd = cpu_get_pgd() + index;
 439        pgd_k = init_mm.pgd + index;
 440
 441        if (pgd_none(*pgd_k))
 442                goto bad_area;
 443        if (!pgd_present(*pgd))
 444                set_pgd(pgd, *pgd_k);
 445
 446        pud = pud_offset(pgd, addr);
 447        pud_k = pud_offset(pgd_k, addr);
 448
 449        if (pud_none(*pud_k))
 450                goto bad_area;
 451        if (!pud_present(*pud))
 452                set_pud(pud, *pud_k);
 453
 454        pmd = pmd_offset(pud, addr);
 455        pmd_k = pmd_offset(pud_k, addr);
 456
 457#ifdef CONFIG_ARM_LPAE
 458        /*
 459         * Only one hardware entry per PMD with LPAE.
 460         */
 461        index = 0;
 462#else
 463        /*
 464         * On ARM one Linux PGD entry contains two hardware entries (see page
 465         * tables layout in pgtable.h). We normally guarantee that we always
 466         * fill both L1 entries. But create_mapping() doesn't follow the rule.
 467         * It can create inidividual L1 entries, so here we have to call
 468         * pmd_none() check for the entry really corresponded to address, not
 469         * for the first of pair.
 470         */
 471        index = (addr >> SECTION_SHIFT) & 1;
 472#endif
 473        if (pmd_none(pmd_k[index]))
 474                goto bad_area;
 475
 476        copy_pmd(pmd, pmd_k);
 477        return 0;
 478
 479bad_area:
 480        do_bad_area(addr, fsr, regs);
 481        return 0;
 482}
 483#else                                   /* CONFIG_MMU */
 484static int
 485do_translation_fault(unsigned long addr, unsigned int fsr,
 486                     struct pt_regs *regs)
 487{
 488        return 0;
 489}
 490#endif                                  /* CONFIG_MMU */
 491
 492/*
 493 * Some section permission faults need to be handled gracefully.
 494 * They can happen due to a __{get,put}_user during an oops.
 495 */
 496#ifndef CONFIG_ARM_LPAE
 497static int
 498do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 499{
 500        do_bad_area(addr, fsr, regs);
 501        return 0;
 502}
 503#endif /* CONFIG_ARM_LPAE */
 504
 505/*
 506 * This abort handler always returns "fault".
 507 */
 508static int
 509do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 510{
 511        return 1;
 512}
 513
 514struct fsr_info {
 515        int     (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
 516        int     sig;
 517        int     code;
 518        const char *name;
 519};
 520
 521/* FSR definition */
 522#ifdef CONFIG_ARM_LPAE
 523#include "fsr-3level.c"
 524#else
 525#include "fsr-2level.c"
 526#endif
 527
 528void __init
 529hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
 530                int sig, int code, const char *name)
 531{
 532        if (nr < 0 || nr >= ARRAY_SIZE(fsr_info))
 533                BUG();
 534
 535        fsr_info[nr].fn   = fn;
 536        fsr_info[nr].sig  = sig;
 537        fsr_info[nr].code = code;
 538        fsr_info[nr].name = name;
 539}
 540
 541/*
 542 * Dispatch a data abort to the relevant handler.
 543 */
 544asmlinkage void __exception
 545do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 546{
 547        const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
 548        struct siginfo info;
 549
 550        if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
 551                return;
 552
 553        pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
 554                inf->name, fsr, addr);
 555        show_pte(current->mm, addr);
 556
 557        info.si_signo = inf->sig;
 558        info.si_errno = 0;
 559        info.si_code  = inf->code;
 560        info.si_addr  = (void __user *)addr;
 561        arm_notify_die("", regs, &info, fsr, 0);
 562}
 563
 564void __init
 565hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
 566                 int sig, int code, const char *name)
 567{
 568        if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info))
 569                BUG();
 570
 571        ifsr_info[nr].fn   = fn;
 572        ifsr_info[nr].sig  = sig;
 573        ifsr_info[nr].code = code;
 574        ifsr_info[nr].name = name;
 575}
 576
 577asmlinkage void __exception
 578do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
 579{
 580        const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
 581        struct siginfo info;
 582
 583        if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
 584                return;
 585
 586        pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
 587                inf->name, ifsr, addr);
 588
 589        info.si_signo = inf->sig;
 590        info.si_errno = 0;
 591        info.si_code  = inf->code;
 592        info.si_addr  = (void __user *)addr;
 593        arm_notify_die("", regs, &info, ifsr, 0);
 594}
 595
 596#ifndef CONFIG_ARM_LPAE
 597static int __init exceptions_init(void)
 598{
 599        if (cpu_architecture() >= CPU_ARCH_ARMv6) {
 600                hook_fault_code(4, do_translation_fault, SIGSEGV, SEGV_MAPERR,
 601                                "I-cache maintenance fault");
 602        }
 603
 604        if (cpu_architecture() >= CPU_ARCH_ARMv7) {
 605                /*
 606                 * TODO: Access flag faults introduced in ARMv6K.
 607                 * Runtime check for 'K' extension is needed
 608                 */
 609                hook_fault_code(3, do_bad, SIGSEGV, SEGV_MAPERR,
 610                                "section access flag fault");
 611                hook_fault_code(6, do_bad, SIGSEGV, SEGV_MAPERR,
 612                                "section access flag fault");
 613        }
 614
 615        return 0;
 616}
 617
 618arch_initcall(exceptions_init);
 619#endif
 620