1
2
3
4
5
6#include <linux/magic.h>
7#include <linux/sched.h>
8#include <linux/kdebug.h>
9#include <linux/module.h>
10#include <linux/bootmem.h>
11#include <linux/kprobes.h>
12#include <linux/mmiotrace.h>
13#include <linux/perf_event.h>
14#include <linux/hugetlb.h>
15
16#include <asm/traps.h>
17#include <asm/pgalloc.h>
18#include <asm/kmemcheck.h>
19
20
21
22
23
24
25
26
27
28
29enum x86_pf_error_code {
30
31 PF_PROT = 1 << 0,
32 PF_WRITE = 1 << 1,
33 PF_USER = 1 << 2,
34 PF_RSVD = 1 << 3,
35 PF_INSTR = 1 << 4,
36};
37
38
39
40
41
42static inline int __kprobes
43kmmio_fault(struct pt_regs *regs, unsigned long addr)
44{
45 if (unlikely(is_kmmio_active()))
46 if (kmmio_handler(regs, addr) == 1)
47 return -1;
48 return 0;
49}
50
51static inline int __kprobes notify_page_fault(struct pt_regs *regs)
52{
53 int ret = 0;
54
55
56 if (kprobes_built_in() && !user_mode_vm(regs)) {
57 preempt_disable();
58 if (kprobe_running() && kprobe_fault_handler(regs, 14))
59 ret = 1;
60 preempt_enable();
61 }
62
63 return ret;
64}
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81static inline int
82check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
83 unsigned char opcode, int *prefetch)
84{
85 unsigned char instr_hi = opcode & 0xf0;
86 unsigned char instr_lo = opcode & 0x0f;
87
88 switch (instr_hi) {
89 case 0x20:
90 case 0x30:
91
92
93
94
95
96
97 return ((instr_lo & 7) == 0x6);
98#ifdef CONFIG_X86_64
99 case 0x40:
100
101
102
103
104
105
106
107 return (!user_mode(regs)) || (regs->cs == __USER_CS);
108#endif
109 case 0x60:
110
111 return (instr_lo & 0xC) == 0x4;
112 case 0xF0:
113
114 return !instr_lo || (instr_lo>>1) == 1;
115 case 0x00:
116
117 if (probe_kernel_address(instr, opcode))
118 return 0;
119
120 *prefetch = (instr_lo == 0xF) &&
121 (opcode == 0x0D || opcode == 0x18);
122 return 0;
123 default:
124 return 0;
125 }
126}
127
128static int
129is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
130{
131 unsigned char *max_instr;
132 unsigned char *instr;
133 int prefetch = 0;
134
135
136
137
138
139 if (error_code & PF_INSTR)
140 return 0;
141
142 instr = (void *)convert_ip_to_linear(current, regs);
143 max_instr = instr + 15;
144
145 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
146 return 0;
147
148 while (instr < max_instr) {
149 unsigned char opcode;
150
151 if (probe_kernel_address(instr, opcode))
152 break;
153
154 instr++;
155
156 if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
157 break;
158 }
159 return prefetch;
160}
161
162static void
163force_sig_info_fault(int si_signo, int si_code, unsigned long address,
164 struct task_struct *tsk, int fault)
165{
166 unsigned lsb = 0;
167 siginfo_t info;
168
169 info.si_signo = si_signo;
170 info.si_errno = 0;
171 info.si_code = si_code;
172 info.si_addr = (void __user *)address;
173 if (fault & VM_FAULT_HWPOISON_LARGE)
174 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
175 if (fault & VM_FAULT_HWPOISON)
176 lsb = PAGE_SHIFT;
177 info.si_addr_lsb = lsb;
178
179 force_sig_info(si_signo, &info, tsk);
180}
181
182DEFINE_SPINLOCK(pgd_lock);
183LIST_HEAD(pgd_list);
184
185#ifdef CONFIG_X86_32
186static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
187{
188 unsigned index = pgd_index(address);
189 pgd_t *pgd_k;
190 pud_t *pud, *pud_k;
191 pmd_t *pmd, *pmd_k;
192
193 pgd += index;
194 pgd_k = init_mm.pgd + index;
195
196 if (!pgd_present(*pgd_k))
197 return NULL;
198
199
200
201
202
203
204 pud = pud_offset(pgd, address);
205 pud_k = pud_offset(pgd_k, address);
206 if (!pud_present(*pud_k))
207 return NULL;
208
209 pmd = pmd_offset(pud, address);
210 pmd_k = pmd_offset(pud_k, address);
211 if (!pmd_present(*pmd_k))
212 return NULL;
213
214 if (!pmd_present(*pmd))
215 set_pmd(pmd, *pmd_k);
216 else
217 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
218
219 return pmd_k;
220}
221
222void vmalloc_sync_all(void)
223{
224 unsigned long address;
225
226 if (SHARED_KERNEL_PMD)
227 return;
228
229 for (address = VMALLOC_START & PMD_MASK;
230 address >= TASK_SIZE && address < FIXADDR_TOP;
231 address += PMD_SIZE) {
232 struct page *page;
233
234 spin_lock(&pgd_lock);
235 list_for_each_entry(page, &pgd_list, lru) {
236 spinlock_t *pgt_lock;
237 pmd_t *ret;
238
239
240 pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
241
242 spin_lock(pgt_lock);
243 ret = vmalloc_sync_one(page_address(page), address);
244 spin_unlock(pgt_lock);
245
246 if (!ret)
247 break;
248 }
249 spin_unlock(&pgd_lock);
250 }
251}
252
253
254
255
256
257
258static noinline __kprobes int vmalloc_fault(unsigned long address)
259{
260 unsigned long pgd_paddr;
261 pmd_t *pmd_k;
262 pte_t *pte_k;
263
264
265 if (!(address >= VMALLOC_START && address < VMALLOC_END))
266 return -1;
267
268 WARN_ON_ONCE(in_nmi());
269
270
271
272
273
274
275
276
277 pgd_paddr = read_cr3();
278 pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
279 if (!pmd_k)
280 return -1;
281
282 pte_k = pte_offset_kernel(pmd_k, address);
283 if (!pte_present(*pte_k))
284 return -1;
285
286 return 0;
287}
288
289
290
291
292static inline void
293check_v8086_mode(struct pt_regs *regs, unsigned long address,
294 struct task_struct *tsk)
295{
296 unsigned long bit;
297
298 if (!v8086_mode(regs))
299 return;
300
301 bit = (address - 0xA0000) >> PAGE_SHIFT;
302 if (bit < 32)
303 tsk->thread.screen_bitmap |= 1 << bit;
304}
305
306static bool low_pfn(unsigned long pfn)
307{
308 return pfn < max_low_pfn;
309}
310
311static void dump_pagetable(unsigned long address)
312{
313 pgd_t *base = __va(read_cr3());
314 pgd_t *pgd = &base[pgd_index(address)];
315 pmd_t *pmd;
316 pte_t *pte;
317
318#ifdef CONFIG_X86_PAE
319 printk("*pdpt = %016Lx ", pgd_val(*pgd));
320 if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
321 goto out;
322#endif
323 pmd = pmd_offset(pud_offset(pgd, address), address);
324 printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
325
326
327
328
329
330
331
332 if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
333 goto out;
334
335 pte = pte_offset_kernel(pmd, address);
336 printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
337out:
338 printk("\n");
339}
340
341#else
342
343void vmalloc_sync_all(void)
344{
345 sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
346}
347
348
349
350
351
352
353
354
355static noinline __kprobes int vmalloc_fault(unsigned long address)
356{
357 pgd_t *pgd, *pgd_ref;
358 pud_t *pud, *pud_ref;
359 pmd_t *pmd, *pmd_ref;
360 pte_t *pte, *pte_ref;
361
362
363 if (!(address >= VMALLOC_START && address < VMALLOC_END))
364 return -1;
365
366 WARN_ON_ONCE(in_nmi());
367
368
369
370
371
372
373 pgd = pgd_offset(current->active_mm, address);
374 pgd_ref = pgd_offset_k(address);
375 if (pgd_none(*pgd_ref))
376 return -1;
377
378 if (pgd_none(*pgd))
379 set_pgd(pgd, *pgd_ref);
380 else
381 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
382
383
384
385
386
387
388 pud = pud_offset(pgd, address);
389 pud_ref = pud_offset(pgd_ref, address);
390 if (pud_none(*pud_ref))
391 return -1;
392
393 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
394 BUG();
395
396 pmd = pmd_offset(pud, address);
397 pmd_ref = pmd_offset(pud_ref, address);
398 if (pmd_none(*pmd_ref))
399 return -1;
400
401 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
402 BUG();
403
404 pte_ref = pte_offset_kernel(pmd_ref, address);
405 if (!pte_present(*pte_ref))
406 return -1;
407
408 pte = pte_offset_kernel(pmd, address);
409
410
411
412
413
414
415 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
416 BUG();
417
418 return 0;
419}
420
421static const char errata93_warning[] =
422KERN_ERR
423"******* Your BIOS seems to not contain a fix for K8 errata #93\n"
424"******* Working around it, but it may cause SEGVs or burn power.\n"
425"******* Please consider a BIOS update.\n"
426"******* Disabling USB legacy in the BIOS may also help.\n";
427
428
429
430
431static inline void
432check_v8086_mode(struct pt_regs *regs, unsigned long address,
433 struct task_struct *tsk)
434{
435}
436
437static int bad_address(void *p)
438{
439 unsigned long dummy;
440
441 return probe_kernel_address((unsigned long *)p, dummy);
442}
443
444static void dump_pagetable(unsigned long address)
445{
446 pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
447 pgd_t *pgd = base + pgd_index(address);
448 pud_t *pud;
449 pmd_t *pmd;
450 pte_t *pte;
451
452 if (bad_address(pgd))
453 goto bad;
454
455 printk("PGD %lx ", pgd_val(*pgd));
456
457 if (!pgd_present(*pgd))
458 goto out;
459
460 pud = pud_offset(pgd, address);
461 if (bad_address(pud))
462 goto bad;
463
464 printk("PUD %lx ", pud_val(*pud));
465 if (!pud_present(*pud) || pud_large(*pud))
466 goto out;
467
468 pmd = pmd_offset(pud, address);
469 if (bad_address(pmd))
470 goto bad;
471
472 printk("PMD %lx ", pmd_val(*pmd));
473 if (!pmd_present(*pmd) || pmd_large(*pmd))
474 goto out;
475
476 pte = pte_offset_kernel(pmd, address);
477 if (bad_address(pte))
478 goto bad;
479
480 printk("PTE %lx", pte_val(*pte));
481out:
482 printk("\n");
483 return;
484bad:
485 printk("BAD\n");
486}
487
488#endif
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504static int is_errata93(struct pt_regs *regs, unsigned long address)
505{
506#ifdef CONFIG_X86_64
507 if (address != regs->ip)
508 return 0;
509
510 if ((address >> 32) != 0)
511 return 0;
512
513 address |= 0xffffffffUL << 32;
514 if ((address >= (u64)_stext && address <= (u64)_etext) ||
515 (address >= MODULES_VADDR && address <= MODULES_END)) {
516 printk_once(errata93_warning);
517 regs->ip = address;
518 return 1;
519 }
520#endif
521 return 0;
522}
523
524
525
526
527
528
529
530
531
532static int is_errata100(struct pt_regs *regs, unsigned long address)
533{
534#ifdef CONFIG_X86_64
535 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
536 return 1;
537#endif
538 return 0;
539}
540
541static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
542{
543#ifdef CONFIG_X86_F00F_BUG
544 unsigned long nr;
545
546
547
548
549 if (boot_cpu_data.f00f_bug) {
550 nr = (address - idt_descr.address) >> 3;
551
552 if (nr == 6) {
553 do_invalid_op(regs, 0);
554 return 1;
555 }
556 }
557#endif
558 return 0;
559}
560
561static const char nx_warning[] = KERN_CRIT
562"kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
563
564static void
565show_fault_oops(struct pt_regs *regs, unsigned long error_code,
566 unsigned long address)
567{
568 if (!oops_may_print())
569 return;
570
571 if (error_code & PF_INSTR) {
572 unsigned int level;
573
574 pte_t *pte = lookup_address(address, &level);
575
576 if (pte && pte_present(*pte) && !pte_exec(*pte))
577 printk(nx_warning, current_uid());
578 }
579
580 printk(KERN_ALERT "BUG: unable to handle kernel ");
581 if (address < PAGE_SIZE)
582 printk(KERN_CONT "NULL pointer dereference");
583 else
584 printk(KERN_CONT "paging request");
585
586 printk(KERN_CONT " at %p\n", (void *) address);
587 printk(KERN_ALERT "IP:");
588 printk_address(regs->ip, 1);
589
590 dump_pagetable(address);
591}
592
593static noinline void
594pgtable_bad(struct pt_regs *regs, unsigned long error_code,
595 unsigned long address)
596{
597 struct task_struct *tsk;
598 unsigned long flags;
599 int sig;
600
601 flags = oops_begin();
602 tsk = current;
603 sig = SIGKILL;
604
605 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
606 tsk->comm, address);
607 dump_pagetable(address);
608
609 tsk->thread.cr2 = address;
610 tsk->thread.trap_no = 14;
611 tsk->thread.error_code = error_code;
612
613 if (__die("Bad pagetable", regs, error_code))
614 sig = 0;
615
616 oops_end(flags, regs, sig);
617}
618
619static noinline void
620no_context(struct pt_regs *regs, unsigned long error_code,
621 unsigned long address)
622{
623 struct task_struct *tsk = current;
624 unsigned long *stackend;
625 unsigned long flags;
626 int sig;
627
628
629 if (fixup_exception(regs))
630 return;
631
632
633
634
635
636
637
638
639
640
641
642
643 if (is_prefetch(regs, error_code, address))
644 return;
645
646 if (is_errata93(regs, address))
647 return;
648
649
650
651
652
653 flags = oops_begin();
654
655 show_fault_oops(regs, error_code, address);
656
657 stackend = end_of_stack(tsk);
658 if (tsk != &init_task && *stackend != STACK_END_MAGIC)
659 printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
660
661 tsk->thread.cr2 = address;
662 tsk->thread.trap_no = 14;
663 tsk->thread.error_code = error_code;
664
665 sig = SIGKILL;
666 if (__die("Oops", regs, error_code))
667 sig = 0;
668
669
670 printk(KERN_EMERG "CR2: %016lx\n", address);
671
672 oops_end(flags, regs, sig);
673}
674
675
676
677
678
679static inline void
680show_signal_msg(struct pt_regs *regs, unsigned long error_code,
681 unsigned long address, struct task_struct *tsk)
682{
683 if (!unhandled_signal(tsk, SIGSEGV))
684 return;
685
686 if (!printk_ratelimit())
687 return;
688
689 printk("%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
690 task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
691 tsk->comm, task_pid_nr(tsk), address,
692 (void *)regs->ip, (void *)regs->sp, error_code);
693
694 print_vma_addr(KERN_CONT " in ", regs->ip);
695
696 printk(KERN_CONT "\n");
697}
698
699static void
700__bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
701 unsigned long address, int si_code)
702{
703 struct task_struct *tsk = current;
704
705
706 if (error_code & PF_USER) {
707
708
709
710 local_irq_enable();
711
712
713
714
715
716 if (is_prefetch(regs, error_code, address))
717 return;
718
719 if (is_errata100(regs, address))
720 return;
721
722 if (unlikely(show_unhandled_signals))
723 show_signal_msg(regs, error_code, address, tsk);
724
725
726 tsk->thread.cr2 = address;
727 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
728 tsk->thread.trap_no = 14;
729
730 force_sig_info_fault(SIGSEGV, si_code, address, tsk, 0);
731
732 return;
733 }
734
735 if (is_f00f_bug(regs, address))
736 return;
737
738 no_context(regs, error_code, address);
739}
740
741static noinline void
742bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
743 unsigned long address)
744{
745 __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
746}
747
748static void
749__bad_area(struct pt_regs *regs, unsigned long error_code,
750 unsigned long address, int si_code)
751{
752 struct mm_struct *mm = current->mm;
753
754
755
756
757
758 up_read(&mm->mmap_sem);
759
760 __bad_area_nosemaphore(regs, error_code, address, si_code);
761}
762
763static noinline void
764bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
765{
766 __bad_area(regs, error_code, address, SEGV_MAPERR);
767}
768
769static noinline void
770bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
771 unsigned long address)
772{
773 __bad_area(regs, error_code, address, SEGV_ACCERR);
774}
775
776
777static void
778out_of_memory(struct pt_regs *regs, unsigned long error_code,
779 unsigned long address)
780{
781
782
783
784
785 up_read(¤t->mm->mmap_sem);
786
787 pagefault_out_of_memory();
788}
789
790static void
791do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
792 unsigned int fault)
793{
794 struct task_struct *tsk = current;
795 struct mm_struct *mm = tsk->mm;
796 int code = BUS_ADRERR;
797
798 up_read(&mm->mmap_sem);
799
800
801 if (!(error_code & PF_USER)) {
802 no_context(regs, error_code, address);
803 return;
804 }
805
806
807 if (is_prefetch(regs, error_code, address))
808 return;
809
810 tsk->thread.cr2 = address;
811 tsk->thread.error_code = error_code;
812 tsk->thread.trap_no = 14;
813
814#ifdef CONFIG_MEMORY_FAILURE
815 if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
816 printk(KERN_ERR
817 "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
818 tsk->comm, tsk->pid, address);
819 code = BUS_MCEERR_AR;
820 }
821#endif
822 force_sig_info_fault(SIGBUS, code, address, tsk, fault);
823}
824
825static noinline void
826mm_fault_error(struct pt_regs *regs, unsigned long error_code,
827 unsigned long address, unsigned int fault)
828{
829 if (fault & VM_FAULT_OOM) {
830
831 if (!(error_code & PF_USER)) {
832 up_read(¤t->mm->mmap_sem);
833 no_context(regs, error_code, address);
834 return;
835 }
836
837 out_of_memory(regs, error_code, address);
838 } else {
839 if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
840 VM_FAULT_HWPOISON_LARGE))
841 do_sigbus(regs, error_code, address, fault);
842 else
843 BUG();
844 }
845}
846
847static int spurious_fault_check(unsigned long error_code, pte_t *pte)
848{
849 if ((error_code & PF_WRITE) && !pte_write(*pte))
850 return 0;
851
852 if ((error_code & PF_INSTR) && !pte_exec(*pte))
853 return 0;
854
855 return 1;
856}
857
858
859
860
861
862
863
864
865
866
867
868
869
870static noinline __kprobes int
871spurious_fault(unsigned long error_code, unsigned long address)
872{
873 pgd_t *pgd;
874 pud_t *pud;
875 pmd_t *pmd;
876 pte_t *pte;
877 int ret;
878
879
880 if (error_code & (PF_USER | PF_RSVD))
881 return 0;
882
883 pgd = init_mm.pgd + pgd_index(address);
884 if (!pgd_present(*pgd))
885 return 0;
886
887 pud = pud_offset(pgd, address);
888 if (!pud_present(*pud))
889 return 0;
890
891 if (pud_large(*pud))
892 return spurious_fault_check(error_code, (pte_t *) pud);
893
894 pmd = pmd_offset(pud, address);
895 if (!pmd_present(*pmd))
896 return 0;
897
898 if (pmd_large(*pmd))
899 return spurious_fault_check(error_code, (pte_t *) pmd);
900
901
902
903
904
905
906
907 pte = pte_offset_kernel(pmd, address);
908 if (!(pte_flags(*pte) & _PAGE_PRESENT))
909 return 0;
910
911 ret = spurious_fault_check(error_code, pte);
912 if (!ret)
913 return 0;
914
915
916
917
918
919 ret = spurious_fault_check(error_code, (pte_t *) pmd);
920 WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
921
922 return ret;
923}
924
925int show_unhandled_signals = 1;
926
927static inline int
928access_error(unsigned long error_code, struct vm_area_struct *vma)
929{
930 if (error_code & PF_WRITE) {
931
932 if (unlikely(!(vma->vm_flags & VM_WRITE)))
933 return 1;
934 return 0;
935 }
936
937
938 if (unlikely(error_code & PF_PROT))
939 return 1;
940
941
942 if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
943 return 1;
944
945 return 0;
946}
947
948static int fault_in_kernel_space(unsigned long address)
949{
950 return address >= TASK_SIZE_MAX;
951}
952
953
954
955
956
957
958dotraplinkage void __kprobes
959do_page_fault(struct pt_regs *regs, unsigned long error_code)
960{
961 struct vm_area_struct *vma;
962 struct task_struct *tsk;
963 unsigned long address;
964 struct mm_struct *mm;
965 int fault;
966 int write = error_code & PF_WRITE;
967 unsigned int flags = FAULT_FLAG_ALLOW_RETRY |
968 (write ? FAULT_FLAG_WRITE : 0);
969
970 tsk = current;
971 mm = tsk->mm;
972
973
974 address = read_cr2();
975
976
977
978
979
980 if (kmemcheck_active(regs))
981 kmemcheck_hide(regs);
982 prefetchw(&mm->mmap_sem);
983
984 if (unlikely(kmmio_fault(regs, address)))
985 return;
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000 if (unlikely(fault_in_kernel_space(address))) {
1001 if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) {
1002 if (vmalloc_fault(address) >= 0)
1003 return;
1004
1005 if (kmemcheck_fault(regs, address, error_code))
1006 return;
1007 }
1008
1009
1010 if (spurious_fault(error_code, address))
1011 return;
1012
1013
1014 if (notify_page_fault(regs))
1015 return;
1016
1017
1018
1019
1020 bad_area_nosemaphore(regs, error_code, address);
1021
1022 return;
1023 }
1024
1025
1026 if (unlikely(notify_page_fault(regs)))
1027 return;
1028
1029
1030
1031
1032
1033
1034
1035 if (user_mode_vm(regs)) {
1036 local_irq_enable();
1037 error_code |= PF_USER;
1038 } else {
1039 if (regs->flags & X86_EFLAGS_IF)
1040 local_irq_enable();
1041 }
1042
1043 if (unlikely(error_code & PF_RSVD))
1044 pgtable_bad(regs, error_code, address);
1045
1046 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
1047
1048
1049
1050
1051
1052 if (unlikely(in_atomic() || !mm)) {
1053 bad_area_nosemaphore(regs, error_code, address);
1054 return;
1055 }
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
1074 if ((error_code & PF_USER) == 0 &&
1075 !search_exception_tables(regs->ip)) {
1076 bad_area_nosemaphore(regs, error_code, address);
1077 return;
1078 }
1079retry:
1080 down_read(&mm->mmap_sem);
1081 } else {
1082
1083
1084
1085
1086
1087 might_sleep();
1088 }
1089
1090 vma = find_vma(mm, address);
1091 if (unlikely(!vma)) {
1092 bad_area(regs, error_code, address);
1093 return;
1094 }
1095 if (likely(vma->vm_start <= address))
1096 goto good_area;
1097 if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
1098 bad_area(regs, error_code, address);
1099 return;
1100 }
1101 if (error_code & PF_USER) {
1102
1103
1104
1105
1106
1107
1108 if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
1109 bad_area(regs, error_code, address);
1110 return;
1111 }
1112 }
1113 if (unlikely(expand_stack(vma, address))) {
1114 bad_area(regs, error_code, address);
1115 return;
1116 }
1117
1118
1119
1120
1121
1122good_area:
1123 if (unlikely(access_error(error_code, vma))) {
1124 bad_area_access_error(regs, error_code, address);
1125 return;
1126 }
1127
1128
1129
1130
1131
1132
1133 fault = handle_mm_fault(mm, vma, address, flags);
1134
1135 if (unlikely(fault & VM_FAULT_ERROR)) {
1136 mm_fault_error(regs, error_code, address, fault);
1137 return;
1138 }
1139
1140
1141
1142
1143
1144
1145 if (flags & FAULT_FLAG_ALLOW_RETRY) {
1146 if (fault & VM_FAULT_MAJOR) {
1147 tsk->maj_flt++;
1148 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
1149 regs, address);
1150 } else {
1151 tsk->min_flt++;
1152 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
1153 regs, address);
1154 }
1155 if (fault & VM_FAULT_RETRY) {
1156
1157
1158 flags &= ~FAULT_FLAG_ALLOW_RETRY;
1159 goto retry;
1160 }
1161 }
1162
1163 check_v8086_mode(regs, address, tsk);
1164
1165 up_read(&mm->mmap_sem);
1166}
1167