1
2
3
4
5
6#include <linux/magic.h>
7#include <linux/sched.h>
8#include <linux/kdebug.h>
9#include <linux/module.h>
10#include <linux/bootmem.h>
11#include <linux/kprobes.h>
12#include <linux/mmiotrace.h>
13#include <linux/perf_event.h>
14#include <linux/hugetlb.h>
15#include <linux/prefetch.h>
16#include <linux/context_tracking.h>
17
18#include <asm/traps.h>
19#include <asm/pgalloc.h>
20#include <asm/kmemcheck.h>
21#include <asm/fixmap.h>
22
23
24
25
26
27
28
29
30
31
32enum x86_pf_error_code {
33
34 PF_PROT = 1 << 0,
35 PF_WRITE = 1 << 1,
36 PF_USER = 1 << 2,
37 PF_RSVD = 1 << 3,
38 PF_INSTR = 1 << 4,
39};
40
41
42
43
44
45static inline int __kprobes
46kmmio_fault(struct pt_regs *regs, unsigned long addr)
47{
48 if (unlikely(is_kmmio_active()))
49 if (kmmio_handler(regs, addr) == 1)
50 return -1;
51 return 0;
52}
53
54static inline int __kprobes notify_page_fault(struct pt_regs *regs)
55{
56 int ret = 0;
57
58
59 if (kprobes_built_in() && !user_mode_vm(regs)) {
60 preempt_disable();
61 if (kprobe_running() && kprobe_fault_handler(regs, 14))
62 ret = 1;
63 preempt_enable();
64 }
65
66 return ret;
67}
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84static inline int
85check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
86 unsigned char opcode, int *prefetch)
87{
88 unsigned char instr_hi = opcode & 0xf0;
89 unsigned char instr_lo = opcode & 0x0f;
90
91 switch (instr_hi) {
92 case 0x20:
93 case 0x30:
94
95
96
97
98
99
100 return ((instr_lo & 7) == 0x6);
101#ifdef CONFIG_X86_64
102 case 0x40:
103
104
105
106
107
108
109
110 return (!user_mode(regs) || user_64bit_mode(regs));
111#endif
112 case 0x60:
113
114 return (instr_lo & 0xC) == 0x4;
115 case 0xF0:
116
117 return !instr_lo || (instr_lo>>1) == 1;
118 case 0x00:
119
120 if (probe_kernel_address(instr, opcode))
121 return 0;
122
123 *prefetch = (instr_lo == 0xF) &&
124 (opcode == 0x0D || opcode == 0x18);
125 return 0;
126 default:
127 return 0;
128 }
129}
130
131static int
132is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
133{
134 unsigned char *max_instr;
135 unsigned char *instr;
136 int prefetch = 0;
137
138
139
140
141
142 if (error_code & PF_INSTR)
143 return 0;
144
145 instr = (void *)convert_ip_to_linear(current, regs);
146 max_instr = instr + 15;
147
148 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
149 return 0;
150
151 while (instr < max_instr) {
152 unsigned char opcode;
153
154 if (probe_kernel_address(instr, opcode))
155 break;
156
157 instr++;
158
159 if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
160 break;
161 }
162 return prefetch;
163}
164
165static void
166force_sig_info_fault(int si_signo, int si_code, unsigned long address,
167 struct task_struct *tsk, int fault)
168{
169 unsigned lsb = 0;
170 siginfo_t info;
171
172 info.si_signo = si_signo;
173 info.si_errno = 0;
174 info.si_code = si_code;
175 info.si_addr = (void __user *)address;
176 if (fault & VM_FAULT_HWPOISON_LARGE)
177 lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
178 if (fault & VM_FAULT_HWPOISON)
179 lsb = PAGE_SHIFT;
180 info.si_addr_lsb = lsb;
181
182 force_sig_info(si_signo, &info, tsk);
183}
184
185DEFINE_SPINLOCK(pgd_lock);
186LIST_HEAD(pgd_list);
187
188#ifdef CONFIG_X86_32
189static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
190{
191 unsigned index = pgd_index(address);
192 pgd_t *pgd_k;
193 pud_t *pud, *pud_k;
194 pmd_t *pmd, *pmd_k;
195
196 pgd += index;
197 pgd_k = init_mm.pgd + index;
198
199 if (!pgd_present(*pgd_k))
200 return NULL;
201
202
203
204
205
206
207 pud = pud_offset(pgd, address);
208 pud_k = pud_offset(pgd_k, address);
209 if (!pud_present(*pud_k))
210 return NULL;
211
212 pmd = pmd_offset(pud, address);
213 pmd_k = pmd_offset(pud_k, address);
214 if (!pmd_present(*pmd_k))
215 return NULL;
216
217 if (!pmd_present(*pmd))
218 set_pmd(pmd, *pmd_k);
219 else
220 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
221
222 return pmd_k;
223}
224
225void vmalloc_sync_all(void)
226{
227 unsigned long address;
228
229 if (SHARED_KERNEL_PMD)
230 return;
231
232 for (address = VMALLOC_START & PMD_MASK;
233 address >= TASK_SIZE && address < FIXADDR_TOP;
234 address += PMD_SIZE) {
235 struct page *page;
236
237 spin_lock(&pgd_lock);
238 list_for_each_entry(page, &pgd_list, lru) {
239 spinlock_t *pgt_lock;
240 pmd_t *ret;
241
242
243 pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
244
245 spin_lock(pgt_lock);
246 ret = vmalloc_sync_one(page_address(page), address);
247 spin_unlock(pgt_lock);
248
249 if (!ret)
250 break;
251 }
252 spin_unlock(&pgd_lock);
253 }
254}
255
256
257
258
259
260
261static noinline __kprobes int vmalloc_fault(unsigned long address)
262{
263 unsigned long pgd_paddr;
264 pmd_t *pmd_k;
265 pte_t *pte_k;
266
267
268 if (!(address >= VMALLOC_START && address < VMALLOC_END))
269 return -1;
270
271 WARN_ON_ONCE(in_nmi());
272
273
274
275
276
277
278
279
280 pgd_paddr = read_cr3();
281 pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
282 if (!pmd_k)
283 return -1;
284
285 pte_k = pte_offset_kernel(pmd_k, address);
286 if (!pte_present(*pte_k))
287 return -1;
288
289 return 0;
290}
291
292
293
294
295static inline void
296check_v8086_mode(struct pt_regs *regs, unsigned long address,
297 struct task_struct *tsk)
298{
299 unsigned long bit;
300
301 if (!v8086_mode(regs))
302 return;
303
304 bit = (address - 0xA0000) >> PAGE_SHIFT;
305 if (bit < 32)
306 tsk->thread.screen_bitmap |= 1 << bit;
307}
308
309static bool low_pfn(unsigned long pfn)
310{
311 return pfn < max_low_pfn;
312}
313
314static void dump_pagetable(unsigned long address)
315{
316 pgd_t *base = __va(read_cr3());
317 pgd_t *pgd = &base[pgd_index(address)];
318 pmd_t *pmd;
319 pte_t *pte;
320
321#ifdef CONFIG_X86_PAE
322 printk("*pdpt = %016Lx ", pgd_val(*pgd));
323 if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
324 goto out;
325#endif
326 pmd = pmd_offset(pud_offset(pgd, address), address);
327 printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
328
329
330
331
332
333
334
335 if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
336 goto out;
337
338 pte = pte_offset_kernel(pmd, address);
339 printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
340out:
341 printk("\n");
342}
343
344#else
345
346void vmalloc_sync_all(void)
347{
348 sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
349}
350
351
352
353
354
355
356
357
358static noinline __kprobes int vmalloc_fault(unsigned long address)
359{
360 pgd_t *pgd, *pgd_ref;
361 pud_t *pud, *pud_ref;
362 pmd_t *pmd, *pmd_ref;
363 pte_t *pte, *pte_ref;
364
365
366 if (!(address >= VMALLOC_START && address < VMALLOC_END))
367 return -1;
368
369 WARN_ON_ONCE(in_nmi());
370
371
372
373
374
375
376 pgd = pgd_offset(current->active_mm, address);
377 pgd_ref = pgd_offset_k(address);
378 if (pgd_none(*pgd_ref))
379 return -1;
380
381 if (pgd_none(*pgd)) {
382 set_pgd(pgd, *pgd_ref);
383 arch_flush_lazy_mmu_mode();
384 } else {
385 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
386 }
387
388
389
390
391
392
393 pud = pud_offset(pgd, address);
394 pud_ref = pud_offset(pgd_ref, address);
395 if (pud_none(*pud_ref))
396 return -1;
397
398 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
399 BUG();
400
401 pmd = pmd_offset(pud, address);
402 pmd_ref = pmd_offset(pud_ref, address);
403 if (pmd_none(*pmd_ref))
404 return -1;
405
406 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
407 BUG();
408
409 pte_ref = pte_offset_kernel(pmd_ref, address);
410 if (!pte_present(*pte_ref))
411 return -1;
412
413 pte = pte_offset_kernel(pmd, address);
414
415
416
417
418
419
420 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
421 BUG();
422
423 return 0;
424}
425
426#ifdef CONFIG_CPU_SUP_AMD
427static const char errata93_warning[] =
428KERN_ERR
429"******* Your BIOS seems to not contain a fix for K8 errata #93\n"
430"******* Working around it, but it may cause SEGVs or burn power.\n"
431"******* Please consider a BIOS update.\n"
432"******* Disabling USB legacy in the BIOS may also help.\n";
433#endif
434
435
436
437
438static inline void
439check_v8086_mode(struct pt_regs *regs, unsigned long address,
440 struct task_struct *tsk)
441{
442}
443
444static int bad_address(void *p)
445{
446 unsigned long dummy;
447
448 return probe_kernel_address((unsigned long *)p, dummy);
449}
450
451static void dump_pagetable(unsigned long address)
452{
453 pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
454 pgd_t *pgd = base + pgd_index(address);
455 pud_t *pud;
456 pmd_t *pmd;
457 pte_t *pte;
458
459 if (bad_address(pgd))
460 goto bad;
461
462 printk("PGD %lx ", pgd_val(*pgd));
463
464 if (!pgd_present(*pgd))
465 goto out;
466
467 pud = pud_offset(pgd, address);
468 if (bad_address(pud))
469 goto bad;
470
471 printk("PUD %lx ", pud_val(*pud));
472 if (!pud_present(*pud) || pud_large(*pud))
473 goto out;
474
475 pmd = pmd_offset(pud, address);
476 if (bad_address(pmd))
477 goto bad;
478
479 printk("PMD %lx ", pmd_val(*pmd));
480 if (!pmd_present(*pmd) || pmd_large(*pmd))
481 goto out;
482
483 pte = pte_offset_kernel(pmd, address);
484 if (bad_address(pte))
485 goto bad;
486
487 printk("PTE %lx", pte_val(*pte));
488out:
489 printk("\n");
490 return;
491bad:
492 printk("BAD\n");
493}
494
495#endif
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511static int is_errata93(struct pt_regs *regs, unsigned long address)
512{
513#if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD)
514 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD
515 || boot_cpu_data.x86 != 0xf)
516 return 0;
517
518 if (address != regs->ip)
519 return 0;
520
521 if ((address >> 32) != 0)
522 return 0;
523
524 address |= 0xffffffffUL << 32;
525 if ((address >= (u64)_stext && address <= (u64)_etext) ||
526 (address >= MODULES_VADDR && address <= MODULES_END)) {
527 printk_once(errata93_warning);
528 regs->ip = address;
529 return 1;
530 }
531#endif
532 return 0;
533}
534
535
536
537
538
539
540
541
542
543static int is_errata100(struct pt_regs *regs, unsigned long address)
544{
545#ifdef CONFIG_X86_64
546 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
547 return 1;
548#endif
549 return 0;
550}
551
552static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
553{
554#ifdef CONFIG_X86_F00F_BUG
555 unsigned long nr;
556
557
558
559
560 if (boot_cpu_has_bug(X86_BUG_F00F)) {
561 nr = (address - idt_descr.address) >> 3;
562
563 if (nr == 6) {
564 do_invalid_op(regs, 0);
565 return 1;
566 }
567 }
568#endif
569 return 0;
570}
571
572static const char nx_warning[] = KERN_CRIT
573"kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
574
575static void
576show_fault_oops(struct pt_regs *regs, unsigned long error_code,
577 unsigned long address)
578{
579 if (!oops_may_print())
580 return;
581
582 if (error_code & PF_INSTR) {
583 unsigned int level;
584
585 pte_t *pte = lookup_address(address, &level);
586
587 if (pte && pte_present(*pte) && !pte_exec(*pte))
588 printk(nx_warning, from_kuid(&init_user_ns, current_uid()));
589 }
590
591 printk(KERN_ALERT "BUG: unable to handle kernel ");
592 if (address < PAGE_SIZE)
593 printk(KERN_CONT "NULL pointer dereference");
594 else
595 printk(KERN_CONT "paging request");
596
597 printk(KERN_CONT " at %p\n", (void *) address);
598 printk(KERN_ALERT "IP:");
599 printk_address(regs->ip, 1);
600
601 dump_pagetable(address);
602}
603
604static noinline void
605pgtable_bad(struct pt_regs *regs, unsigned long error_code,
606 unsigned long address)
607{
608 struct task_struct *tsk;
609 unsigned long flags;
610 int sig;
611
612 flags = oops_begin();
613 tsk = current;
614 sig = SIGKILL;
615
616 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
617 tsk->comm, address);
618 dump_pagetable(address);
619
620 tsk->thread.cr2 = address;
621 tsk->thread.trap_nr = X86_TRAP_PF;
622 tsk->thread.error_code = error_code;
623
624 if (__die("Bad pagetable", regs, error_code))
625 sig = 0;
626
627 oops_end(flags, regs, sig);
628}
629
630static noinline void
631no_context(struct pt_regs *regs, unsigned long error_code,
632 unsigned long address, int signal, int si_code)
633{
634 struct task_struct *tsk = current;
635 unsigned long *stackend;
636 unsigned long flags;
637 int sig;
638
639
640 if (fixup_exception(regs)) {
641 if (current_thread_info()->sig_on_uaccess_error && signal) {
642 tsk->thread.trap_nr = X86_TRAP_PF;
643 tsk->thread.error_code = error_code | PF_USER;
644 tsk->thread.cr2 = address;
645
646
647 force_sig_info_fault(signal, si_code, address, tsk, 0);
648 }
649 return;
650 }
651
652
653
654
655
656
657
658
659
660
661
662
663 if (is_prefetch(regs, error_code, address))
664 return;
665
666 if (is_errata93(regs, address))
667 return;
668
669
670
671
672
673 flags = oops_begin();
674
675 show_fault_oops(regs, error_code, address);
676
677 stackend = end_of_stack(tsk);
678 if (tsk != &init_task && *stackend != STACK_END_MAGIC)
679 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
680
681 tsk->thread.cr2 = address;
682 tsk->thread.trap_nr = X86_TRAP_PF;
683 tsk->thread.error_code = error_code;
684
685 sig = SIGKILL;
686 if (__die("Oops", regs, error_code))
687 sig = 0;
688
689
690 printk(KERN_DEFAULT "CR2: %016lx\n", address);
691
692 oops_end(flags, regs, sig);
693}
694
695
696
697
698
699static inline void
700show_signal_msg(struct pt_regs *regs, unsigned long error_code,
701 unsigned long address, struct task_struct *tsk)
702{
703 if (!unhandled_signal(tsk, SIGSEGV))
704 return;
705
706 if (!printk_ratelimit())
707 return;
708
709 printk("%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
710 task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
711 tsk->comm, task_pid_nr(tsk), address,
712 (void *)regs->ip, (void *)regs->sp, error_code);
713
714 print_vma_addr(KERN_CONT " in ", regs->ip);
715
716 printk(KERN_CONT "\n");
717}
718
719static void
720__bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
721 unsigned long address, int si_code)
722{
723 struct task_struct *tsk = current;
724
725
726 if (error_code & PF_USER) {
727
728
729
730 local_irq_enable();
731
732
733
734
735
736 if (is_prefetch(regs, error_code, address))
737 return;
738
739 if (is_errata100(regs, address))
740 return;
741
742#ifdef CONFIG_X86_64
743
744
745
746
747 if (unlikely((error_code & PF_INSTR) &&
748 ((address & ~0xfff) == VSYSCALL_START))) {
749 if (emulate_vsyscall(regs, address))
750 return;
751 }
752#endif
753
754 if (address >= TASK_SIZE)
755 error_code |= PF_PROT;
756
757 if (likely(show_unhandled_signals))
758 show_signal_msg(regs, error_code, address, tsk);
759
760 tsk->thread.cr2 = address;
761 tsk->thread.error_code = error_code;
762 tsk->thread.trap_nr = X86_TRAP_PF;
763
764 force_sig_info_fault(SIGSEGV, si_code, address, tsk, 0);
765
766 return;
767 }
768
769 if (is_f00f_bug(regs, address))
770 return;
771
772 no_context(regs, error_code, address, SIGSEGV, si_code);
773}
774
775static noinline void
776bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
777 unsigned long address)
778{
779 __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
780}
781
782static void
783__bad_area(struct pt_regs *regs, unsigned long error_code,
784 unsigned long address, int si_code)
785{
786 struct mm_struct *mm = current->mm;
787
788
789
790
791
792 up_read(&mm->mmap_sem);
793
794 __bad_area_nosemaphore(regs, error_code, address, si_code);
795}
796
797static noinline void
798bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
799{
800 __bad_area(regs, error_code, address, SEGV_MAPERR);
801}
802
803static noinline void
804bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
805 unsigned long address)
806{
807 __bad_area(regs, error_code, address, SEGV_ACCERR);
808}
809
810static void
811do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
812 unsigned int fault)
813{
814 struct task_struct *tsk = current;
815 struct mm_struct *mm = tsk->mm;
816 int code = BUS_ADRERR;
817
818 up_read(&mm->mmap_sem);
819
820
821 if (!(error_code & PF_USER)) {
822 no_context(regs, error_code, address, SIGBUS, BUS_ADRERR);
823 return;
824 }
825
826
827 if (is_prefetch(regs, error_code, address))
828 return;
829
830 tsk->thread.cr2 = address;
831 tsk->thread.error_code = error_code;
832 tsk->thread.trap_nr = X86_TRAP_PF;
833
834#ifdef CONFIG_MEMORY_FAILURE
835 if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
836 printk(KERN_ERR
837 "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
838 tsk->comm, tsk->pid, address);
839 code = BUS_MCEERR_AR;
840 }
841#endif
842 force_sig_info_fault(SIGBUS, code, address, tsk, fault);
843}
844
845static noinline void
846mm_fault_error(struct pt_regs *regs, unsigned long error_code,
847 unsigned long address, unsigned int fault)
848{
849 if (fatal_signal_pending(current) && !(error_code & PF_USER)) {
850 up_read(¤t->mm->mmap_sem);
851 no_context(regs, error_code, address, 0, 0);
852 return;
853 }
854
855 if (fault & VM_FAULT_OOM) {
856
857 if (!(error_code & PF_USER)) {
858 up_read(¤t->mm->mmap_sem);
859 no_context(regs, error_code, address,
860 SIGSEGV, SEGV_MAPERR);
861 return;
862 }
863
864 up_read(¤t->mm->mmap_sem);
865
866
867
868
869
870
871 pagefault_out_of_memory();
872 } else {
873 if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
874 VM_FAULT_HWPOISON_LARGE))
875 do_sigbus(regs, error_code, address, fault);
876 else
877 BUG();
878 }
879}
880
881static int spurious_fault_check(unsigned long error_code, pte_t *pte)
882{
883 if ((error_code & PF_WRITE) && !pte_write(*pte))
884 return 0;
885
886 if ((error_code & PF_INSTR) && !pte_exec(*pte))
887 return 0;
888
889 return 1;
890}
891
892
893
894
895
896
897
898
899
900
901
902
903
904static noinline __kprobes int
905spurious_fault(unsigned long error_code, unsigned long address)
906{
907 pgd_t *pgd;
908 pud_t *pud;
909 pmd_t *pmd;
910 pte_t *pte;
911 int ret;
912
913
914 if (error_code & (PF_USER | PF_RSVD))
915 return 0;
916
917 pgd = init_mm.pgd + pgd_index(address);
918 if (!pgd_present(*pgd))
919 return 0;
920
921 pud = pud_offset(pgd, address);
922 if (!pud_present(*pud))
923 return 0;
924
925 if (pud_large(*pud))
926 return spurious_fault_check(error_code, (pte_t *) pud);
927
928 pmd = pmd_offset(pud, address);
929 if (!pmd_present(*pmd))
930 return 0;
931
932 if (pmd_large(*pmd))
933 return spurious_fault_check(error_code, (pte_t *) pmd);
934
935 pte = pte_offset_kernel(pmd, address);
936 if (!pte_present(*pte))
937 return 0;
938
939 ret = spurious_fault_check(error_code, pte);
940 if (!ret)
941 return 0;
942
943
944
945
946
947 ret = spurious_fault_check(error_code, (pte_t *) pmd);
948 WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
949
950 return ret;
951}
952
953int show_unhandled_signals = 1;
954
955static inline int
956access_error(unsigned long error_code, struct vm_area_struct *vma)
957{
958 if (error_code & PF_WRITE) {
959
960 if (unlikely(!(vma->vm_flags & VM_WRITE)))
961 return 1;
962 return 0;
963 }
964
965
966 if (unlikely(error_code & PF_PROT))
967 return 1;
968
969
970 if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
971 return 1;
972
973 return 0;
974}
975
976static int fault_in_kernel_space(unsigned long address)
977{
978 return address >= TASK_SIZE_MAX;
979}
980
981static inline bool smap_violation(int error_code, struct pt_regs *regs)
982{
983 if (error_code & PF_USER)
984 return false;
985
986 if (!user_mode_vm(regs) && (regs->flags & X86_EFLAGS_AC))
987 return false;
988
989 return true;
990}
991
992
993
994
995
996
997static void __kprobes
998__do_page_fault(struct pt_regs *regs, unsigned long error_code)
999{
1000 struct vm_area_struct *vma;
1001 struct task_struct *tsk;
1002 unsigned long address;
1003 struct mm_struct *mm;
1004 int fault;
1005 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
1006
1007 tsk = current;
1008 mm = tsk->mm;
1009
1010
1011 address = read_cr2();
1012
1013
1014
1015
1016
1017 if (kmemcheck_active(regs))
1018 kmemcheck_hide(regs);
1019 prefetchw(&mm->mmap_sem);
1020
1021 if (unlikely(kmmio_fault(regs, address)))
1022 return;
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037 if (unlikely(fault_in_kernel_space(address))) {
1038 if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) {
1039 if (vmalloc_fault(address) >= 0)
1040 return;
1041
1042 if (kmemcheck_fault(regs, address, error_code))
1043 return;
1044 }
1045
1046
1047 if (spurious_fault(error_code, address))
1048 return;
1049
1050
1051 if (notify_page_fault(regs))
1052 return;
1053
1054
1055
1056
1057 bad_area_nosemaphore(regs, error_code, address);
1058
1059 return;
1060 }
1061
1062
1063 if (unlikely(notify_page_fault(regs)))
1064 return;
1065
1066
1067
1068
1069
1070
1071
1072 if (user_mode_vm(regs)) {
1073 local_irq_enable();
1074 error_code |= PF_USER;
1075 flags |= FAULT_FLAG_USER;
1076 } else {
1077 if (regs->flags & X86_EFLAGS_IF)
1078 local_irq_enable();
1079 }
1080
1081 if (unlikely(error_code & PF_RSVD))
1082 pgtable_bad(regs, error_code, address);
1083
1084 if (static_cpu_has(X86_FEATURE_SMAP)) {
1085 if (unlikely(smap_violation(error_code, regs))) {
1086 bad_area_nosemaphore(regs, error_code, address);
1087 return;
1088 }
1089 }
1090
1091 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
1092
1093
1094
1095
1096
1097 if (unlikely(in_atomic() || !mm)) {
1098 bad_area_nosemaphore(regs, error_code, address);
1099 return;
1100 }
1101
1102 if (error_code & PF_WRITE)
1103 flags |= FAULT_FLAG_WRITE;
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
1122 if ((error_code & PF_USER) == 0 &&
1123 !search_exception_tables(regs->ip)) {
1124 bad_area_nosemaphore(regs, error_code, address);
1125 return;
1126 }
1127retry:
1128 down_read(&mm->mmap_sem);
1129 } else {
1130
1131
1132
1133
1134
1135 might_sleep();
1136 }
1137
1138 vma = find_vma(mm, address);
1139 if (unlikely(!vma)) {
1140 bad_area(regs, error_code, address);
1141 return;
1142 }
1143 if (likely(vma->vm_start <= address))
1144 goto good_area;
1145 if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
1146 bad_area(regs, error_code, address);
1147 return;
1148 }
1149 if (error_code & PF_USER) {
1150
1151
1152
1153
1154
1155
1156 if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
1157 bad_area(regs, error_code, address);
1158 return;
1159 }
1160 }
1161 if (unlikely(expand_stack(vma, address))) {
1162 bad_area(regs, error_code, address);
1163 return;
1164 }
1165
1166
1167
1168
1169
1170good_area:
1171 if (unlikely(access_error(error_code, vma))) {
1172 bad_area_access_error(regs, error_code, address);
1173 return;
1174 }
1175
1176
1177
1178
1179
1180
1181 fault = handle_mm_fault(mm, vma, address, flags);
1182
1183
1184
1185
1186
1187
1188 if (unlikely((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)))
1189 return;
1190
1191 if (unlikely(fault & VM_FAULT_ERROR)) {
1192 mm_fault_error(regs, error_code, address, fault);
1193 return;
1194 }
1195
1196
1197
1198
1199
1200
1201 if (flags & FAULT_FLAG_ALLOW_RETRY) {
1202 if (fault & VM_FAULT_MAJOR) {
1203 tsk->maj_flt++;
1204 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
1205 regs, address);
1206 } else {
1207 tsk->min_flt++;
1208 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
1209 regs, address);
1210 }
1211 if (fault & VM_FAULT_RETRY) {
1212
1213
1214 flags &= ~FAULT_FLAG_ALLOW_RETRY;
1215 flags |= FAULT_FLAG_TRIED;
1216 goto retry;
1217 }
1218 }
1219
1220 check_v8086_mode(regs, address, tsk);
1221
1222 up_read(&mm->mmap_sem);
1223}
1224
1225dotraplinkage void __kprobes
1226do_page_fault(struct pt_regs *regs, unsigned long error_code)
1227{
1228 enum ctx_state prev_state;
1229
1230 prev_state = exception_enter();
1231 __do_page_fault(regs, error_code);
1232 exception_exit(prev_state);
1233}
1234