1
2
3
4
5
6#include <linux/magic.h>
7#include <linux/sched.h>
8#include <linux/kdebug.h>
9#include <linux/module.h>
10#include <linux/bootmem.h>
11#include <linux/kprobes.h>
12#include <linux/mmiotrace.h>
13#include <linux/perf_event.h>
14
15#include <asm/traps.h>
16#include <asm/pgalloc.h>
17#include <asm/kmemcheck.h>
18
19
20
21
22
23
24
25
26
27
28enum x86_pf_error_code {
29
30 PF_PROT = 1 << 0,
31 PF_WRITE = 1 << 1,
32 PF_USER = 1 << 2,
33 PF_RSVD = 1 << 3,
34 PF_INSTR = 1 << 4,
35};
36
37
38
39
40
41static inline int kmmio_fault(struct pt_regs *regs, unsigned long addr)
42{
43 if (unlikely(is_kmmio_active()))
44 if (kmmio_handler(regs, addr) == 1)
45 return -1;
46 return 0;
47}
48
49static inline int notify_page_fault(struct pt_regs *regs)
50{
51 int ret = 0;
52
53
54 if (kprobes_built_in() && !user_mode_vm(regs)) {
55 preempt_disable();
56 if (kprobe_running() && kprobe_fault_handler(regs, 14))
57 ret = 1;
58 preempt_enable();
59 }
60
61 return ret;
62}
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79static inline int
80check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
81 unsigned char opcode, int *prefetch)
82{
83 unsigned char instr_hi = opcode & 0xf0;
84 unsigned char instr_lo = opcode & 0x0f;
85
86 switch (instr_hi) {
87 case 0x20:
88 case 0x30:
89
90
91
92
93
94
95 return ((instr_lo & 7) == 0x6);
96#ifdef CONFIG_X86_64
97 case 0x40:
98
99
100
101
102
103
104
105 return (!user_mode(regs)) || (regs->cs == __USER_CS);
106#endif
107 case 0x60:
108
109 return (instr_lo & 0xC) == 0x4;
110 case 0xF0:
111
112 return !instr_lo || (instr_lo>>1) == 1;
113 case 0x00:
114
115 if (probe_kernel_address(instr, opcode))
116 return 0;
117
118 *prefetch = (instr_lo == 0xF) &&
119 (opcode == 0x0D || opcode == 0x18);
120 return 0;
121 default:
122 return 0;
123 }
124}
125
126static int
127is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
128{
129 unsigned char *max_instr;
130 unsigned char *instr;
131 int prefetch = 0;
132
133
134
135
136
137 if (error_code & PF_INSTR)
138 return 0;
139
140 instr = (void *)convert_ip_to_linear(current, regs);
141 max_instr = instr + 15;
142
143 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
144 return 0;
145
146 while (instr < max_instr) {
147 unsigned char opcode;
148
149 if (probe_kernel_address(instr, opcode))
150 break;
151
152 instr++;
153
154 if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
155 break;
156 }
157 return prefetch;
158}
159
160static void
161force_sig_info_fault(int si_signo, int si_code, unsigned long address,
162 struct task_struct *tsk)
163{
164 siginfo_t info;
165
166 info.si_signo = si_signo;
167 info.si_errno = 0;
168 info.si_code = si_code;
169 info.si_addr = (void __user *)address;
170 info.si_addr_lsb = si_code == BUS_MCEERR_AR ? PAGE_SHIFT : 0;
171
172 force_sig_info(si_signo, &info, tsk);
173}
174
175DEFINE_SPINLOCK(pgd_lock);
176LIST_HEAD(pgd_list);
177
178#ifdef CONFIG_X86_32
179static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
180{
181 unsigned index = pgd_index(address);
182 pgd_t *pgd_k;
183 pud_t *pud, *pud_k;
184 pmd_t *pmd, *pmd_k;
185
186 pgd += index;
187 pgd_k = init_mm.pgd + index;
188
189 if (!pgd_present(*pgd_k))
190 return NULL;
191
192
193
194
195
196
197 pud = pud_offset(pgd, address);
198 pud_k = pud_offset(pgd_k, address);
199 if (!pud_present(*pud_k))
200 return NULL;
201
202 pmd = pmd_offset(pud, address);
203 pmd_k = pmd_offset(pud_k, address);
204 if (!pmd_present(*pmd_k))
205 return NULL;
206
207 if (!pmd_present(*pmd))
208 set_pmd(pmd, *pmd_k);
209 else
210 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
211
212 return pmd_k;
213}
214
215void vmalloc_sync_all(void)
216{
217 unsigned long address;
218
219 if (SHARED_KERNEL_PMD)
220 return;
221
222 for (address = VMALLOC_START & PMD_MASK;
223 address >= TASK_SIZE && address < FIXADDR_TOP;
224 address += PMD_SIZE) {
225
226 unsigned long flags;
227 struct page *page;
228
229 spin_lock_irqsave(&pgd_lock, flags);
230 list_for_each_entry(page, &pgd_list, lru) {
231 if (!vmalloc_sync_one(page_address(page), address))
232 break;
233 }
234 spin_unlock_irqrestore(&pgd_lock, flags);
235 }
236}
237
238
239
240
241
242
243static noinline int vmalloc_fault(unsigned long address)
244{
245 unsigned long pgd_paddr;
246 pmd_t *pmd_k;
247 pte_t *pte_k;
248
249
250 if (!(address >= VMALLOC_START && address < VMALLOC_END))
251 return -1;
252
253
254
255
256
257
258
259
260 pgd_paddr = read_cr3();
261 pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
262 if (!pmd_k)
263 return -1;
264
265 pte_k = pte_offset_kernel(pmd_k, address);
266 if (!pte_present(*pte_k))
267 return -1;
268
269 return 0;
270}
271
272
273
274
275static inline void
276check_v8086_mode(struct pt_regs *regs, unsigned long address,
277 struct task_struct *tsk)
278{
279 unsigned long bit;
280
281 if (!v8086_mode(regs))
282 return;
283
284 bit = (address - 0xA0000) >> PAGE_SHIFT;
285 if (bit < 32)
286 tsk->thread.screen_bitmap |= 1 << bit;
287}
288
289static bool low_pfn(unsigned long pfn)
290{
291 return pfn < max_low_pfn;
292}
293
294static void dump_pagetable(unsigned long address)
295{
296 pgd_t *base = __va(read_cr3());
297 pgd_t *pgd = &base[pgd_index(address)];
298 pmd_t *pmd;
299 pte_t *pte;
300
301#ifdef CONFIG_X86_PAE
302 printk("*pdpt = %016Lx ", pgd_val(*pgd));
303 if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
304 goto out;
305#endif
306 pmd = pmd_offset(pud_offset(pgd, address), address);
307 printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
308
309
310
311
312
313
314
315 if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
316 goto out;
317
318 pte = pte_offset_kernel(pmd, address);
319 printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
320out:
321 printk("\n");
322}
323
324#else
325
326void vmalloc_sync_all(void)
327{
328 unsigned long address;
329
330 for (address = VMALLOC_START & PGDIR_MASK; address <= VMALLOC_END;
331 address += PGDIR_SIZE) {
332
333 const pgd_t *pgd_ref = pgd_offset_k(address);
334 unsigned long flags;
335 struct page *page;
336
337 if (pgd_none(*pgd_ref))
338 continue;
339
340 spin_lock_irqsave(&pgd_lock, flags);
341 list_for_each_entry(page, &pgd_list, lru) {
342 pgd_t *pgd;
343 pgd = (pgd_t *)page_address(page) + pgd_index(address);
344 if (pgd_none(*pgd))
345 set_pgd(pgd, *pgd_ref);
346 else
347 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
348 }
349 spin_unlock_irqrestore(&pgd_lock, flags);
350 }
351}
352
353
354
355
356
357
358
359
360static noinline int vmalloc_fault(unsigned long address)
361{
362 pgd_t *pgd, *pgd_ref;
363 pud_t *pud, *pud_ref;
364 pmd_t *pmd, *pmd_ref;
365 pte_t *pte, *pte_ref;
366
367
368 if (!(address >= VMALLOC_START && address < VMALLOC_END))
369 return -1;
370
371
372
373
374
375
376 pgd = pgd_offset(current->active_mm, address);
377 pgd_ref = pgd_offset_k(address);
378 if (pgd_none(*pgd_ref))
379 return -1;
380
381 if (pgd_none(*pgd))
382 set_pgd(pgd, *pgd_ref);
383 else
384 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
385
386
387
388
389
390
391 pud = pud_offset(pgd, address);
392 pud_ref = pud_offset(pgd_ref, address);
393 if (pud_none(*pud_ref))
394 return -1;
395
396 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
397 BUG();
398
399 pmd = pmd_offset(pud, address);
400 pmd_ref = pmd_offset(pud_ref, address);
401 if (pmd_none(*pmd_ref))
402 return -1;
403
404 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
405 BUG();
406
407 pte_ref = pte_offset_kernel(pmd_ref, address);
408 if (!pte_present(*pte_ref))
409 return -1;
410
411 pte = pte_offset_kernel(pmd, address);
412
413
414
415
416
417
418 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
419 BUG();
420
421 return 0;
422}
423
424static const char errata93_warning[] =
425KERN_ERR
426"******* Your BIOS seems to not contain a fix for K8 errata #93\n"
427"******* Working around it, but it may cause SEGVs or burn power.\n"
428"******* Please consider a BIOS update.\n"
429"******* Disabling USB legacy in the BIOS may also help.\n";
430
431
432
433
434static inline void
435check_v8086_mode(struct pt_regs *regs, unsigned long address,
436 struct task_struct *tsk)
437{
438}
439
440static int bad_address(void *p)
441{
442 unsigned long dummy;
443
444 return probe_kernel_address((unsigned long *)p, dummy);
445}
446
447static void dump_pagetable(unsigned long address)
448{
449 pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
450 pgd_t *pgd = base + pgd_index(address);
451 pud_t *pud;
452 pmd_t *pmd;
453 pte_t *pte;
454
455 if (bad_address(pgd))
456 goto bad;
457
458 printk("PGD %lx ", pgd_val(*pgd));
459
460 if (!pgd_present(*pgd))
461 goto out;
462
463 pud = pud_offset(pgd, address);
464 if (bad_address(pud))
465 goto bad;
466
467 printk("PUD %lx ", pud_val(*pud));
468 if (!pud_present(*pud) || pud_large(*pud))
469 goto out;
470
471 pmd = pmd_offset(pud, address);
472 if (bad_address(pmd))
473 goto bad;
474
475 printk("PMD %lx ", pmd_val(*pmd));
476 if (!pmd_present(*pmd) || pmd_large(*pmd))
477 goto out;
478
479 pte = pte_offset_kernel(pmd, address);
480 if (bad_address(pte))
481 goto bad;
482
483 printk("PTE %lx", pte_val(*pte));
484out:
485 printk("\n");
486 return;
487bad:
488 printk("BAD\n");
489}
490
491#endif
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507static int is_errata93(struct pt_regs *regs, unsigned long address)
508{
509#ifdef CONFIG_X86_64
510 if (address != regs->ip)
511 return 0;
512
513 if ((address >> 32) != 0)
514 return 0;
515
516 address |= 0xffffffffUL << 32;
517 if ((address >= (u64)_stext && address <= (u64)_etext) ||
518 (address >= MODULES_VADDR && address <= MODULES_END)) {
519 printk_once(errata93_warning);
520 regs->ip = address;
521 return 1;
522 }
523#endif
524 return 0;
525}
526
527
528
529
530
531
532
533
534
535static int is_errata100(struct pt_regs *regs, unsigned long address)
536{
537#ifdef CONFIG_X86_64
538 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
539 return 1;
540#endif
541 return 0;
542}
543
544static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
545{
546#ifdef CONFIG_X86_F00F_BUG
547 unsigned long nr;
548
549
550
551
552 if (boot_cpu_data.f00f_bug) {
553 nr = (address - idt_descr.address) >> 3;
554
555 if (nr == 6) {
556 do_invalid_op(regs, 0);
557 return 1;
558 }
559 }
560#endif
561 return 0;
562}
563
564static const char nx_warning[] = KERN_CRIT
565"kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
566
567static void
568show_fault_oops(struct pt_regs *regs, unsigned long error_code,
569 unsigned long address)
570{
571 if (!oops_may_print())
572 return;
573
574 if (error_code & PF_INSTR) {
575 unsigned int level;
576
577 pte_t *pte = lookup_address(address, &level);
578
579 if (pte && pte_present(*pte) && !pte_exec(*pte))
580 printk(nx_warning, current_uid());
581 }
582
583 printk(KERN_ALERT "BUG: unable to handle kernel ");
584 if (address < PAGE_SIZE)
585 printk(KERN_CONT "NULL pointer dereference");
586 else
587 printk(KERN_CONT "paging request");
588
589 printk(KERN_CONT " at %p\n", (void *) address);
590 printk(KERN_ALERT "IP:");
591 printk_address(regs->ip, 1);
592
593 dump_pagetable(address);
594}
595
596static noinline void
597pgtable_bad(struct pt_regs *regs, unsigned long error_code,
598 unsigned long address)
599{
600 struct task_struct *tsk;
601 unsigned long flags;
602 int sig;
603
604 flags = oops_begin();
605 tsk = current;
606 sig = SIGKILL;
607
608 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
609 tsk->comm, address);
610 dump_pagetable(address);
611
612 tsk->thread.cr2 = address;
613 tsk->thread.trap_no = 14;
614 tsk->thread.error_code = error_code;
615
616 if (__die("Bad pagetable", regs, error_code))
617 sig = 0;
618
619 oops_end(flags, regs, sig);
620}
621
622static noinline void
623no_context(struct pt_regs *regs, unsigned long error_code,
624 unsigned long address)
625{
626 struct task_struct *tsk = current;
627 unsigned long *stackend;
628 unsigned long flags;
629 int sig;
630
631
632 if (fixup_exception(regs))
633 return;
634
635
636
637
638
639
640
641
642
643
644
645
646 if (is_prefetch(regs, error_code, address))
647 return;
648
649 if (is_errata93(regs, address))
650 return;
651
652
653
654
655
656 flags = oops_begin();
657
658 show_fault_oops(regs, error_code, address);
659
660 stackend = end_of_stack(tsk);
661 if (*stackend != STACK_END_MAGIC)
662 printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
663
664 tsk->thread.cr2 = address;
665 tsk->thread.trap_no = 14;
666 tsk->thread.error_code = error_code;
667
668 sig = SIGKILL;
669 if (__die("Oops", regs, error_code))
670 sig = 0;
671
672
673 printk(KERN_EMERG "CR2: %016lx\n", address);
674
675 oops_end(flags, regs, sig);
676}
677
678
679
680
681
682static inline void
683show_signal_msg(struct pt_regs *regs, unsigned long error_code,
684 unsigned long address, struct task_struct *tsk)
685{
686 if (!unhandled_signal(tsk, SIGSEGV))
687 return;
688
689 if (!printk_ratelimit())
690 return;
691
692 printk("%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
693 task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
694 tsk->comm, task_pid_nr(tsk), address,
695 (void *)regs->ip, (void *)regs->sp, error_code);
696
697 print_vma_addr(KERN_CONT " in ", regs->ip);
698
699 printk(KERN_CONT "\n");
700}
701
702static void
703__bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
704 unsigned long address, int si_code)
705{
706 struct task_struct *tsk = current;
707
708
709 if (error_code & PF_USER) {
710
711
712
713 local_irq_enable();
714
715
716
717
718
719 if (is_prefetch(regs, error_code, address))
720 return;
721
722 if (is_errata100(regs, address))
723 return;
724
725 if (unlikely(show_unhandled_signals))
726 show_signal_msg(regs, error_code, address, tsk);
727
728
729 tsk->thread.cr2 = address;
730 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
731 tsk->thread.trap_no = 14;
732
733 force_sig_info_fault(SIGSEGV, si_code, address, tsk);
734
735 return;
736 }
737
738 if (is_f00f_bug(regs, address))
739 return;
740
741 no_context(regs, error_code, address);
742}
743
744static noinline void
745bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
746 unsigned long address)
747{
748 __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
749}
750
751static void
752__bad_area(struct pt_regs *regs, unsigned long error_code,
753 unsigned long address, int si_code)
754{
755 struct mm_struct *mm = current->mm;
756
757
758
759
760
761 up_read(&mm->mmap_sem);
762
763 __bad_area_nosemaphore(regs, error_code, address, si_code);
764}
765
766static noinline void
767bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
768{
769 __bad_area(regs, error_code, address, SEGV_MAPERR);
770}
771
772static noinline void
773bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
774 unsigned long address)
775{
776 __bad_area(regs, error_code, address, SEGV_ACCERR);
777}
778
779
780static void
781out_of_memory(struct pt_regs *regs, unsigned long error_code,
782 unsigned long address)
783{
784
785
786
787
788 up_read(¤t->mm->mmap_sem);
789
790 pagefault_out_of_memory();
791}
792
793static void
794do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
795 unsigned int fault)
796{
797 struct task_struct *tsk = current;
798 struct mm_struct *mm = tsk->mm;
799 int code = BUS_ADRERR;
800
801 up_read(&mm->mmap_sem);
802
803
804 if (!(error_code & PF_USER))
805 no_context(regs, error_code, address);
806
807
808 if (is_prefetch(regs, error_code, address))
809 return;
810
811 tsk->thread.cr2 = address;
812 tsk->thread.error_code = error_code;
813 tsk->thread.trap_no = 14;
814
815#ifdef CONFIG_MEMORY_FAILURE
816 if (fault & VM_FAULT_HWPOISON) {
817 printk(KERN_ERR
818 "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
819 tsk->comm, tsk->pid, address);
820 code = BUS_MCEERR_AR;
821 }
822#endif
823 force_sig_info_fault(SIGBUS, code, address, tsk);
824}
825
826static noinline void
827mm_fault_error(struct pt_regs *regs, unsigned long error_code,
828 unsigned long address, unsigned int fault)
829{
830 if (fault & VM_FAULT_OOM) {
831 out_of_memory(regs, error_code, address);
832 } else {
833 if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON))
834 do_sigbus(regs, error_code, address, fault);
835 else
836 BUG();
837 }
838}
839
840static int spurious_fault_check(unsigned long error_code, pte_t *pte)
841{
842 if ((error_code & PF_WRITE) && !pte_write(*pte))
843 return 0;
844
845 if ((error_code & PF_INSTR) && !pte_exec(*pte))
846 return 0;
847
848 return 1;
849}
850
851
852
853
854
855
856
857
858
859
860
861
862
863static noinline int
864spurious_fault(unsigned long error_code, unsigned long address)
865{
866 pgd_t *pgd;
867 pud_t *pud;
868 pmd_t *pmd;
869 pte_t *pte;
870 int ret;
871
872
873 if (error_code & (PF_USER | PF_RSVD))
874 return 0;
875
876 pgd = init_mm.pgd + pgd_index(address);
877 if (!pgd_present(*pgd))
878 return 0;
879
880 pud = pud_offset(pgd, address);
881 if (!pud_present(*pud))
882 return 0;
883
884 if (pud_large(*pud))
885 return spurious_fault_check(error_code, (pte_t *) pud);
886
887 pmd = pmd_offset(pud, address);
888 if (!pmd_present(*pmd))
889 return 0;
890
891 if (pmd_large(*pmd))
892 return spurious_fault_check(error_code, (pte_t *) pmd);
893
894 pte = pte_offset_kernel(pmd, address);
895 if (!pte_present(*pte))
896 return 0;
897
898 ret = spurious_fault_check(error_code, pte);
899 if (!ret)
900 return 0;
901
902
903
904
905
906 ret = spurious_fault_check(error_code, (pte_t *) pmd);
907 WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
908
909 return ret;
910}
911
912int show_unhandled_signals = 1;
913
914static inline int
915access_error(unsigned long error_code, int write, struct vm_area_struct *vma)
916{
917 if (write) {
918
919 if (unlikely(!(vma->vm_flags & VM_WRITE)))
920 return 1;
921 return 0;
922 }
923
924
925 if (unlikely(error_code & PF_PROT))
926 return 1;
927
928
929 if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
930 return 1;
931
932 return 0;
933}
934
935static int fault_in_kernel_space(unsigned long address)
936{
937 return address >= TASK_SIZE_MAX;
938}
939
940
941
942
943
944
945dotraplinkage void __kprobes
946do_page_fault(struct pt_regs *regs, unsigned long error_code)
947{
948 struct vm_area_struct *vma;
949 struct task_struct *tsk;
950 unsigned long address;
951 struct mm_struct *mm;
952 int write;
953 int fault;
954
955 tsk = current;
956 mm = tsk->mm;
957
958
959 address = read_cr2();
960
961
962
963
964
965 if (kmemcheck_active(regs))
966 kmemcheck_hide(regs);
967 prefetchw(&mm->mmap_sem);
968
969 if (unlikely(kmmio_fault(regs, address)))
970 return;
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985 if (unlikely(fault_in_kernel_space(address))) {
986 if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) {
987 if (vmalloc_fault(address) >= 0)
988 return;
989
990 if (kmemcheck_fault(regs, address, error_code))
991 return;
992 }
993
994
995 if (spurious_fault(error_code, address))
996 return;
997
998
999 if (notify_page_fault(regs))
1000 return;
1001
1002
1003
1004
1005 bad_area_nosemaphore(regs, error_code, address);
1006
1007 return;
1008 }
1009
1010
1011 if (unlikely(notify_page_fault(regs)))
1012 return;
1013
1014
1015
1016
1017
1018
1019
1020 if (user_mode_vm(regs)) {
1021 local_irq_enable();
1022 error_code |= PF_USER;
1023 } else {
1024 if (regs->flags & X86_EFLAGS_IF)
1025 local_irq_enable();
1026 }
1027
1028 if (unlikely(error_code & PF_RSVD))
1029 pgtable_bad(regs, error_code, address);
1030
1031 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
1032
1033
1034
1035
1036
1037 if (unlikely(in_atomic() || !mm)) {
1038 bad_area_nosemaphore(regs, error_code, address);
1039 return;
1040 }
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
1059 if ((error_code & PF_USER) == 0 &&
1060 !search_exception_tables(regs->ip)) {
1061 bad_area_nosemaphore(regs, error_code, address);
1062 return;
1063 }
1064 down_read(&mm->mmap_sem);
1065 } else {
1066
1067
1068
1069
1070
1071 might_sleep();
1072 }
1073
1074 vma = find_vma(mm, address);
1075 if (unlikely(!vma)) {
1076 bad_area(regs, error_code, address);
1077 return;
1078 }
1079 if (likely(vma->vm_start <= address))
1080 goto good_area;
1081 if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
1082 bad_area(regs, error_code, address);
1083 return;
1084 }
1085 if (error_code & PF_USER) {
1086
1087
1088
1089
1090
1091
1092 if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
1093 bad_area(regs, error_code, address);
1094 return;
1095 }
1096 }
1097 if (unlikely(expand_stack(vma, address))) {
1098 bad_area(regs, error_code, address);
1099 return;
1100 }
1101
1102
1103
1104
1105
1106good_area:
1107 write = error_code & PF_WRITE;
1108
1109 if (unlikely(access_error(error_code, write, vma))) {
1110 bad_area_access_error(regs, error_code, address);
1111 return;
1112 }
1113
1114
1115
1116
1117
1118
1119 fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0);
1120
1121 if (unlikely(fault & VM_FAULT_ERROR)) {
1122 mm_fault_error(regs, error_code, address, fault);
1123 return;
1124 }
1125
1126 if (fault & VM_FAULT_MAJOR) {
1127 tsk->maj_flt++;
1128 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
1129 regs, address);
1130 } else {
1131 tsk->min_flt++;
1132 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
1133 regs, address);
1134 }
1135
1136 check_v8086_mode(regs, address, tsk);
1137
1138 up_read(&mm->mmap_sem);
1139}
1140