1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/signal.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/errno.h>
21#include <linux/string.h>
22#include <linux/types.h>
23#include <linux/ptrace.h>
24#include <linux/mman.h>
25#include <linux/mm.h>
26#include <linux/smp.h>
27#include <linux/interrupt.h>
28#include <linux/init.h>
29#include <linux/tty.h>
30#include <linux/vt_kern.h>
31#include <linux/highmem.h>
32#include <linux/module.h>
33#include <linux/kprobes.h>
34#include <linux/hugetlb.h>
35#include <linux/syscalls.h>
36#include <linux/uaccess.h>
37#include <linux/kdebug.h>
38
39#include <asm/pgalloc.h>
40#include <asm/sections.h>
41#include <asm/traps.h>
42#include <asm/syscalls.h>
43
44#include <arch/interrupts.h>
45
46static noinline void force_sig_info_fault(const char *type, int si_signo,
47 int si_code, unsigned long address,
48 int fault_num,
49 struct task_struct *tsk,
50 struct pt_regs *regs)
51{
52 siginfo_t info;
53
54 if (unlikely(tsk->pid < 2)) {
55 panic("Signal %d (code %d) at %#lx sent to %s!",
56 si_signo, si_code & 0xffff, address,
57 is_idle_task(tsk) ? "the idle task" : "init");
58 }
59
60 info.si_signo = si_signo;
61 info.si_errno = 0;
62 info.si_code = si_code;
63 info.si_addr = (void __user *)address;
64 info.si_trapno = fault_num;
65 trace_unhandled_signal(type, regs, address, si_signo);
66 force_sig_info(si_signo, &info, tsk);
67}
68
69#ifndef __tilegx__
70
71
72
73
74SYSCALL_DEFINE1(cmpxchg_badaddr, unsigned long, address)
75{
76 struct pt_regs *regs = current_pt_regs();
77
78 if (address >= PAGE_OFFSET)
79 force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR,
80 address, INT_DTLB_MISS, current, regs);
81 else
82 force_sig_info_fault("atomic alignment fault", SIGBUS,
83 BUS_ADRALN, address,
84 INT_UNALIGN_DATA, current, regs);
85
86
87
88
89
90
91 regs->pc -= 8;
92
93
94
95
96
97
98 regs->flags |= PT_FLAGS_CALLER_SAVES;
99
100 return 0;
101}
102#endif
103
104static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
105{
106 unsigned index = pgd_index(address);
107 pgd_t *pgd_k;
108 pud_t *pud, *pud_k;
109 pmd_t *pmd, *pmd_k;
110
111 pgd += index;
112 pgd_k = init_mm.pgd + index;
113
114 if (!pgd_present(*pgd_k))
115 return NULL;
116
117 pud = pud_offset(pgd, address);
118 pud_k = pud_offset(pgd_k, address);
119 if (!pud_present(*pud_k))
120 return NULL;
121
122 pmd = pmd_offset(pud, address);
123 pmd_k = pmd_offset(pud_k, address);
124 if (!pmd_present(*pmd_k))
125 return NULL;
126 if (!pmd_present(*pmd))
127 set_pmd(pmd, *pmd_k);
128 else
129 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
130 return pmd_k;
131}
132
133
134
135
136static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
137{
138 pmd_t *pmd_k;
139 pte_t *pte_k;
140
141
142 if (!(address >= VMALLOC_START && address < VMALLOC_END))
143 return -1;
144
145
146
147
148
149 pmd_k = vmalloc_sync_one(pgd, address);
150 if (!pmd_k)
151 return -1;
152 pte_k = pte_offset_kernel(pmd_k, address);
153 if (!pte_present(*pte_k))
154 return -1;
155 return 0;
156}
157
158
159static void wait_for_migration(pte_t *pte)
160{
161 if (pte_migrating(*pte)) {
162
163
164
165
166
167 int retries = 0;
168 int bound = get_clock_rate();
169 while (pte_migrating(*pte)) {
170 barrier();
171 if (++retries > bound)
172 panic("Hit migrating PTE (%#llx) and page PFN %#lx still migrating",
173 pte->val, pte_pfn(*pte));
174 }
175 }
176}
177
178
179
180
181
182
183static pgd_t *get_current_pgd(void)
184{
185 HV_Context ctx = hv_inquire_context();
186 unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
187 struct page *pgd_page = pfn_to_page(pgd_pfn);
188 BUG_ON(PageHighMem(pgd_page));
189 return (pgd_t *) __va(ctx.page_table);
190}
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210static int handle_migrating_pte(pgd_t *pgd, int fault_num,
211 unsigned long address, unsigned long pc,
212 int is_kernel_mode, int write)
213{
214 pud_t *pud;
215 pmd_t *pmd;
216 pte_t *pte;
217 pte_t pteval;
218
219 if (pgd_addr_invalid(address))
220 return 0;
221
222 pgd += pgd_index(address);
223 pud = pud_offset(pgd, address);
224 if (!pud || !pud_present(*pud))
225 return 0;
226 pmd = pmd_offset(pud, address);
227 if (!pmd || !pmd_present(*pmd))
228 return 0;
229 pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
230 pte_offset_kernel(pmd, address);
231 pteval = *pte;
232 if (pte_migrating(pteval)) {
233 if (in_nmi() && search_exception_tables(pc))
234 return 0;
235 wait_for_migration(pte);
236 return 1;
237 }
238
239 if (!is_kernel_mode || !pte_present(pteval))
240 return 0;
241 if (fault_num == INT_ITLB_MISS) {
242 if (pte_exec(pteval))
243 return 1;
244 } else if (write) {
245 if (pte_write(pteval))
246 return 1;
247 } else {
248 if (pte_read(pteval))
249 return 1;
250 }
251
252 return 0;
253}
254
255
256
257
258
259
260static int handle_page_fault(struct pt_regs *regs,
261 int fault_num,
262 int is_page_fault,
263 unsigned long address,
264 int write)
265{
266 struct task_struct *tsk;
267 struct mm_struct *mm;
268 struct vm_area_struct *vma;
269 unsigned long stack_offset;
270 int fault;
271 int si_code;
272 int is_kernel_mode;
273 pgd_t *pgd;
274 unsigned int flags;
275
276
277 if (!is_page_fault)
278 write = 1;
279
280 flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
281
282 is_kernel_mode = !user_mode(regs);
283
284 tsk = validate_current();
285
286
287
288
289
290
291
292 stack_offset = stack_pointer & (THREAD_SIZE-1);
293 if (stack_offset < THREAD_SIZE / 8) {
294 pr_alert("Potential stack overrun: sp %#lx\n", stack_pointer);
295 show_regs(regs);
296 pr_alert("Killing current process %d/%s\n",
297 tsk->pid, tsk->comm);
298 do_group_exit(SIGKILL);
299 }
300
301
302
303
304
305
306
307
308
309 pgd = get_current_pgd();
310 if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
311 is_kernel_mode, write))
312 return 1;
313
314 si_code = SEGV_MAPERR;
315
316
317
318
319
320
321
322
323
324
325
326
327
328 if (unlikely(address >= TASK_SIZE &&
329 !is_arch_mappable_range(address, 0))) {
330 if (is_kernel_mode && is_page_fault &&
331 vmalloc_fault(pgd, address) >= 0)
332 return 1;
333
334
335
336
337 mm = NULL;
338 vma = NULL;
339 goto bad_area_nosemaphore;
340 }
341
342
343
344
345
346
347
348
349 if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
350 local_irq_enable();
351
352 mm = tsk->mm;
353
354
355
356
357
358 if (pagefault_disabled() || !mm) {
359 vma = NULL;
360 goto bad_area_nosemaphore;
361 }
362
363 if (!is_kernel_mode)
364 flags |= FAULT_FLAG_USER;
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382 if (!down_read_trylock(&mm->mmap_sem)) {
383 if (is_kernel_mode &&
384 !search_exception_tables(regs->pc)) {
385 vma = NULL;
386 goto bad_area_nosemaphore;
387 }
388
389retry:
390 down_read(&mm->mmap_sem);
391 }
392
393 vma = find_vma(mm, address);
394 if (!vma)
395 goto bad_area;
396 if (vma->vm_start <= address)
397 goto good_area;
398 if (!(vma->vm_flags & VM_GROWSDOWN))
399 goto bad_area;
400 if (regs->sp < PAGE_OFFSET) {
401
402
403
404 if (address < regs->sp)
405 goto bad_area;
406 }
407 if (expand_stack(vma, address))
408 goto bad_area;
409
410
411
412
413
414good_area:
415 si_code = SEGV_ACCERR;
416 if (fault_num == INT_ITLB_MISS) {
417 if (!(vma->vm_flags & VM_EXEC))
418 goto bad_area;
419 } else if (write) {
420#ifdef TEST_VERIFY_AREA
421 if (!is_page_fault && regs->cs == KERNEL_CS)
422 pr_err("WP fault at " REGFMT "\n", regs->eip);
423#endif
424 if (!(vma->vm_flags & VM_WRITE))
425 goto bad_area;
426 flags |= FAULT_FLAG_WRITE;
427 } else {
428 if (!is_page_fault || !(vma->vm_flags & VM_READ))
429 goto bad_area;
430 }
431
432
433
434
435
436
437 fault = handle_mm_fault(mm, vma, address, flags);
438
439 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
440 return 0;
441
442 if (unlikely(fault & VM_FAULT_ERROR)) {
443 if (fault & VM_FAULT_OOM)
444 goto out_of_memory;
445 else if (fault & VM_FAULT_SIGSEGV)
446 goto bad_area;
447 else if (fault & VM_FAULT_SIGBUS)
448 goto do_sigbus;
449 BUG();
450 }
451 if (flags & FAULT_FLAG_ALLOW_RETRY) {
452 if (fault & VM_FAULT_MAJOR)
453 tsk->maj_flt++;
454 else
455 tsk->min_flt++;
456 if (fault & VM_FAULT_RETRY) {
457 flags &= ~FAULT_FLAG_ALLOW_RETRY;
458 flags |= FAULT_FLAG_TRIED;
459
460
461
462
463
464
465 goto retry;
466 }
467 }
468
469#if CHIP_HAS_TILE_DMA()
470
471 switch (fault_num) {
472 case INT_DMATLB_MISS:
473 case INT_DMATLB_MISS_DWNCL:
474 case INT_DMATLB_ACCESS:
475 case INT_DMATLB_ACCESS_DWNCL:
476 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
477 break;
478 }
479#endif
480
481 up_read(&mm->mmap_sem);
482 return 1;
483
484
485
486
487
488bad_area:
489 up_read(&mm->mmap_sem);
490
491bad_area_nosemaphore:
492
493 if (!is_kernel_mode) {
494
495
496
497 local_irq_enable();
498
499 force_sig_info_fault("segfault", SIGSEGV, si_code, address,
500 fault_num, tsk, regs);
501 return 0;
502 }
503
504no_context:
505
506 if (fixup_exception(regs))
507 return 0;
508
509
510
511
512
513
514 bust_spinlocks(1);
515
516
517#ifdef SUPPORT_LOOKUP_ADDRESS
518 if (fault_num == INT_ITLB_MISS) {
519 pte_t *pte = lookup_address(address);
520
521 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
522 pr_crit("kernel tried to execute non-executable page - exploit attempt? (uid: %d)\n",
523 current->uid);
524 }
525#endif
526 if (address < PAGE_SIZE)
527 pr_alert("Unable to handle kernel NULL pointer dereference\n");
528 else
529 pr_alert("Unable to handle kernel paging request\n");
530 pr_alert(" at virtual address " REGFMT ", pc " REGFMT "\n",
531 address, regs->pc);
532
533 show_regs(regs);
534
535 if (unlikely(tsk->pid < 2)) {
536 panic("Kernel page fault running %s!",
537 is_idle_task(tsk) ? "the idle task" : "init");
538 }
539
540
541
542
543
544#ifdef SUPPORT_DIE
545 die("Oops", regs);
546#endif
547 bust_spinlocks(1);
548
549 do_group_exit(SIGKILL);
550
551
552
553
554
555out_of_memory:
556 up_read(&mm->mmap_sem);
557 if (is_kernel_mode)
558 goto no_context;
559 pagefault_out_of_memory();
560 return 0;
561
562do_sigbus:
563 up_read(&mm->mmap_sem);
564
565
566 if (is_kernel_mode)
567 goto no_context;
568
569 force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
570 fault_num, tsk, regs);
571 return 0;
572}
573
574#ifndef __tilegx__
575
576
577#define ics_panic(fmt, ...) \
578do { \
579 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \
580 panic(fmt, ##__VA_ARGS__); \
581} while (0)
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num,
600 unsigned long address,
601 unsigned long info)
602{
603 unsigned long pc = info & ~1;
604 int write = info & 1;
605 pgd_t *pgd = get_current_pgd();
606
607
608 struct intvec_state state = {
609 do_page_fault, fault_num, address, write, 1
610 };
611
612
613 if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
614 (fault_num != INT_DTLB_MISS &&
615 fault_num != INT_DTLB_ACCESS)) {
616 unsigned long old_pc = regs->pc;
617 regs->pc = pc;
618 ics_panic("Bad ICS page fault args: old PC %#lx, fault %d/%d at %#lx",
619 old_pc, fault_num, write, address);
620 }
621
622
623 if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
624 return state;
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642 if (pc >= (unsigned long) sys_cmpxchg &&
643 pc < (unsigned long) __sys_cmpxchg_end) {
644#ifdef CONFIG_SMP
645
646 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
647 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
648 __atomic_fault_unlock(lock_ptr);
649 }
650#endif
651 regs->sp = regs->regs[27];
652 }
653
654
655
656
657
658
659
660
661 else if (pc >= (unsigned long) __start_atomic_asm_code &&
662 pc < (unsigned long) __end_atomic_asm_code) {
663 const struct exception_table_entry *fixup;
664#ifdef CONFIG_SMP
665
666 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
667 __atomic_fault_unlock(lock_ptr);
668#endif
669 fixup = search_exception_tables(pc);
670 if (!fixup)
671 ics_panic("ICS atomic fault not in table: PC %#lx, fault %d",
672 pc, fault_num);
673 regs->pc = fixup->fixup;
674 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
675 }
676
677
678
679
680
681 if (fault_num == INT_DTLB_ACCESS)
682 write = 1;
683 if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write))
684 return state;
685
686
687 state.retval = 0;
688 return state;
689}
690
691#endif
692
693
694
695
696
697
698
699
700
701static inline void __do_page_fault(struct pt_regs *regs, int fault_num,
702 unsigned long address, unsigned long write)
703{
704 int is_page_fault;
705
706#ifdef CONFIG_KPROBES
707
708
709
710
711
712 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1,
713 regs->faultnum, SIGSEGV) == NOTIFY_STOP)
714 return;
715#endif
716
717#ifdef __tilegx__
718
719
720
721
722
723
724
725
726 if (write & ~1) {
727 unsigned long pc = write & ~1;
728 if (pc >= (unsigned long) __start_unalign_asm_code &&
729 pc < (unsigned long) __end_unalign_asm_code) {
730 struct thread_info *ti = current_thread_info();
731
732
733
734
735
736
737
738
739
740
741
742 regs->sp &= ~1UL;
743 regs->regs[0] = ti->unalign_jit_tmp[0];
744 regs->regs[1] = ti->unalign_jit_tmp[1];
745 regs->regs[2] = ti->unalign_jit_tmp[2];
746 regs->regs[3] = ti->unalign_jit_tmp[3];
747 write &= 1;
748 } else {
749 pr_alert("%s/%d: ICS set at page fault at %#lx: %#lx\n",
750 current->comm, current->pid, pc, address);
751 show_regs(regs);
752 do_group_exit(SIGKILL);
753 }
754 }
755#else
756
757 BUG_ON(write & ~1);
758#endif
759
760#if CHIP_HAS_TILE_DMA()
761
762
763
764
765
766
767
768 if (fault_num == INT_DMATLB_MISS ||
769 fault_num == INT_DMATLB_ACCESS ||
770 fault_num == INT_DMATLB_MISS_DWNCL ||
771 fault_num == INT_DMATLB_ACCESS_DWNCL) {
772 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
773 while (__insn_mfspr(SPR_DMA_USER_STATUS) &
774 SPR_DMA_STATUS__BUSY_MASK)
775 ;
776 }
777#endif
778
779
780 switch (fault_num) {
781 case INT_ITLB_MISS:
782 case INT_DTLB_MISS:
783#if CHIP_HAS_TILE_DMA()
784 case INT_DMATLB_MISS:
785 case INT_DMATLB_MISS_DWNCL:
786#endif
787 is_page_fault = 1;
788 break;
789
790 case INT_DTLB_ACCESS:
791#if CHIP_HAS_TILE_DMA()
792 case INT_DMATLB_ACCESS:
793 case INT_DMATLB_ACCESS_DWNCL:
794#endif
795 is_page_fault = 0;
796 break;
797
798 default:
799 panic("Bad fault number %d in do_page_fault", fault_num);
800 }
801
802#if CHIP_HAS_TILE_DMA()
803 if (!user_mode(regs)) {
804 struct async_tlb *async;
805 switch (fault_num) {
806#if CHIP_HAS_TILE_DMA()
807 case INT_DMATLB_MISS:
808 case INT_DMATLB_ACCESS:
809 case INT_DMATLB_MISS_DWNCL:
810 case INT_DMATLB_ACCESS_DWNCL:
811 async = ¤t->thread.dma_async_tlb;
812 break;
813#endif
814 default:
815 async = NULL;
816 }
817 if (async) {
818
819
820
821
822
823 local_irq_enable();
824
825 set_thread_flag(TIF_ASYNC_TLB);
826 if (async->fault_num != 0) {
827 panic("Second async fault %d; old fault was %d (%#lx/%ld)",
828 fault_num, async->fault_num,
829 address, write);
830 }
831 BUG_ON(fault_num == 0);
832 async->fault_num = fault_num;
833 async->is_fault = is_page_fault;
834 async->is_write = write;
835 async->address = address;
836 return;
837 }
838 }
839#endif
840
841 handle_page_fault(regs, fault_num, is_page_fault, address, write);
842}
843
844void do_page_fault(struct pt_regs *regs, int fault_num,
845 unsigned long address, unsigned long write)
846{
847 __do_page_fault(regs, fault_num, address, write);
848}
849
850#if CHIP_HAS_TILE_DMA()
851
852
853
854
855void do_async_page_fault(struct pt_regs *regs)
856{
857 struct async_tlb *async = ¤t->thread.dma_async_tlb;
858
859
860
861
862
863
864 clear_thread_flag(TIF_ASYNC_TLB);
865
866 if (async->fault_num) {
867
868
869
870
871
872
873 int fault_num = async->fault_num;
874 async->fault_num = 0;
875 handle_page_fault(regs, fault_num, async->is_fault,
876 async->address, async->is_write);
877 }
878}
879#endif
880
881
882void vmalloc_sync_all(void)
883{
884#ifdef __tilegx__
885
886 BUILD_BUG_ON(pgd_index(VMALLOC_END - PAGE_SIZE) !=
887 pgd_index(VMALLOC_START));
888#else
889
890
891
892
893
894
895 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
896 static unsigned long start = PAGE_OFFSET;
897 unsigned long address;
898
899 BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
900 for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
901 if (!test_bit(pgd_index(address), insync)) {
902 unsigned long flags;
903 struct list_head *pos;
904
905 spin_lock_irqsave(&pgd_lock, flags);
906 list_for_each(pos, &pgd_list)
907 if (!vmalloc_sync_one(list_to_pgd(pos),
908 address)) {
909
910 BUG_ON(pos != pgd_list.next);
911 break;
912 }
913 spin_unlock_irqrestore(&pgd_lock, flags);
914 if (pos != pgd_list.next)
915 set_bit(pgd_index(address), insync);
916 }
917 if (address == start && test_bit(pgd_index(address), insync))
918 start = address + PGDIR_SIZE;
919 }
920#endif
921}
922