1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/signal.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/errno.h>
21#include <linux/string.h>
22#include <linux/types.h>
23#include <linux/ptrace.h>
24#include <linux/mman.h>
25#include <linux/mm.h>
26#include <linux/smp.h>
27#include <linux/interrupt.h>
28#include <linux/init.h>
29#include <linux/tty.h>
30#include <linux/vt_kern.h>
31#include <linux/highmem.h>
32#include <linux/module.h>
33#include <linux/kprobes.h>
34#include <linux/hugetlb.h>
35#include <linux/syscalls.h>
36#include <linux/uaccess.h>
37
38#include <asm/system.h>
39#include <asm/pgalloc.h>
40#include <asm/sections.h>
41#include <asm/traps.h>
42#include <asm/syscalls.h>
43
44#include <arch/interrupts.h>
45
46static noinline void force_sig_info_fault(int si_signo, int si_code,
47 unsigned long address, int fault_num, struct task_struct *tsk)
48{
49 siginfo_t info;
50
51 if (unlikely(tsk->pid < 2)) {
52 panic("Signal %d (code %d) at %#lx sent to %s!",
53 si_signo, si_code & 0xffff, address,
54 tsk->pid ? "init" : "the idle task");
55 }
56
57 info.si_signo = si_signo;
58 info.si_errno = 0;
59 info.si_code = si_code;
60 info.si_addr = (void __user *)address;
61 info.si_trapno = fault_num;
62 force_sig_info(si_signo, &info, tsk);
63}
64
65#ifndef __tilegx__
66
67
68
69
70SYSCALL_DEFINE2(cmpxchg_badaddr, unsigned long, address,
71 struct pt_regs *, regs)
72{
73 if (address >= PAGE_OFFSET)
74 force_sig_info_fault(SIGSEGV, SEGV_MAPERR, address,
75 INT_DTLB_MISS, current);
76 else
77 force_sig_info_fault(SIGBUS, BUS_ADRALN, address,
78 INT_UNALIGN_DATA, current);
79
80
81
82
83
84
85 regs->pc -= 8;
86
87
88
89
90
91
92 regs->flags |= PT_FLAGS_CALLER_SAVES;
93
94 return 0;
95}
96#endif
97
98static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
99{
100 unsigned index = pgd_index(address);
101 pgd_t *pgd_k;
102 pud_t *pud, *pud_k;
103 pmd_t *pmd, *pmd_k;
104
105 pgd += index;
106 pgd_k = init_mm.pgd + index;
107
108 if (!pgd_present(*pgd_k))
109 return NULL;
110
111 pud = pud_offset(pgd, address);
112 pud_k = pud_offset(pgd_k, address);
113 if (!pud_present(*pud_k))
114 return NULL;
115
116 pmd = pmd_offset(pud, address);
117 pmd_k = pmd_offset(pud_k, address);
118 if (!pmd_present(*pmd_k))
119 return NULL;
120 if (!pmd_present(*pmd)) {
121 set_pmd(pmd, *pmd_k);
122 arch_flush_lazy_mmu_mode();
123 } else
124 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
125 return pmd_k;
126}
127
128
129
130
131static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
132{
133 pmd_t *pmd_k;
134 pte_t *pte_k;
135
136
137 if (!(address >= VMALLOC_START && address < VMALLOC_END))
138 return -1;
139
140
141
142
143
144 pmd_k = vmalloc_sync_one(pgd, address);
145 if (!pmd_k)
146 return -1;
147 if (pmd_huge(*pmd_k))
148 return 0;
149 pte_k = pte_offset_kernel(pmd_k, address);
150 if (!pte_present(*pte_k))
151 return -1;
152 return 0;
153}
154
155
156static void wait_for_migration(pte_t *pte)
157{
158 if (pte_migrating(*pte)) {
159
160
161
162
163
164 int retries = 0;
165 int bound = get_clock_rate();
166 while (pte_migrating(*pte)) {
167 barrier();
168 if (++retries > bound)
169 panic("Hit migrating PTE (%#llx) and"
170 " page PFN %#lx still migrating",
171 pte->val, pte_pfn(*pte));
172 }
173 }
174}
175
176
177
178
179
180
181static pgd_t *get_current_pgd(void)
182{
183 HV_Context ctx = hv_inquire_context();
184 unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
185 struct page *pgd_page = pfn_to_page(pgd_pfn);
186 BUG_ON(PageHighMem(pgd_page));
187 return (pgd_t *) __va(ctx.page_table);
188}
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203static int handle_migrating_pte(pgd_t *pgd, int fault_num,
204 unsigned long address,
205 int is_kernel_mode, int write)
206{
207 pud_t *pud;
208 pmd_t *pmd;
209 pte_t *pte;
210 pte_t pteval;
211
212 if (pgd_addr_invalid(address))
213 return 0;
214
215 pgd += pgd_index(address);
216 pud = pud_offset(pgd, address);
217 if (!pud || !pud_present(*pud))
218 return 0;
219 pmd = pmd_offset(pud, address);
220 if (!pmd || !pmd_present(*pmd))
221 return 0;
222 pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
223 pte_offset_kernel(pmd, address);
224 pteval = *pte;
225 if (pte_migrating(pteval)) {
226 wait_for_migration(pte);
227 return 1;
228 }
229
230 if (!is_kernel_mode || !pte_present(pteval))
231 return 0;
232 if (fault_num == INT_ITLB_MISS) {
233 if (pte_exec(pteval))
234 return 1;
235 } else if (write) {
236 if (pte_write(pteval))
237 return 1;
238 } else {
239 if (pte_read(pteval))
240 return 1;
241 }
242
243 return 0;
244}
245
246
247
248
249
250
251static int handle_page_fault(struct pt_regs *regs,
252 int fault_num,
253 int is_page_fault,
254 unsigned long address,
255 int write)
256{
257 struct task_struct *tsk;
258 struct mm_struct *mm;
259 struct vm_area_struct *vma;
260 unsigned long stack_offset;
261 int fault;
262 int si_code;
263 int is_kernel_mode;
264 pgd_t *pgd;
265
266
267 if (!is_page_fault)
268 write = 1;
269
270 is_kernel_mode = (EX1_PL(regs->ex1) != USER_PL);
271
272 tsk = validate_current();
273
274
275
276
277
278
279
280 stack_offset = stack_pointer & (THREAD_SIZE-1);
281 if (stack_offset < THREAD_SIZE / 8) {
282 pr_alert("Potential stack overrun: sp %#lx\n",
283 stack_pointer);
284 show_regs(regs);
285 pr_alert("Killing current process %d/%s\n",
286 tsk->pid, tsk->comm);
287 do_group_exit(SIGKILL);
288 }
289
290
291
292
293
294
295
296
297
298 pgd = get_current_pgd();
299 if (handle_migrating_pte(pgd, fault_num, address,
300 is_kernel_mode, write))
301 return 1;
302
303 si_code = SEGV_MAPERR;
304
305
306
307
308
309
310
311
312
313
314
315
316
317 if (unlikely(address >= TASK_SIZE &&
318 !is_arch_mappable_range(address, 0))) {
319 if (is_kernel_mode && is_page_fault &&
320 vmalloc_fault(pgd, address) >= 0)
321 return 1;
322
323
324
325
326 mm = NULL;
327 vma = NULL;
328 goto bad_area_nosemaphore;
329 }
330
331
332
333
334
335
336 local_irq_enable();
337
338 mm = tsk->mm;
339
340
341
342
343
344 if (in_atomic() || !mm) {
345 vma = NULL;
346 goto bad_area_nosemaphore;
347 }
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365 if (!down_read_trylock(&mm->mmap_sem)) {
366 if (is_kernel_mode &&
367 !search_exception_tables(regs->pc)) {
368 vma = NULL;
369 goto bad_area_nosemaphore;
370 }
371 down_read(&mm->mmap_sem);
372 }
373
374 vma = find_vma(mm, address);
375 if (!vma)
376 goto bad_area;
377 if (vma->vm_start <= address)
378 goto good_area;
379 if (!(vma->vm_flags & VM_GROWSDOWN))
380 goto bad_area;
381 if (regs->sp < PAGE_OFFSET) {
382
383
384
385 if (address < regs->sp)
386 goto bad_area;
387 }
388 if (expand_stack(vma, address))
389 goto bad_area;
390
391
392
393
394
395good_area:
396 si_code = SEGV_ACCERR;
397 if (fault_num == INT_ITLB_MISS) {
398 if (!(vma->vm_flags & VM_EXEC))
399 goto bad_area;
400 } else if (write) {
401#ifdef TEST_VERIFY_AREA
402 if (!is_page_fault && regs->cs == KERNEL_CS)
403 pr_err("WP fault at "REGFMT"\n", regs->eip);
404#endif
405 if (!(vma->vm_flags & VM_WRITE))
406 goto bad_area;
407 } else {
408 if (!is_page_fault || !(vma->vm_flags & VM_READ))
409 goto bad_area;
410 }
411
412 survive:
413
414
415
416
417
418 fault = handle_mm_fault(mm, vma, address, write);
419 if (unlikely(fault & VM_FAULT_ERROR)) {
420 if (fault & VM_FAULT_OOM)
421 goto out_of_memory;
422 else if (fault & VM_FAULT_SIGBUS)
423 goto do_sigbus;
424 BUG();
425 }
426 if (fault & VM_FAULT_MAJOR)
427 tsk->maj_flt++;
428 else
429 tsk->min_flt++;
430
431#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
432
433
434
435
436 switch (fault_num) {
437#if CHIP_HAS_TILE_DMA()
438 case INT_DMATLB_MISS:
439 case INT_DMATLB_MISS_DWNCL:
440 case INT_DMATLB_ACCESS:
441 case INT_DMATLB_ACCESS_DWNCL:
442 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
443 break;
444#endif
445#if CHIP_HAS_SN_PROC()
446 case INT_SNITLB_MISS:
447 case INT_SNITLB_MISS_DWNCL:
448 __insn_mtspr(SPR_SNCTL,
449 __insn_mfspr(SPR_SNCTL) &
450 ~SPR_SNCTL__FRZPROC_MASK);
451 break;
452#endif
453 }
454#endif
455
456 up_read(&mm->mmap_sem);
457 return 1;
458
459
460
461
462
463bad_area:
464 up_read(&mm->mmap_sem);
465
466bad_area_nosemaphore:
467
468 if (!is_kernel_mode) {
469
470
471
472 local_irq_enable();
473
474 force_sig_info_fault(SIGSEGV, si_code, address,
475 fault_num, tsk);
476 return 0;
477 }
478
479no_context:
480
481 if (fixup_exception(regs))
482 return 0;
483
484
485
486
487
488
489 bust_spinlocks(1);
490
491
492#ifdef SUPPORT_LOOKUP_ADDRESS
493 if (fault_num == INT_ITLB_MISS) {
494 pte_t *pte = lookup_address(address);
495
496 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
497 pr_crit("kernel tried to execute"
498 " non-executable page - exploit attempt?"
499 " (uid: %d)\n", current->uid);
500 }
501#endif
502 if (address < PAGE_SIZE)
503 pr_alert("Unable to handle kernel NULL pointer dereference\n");
504 else
505 pr_alert("Unable to handle kernel paging request\n");
506 pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n",
507 address, regs->pc);
508
509 show_regs(regs);
510
511 if (unlikely(tsk->pid < 2)) {
512 panic("Kernel page fault running %s!",
513 tsk->pid ? "init" : "the idle task");
514 }
515
516
517
518
519
520#ifdef SUPPORT_DIE
521 die("Oops", regs);
522#endif
523 bust_spinlocks(1);
524
525 do_group_exit(SIGKILL);
526
527
528
529
530
531out_of_memory:
532 up_read(&mm->mmap_sem);
533 if (is_global_init(tsk)) {
534 yield();
535 down_read(&mm->mmap_sem);
536 goto survive;
537 }
538 pr_alert("VM: killing process %s\n", tsk->comm);
539 if (!is_kernel_mode)
540 do_group_exit(SIGKILL);
541 goto no_context;
542
543do_sigbus:
544 up_read(&mm->mmap_sem);
545
546
547 if (is_kernel_mode)
548 goto no_context;
549
550 force_sig_info_fault(SIGBUS, BUS_ADRERR, address, fault_num, tsk);
551 return 0;
552}
553
554#ifndef __tilegx__
555
556
557#define ics_panic(fmt, ...) do { \
558 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \
559 panic(fmt, __VA_ARGS__); \
560} while (0)
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num,
579 unsigned long address,
580 unsigned long info)
581{
582 unsigned long pc = info & ~1;
583 int write = info & 1;
584 pgd_t *pgd = get_current_pgd();
585
586
587 struct intvec_state state = {
588 do_page_fault, fault_num, address, write, 1
589 };
590
591
592 if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
593 (fault_num != INT_DTLB_MISS &&
594 fault_num != INT_DTLB_ACCESS)) {
595 unsigned long old_pc = regs->pc;
596 regs->pc = pc;
597 ics_panic("Bad ICS page fault args:"
598 " old PC %#lx, fault %d/%d at %#lx\n",
599 old_pc, fault_num, write, address);
600 }
601
602
603 if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
604 return state;
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622 if (pc >= (unsigned long) sys_cmpxchg &&
623 pc < (unsigned long) __sys_cmpxchg_end) {
624#ifdef CONFIG_SMP
625
626 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
627 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
628 __atomic_fault_unlock(lock_ptr);
629 }
630#endif
631 regs->sp = regs->regs[27];
632 }
633
634
635
636
637
638
639
640
641 else if (pc >= (unsigned long) __start_atomic_asm_code &&
642 pc < (unsigned long) __end_atomic_asm_code) {
643 const struct exception_table_entry *fixup;
644#ifdef CONFIG_SMP
645
646 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
647 __atomic_fault_unlock(lock_ptr);
648#endif
649 fixup = search_exception_tables(pc);
650 if (!fixup)
651 ics_panic("ICS atomic fault not in table:"
652 " PC %#lx, fault %d", pc, fault_num);
653 regs->pc = fixup->fixup;
654 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
655 }
656
657
658
659
660
661
662
663
664
665
666
667
668
669 if (fault_num == INT_DTLB_ACCESS)
670 write = 1;
671 if (handle_migrating_pte(pgd, fault_num, address, 1, write))
672 return state;
673
674
675 state.retval = 0;
676 return state;
677}
678
679#endif
680
681
682
683
684
685
686
687
688
689void do_page_fault(struct pt_regs *regs, int fault_num,
690 unsigned long address, unsigned long write)
691{
692 int is_page_fault;
693
694
695 BUG_ON(write & ~1);
696
697#if CHIP_HAS_TILE_DMA()
698
699
700
701
702
703
704
705 if (fault_num == INT_DMATLB_MISS ||
706 fault_num == INT_DMATLB_ACCESS ||
707 fault_num == INT_DMATLB_MISS_DWNCL ||
708 fault_num == INT_DMATLB_ACCESS_DWNCL) {
709 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
710 while (__insn_mfspr(SPR_DMA_USER_STATUS) &
711 SPR_DMA_STATUS__BUSY_MASK)
712 ;
713 }
714#endif
715
716
717 switch (fault_num) {
718 case INT_ITLB_MISS:
719 case INT_DTLB_MISS:
720#if CHIP_HAS_TILE_DMA()
721 case INT_DMATLB_MISS:
722 case INT_DMATLB_MISS_DWNCL:
723#endif
724#if CHIP_HAS_SN_PROC()
725 case INT_SNITLB_MISS:
726 case INT_SNITLB_MISS_DWNCL:
727#endif
728 is_page_fault = 1;
729 break;
730
731 case INT_DTLB_ACCESS:
732#if CHIP_HAS_TILE_DMA()
733 case INT_DMATLB_ACCESS:
734 case INT_DMATLB_ACCESS_DWNCL:
735#endif
736 is_page_fault = 0;
737 break;
738
739 default:
740 panic("Bad fault number %d in do_page_fault", fault_num);
741 }
742
743 if (EX1_PL(regs->ex1) != USER_PL) {
744 struct async_tlb *async;
745 switch (fault_num) {
746#if CHIP_HAS_TILE_DMA()
747 case INT_DMATLB_MISS:
748 case INT_DMATLB_ACCESS:
749 case INT_DMATLB_MISS_DWNCL:
750 case INT_DMATLB_ACCESS_DWNCL:
751 async = ¤t->thread.dma_async_tlb;
752 break;
753#endif
754#if CHIP_HAS_SN_PROC()
755 case INT_SNITLB_MISS:
756 case INT_SNITLB_MISS_DWNCL:
757 async = ¤t->thread.sn_async_tlb;
758 break;
759#endif
760 default:
761 async = NULL;
762 }
763 if (async) {
764
765
766
767
768
769 local_irq_enable();
770
771 set_thread_flag(TIF_ASYNC_TLB);
772 if (async->fault_num != 0) {
773 panic("Second async fault %d;"
774 " old fault was %d (%#lx/%ld)",
775 fault_num, async->fault_num,
776 address, write);
777 }
778 BUG_ON(fault_num == 0);
779 async->fault_num = fault_num;
780 async->is_fault = is_page_fault;
781 async->is_write = write;
782 async->address = address;
783 return;
784 }
785 }
786
787 handle_page_fault(regs, fault_num, is_page_fault, address, write);
788}
789
790
791#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
792
793
794
795
796static void handle_async_page_fault(struct pt_regs *regs,
797 struct async_tlb *async)
798{
799 if (async->fault_num) {
800
801
802
803
804
805
806 int fault_num = async->fault_num;
807 async->fault_num = 0;
808 handle_page_fault(regs, fault_num, async->is_fault,
809 async->address, async->is_write);
810 }
811}
812#endif
813
814
815
816
817
818
819void do_async_page_fault(struct pt_regs *regs)
820{
821
822
823
824
825
826 clear_thread_flag(TIF_ASYNC_TLB);
827
828#if CHIP_HAS_TILE_DMA()
829 handle_async_page_fault(regs, ¤t->thread.dma_async_tlb);
830#endif
831#if CHIP_HAS_SN_PROC()
832 handle_async_page_fault(regs, ¤t->thread.sn_async_tlb);
833#endif
834}
835
836void vmalloc_sync_all(void)
837{
838#ifdef __tilegx__
839
840 BUG_ON(pgd_index(VMALLOC_END) != pgd_index(VMALLOC_START));
841#else
842
843
844
845
846
847
848 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
849 static unsigned long start = PAGE_OFFSET;
850 unsigned long address;
851
852 BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
853 for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
854 if (!test_bit(pgd_index(address), insync)) {
855 unsigned long flags;
856 struct list_head *pos;
857
858 spin_lock_irqsave(&pgd_lock, flags);
859 list_for_each(pos, &pgd_list)
860 if (!vmalloc_sync_one(list_to_pgd(pos),
861 address)) {
862
863 BUG_ON(pos != pgd_list.next);
864 break;
865 }
866 spin_unlock_irqrestore(&pgd_lock, flags);
867 if (pos != pgd_list.next)
868 set_bit(pgd_index(address), insync);
869 }
870 if (address == start && test_bit(pgd_index(address), insync))
871 start = address + PGDIR_SIZE;
872 }
873#endif
874}
875