1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/signal.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/errno.h>
21#include <linux/string.h>
22#include <linux/types.h>
23#include <linux/ptrace.h>
24#include <linux/mman.h>
25#include <linux/mm.h>
26#include <linux/smp.h>
27#include <linux/interrupt.h>
28#include <linux/init.h>
29#include <linux/tty.h>
30#include <linux/vt_kern.h>
31#include <linux/highmem.h>
32#include <linux/module.h>
33#include <linux/kprobes.h>
34#include <linux/hugetlb.h>
35#include <linux/syscalls.h>
36#include <linux/uaccess.h>
37
38#include <asm/pgalloc.h>
39#include <asm/sections.h>
40#include <asm/traps.h>
41#include <asm/syscalls.h>
42
43#include <arch/interrupts.h>
44
45static noinline void force_sig_info_fault(const char *type, int si_signo,
46 int si_code, unsigned long address,
47 int fault_num,
48 struct task_struct *tsk,
49 struct pt_regs *regs)
50{
51 siginfo_t info;
52
53 if (unlikely(tsk->pid < 2)) {
54 panic("Signal %d (code %d) at %#lx sent to %s!",
55 si_signo, si_code & 0xffff, address,
56 is_idle_task(tsk) ? "the idle task" : "init");
57 }
58
59 info.si_signo = si_signo;
60 info.si_errno = 0;
61 info.si_code = si_code;
62 info.si_addr = (void __user *)address;
63 info.si_trapno = fault_num;
64 trace_unhandled_signal(type, regs, address, si_signo);
65 force_sig_info(si_signo, &info, tsk);
66}
67
68#ifndef __tilegx__
69
70
71
72
73SYSCALL_DEFINE1(cmpxchg_badaddr, unsigned long, address)
74{
75 struct pt_regs *regs = current_pt_regs();
76
77 if (address >= PAGE_OFFSET)
78 force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR,
79 address, INT_DTLB_MISS, current, regs);
80 else
81 force_sig_info_fault("atomic alignment fault", SIGBUS,
82 BUS_ADRALN, address,
83 INT_UNALIGN_DATA, current, regs);
84
85
86
87
88
89
90 regs->pc -= 8;
91
92
93
94
95
96
97 regs->flags |= PT_FLAGS_CALLER_SAVES;
98
99 return 0;
100}
101#endif
102
103static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
104{
105 unsigned index = pgd_index(address);
106 pgd_t *pgd_k;
107 pud_t *pud, *pud_k;
108 pmd_t *pmd, *pmd_k;
109
110 pgd += index;
111 pgd_k = init_mm.pgd + index;
112
113 if (!pgd_present(*pgd_k))
114 return NULL;
115
116 pud = pud_offset(pgd, address);
117 pud_k = pud_offset(pgd_k, address);
118 if (!pud_present(*pud_k))
119 return NULL;
120
121 pmd = pmd_offset(pud, address);
122 pmd_k = pmd_offset(pud_k, address);
123 if (!pmd_present(*pmd_k))
124 return NULL;
125 if (!pmd_present(*pmd)) {
126 set_pmd(pmd, *pmd_k);
127 arch_flush_lazy_mmu_mode();
128 } else
129 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
130 return pmd_k;
131}
132
133
134
135
136static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
137{
138 pmd_t *pmd_k;
139 pte_t *pte_k;
140
141
142 if (!(address >= VMALLOC_START && address < VMALLOC_END))
143 return -1;
144
145
146
147
148
149 pmd_k = vmalloc_sync_one(pgd, address);
150 if (!pmd_k)
151 return -1;
152 if (pmd_huge(*pmd_k))
153 return 0;
154 pte_k = pte_offset_kernel(pmd_k, address);
155 if (!pte_present(*pte_k))
156 return -1;
157 return 0;
158}
159
160
161static void wait_for_migration(pte_t *pte)
162{
163 if (pte_migrating(*pte)) {
164
165
166
167
168
169 int retries = 0;
170 int bound = get_clock_rate();
171 while (pte_migrating(*pte)) {
172 barrier();
173 if (++retries > bound)
174 panic("Hit migrating PTE (%#llx) and"
175 " page PFN %#lx still migrating",
176 pte->val, pte_pfn(*pte));
177 }
178 }
179}
180
181
182
183
184
185
186static pgd_t *get_current_pgd(void)
187{
188 HV_Context ctx = hv_inquire_context();
189 unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
190 struct page *pgd_page = pfn_to_page(pgd_pfn);
191 BUG_ON(PageHighMem(pgd_page));
192 return (pgd_t *) __va(ctx.page_table);
193}
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213static int handle_migrating_pte(pgd_t *pgd, int fault_num,
214 unsigned long address, unsigned long pc,
215 int is_kernel_mode, int write)
216{
217 pud_t *pud;
218 pmd_t *pmd;
219 pte_t *pte;
220 pte_t pteval;
221
222 if (pgd_addr_invalid(address))
223 return 0;
224
225 pgd += pgd_index(address);
226 pud = pud_offset(pgd, address);
227 if (!pud || !pud_present(*pud))
228 return 0;
229 pmd = pmd_offset(pud, address);
230 if (!pmd || !pmd_present(*pmd))
231 return 0;
232 pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
233 pte_offset_kernel(pmd, address);
234 pteval = *pte;
235 if (pte_migrating(pteval)) {
236 if (in_nmi() && search_exception_tables(pc))
237 return 0;
238 wait_for_migration(pte);
239 return 1;
240 }
241
242 if (!is_kernel_mode || !pte_present(pteval))
243 return 0;
244 if (fault_num == INT_ITLB_MISS) {
245 if (pte_exec(pteval))
246 return 1;
247 } else if (write) {
248 if (pte_write(pteval))
249 return 1;
250 } else {
251 if (pte_read(pteval))
252 return 1;
253 }
254
255 return 0;
256}
257
258
259
260
261
262
263static int handle_page_fault(struct pt_regs *regs,
264 int fault_num,
265 int is_page_fault,
266 unsigned long address,
267 int write)
268{
269 struct task_struct *tsk;
270 struct mm_struct *mm;
271 struct vm_area_struct *vma;
272 unsigned long stack_offset;
273 int fault;
274 int si_code;
275 int is_kernel_mode;
276 pgd_t *pgd;
277 unsigned int flags;
278
279
280 if (!is_page_fault)
281 write = 1;
282
283 flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
284 (write ? FAULT_FLAG_WRITE : 0));
285
286 is_kernel_mode = (EX1_PL(regs->ex1) != USER_PL);
287
288 tsk = validate_current();
289
290
291
292
293
294
295
296 stack_offset = stack_pointer & (THREAD_SIZE-1);
297 if (stack_offset < THREAD_SIZE / 8) {
298 pr_alert("Potential stack overrun: sp %#lx\n",
299 stack_pointer);
300 show_regs(regs);
301 pr_alert("Killing current process %d/%s\n",
302 tsk->pid, tsk->comm);
303 do_group_exit(SIGKILL);
304 }
305
306
307
308
309
310
311
312
313
314 pgd = get_current_pgd();
315 if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
316 is_kernel_mode, write))
317 return 1;
318
319 si_code = SEGV_MAPERR;
320
321
322
323
324
325
326
327
328
329
330
331
332
333 if (unlikely(address >= TASK_SIZE &&
334 !is_arch_mappable_range(address, 0))) {
335 if (is_kernel_mode && is_page_fault &&
336 vmalloc_fault(pgd, address) >= 0)
337 return 1;
338
339
340
341
342 mm = NULL;
343 vma = NULL;
344 goto bad_area_nosemaphore;
345 }
346
347
348
349
350
351
352
353
354 if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
355 local_irq_enable();
356
357 mm = tsk->mm;
358
359
360
361
362
363 if (in_atomic() || !mm) {
364 vma = NULL;
365 goto bad_area_nosemaphore;
366 }
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384 if (!down_read_trylock(&mm->mmap_sem)) {
385 if (is_kernel_mode &&
386 !search_exception_tables(regs->pc)) {
387 vma = NULL;
388 goto bad_area_nosemaphore;
389 }
390
391retry:
392 down_read(&mm->mmap_sem);
393 }
394
395 vma = find_vma(mm, address);
396 if (!vma)
397 goto bad_area;
398 if (vma->vm_start <= address)
399 goto good_area;
400 if (!(vma->vm_flags & VM_GROWSDOWN))
401 goto bad_area;
402 if (regs->sp < PAGE_OFFSET) {
403
404
405
406 if (address < regs->sp)
407 goto bad_area;
408 }
409 if (expand_stack(vma, address))
410 goto bad_area;
411
412
413
414
415
416good_area:
417 si_code = SEGV_ACCERR;
418 if (fault_num == INT_ITLB_MISS) {
419 if (!(vma->vm_flags & VM_EXEC))
420 goto bad_area;
421 } else if (write) {
422#ifdef TEST_VERIFY_AREA
423 if (!is_page_fault && regs->cs == KERNEL_CS)
424 pr_err("WP fault at "REGFMT"\n", regs->eip);
425#endif
426 if (!(vma->vm_flags & VM_WRITE))
427 goto bad_area;
428 } else {
429 if (!is_page_fault || !(vma->vm_flags & VM_READ))
430 goto bad_area;
431 }
432
433 survive:
434
435
436
437
438
439 fault = handle_mm_fault(mm, vma, address, flags);
440
441 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
442 return 0;
443
444 if (unlikely(fault & VM_FAULT_ERROR)) {
445 if (fault & VM_FAULT_OOM)
446 goto out_of_memory;
447 else if (fault & VM_FAULT_SIGBUS)
448 goto do_sigbus;
449 BUG();
450 }
451 if (flags & FAULT_FLAG_ALLOW_RETRY) {
452 if (fault & VM_FAULT_MAJOR)
453 tsk->maj_flt++;
454 else
455 tsk->min_flt++;
456 if (fault & VM_FAULT_RETRY) {
457 flags &= ~FAULT_FLAG_ALLOW_RETRY;
458 flags |= FAULT_FLAG_TRIED;
459
460
461
462
463
464
465 goto retry;
466 }
467 }
468
469#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
470
471
472
473
474 switch (fault_num) {
475#if CHIP_HAS_TILE_DMA()
476 case INT_DMATLB_MISS:
477 case INT_DMATLB_MISS_DWNCL:
478 case INT_DMATLB_ACCESS:
479 case INT_DMATLB_ACCESS_DWNCL:
480 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
481 break;
482#endif
483#if CHIP_HAS_SN_PROC()
484 case INT_SNITLB_MISS:
485 case INT_SNITLB_MISS_DWNCL:
486 __insn_mtspr(SPR_SNCTL,
487 __insn_mfspr(SPR_SNCTL) &
488 ~SPR_SNCTL__FRZPROC_MASK);
489 break;
490#endif
491 }
492#endif
493
494 up_read(&mm->mmap_sem);
495 return 1;
496
497
498
499
500
501bad_area:
502 up_read(&mm->mmap_sem);
503
504bad_area_nosemaphore:
505
506 if (!is_kernel_mode) {
507
508
509
510 local_irq_enable();
511
512 force_sig_info_fault("segfault", SIGSEGV, si_code, address,
513 fault_num, tsk, regs);
514 return 0;
515 }
516
517no_context:
518
519 if (fixup_exception(regs))
520 return 0;
521
522
523
524
525
526
527 bust_spinlocks(1);
528
529
530#ifdef SUPPORT_LOOKUP_ADDRESS
531 if (fault_num == INT_ITLB_MISS) {
532 pte_t *pte = lookup_address(address);
533
534 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
535 pr_crit("kernel tried to execute"
536 " non-executable page - exploit attempt?"
537 " (uid: %d)\n", current->uid);
538 }
539#endif
540 if (address < PAGE_SIZE)
541 pr_alert("Unable to handle kernel NULL pointer dereference\n");
542 else
543 pr_alert("Unable to handle kernel paging request\n");
544 pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n",
545 address, regs->pc);
546
547 show_regs(regs);
548
549 if (unlikely(tsk->pid < 2)) {
550 panic("Kernel page fault running %s!",
551 is_idle_task(tsk) ? "the idle task" : "init");
552 }
553
554
555
556
557
558#ifdef SUPPORT_DIE
559 die("Oops", regs);
560#endif
561 bust_spinlocks(1);
562
563 do_group_exit(SIGKILL);
564
565
566
567
568
569out_of_memory:
570 up_read(&mm->mmap_sem);
571 if (is_global_init(tsk)) {
572 yield();
573 down_read(&mm->mmap_sem);
574 goto survive;
575 }
576 if (is_kernel_mode)
577 goto no_context;
578 pagefault_out_of_memory();
579 return 0;
580
581do_sigbus:
582 up_read(&mm->mmap_sem);
583
584
585 if (is_kernel_mode)
586 goto no_context;
587
588 force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
589 fault_num, tsk, regs);
590 return 0;
591}
592
593#ifndef __tilegx__
594
595
596#define ics_panic(fmt, ...) do { \
597 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \
598 panic(fmt, __VA_ARGS__); \
599} while (0)
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num,
618 unsigned long address,
619 unsigned long info)
620{
621 unsigned long pc = info & ~1;
622 int write = info & 1;
623 pgd_t *pgd = get_current_pgd();
624
625
626 struct intvec_state state = {
627 do_page_fault, fault_num, address, write, 1
628 };
629
630
631 if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
632 (fault_num != INT_DTLB_MISS &&
633 fault_num != INT_DTLB_ACCESS)) {
634 unsigned long old_pc = regs->pc;
635 regs->pc = pc;
636 ics_panic("Bad ICS page fault args:"
637 " old PC %#lx, fault %d/%d at %#lx\n",
638 old_pc, fault_num, write, address);
639 }
640
641
642 if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
643 return state;
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661 if (pc >= (unsigned long) sys_cmpxchg &&
662 pc < (unsigned long) __sys_cmpxchg_end) {
663#ifdef CONFIG_SMP
664
665 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
666 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
667 __atomic_fault_unlock(lock_ptr);
668 }
669#endif
670 regs->sp = regs->regs[27];
671 }
672
673
674
675
676
677
678
679
680 else if (pc >= (unsigned long) __start_atomic_asm_code &&
681 pc < (unsigned long) __end_atomic_asm_code) {
682 const struct exception_table_entry *fixup;
683#ifdef CONFIG_SMP
684
685 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
686 __atomic_fault_unlock(lock_ptr);
687#endif
688 fixup = search_exception_tables(pc);
689 if (!fixup)
690 ics_panic("ICS atomic fault not in table:"
691 " PC %#lx, fault %d", pc, fault_num);
692 regs->pc = fixup->fixup;
693 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
694 }
695
696
697
698
699
700 if (fault_num == INT_DTLB_ACCESS)
701 write = 1;
702 if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write))
703 return state;
704
705
706 state.retval = 0;
707 return state;
708}
709
710#endif
711
712
713
714
715
716
717
718
719
720void do_page_fault(struct pt_regs *regs, int fault_num,
721 unsigned long address, unsigned long write)
722{
723 int is_page_fault;
724
725
726 BUG_ON(write & ~1);
727
728#if CHIP_HAS_TILE_DMA()
729
730
731
732
733
734
735
736 if (fault_num == INT_DMATLB_MISS ||
737 fault_num == INT_DMATLB_ACCESS ||
738 fault_num == INT_DMATLB_MISS_DWNCL ||
739 fault_num == INT_DMATLB_ACCESS_DWNCL) {
740 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
741 while (__insn_mfspr(SPR_DMA_USER_STATUS) &
742 SPR_DMA_STATUS__BUSY_MASK)
743 ;
744 }
745#endif
746
747
748 switch (fault_num) {
749 case INT_ITLB_MISS:
750 case INT_DTLB_MISS:
751#if CHIP_HAS_TILE_DMA()
752 case INT_DMATLB_MISS:
753 case INT_DMATLB_MISS_DWNCL:
754#endif
755#if CHIP_HAS_SN_PROC()
756 case INT_SNITLB_MISS:
757 case INT_SNITLB_MISS_DWNCL:
758#endif
759 is_page_fault = 1;
760 break;
761
762 case INT_DTLB_ACCESS:
763#if CHIP_HAS_TILE_DMA()
764 case INT_DMATLB_ACCESS:
765 case INT_DMATLB_ACCESS_DWNCL:
766#endif
767 is_page_fault = 0;
768 break;
769
770 default:
771 panic("Bad fault number %d in do_page_fault", fault_num);
772 }
773
774#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
775 if (EX1_PL(regs->ex1) != USER_PL) {
776 struct async_tlb *async;
777 switch (fault_num) {
778#if CHIP_HAS_TILE_DMA()
779 case INT_DMATLB_MISS:
780 case INT_DMATLB_ACCESS:
781 case INT_DMATLB_MISS_DWNCL:
782 case INT_DMATLB_ACCESS_DWNCL:
783 async = ¤t->thread.dma_async_tlb;
784 break;
785#endif
786#if CHIP_HAS_SN_PROC()
787 case INT_SNITLB_MISS:
788 case INT_SNITLB_MISS_DWNCL:
789 async = ¤t->thread.sn_async_tlb;
790 break;
791#endif
792 default:
793 async = NULL;
794 }
795 if (async) {
796
797
798
799
800
801 local_irq_enable();
802
803 set_thread_flag(TIF_ASYNC_TLB);
804 if (async->fault_num != 0) {
805 panic("Second async fault %d;"
806 " old fault was %d (%#lx/%ld)",
807 fault_num, async->fault_num,
808 address, write);
809 }
810 BUG_ON(fault_num == 0);
811 async->fault_num = fault_num;
812 async->is_fault = is_page_fault;
813 async->is_write = write;
814 async->address = address;
815 return;
816 }
817 }
818#endif
819
820 handle_page_fault(regs, fault_num, is_page_fault, address, write);
821}
822
823
824#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
825
826
827
828
829static void handle_async_page_fault(struct pt_regs *regs,
830 struct async_tlb *async)
831{
832 if (async->fault_num) {
833
834
835
836
837
838
839 int fault_num = async->fault_num;
840 async->fault_num = 0;
841 handle_page_fault(regs, fault_num, async->is_fault,
842 async->address, async->is_write);
843 }
844}
845
846
847
848
849
850void do_async_page_fault(struct pt_regs *regs)
851{
852
853
854
855
856
857 clear_thread_flag(TIF_ASYNC_TLB);
858
859#if CHIP_HAS_TILE_DMA()
860 handle_async_page_fault(regs, ¤t->thread.dma_async_tlb);
861#endif
862#if CHIP_HAS_SN_PROC()
863 handle_async_page_fault(regs, ¤t->thread.sn_async_tlb);
864#endif
865}
866#endif
867
868
869void vmalloc_sync_all(void)
870{
871#ifdef __tilegx__
872
873 BUG_ON(pgd_index(VMALLOC_END) != pgd_index(VMALLOC_START));
874#else
875
876
877
878
879
880
881 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
882 static unsigned long start = PAGE_OFFSET;
883 unsigned long address;
884
885 BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
886 for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
887 if (!test_bit(pgd_index(address), insync)) {
888 unsigned long flags;
889 struct list_head *pos;
890
891 spin_lock_irqsave(&pgd_lock, flags);
892 list_for_each(pos, &pgd_list)
893 if (!vmalloc_sync_one(list_to_pgd(pos),
894 address)) {
895
896 BUG_ON(pos != pgd_list.next);
897 break;
898 }
899 spin_unlock_irqrestore(&pgd_lock, flags);
900 if (pos != pgd_list.next)
901 set_bit(pgd_index(address), insync);
902 }
903 if (address == start && test_bit(pgd_index(address), insync))
904 start = address + PGDIR_SIZE;
905 }
906#endif
907}
908