1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/kernel_stat.h>
14#include <linux/perf_event.h>
15#include <linux/signal.h>
16#include <linux/sched.h>
17#include <linux/kernel.h>
18#include <linux/errno.h>
19#include <linux/string.h>
20#include <linux/types.h>
21#include <linux/ptrace.h>
22#include <linux/mman.h>
23#include <linux/mm.h>
24#include <linux/compat.h>
25#include <linux/smp.h>
26#include <linux/kdebug.h>
27#include <linux/init.h>
28#include <linux/console.h>
29#include <linux/module.h>
30#include <linux/hardirq.h>
31#include <linux/kprobes.h>
32#include <linux/uaccess.h>
33#include <linux/hugetlb.h>
34#include <asm/asm-offsets.h>
35#include <asm/system.h>
36#include <asm/pgtable.h>
37#include <asm/s390_ext.h>
38#include <asm/mmu_context.h>
39#include <asm/compat.h>
40#include "../kernel/entry.h"
41
42#ifndef CONFIG_64BIT
43#define __FAIL_ADDR_MASK 0x7ffff000
44#define __SUBCODE_MASK 0x0200
45#define __PF_RES_FIELD 0ULL
46#else
47#define __FAIL_ADDR_MASK -4096L
48#define __SUBCODE_MASK 0x0600
49#define __PF_RES_FIELD 0x8000000000000000ULL
50#endif
51
52#define VM_FAULT_BADCONTEXT 0x010000
53#define VM_FAULT_BADMAP 0x020000
54#define VM_FAULT_BADACCESS 0x040000
55
56static unsigned long store_indication;
57
58void fault_init(void)
59{
60 if (test_facility(2) && test_facility(75))
61 store_indication = 0xc00;
62}
63
64static inline int notify_page_fault(struct pt_regs *regs)
65{
66 int ret = 0;
67
68
69 if (kprobes_built_in() && !user_mode(regs)) {
70 preempt_disable();
71 if (kprobe_running() && kprobe_fault_handler(regs, 14))
72 ret = 1;
73 preempt_enable();
74 }
75 return ret;
76}
77
78
79
80
81
82
83void bust_spinlocks(int yes)
84{
85 if (yes) {
86 oops_in_progress = 1;
87 } else {
88 int loglevel_save = console_loglevel;
89 console_unblank();
90 oops_in_progress = 0;
91
92
93
94
95
96 console_loglevel = 15;
97 printk(" ");
98 console_loglevel = loglevel_save;
99 }
100}
101
102
103
104
105
106static inline int user_space_fault(unsigned long trans_exc_code)
107{
108
109
110
111
112 trans_exc_code &= 3;
113 if (trans_exc_code == 2)
114
115 return current->thread.mm_segment.ar4;
116 if (user_mode == HOME_SPACE_MODE)
117
118 return trans_exc_code == 3;
119
120
121
122
123
124
125 return trans_exc_code != 3;
126}
127
128static inline void report_user_fault(struct pt_regs *regs, long int_code,
129 int signr, unsigned long address)
130{
131 if ((task_pid_nr(current) > 1) && !show_unhandled_signals)
132 return;
133 if (!unhandled_signal(current, signr))
134 return;
135 if (!printk_ratelimit())
136 return;
137 printk("User process fault: interruption code 0x%lX ", int_code);
138 print_vma_addr(KERN_CONT "in ", regs->psw.addr & PSW_ADDR_INSN);
139 printk("\n");
140 printk("failing address: %lX\n", address);
141 show_regs(regs);
142}
143
144
145
146
147
148static noinline void do_sigsegv(struct pt_regs *regs, long int_code,
149 int si_code, unsigned long trans_exc_code)
150{
151 struct siginfo si;
152 unsigned long address;
153
154 address = trans_exc_code & __FAIL_ADDR_MASK;
155 current->thread.prot_addr = address;
156 current->thread.trap_no = int_code;
157 report_user_fault(regs, int_code, SIGSEGV, address);
158 si.si_signo = SIGSEGV;
159 si.si_code = si_code;
160 si.si_addr = (void __user *) address;
161 force_sig_info(SIGSEGV, &si, current);
162}
163
164static noinline void do_no_context(struct pt_regs *regs, long int_code,
165 unsigned long trans_exc_code)
166{
167 const struct exception_table_entry *fixup;
168 unsigned long address;
169
170
171 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
172 if (fixup) {
173 regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
174 return;
175 }
176
177
178
179
180
181 address = trans_exc_code & __FAIL_ADDR_MASK;
182 if (!user_space_fault(trans_exc_code))
183 printk(KERN_ALERT "Unable to handle kernel pointer dereference"
184 " at virtual kernel address %p\n", (void *)address);
185 else
186 printk(KERN_ALERT "Unable to handle kernel paging request"
187 " at virtual user address %p\n", (void *)address);
188
189 die("Oops", regs, int_code);
190 do_exit(SIGKILL);
191}
192
193static noinline void do_low_address(struct pt_regs *regs, long int_code,
194 unsigned long trans_exc_code)
195{
196
197
198 if (regs->psw.mask & PSW_MASK_PSTATE) {
199
200 die ("Low-address protection", regs, int_code);
201 do_exit(SIGKILL);
202 }
203
204 do_no_context(regs, int_code, trans_exc_code);
205}
206
207static noinline void do_sigbus(struct pt_regs *regs, long int_code,
208 unsigned long trans_exc_code)
209{
210 struct task_struct *tsk = current;
211 unsigned long address;
212 struct siginfo si;
213
214
215
216
217
218 address = trans_exc_code & __FAIL_ADDR_MASK;
219 tsk->thread.prot_addr = address;
220 tsk->thread.trap_no = int_code;
221 si.si_signo = SIGBUS;
222 si.si_errno = 0;
223 si.si_code = BUS_ADRERR;
224 si.si_addr = (void __user *) address;
225 force_sig_info(SIGBUS, &si, tsk);
226}
227
228#ifdef CONFIG_S390_EXEC_PROTECT
229static noinline int signal_return(struct pt_regs *regs, long int_code,
230 unsigned long trans_exc_code)
231{
232 u16 instruction;
233 int rc;
234
235 rc = __get_user(instruction, (u16 __user *) regs->psw.addr);
236
237 if (!rc && instruction == 0x0a77) {
238 clear_tsk_thread_flag(current, TIF_PER_TRAP);
239 if (is_compat_task())
240 sys32_sigreturn();
241 else
242 sys_sigreturn();
243 } else if (!rc && instruction == 0x0aad) {
244 clear_tsk_thread_flag(current, TIF_PER_TRAP);
245 if (is_compat_task())
246 sys32_rt_sigreturn();
247 else
248 sys_rt_sigreturn();
249 } else
250 do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code);
251 return 0;
252}
253#endif
254
255static noinline void do_fault_error(struct pt_regs *regs, long int_code,
256 unsigned long trans_exc_code, int fault)
257{
258 int si_code;
259
260 switch (fault) {
261 case VM_FAULT_BADACCESS:
262#ifdef CONFIG_S390_EXEC_PROTECT
263 if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY &&
264 (trans_exc_code & 3) == 0) {
265 signal_return(regs, int_code, trans_exc_code);
266 break;
267 }
268#endif
269 case VM_FAULT_BADMAP:
270
271 if (regs->psw.mask & PSW_MASK_PSTATE) {
272
273 si_code = (fault == VM_FAULT_BADMAP) ?
274 SEGV_MAPERR : SEGV_ACCERR;
275 do_sigsegv(regs, int_code, si_code, trans_exc_code);
276 return;
277 }
278 case VM_FAULT_BADCONTEXT:
279 do_no_context(regs, int_code, trans_exc_code);
280 break;
281 default:
282 if (fault & VM_FAULT_OOM)
283 pagefault_out_of_memory();
284 else if (fault & VM_FAULT_SIGBUS) {
285
286 if (!(regs->psw.mask & PSW_MASK_PSTATE))
287 do_no_context(regs, int_code, trans_exc_code);
288 else
289 do_sigbus(regs, int_code, trans_exc_code);
290 } else
291 BUG();
292 break;
293 }
294}
295
296
297
298
299
300
301
302
303
304
305
306
307static inline int do_exception(struct pt_regs *regs, int access,
308 unsigned long trans_exc_code)
309{
310 struct task_struct *tsk;
311 struct mm_struct *mm;
312 struct vm_area_struct *vma;
313 unsigned long address;
314 int fault, write;
315
316 if (notify_page_fault(regs))
317 return 0;
318
319 tsk = current;
320 mm = tsk->mm;
321
322
323
324
325
326
327 fault = VM_FAULT_BADCONTEXT;
328 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
329 goto out;
330
331 address = trans_exc_code & __FAIL_ADDR_MASK;
332 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
333 down_read(&mm->mmap_sem);
334
335 fault = VM_FAULT_BADMAP;
336 vma = find_vma(mm, address);
337 if (!vma)
338 goto out_up;
339
340 if (unlikely(vma->vm_start > address)) {
341 if (!(vma->vm_flags & VM_GROWSDOWN))
342 goto out_up;
343 if (expand_stack(vma, address))
344 goto out_up;
345 }
346
347
348
349
350
351 fault = VM_FAULT_BADACCESS;
352 if (unlikely(!(vma->vm_flags & access)))
353 goto out_up;
354
355 if (is_vm_hugetlb_page(vma))
356 address &= HPAGE_MASK;
357
358
359
360
361
362 write = (access == VM_WRITE ||
363 (trans_exc_code & store_indication) == 0x400) ?
364 FAULT_FLAG_WRITE : 0;
365 fault = handle_mm_fault(mm, vma, address, write);
366 if (unlikely(fault & VM_FAULT_ERROR))
367 goto out_up;
368
369 if (fault & VM_FAULT_MAJOR) {
370 tsk->maj_flt++;
371 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
372 regs, address);
373 } else {
374 tsk->min_flt++;
375 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
376 regs, address);
377 }
378
379
380
381
382 clear_tsk_thread_flag(tsk, TIF_PER_TRAP);
383 fault = 0;
384out_up:
385 up_read(&mm->mmap_sem);
386out:
387 return fault;
388}
389
390void __kprobes do_protection_exception(struct pt_regs *regs, long pgm_int_code,
391 unsigned long trans_exc_code)
392{
393 int fault;
394
395
396 regs->psw.addr -= (pgm_int_code >> 16);
397
398
399
400
401
402 if (unlikely(!(trans_exc_code & 4))) {
403 do_low_address(regs, pgm_int_code, trans_exc_code);
404 return;
405 }
406 fault = do_exception(regs, VM_WRITE, trans_exc_code);
407 if (unlikely(fault))
408 do_fault_error(regs, 4, trans_exc_code, fault);
409}
410
411void __kprobes do_dat_exception(struct pt_regs *regs, long pgm_int_code,
412 unsigned long trans_exc_code)
413{
414 int access, fault;
415
416 access = VM_READ | VM_EXEC | VM_WRITE;
417#ifdef CONFIG_S390_EXEC_PROTECT
418 if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY &&
419 (trans_exc_code & 3) == 0)
420 access = VM_EXEC;
421#endif
422 fault = do_exception(regs, access, trans_exc_code);
423 if (unlikely(fault))
424 do_fault_error(regs, pgm_int_code & 255, trans_exc_code, fault);
425}
426
427#ifdef CONFIG_64BIT
428void __kprobes do_asce_exception(struct pt_regs *regs, long pgm_int_code,
429 unsigned long trans_exc_code)
430{
431 struct mm_struct *mm = current->mm;
432 struct vm_area_struct *vma;
433
434 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
435 goto no_context;
436
437 down_read(&mm->mmap_sem);
438 vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK);
439 up_read(&mm->mmap_sem);
440
441 if (vma) {
442 update_mm(mm, current);
443 return;
444 }
445
446
447 if (regs->psw.mask & PSW_MASK_PSTATE) {
448 do_sigsegv(regs, pgm_int_code, SEGV_MAPERR, trans_exc_code);
449 return;
450 }
451
452no_context:
453 do_no_context(regs, pgm_int_code, trans_exc_code);
454}
455#endif
456
457int __handle_fault(unsigned long uaddr, unsigned long pgm_int_code, int write)
458{
459 struct pt_regs regs;
460 int access, fault;
461
462 regs.psw.mask = psw_kernel_bits;
463 if (!irqs_disabled())
464 regs.psw.mask |= PSW_MASK_IO | PSW_MASK_EXT;
465 regs.psw.addr = (unsigned long) __builtin_return_address(0);
466 regs.psw.addr |= PSW_ADDR_AMODE;
467 uaddr &= PAGE_MASK;
468 access = write ? VM_WRITE : VM_READ;
469 fault = do_exception(®s, access, uaddr | 2);
470 if (unlikely(fault)) {
471 if (fault & VM_FAULT_OOM) {
472 pagefault_out_of_memory();
473 fault = 0;
474 } else if (fault & VM_FAULT_SIGBUS)
475 do_sigbus(®s, pgm_int_code, uaddr);
476 }
477 return fault ? -EFAULT : 0;
478}
479
480#ifdef CONFIG_PFAULT
481
482
483
484static int pfault_disable;
485
486static int __init nopfault(char *str)
487{
488 pfault_disable = 1;
489 return 1;
490}
491
492__setup("nopfault", nopfault);
493
494typedef struct {
495 __u16 refdiagc;
496 __u16 reffcode;
497 __u16 refdwlen;
498 __u16 refversn;
499 __u64 refgaddr;
500 __u64 refselmk;
501 __u64 refcmpmk;
502 __u64 reserved;
503} __attribute__ ((packed, aligned(8))) pfault_refbk_t;
504
505int pfault_init(void)
506{
507 pfault_refbk_t refbk =
508 { 0x258, 0, 5, 2, __LC_CURRENT, 1ULL << 48, 1ULL << 48,
509 __PF_RES_FIELD };
510 int rc;
511
512 if (!MACHINE_IS_VM || pfault_disable)
513 return -1;
514 asm volatile(
515 " diag %1,%0,0x258\n"
516 "0: j 2f\n"
517 "1: la %0,8\n"
518 "2:\n"
519 EX_TABLE(0b,1b)
520 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc");
521 __ctl_set_bit(0, 9);
522 return rc;
523}
524
525void pfault_fini(void)
526{
527 pfault_refbk_t refbk =
528 { 0x258, 1, 5, 2, 0ULL, 0ULL, 0ULL, 0ULL };
529
530 if (!MACHINE_IS_VM || pfault_disable)
531 return;
532 __ctl_clear_bit(0,9);
533 asm volatile(
534 " diag %0,0,0x258\n"
535 "0:\n"
536 EX_TABLE(0b,0b)
537 : : "a" (&refbk), "m" (refbk) : "cc");
538}
539
540static void pfault_interrupt(unsigned int ext_int_code,
541 unsigned int param32, unsigned long param64)
542{
543 struct task_struct *tsk;
544 __u16 subcode;
545
546 kstat_cpu(smp_processor_id()).irqs[EXTINT_PFL]++;
547
548
549
550
551
552
553 subcode = ext_int_code >> 16;
554 if ((subcode & 0xff00) != __SUBCODE_MASK)
555 return;
556
557
558
559
560#ifdef CONFIG_64BIT
561 tsk = *(struct task_struct **) param64;
562#else
563 tsk = *(struct task_struct **) param32;
564#endif
565
566 if (subcode & 0x0080) {
567
568 if (xchg(&tsk->thread.pfault_wait, -1) != 0) {
569
570
571
572
573
574 tsk->thread.pfault_wait = 0;
575 wake_up_process(tsk);
576 put_task_struct(tsk);
577 }
578 } else {
579
580 get_task_struct(tsk);
581 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
582 if (xchg(&tsk->thread.pfault_wait, 1) != 0) {
583
584
585
586
587
588 tsk->thread.pfault_wait = 0;
589 set_task_state(tsk, TASK_RUNNING);
590 put_task_struct(tsk);
591 } else
592 set_tsk_need_resched(tsk);
593 }
594}
595
596static int __init pfault_irq_init(void)
597{
598 int rc;
599
600 if (!MACHINE_IS_VM)
601 return 0;
602
603
604
605 rc = register_external_interrupt(0x2603, pfault_interrupt);
606 if (rc) {
607 pfault_disable = 1;
608 return rc;
609 }
610 if (pfault_init() == 0)
611 return 0;
612
613
614 pfault_disable = 1;
615 unregister_external_interrupt(0x2603, pfault_interrupt);
616 return 0;
617}
618early_initcall(pfault_irq_init);
619
620#endif
621