1
2
3
4
5
6
7
8
9
10
11
12
13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14
15#include <linux/context_tracking.h>
16#include <linux/interrupt.h>
17#include <linux/kallsyms.h>
18#include <linux/spinlock.h>
19#include <linux/kprobes.h>
20#include <linux/uaccess.h>
21#include <linux/kdebug.h>
22#include <linux/kgdb.h>
23#include <linux/kernel.h>
24#include <linux/export.h>
25#include <linux/ptrace.h>
26#include <linux/uprobes.h>
27#include <linux/string.h>
28#include <linux/delay.h>
29#include <linux/errno.h>
30#include <linux/kexec.h>
31#include <linux/sched.h>
32#include <linux/sched/task_stack.h>
33#include <linux/timer.h>
34#include <linux/init.h>
35#include <linux/bug.h>
36#include <linux/nmi.h>
37#include <linux/mm.h>
38#include <linux/smp.h>
39#include <linux/io.h>
40
41#ifdef CONFIG_EISA
42#include <linux/ioport.h>
43#include <linux/eisa.h>
44#endif
45
46#if defined(CONFIG_EDAC)
47#include <linux/edac.h>
48#endif
49
50#include <asm/kmemcheck.h>
51#include <asm/stacktrace.h>
52#include <asm/processor.h>
53#include <asm/debugreg.h>
54#include <linux/atomic.h>
55#include <asm/text-patching.h>
56#include <asm/ftrace.h>
57#include <asm/traps.h>
58#include <asm/desc.h>
59#include <asm/fpu/internal.h>
60#include <asm/mce.h>
61#include <asm/fixmap.h>
62#include <asm/mach_traps.h>
63#include <asm/alternative.h>
64#include <asm/fpu/xstate.h>
65#include <asm/trace/mpx.h>
66#include <asm/mpx.h>
67#include <asm/vm86.h>
68
69#ifdef CONFIG_X86_64
70#include <asm/x86_init.h>
71#include <asm/pgalloc.h>
72#include <asm/proto.h>
73
74
75gate_desc debug_idt_table[NR_VECTORS] __page_aligned_bss;
76#else
77#include <asm/processor-flags.h>
78#include <asm/setup.h>
79#include <asm/proto.h>
80#endif
81
82
83gate_desc idt_table[NR_VECTORS] __page_aligned_bss;
84
85DECLARE_BITMAP(used_vectors, NR_VECTORS);
86EXPORT_SYMBOL_GPL(used_vectors);
87
88static inline void cond_local_irq_enable(struct pt_regs *regs)
89{
90 if (regs->flags & X86_EFLAGS_IF)
91 local_irq_enable();
92}
93
94static inline void cond_local_irq_disable(struct pt_regs *regs)
95{
96 if (regs->flags & X86_EFLAGS_IF)
97 local_irq_disable();
98}
99
100
101
102
103
104
105
106void ist_enter(struct pt_regs *regs)
107{
108 if (user_mode(regs)) {
109 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
110 } else {
111
112
113
114
115
116
117 rcu_nmi_enter();
118 }
119
120 preempt_disable();
121
122
123 RCU_LOCKDEP_WARN(!rcu_is_watching(), "ist_enter didn't work");
124}
125
126void ist_exit(struct pt_regs *regs)
127{
128 preempt_enable_no_resched();
129
130 if (!user_mode(regs))
131 rcu_nmi_exit();
132}
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147void ist_begin_non_atomic(struct pt_regs *regs)
148{
149 BUG_ON(!user_mode(regs));
150
151
152
153
154
155
156 BUG_ON((unsigned long)(current_top_of_stack() -
157 current_stack_pointer()) >= THREAD_SIZE);
158
159 preempt_enable_no_resched();
160}
161
162
163
164
165
166
167void ist_end_non_atomic(void)
168{
169 preempt_disable();
170}
171
172int is_valid_bugaddr(unsigned long addr)
173{
174 unsigned short ud;
175
176 if (addr < TASK_SIZE_MAX)
177 return 0;
178
179 if (probe_kernel_address((unsigned short *)addr, ud))
180 return 0;
181
182 return ud == INSN_UD0 || ud == INSN_UD2;
183}
184
185int fixup_bug(struct pt_regs *regs, int trapnr)
186{
187 if (trapnr != X86_TRAP_UD)
188 return 0;
189
190 switch (report_bug(regs->ip, regs)) {
191 case BUG_TRAP_TYPE_NONE:
192 case BUG_TRAP_TYPE_BUG:
193 break;
194
195 case BUG_TRAP_TYPE_WARN:
196 regs->ip += LEN_UD0;
197 return 1;
198 }
199
200 return 0;
201}
202
203static nokprobe_inline int
204do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
205 struct pt_regs *regs, long error_code)
206{
207 if (v8086_mode(regs)) {
208
209
210
211
212 if (trapnr < X86_TRAP_UD) {
213 if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,
214 error_code, trapnr))
215 return 0;
216 }
217 return -1;
218 }
219
220 if (!user_mode(regs)) {
221 if (fixup_exception(regs, trapnr))
222 return 0;
223
224 if (fixup_bug(regs, trapnr))
225 return 0;
226
227 tsk->thread.error_code = error_code;
228 tsk->thread.trap_nr = trapnr;
229 die(str, regs, error_code);
230 }
231
232 return -1;
233}
234
235static siginfo_t *fill_trap_info(struct pt_regs *regs, int signr, int trapnr,
236 siginfo_t *info)
237{
238 unsigned long siaddr;
239 int sicode;
240
241 switch (trapnr) {
242 default:
243 return SEND_SIG_PRIV;
244
245 case X86_TRAP_DE:
246 sicode = FPE_INTDIV;
247 siaddr = uprobe_get_trap_addr(regs);
248 break;
249 case X86_TRAP_UD:
250 sicode = ILL_ILLOPN;
251 siaddr = uprobe_get_trap_addr(regs);
252 break;
253 case X86_TRAP_AC:
254 sicode = BUS_ADRALN;
255 siaddr = 0;
256 break;
257 }
258
259 info->si_signo = signr;
260 info->si_errno = 0;
261 info->si_code = sicode;
262 info->si_addr = (void __user *)siaddr;
263 return info;
264}
265
266static void
267do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
268 long error_code, siginfo_t *info)
269{
270 struct task_struct *tsk = current;
271
272
273 if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))
274 return;
275
276
277
278
279
280
281
282
283
284 tsk->thread.error_code = error_code;
285 tsk->thread.trap_nr = trapnr;
286
287 if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
288 printk_ratelimit()) {
289 pr_info("%s[%d] trap %s ip:%lx sp:%lx error:%lx",
290 tsk->comm, tsk->pid, str,
291 regs->ip, regs->sp, error_code);
292 print_vma_addr(KERN_CONT " in ", regs->ip);
293 pr_cont("\n");
294 }
295
296 force_sig_info(signr, info ?: SEND_SIG_PRIV, tsk);
297}
298NOKPROBE_SYMBOL(do_trap);
299
300static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
301 unsigned long trapnr, int signr)
302{
303 siginfo_t info;
304
305 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
306
307 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
308 NOTIFY_STOP) {
309 cond_local_irq_enable(regs);
310 do_trap(trapnr, signr, str, regs, error_code,
311 fill_trap_info(regs, signr, trapnr, &info));
312 }
313}
314
315#define DO_ERROR(trapnr, signr, str, name) \
316dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
317{ \
318 do_error_trap(regs, error_code, str, trapnr, signr); \
319}
320
321DO_ERROR(X86_TRAP_DE, SIGFPE, "divide error", divide_error)
322DO_ERROR(X86_TRAP_OF, SIGSEGV, "overflow", overflow)
323DO_ERROR(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op)
324DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun",coprocessor_segment_overrun)
325DO_ERROR(X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS)
326DO_ERROR(X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present)
327DO_ERROR(X86_TRAP_SS, SIGBUS, "stack segment", stack_segment)
328DO_ERROR(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check)
329
330#ifdef CONFIG_VMAP_STACK
331__visible void __noreturn handle_stack_overflow(const char *message,
332 struct pt_regs *regs,
333 unsigned long fault_address)
334{
335 printk(KERN_EMERG "BUG: stack guard page was hit at %p (stack is %p..%p)\n",
336 (void *)fault_address, current->stack,
337 (char *)current->stack + THREAD_SIZE - 1);
338 die(message, regs, 0);
339
340
341 panic(message);
342}
343#endif
344
345#ifdef CONFIG_X86_64
346
347dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
348{
349 static const char str[] = "double fault";
350 struct task_struct *tsk = current;
351#ifdef CONFIG_VMAP_STACK
352 unsigned long cr2;
353#endif
354
355#ifdef CONFIG_X86_ESPFIX64
356 extern unsigned char native_irq_return_iret[];
357
358
359
360
361
362
363
364
365
366 if (((long)regs->sp >> PGDIR_SHIFT) == ESPFIX_PGD_ENTRY &&
367 regs->cs == __KERNEL_CS &&
368 regs->ip == (unsigned long)native_irq_return_iret)
369 {
370 struct pt_regs *normal_regs = task_pt_regs(current);
371
372
373 memmove(&normal_regs->ip, (void *)regs->sp, 5*8);
374 normal_regs->orig_ax = 0;
375 regs->ip = (unsigned long)general_protection;
376 regs->sp = (unsigned long)&normal_regs->orig_ax;
377
378 return;
379 }
380#endif
381
382 ist_enter(regs);
383 notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
384
385 tsk->thread.error_code = error_code;
386 tsk->thread.trap_nr = X86_TRAP_DF;
387
388#ifdef CONFIG_VMAP_STACK
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426 cr2 = read_cr2();
427 if ((unsigned long)task_stack_page(tsk) - 1 - cr2 < PAGE_SIZE)
428 handle_stack_overflow("kernel stack overflow (double-fault)", regs, cr2);
429#endif
430
431#ifdef CONFIG_DOUBLEFAULT
432 df_debug(regs, error_code);
433#endif
434
435
436
437
438 for (;;)
439 die(str, regs, error_code);
440}
441#endif
442
443dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)
444{
445 const struct mpx_bndcsr *bndcsr;
446 siginfo_t *info;
447
448 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
449 if (notify_die(DIE_TRAP, "bounds", regs, error_code,
450 X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
451 return;
452 cond_local_irq_enable(regs);
453
454 if (!user_mode(regs))
455 die("bounds", regs, error_code);
456
457 if (!cpu_feature_enabled(X86_FEATURE_MPX)) {
458
459 goto exit_trap;
460 }
461
462
463
464
465
466
467
468 bndcsr = get_xsave_field_ptr(XFEATURE_MASK_BNDCSR);
469 if (!bndcsr)
470 goto exit_trap;
471
472 trace_bounds_exception_mpx(bndcsr);
473
474
475
476
477
478 switch (bndcsr->bndstatus & MPX_BNDSTA_ERROR_CODE) {
479 case 2:
480 if (mpx_handle_bd_fault())
481 goto exit_trap;
482 break;
483 case 1:
484 info = mpx_generate_siginfo(regs);
485 if (IS_ERR(info)) {
486
487
488
489
490 goto exit_trap;
491 }
492
493
494
495
496
497
498
499 do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, info);
500 kfree(info);
501 break;
502 case 0:
503 goto exit_trap;
504 default:
505 die("bounds", regs, error_code);
506 }
507
508 return;
509
510exit_trap:
511
512
513
514
515
516
517
518 do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, NULL);
519}
520
521dotraplinkage void
522do_general_protection(struct pt_regs *regs, long error_code)
523{
524 struct task_struct *tsk;
525
526 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
527 cond_local_irq_enable(regs);
528
529 if (v8086_mode(regs)) {
530 local_irq_enable();
531 handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
532 return;
533 }
534
535 tsk = current;
536 if (!user_mode(regs)) {
537 if (fixup_exception(regs, X86_TRAP_GP))
538 return;
539
540 tsk->thread.error_code = error_code;
541 tsk->thread.trap_nr = X86_TRAP_GP;
542 if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
543 X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)
544 die("general protection fault", regs, error_code);
545 return;
546 }
547
548 tsk->thread.error_code = error_code;
549 tsk->thread.trap_nr = X86_TRAP_GP;
550
551 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
552 printk_ratelimit()) {
553 pr_info("%s[%d] general protection ip:%lx sp:%lx error:%lx",
554 tsk->comm, task_pid_nr(tsk),
555 regs->ip, regs->sp, error_code);
556 print_vma_addr(KERN_CONT " in ", regs->ip);
557 pr_cont("\n");
558 }
559
560 force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
561}
562NOKPROBE_SYMBOL(do_general_protection);
563
564
565dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
566{
567#ifdef CONFIG_DYNAMIC_FTRACE
568
569
570
571
572 if (unlikely(atomic_read(&modifying_ftrace_code)) &&
573 ftrace_int3_handler(regs))
574 return;
575#endif
576 if (poke_int3_handler(regs))
577 return;
578
579 ist_enter(regs);
580 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
581#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
582 if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
583 SIGTRAP) == NOTIFY_STOP)
584 goto exit;
585#endif
586
587#ifdef CONFIG_KPROBES
588 if (kprobe_int3_handler(regs))
589 goto exit;
590#endif
591
592 if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
593 SIGTRAP) == NOTIFY_STOP)
594 goto exit;
595
596
597
598
599
600 debug_stack_usage_inc();
601 cond_local_irq_enable(regs);
602 do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
603 cond_local_irq_disable(regs);
604 debug_stack_usage_dec();
605exit:
606 ist_exit(regs);
607}
608NOKPROBE_SYMBOL(do_int3);
609
610#ifdef CONFIG_X86_64
611
612
613
614
615
616asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs)
617{
618 struct pt_regs *regs = task_pt_regs(current);
619 *regs = *eregs;
620 return regs;
621}
622NOKPROBE_SYMBOL(sync_regs);
623
624struct bad_iret_stack {
625 void *error_entry_ret;
626 struct pt_regs regs;
627};
628
629asmlinkage __visible notrace
630struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s)
631{
632
633
634
635
636
637
638
639 struct bad_iret_stack *new_stack =
640 container_of(task_pt_regs(current),
641 struct bad_iret_stack, regs);
642
643
644 memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);
645
646
647 memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));
648
649 BUG_ON(!user_mode(&new_stack->regs));
650 return new_stack;
651}
652NOKPROBE_SYMBOL(fixup_bad_iret);
653#endif
654
655static bool is_sysenter_singlestep(struct pt_regs *regs)
656{
657
658
659
660
661
662
663
664
665#ifdef CONFIG_X86_32
666 return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <
667 (unsigned long)__end_SYSENTER_singlestep_region -
668 (unsigned long)__begin_SYSENTER_singlestep_region;
669#elif defined(CONFIG_IA32_EMULATION)
670 return (regs->ip - (unsigned long)entry_SYSENTER_compat) <
671 (unsigned long)__end_entry_SYSENTER_compat -
672 (unsigned long)entry_SYSENTER_compat;
673#else
674 return false;
675#endif
676}
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
703{
704 struct task_struct *tsk = current;
705 int user_icebp = 0;
706 unsigned long dr6;
707 int si_code;
708
709 ist_enter(regs);
710
711 get_debugreg(dr6, 6);
712
713
714
715
716
717
718
719
720
721
722
723 set_debugreg(0, 6);
724
725
726 dr6 &= ~DR6_RESERVED;
727
728
729
730
731
732
733 clear_tsk_thread_flag(tsk, TIF_BLOCKSTEP);
734
735 if (unlikely(!user_mode(regs) && (dr6 & DR_STEP) &&
736 is_sysenter_singlestep(regs))) {
737 dr6 &= ~DR_STEP;
738 if (!dr6)
739 goto exit;
740
741
742
743
744
745 }
746
747
748
749
750
751
752 if (!dr6 && user_mode(regs))
753 user_icebp = 1;
754
755
756 if ((dr6 & DR_STEP) && kmemcheck_trap(regs))
757 goto exit;
758
759
760 tsk->thread.debugreg6 = dr6;
761
762#ifdef CONFIG_KPROBES
763 if (kprobe_debug_handler(regs))
764 goto exit;
765#endif
766
767 if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code,
768 SIGTRAP) == NOTIFY_STOP)
769 goto exit;
770
771
772
773
774
775 debug_stack_usage_inc();
776
777
778 cond_local_irq_enable(regs);
779
780 if (v8086_mode(regs)) {
781 handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,
782 X86_TRAP_DB);
783 cond_local_irq_disable(regs);
784 debug_stack_usage_dec();
785 goto exit;
786 }
787
788 if (WARN_ON_ONCE((dr6 & DR_STEP) && !user_mode(regs))) {
789
790
791
792
793
794
795 tsk->thread.debugreg6 &= ~DR_STEP;
796 set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
797 regs->flags &= ~X86_EFLAGS_TF;
798 }
799 si_code = get_si_code(tsk->thread.debugreg6);
800 if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp)
801 send_sigtrap(tsk, regs, error_code, si_code);
802 cond_local_irq_disable(regs);
803 debug_stack_usage_dec();
804
805exit:
806#if defined(CONFIG_X86_32)
807
808
809
810
811 WARN(this_cpu_read(cpu_tss.SYSENTER_stack_canary) != STACK_END_MAGIC,
812 "Overran or corrupted SYSENTER stack\n");
813#endif
814 ist_exit(regs);
815}
816NOKPROBE_SYMBOL(do_debug);
817
818
819
820
821
822
823static void math_error(struct pt_regs *regs, int error_code, int trapnr)
824{
825 struct task_struct *task = current;
826 struct fpu *fpu = &task->thread.fpu;
827 siginfo_t info;
828 char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
829 "simd exception";
830
831 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP)
832 return;
833 cond_local_irq_enable(regs);
834
835 if (!user_mode(regs)) {
836 if (!fixup_exception(regs, trapnr)) {
837 task->thread.error_code = error_code;
838 task->thread.trap_nr = trapnr;
839 die(str, regs, error_code);
840 }
841 return;
842 }
843
844
845
846
847 fpu__save(fpu);
848
849 task->thread.trap_nr = trapnr;
850 task->thread.error_code = error_code;
851 info.si_signo = SIGFPE;
852 info.si_errno = 0;
853 info.si_addr = (void __user *)uprobe_get_trap_addr(regs);
854
855 info.si_code = fpu__exception_code(fpu, trapnr);
856
857
858 if (!info.si_code)
859 return;
860
861 force_sig_info(SIGFPE, &info, task);
862}
863
864dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
865{
866 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
867 math_error(regs, error_code, X86_TRAP_MF);
868}
869
870dotraplinkage void
871do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
872{
873 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
874 math_error(regs, error_code, X86_TRAP_XF);
875}
876
877dotraplinkage void
878do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
879{
880 cond_local_irq_enable(regs);
881}
882
883dotraplinkage void
884do_device_not_available(struct pt_regs *regs, long error_code)
885{
886 unsigned long cr0;
887
888 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
889
890#ifdef CONFIG_MATH_EMULATION
891 if (!boot_cpu_has(X86_FEATURE_FPU) && (read_cr0() & X86_CR0_EM)) {
892 struct math_emu_info info = { };
893
894 cond_local_irq_enable(regs);
895
896 info.regs = regs;
897 math_emulate(&info);
898 return;
899 }
900#endif
901
902
903 cr0 = read_cr0();
904 if (WARN(cr0 & X86_CR0_TS, "CR0.TS was set")) {
905
906 write_cr0(cr0 & ~X86_CR0_TS);
907 } else {
908
909
910
911
912
913 die("unexpected #NM exception", regs, error_code);
914 }
915}
916NOKPROBE_SYMBOL(do_device_not_available);
917
918#ifdef CONFIG_X86_32
919dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
920{
921 siginfo_t info;
922
923 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
924 local_irq_enable();
925
926 info.si_signo = SIGILL;
927 info.si_errno = 0;
928 info.si_code = ILL_BADSTK;
929 info.si_addr = NULL;
930 if (notify_die(DIE_TRAP, "iret exception", regs, error_code,
931 X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) {
932 do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
933 &info);
934 }
935}
936#endif
937
938
939void __init early_trap_init(void)
940{
941
942
943
944
945
946
947
948
949
950
951
952
953 set_intr_gate_notrace(X86_TRAP_DB, debug);
954
955 set_system_intr_gate(X86_TRAP_BP, &int3);
956#ifdef CONFIG_X86_32
957 set_intr_gate(X86_TRAP_PF, page_fault);
958#endif
959 load_idt(&idt_descr);
960}
961
962void __init early_trap_pf_init(void)
963{
964#ifdef CONFIG_X86_64
965 set_intr_gate(X86_TRAP_PF, page_fault);
966#endif
967}
968
969void __init trap_init(void)
970{
971 int i;
972
973#ifdef CONFIG_EISA
974 void __iomem *p = early_ioremap(0x0FFFD9, 4);
975
976 if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
977 EISA_bus = 1;
978 early_iounmap(p, 4);
979#endif
980
981 set_intr_gate(X86_TRAP_DE, divide_error);
982 set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK);
983
984 set_system_intr_gate(X86_TRAP_OF, &overflow);
985 set_intr_gate(X86_TRAP_BR, bounds);
986 set_intr_gate(X86_TRAP_UD, invalid_op);
987 set_intr_gate(X86_TRAP_NM, device_not_available);
988#ifdef CONFIG_X86_32
989 set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS);
990#else
991 set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK);
992#endif
993 set_intr_gate(X86_TRAP_OLD_MF, coprocessor_segment_overrun);
994 set_intr_gate(X86_TRAP_TS, invalid_TSS);
995 set_intr_gate(X86_TRAP_NP, segment_not_present);
996 set_intr_gate(X86_TRAP_SS, stack_segment);
997 set_intr_gate(X86_TRAP_GP, general_protection);
998 set_intr_gate(X86_TRAP_SPURIOUS, spurious_interrupt_bug);
999 set_intr_gate(X86_TRAP_MF, coprocessor_error);
1000 set_intr_gate(X86_TRAP_AC, alignment_check);
1001#ifdef CONFIG_X86_MCE
1002 set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK);
1003#endif
1004 set_intr_gate(X86_TRAP_XF, simd_coprocessor_error);
1005
1006
1007 for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
1008 set_bit(i, used_vectors);
1009
1010#ifdef CONFIG_IA32_EMULATION
1011 set_system_intr_gate(IA32_SYSCALL_VECTOR, entry_INT80_compat);
1012 set_bit(IA32_SYSCALL_VECTOR, used_vectors);
1013#endif
1014
1015#ifdef CONFIG_X86_32
1016 set_system_intr_gate(IA32_SYSCALL_VECTOR, entry_INT80_32);
1017 set_bit(IA32_SYSCALL_VECTOR, used_vectors);
1018#endif
1019
1020
1021
1022
1023
1024
1025 __set_fixmap(FIX_RO_IDT, __pa_symbol(idt_table), PAGE_KERNEL_RO);
1026 idt_descr.address = fix_to_virt(FIX_RO_IDT);
1027
1028
1029
1030
1031 cpu_init();
1032
1033
1034
1035
1036
1037
1038 set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
1039
1040 set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
1041
1042 x86_init.irqs.trap_init();
1043
1044#ifdef CONFIG_X86_64
1045 memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16);
1046 set_nmi_gate(X86_TRAP_DB, &debug);
1047 set_nmi_gate(X86_TRAP_BP, &int3);
1048#endif
1049}
1050