1
2#ifndef _ASM_X86_IDTENTRY_H
3#define _ASM_X86_IDTENTRY_H
4
5
6#include <asm/trapnr.h>
7
8#ifndef __ASSEMBLY__
9#include <linux/entry-common.h>
10#include <linux/hardirq.h>
11
12#include <asm/irq_stack.h>
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#define DECLARE_IDTENTRY(vector, func) \
30 asmlinkage void asm_##func(void); \
31 asmlinkage void xen_asm_##func(void); \
32 __visible void func(struct pt_regs *regs)
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#define DEFINE_IDTENTRY(func) \
48static __always_inline void __##func(struct pt_regs *regs); \
49 \
50__visible noinstr void func(struct pt_regs *regs) \
51{ \
52 irqentry_state_t state = irqentry_enter(regs); \
53 \
54 instrumentation_begin(); \
55 __##func (regs); \
56 instrumentation_end(); \
57 irqentry_exit(regs, state); \
58} \
59 \
60static __always_inline void __##func(struct pt_regs *regs)
61
62
63#define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY
64#define DEFINE_IDTENTRY_SW DEFINE_IDTENTRY
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
81 asmlinkage void asm_##func(void); \
82 asmlinkage void xen_asm_##func(void); \
83 __visible void func(struct pt_regs *regs, unsigned long error_code)
84
85
86
87
88
89
90
91
92#define DEFINE_IDTENTRY_ERRORCODE(func) \
93static __always_inline void __##func(struct pt_regs *regs, \
94 unsigned long error_code); \
95 \
96__visible noinstr void func(struct pt_regs *regs, \
97 unsigned long error_code) \
98{ \
99 irqentry_state_t state = irqentry_enter(regs); \
100 \
101 instrumentation_begin(); \
102 __##func (regs, error_code); \
103 instrumentation_end(); \
104 irqentry_exit(regs, state); \
105} \
106 \
107static __always_inline void __##func(struct pt_regs *regs, \
108 unsigned long error_code)
109
110
111
112
113
114
115
116
117
118#define DECLARE_IDTENTRY_RAW(vector, func) \
119 DECLARE_IDTENTRY(vector, func)
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135#define DEFINE_IDTENTRY_RAW(func) \
136__visible noinstr void func(struct pt_regs *regs)
137
138
139
140
141
142
143
144
145
146#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \
147 DECLARE_IDTENTRY_ERRORCODE(vector, func)
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163#define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \
164__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
165
166
167
168
169
170
171
172
173
174#define DECLARE_IDTENTRY_IRQ(vector, func) \
175 DECLARE_IDTENTRY_ERRORCODE(vector, func)
176
177
178
179
180
181
182
183
184
185
186
187
188
189#define DEFINE_IDTENTRY_IRQ(func) \
190static __always_inline void __##func(struct pt_regs *regs, u8 vector); \
191 \
192__visible noinstr void func(struct pt_regs *regs, \
193 unsigned long error_code) \
194{ \
195 irqentry_state_t state = irqentry_enter(regs); \
196 \
197 instrumentation_begin(); \
198 irq_enter_rcu(); \
199 kvm_set_cpu_l1tf_flush_l1d(); \
200 __##func (regs, (u8)error_code); \
201 irq_exit_rcu(); \
202 instrumentation_end(); \
203 irqentry_exit(regs, state); \
204} \
205 \
206static __always_inline void __##func(struct pt_regs *regs, u8 vector)
207
208
209
210
211
212
213
214
215
216
217
218
219
220#define DECLARE_IDTENTRY_SYSVEC(vector, func) \
221 DECLARE_IDTENTRY(vector, func)
222
223
224
225
226
227
228
229
230
231
232#define DEFINE_IDTENTRY_SYSVEC(func) \
233static void __##func(struct pt_regs *regs); \
234 \
235__visible noinstr void func(struct pt_regs *regs) \
236{ \
237 irqentry_state_t state = irqentry_enter(regs); \
238 \
239 instrumentation_begin(); \
240 irq_enter_rcu(); \
241 kvm_set_cpu_l1tf_flush_l1d(); \
242 run_sysvec_on_irqstack_cond(__##func, regs); \
243 irq_exit_rcu(); \
244 instrumentation_end(); \
245 irqentry_exit(regs, state); \
246} \
247 \
248static noinline void __##func(struct pt_regs *regs)
249
250
251
252
253
254
255
256
257
258
259
260
261#define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \
262static __always_inline void __##func(struct pt_regs *regs); \
263 \
264__visible noinstr void func(struct pt_regs *regs) \
265{ \
266 irqentry_state_t state = irqentry_enter(regs); \
267 \
268 instrumentation_begin(); \
269 __irq_enter_raw(); \
270 kvm_set_cpu_l1tf_flush_l1d(); \
271 __##func (regs); \
272 __irq_exit_raw(); \
273 instrumentation_end(); \
274 irqentry_exit(regs, state); \
275} \
276 \
277static __always_inline void __##func(struct pt_regs *regs)
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292#define DECLARE_IDTENTRY_XENCB(vector, func) \
293 DECLARE_IDTENTRY(vector, func)
294
295#ifdef CONFIG_X86_64
296
297
298
299
300
301
302
303
304#define DECLARE_IDTENTRY_IST(vector, func) \
305 DECLARE_IDTENTRY_RAW(vector, func); \
306 __visible void noist_##func(struct pt_regs *regs)
307
308
309
310
311
312
313
314
315
316#define DECLARE_IDTENTRY_VC(vector, func) \
317 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \
318 __visible noinstr void ist_##func(struct pt_regs *regs, unsigned long error_code); \
319 __visible noinstr void safe_stack_##func(struct pt_regs *regs, unsigned long error_code)
320
321
322
323
324
325
326
327#define DEFINE_IDTENTRY_IST(func) \
328 DEFINE_IDTENTRY_RAW(func)
329
330
331
332
333
334
335
336
337
338#define DEFINE_IDTENTRY_NOIST(func) \
339 DEFINE_IDTENTRY_RAW(noist_##func)
340
341
342
343
344
345
346
347
348#define DECLARE_IDTENTRY_DF(vector, func) \
349 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)
350
351
352
353
354
355
356
357#define DEFINE_IDTENTRY_DF(func) \
358 DEFINE_IDTENTRY_RAW_ERRORCODE(func)
359
360
361
362
363
364
365
366
367#define DEFINE_IDTENTRY_VC_SAFE_STACK(func) \
368 DEFINE_IDTENTRY_RAW_ERRORCODE(safe_stack_##func)
369
370
371
372
373
374
375
376
377#define DEFINE_IDTENTRY_VC_IST(func) \
378 DEFINE_IDTENTRY_RAW_ERRORCODE(ist_##func)
379
380
381
382
383
384
385
386#define DEFINE_IDTENTRY_VC(func) \
387 DEFINE_IDTENTRY_RAW_ERRORCODE(func)
388
389#else
390
391
392
393
394
395
396
397
398
399
400#define DECLARE_IDTENTRY_DF(vector, func) \
401 asmlinkage void asm_##func(void); \
402 __visible void func(struct pt_regs *regs, \
403 unsigned long error_code, \
404 unsigned long address)
405
406
407
408
409
410
411
412
413#define DEFINE_IDTENTRY_DF(func) \
414__visible noinstr void func(struct pt_regs *regs, \
415 unsigned long error_code, \
416 unsigned long address)
417
418#endif
419
420
421#define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_RAW
422#define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_RAW
423
424#ifdef CONFIG_X86_64
425#define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST
426#define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST
427#define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST
428
429#define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST
430#define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST
431#define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST
432#endif
433
434#else
435
436
437
438
439#define DECLARE_IDTENTRY(vector, func) \
440 idtentry vector asm_##func func has_error_code=0
441
442#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
443 idtentry vector asm_##func func has_error_code=1
444
445
446#define DECLARE_IDTENTRY_SW(vector, func)
447
448#define DECLARE_IDTENTRY_RAW(vector, func) \
449 DECLARE_IDTENTRY(vector, func)
450
451#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \
452 DECLARE_IDTENTRY_ERRORCODE(vector, func)
453
454
455#define DECLARE_IDTENTRY_IRQ(vector, func) \
456 idtentry_irq vector func
457
458
459#define DECLARE_IDTENTRY_SYSVEC(vector, func) \
460 idtentry_sysvec vector func
461
462#ifdef CONFIG_X86_64
463# define DECLARE_IDTENTRY_MCE(vector, func) \
464 idtentry_mce_db vector asm_##func func
465
466# define DECLARE_IDTENTRY_DEBUG(vector, func) \
467 idtentry_mce_db vector asm_##func func
468
469# define DECLARE_IDTENTRY_DF(vector, func) \
470 idtentry_df vector asm_##func func
471
472# define DECLARE_IDTENTRY_XENCB(vector, func) \
473 DECLARE_IDTENTRY(vector, func)
474
475# define DECLARE_IDTENTRY_VC(vector, func) \
476 idtentry_vc vector asm_##func func
477
478#else
479# define DECLARE_IDTENTRY_MCE(vector, func) \
480 DECLARE_IDTENTRY(vector, func)
481
482
483# define DECLARE_IDTENTRY_DF(vector, func)
484
485
486# define DECLARE_IDTENTRY_XENCB(vector, func)
487
488#endif
489
490
491#define DECLARE_IDTENTRY_NMI(vector, func)
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507 .align 8
508SYM_CODE_START(irq_entries_start)
509 vector=FIRST_EXTERNAL_VECTOR
510 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
511 UNWIND_HINT_IRET_REGS
5120 :
513 .byte 0x6a, vector
514 jmp asm_common_interrupt
515 nop
516
517 . = 0b + 8
518 vector = vector+1
519 .endr
520SYM_CODE_END(irq_entries_start)
521
522#ifdef CONFIG_X86_LOCAL_APIC
523 .align 8
524SYM_CODE_START(spurious_entries_start)
525 vector=FIRST_SYSTEM_VECTOR
526 .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR)
527 UNWIND_HINT_IRET_REGS
5280 :
529 .byte 0x6a, vector
530 jmp asm_spurious_interrupt
531 nop
532
533 . = 0b + 8
534 vector = vector+1
535 .endr
536SYM_CODE_END(spurious_entries_start)
537#endif
538
539#endif
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555#define X86_TRAP_OTHER 0xFFFF
556
557
558DECLARE_IDTENTRY(X86_TRAP_DE, exc_divide_error);
559DECLARE_IDTENTRY(X86_TRAP_OF, exc_overflow);
560DECLARE_IDTENTRY(X86_TRAP_BR, exc_bounds);
561DECLARE_IDTENTRY(X86_TRAP_NM, exc_device_not_available);
562DECLARE_IDTENTRY(X86_TRAP_OLD_MF, exc_coproc_segment_overrun);
563DECLARE_IDTENTRY(X86_TRAP_SPURIOUS, exc_spurious_interrupt_bug);
564DECLARE_IDTENTRY(X86_TRAP_MF, exc_coprocessor_error);
565DECLARE_IDTENTRY(X86_TRAP_XF, exc_simd_coprocessor_error);
566
567
568DECLARE_IDTENTRY_SW(X86_TRAP_IRET, iret_error);
569
570
571DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS, exc_invalid_tss);
572DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP, exc_segment_not_present);
573DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment);
574DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP, exc_general_protection);
575DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC, exc_alignment_check);
576
577
578DECLARE_IDTENTRY_RAW(X86_TRAP_UD, exc_invalid_op);
579DECLARE_IDTENTRY_RAW(X86_TRAP_BP, exc_int3);
580DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF, exc_page_fault);
581
582#ifdef CONFIG_X86_MCE
583#ifdef CONFIG_X86_64
584DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check);
585#else
586DECLARE_IDTENTRY_RAW(X86_TRAP_MC, exc_machine_check);
587#endif
588#endif
589
590
591DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi);
592#ifdef CONFIG_XEN_PV
593DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi);
594#endif
595
596
597#ifdef CONFIG_X86_64
598DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB, exc_debug);
599#else
600DECLARE_IDTENTRY_RAW(X86_TRAP_DB, exc_debug);
601#endif
602#ifdef CONFIG_XEN_PV
603DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug);
604#endif
605
606
607DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault);
608
609
610#ifdef CONFIG_AMD_MEM_ENCRYPT
611DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication);
612#endif
613
614#ifdef CONFIG_XEN_PV
615DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback);
616DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER, exc_xen_unknown_trap);
617#endif
618
619
620DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, common_interrupt);
621#ifdef CONFIG_X86_LOCAL_APIC
622DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, spurious_interrupt);
623#endif
624
625
626#ifdef CONFIG_X86_LOCAL_APIC
627DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR, sysvec_error_interrupt);
628DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR, sysvec_spurious_apic_interrupt);
629DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt);
630DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi);
631#endif
632
633#ifdef CONFIG_SMP
634DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi);
635DECLARE_IDTENTRY_SYSVEC(IRQ_MOVE_CLEANUP_VECTOR, sysvec_irq_move_cleanup);
636DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot);
637DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single);
638DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function);
639#endif
640
641#ifdef CONFIG_X86_LOCAL_APIC
642# ifdef CONFIG_X86_MCE_THRESHOLD
643DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold);
644# endif
645
646# ifdef CONFIG_X86_MCE_AMD
647DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR, sysvec_deferred_error);
648# endif
649
650# ifdef CONFIG_X86_THERMAL_VECTOR
651DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR, sysvec_thermal);
652# endif
653
654# ifdef CONFIG_IRQ_WORK
655DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work);
656# endif
657#endif
658
659#ifdef CONFIG_HAVE_KVM
660DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi);
661DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi);
662DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi);
663#endif
664
665#if IS_ENABLED(CONFIG_HYPERV)
666DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback);
667DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment);
668DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0);
669#endif
670
671#if IS_ENABLED(CONFIG_ACRN_GUEST)
672DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_acrn_hv_callback);
673#endif
674
675#ifdef CONFIG_XEN_PVHVM
676DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback);
677#endif
678
679#ifdef CONFIG_KVM_GUEST
680DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt);
681#endif
682
683#undef X86_TRAP_OTHER
684
685#endif
686