1
2#include <linux/mm.h>
3#include <linux/sched.h>
4#include <linux/sched/debug.h>
5#include <linux/init_task.h>
6#include <linux/fs.h>
7
8#include <linux/uaccess.h>
9#include <asm/processor.h>
10#include <asm/desc.h>
11#include <asm/traps.h>
12
13#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
14
15#define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x)
16
17static void set_df_gdt_entry(unsigned int cpu);
18
19
20
21
22
23asmlinkage noinstr void __noreturn doublefault_shim(void)
24{
25 unsigned long cr2;
26 struct pt_regs regs;
27
28 BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE);
29
30 cr2 = native_read_cr2();
31
32
33 force_reload_TR();
34 set_df_gdt_entry(smp_processor_id());
35
36 trace_hardirqs_off();
37
38
39
40
41
42
43
44
45
46
47 regs.ss = TSS(ss);
48 regs.__ssh = 0;
49 regs.sp = TSS(sp);
50 regs.flags = TSS(flags);
51 regs.cs = TSS(cs);
52
53 regs.__csh = 0;
54 regs.ip = TSS(ip);
55 regs.orig_ax = 0;
56 regs.gs = TSS(gs);
57 regs.__gsh = 0;
58 regs.fs = TSS(fs);
59 regs.__fsh = 0;
60 regs.es = TSS(es);
61 regs.__esh = 0;
62 regs.ds = TSS(ds);
63 regs.__dsh = 0;
64 regs.ax = TSS(ax);
65 regs.bp = TSS(bp);
66 regs.di = TSS(di);
67 regs.si = TSS(si);
68 regs.dx = TSS(dx);
69 regs.cx = TSS(cx);
70 regs.bx = TSS(bx);
71
72 exc_double_fault(®s, 0, cr2);
73
74
75
76
77
78
79
80
81
82
83
84 panic("cannot return from double fault\n");
85}
86
87DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = {
88 .tss = {
89
90
91
92
93 .ldt = 0,
94 .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
95
96 .ip = (unsigned long) asm_exc_double_fault,
97 .flags = X86_EFLAGS_FIXED,
98 .es = __USER_DS,
99 .cs = __KERNEL_CS,
100 .ss = __KERNEL_DS,
101 .ds = __USER_DS,
102 .fs = __KERNEL_PERCPU,
103 .gs = 0,
104
105 .__cr3 = __pa_nodebug(swapper_pg_dir),
106 },
107};
108
109static void set_df_gdt_entry(unsigned int cpu)
110{
111
112 __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS,
113 &get_cpu_entry_area(cpu)->doublefault_stack.tss);
114
115}
116
117void doublefault_init_cpu_tss(void)
118{
119 unsigned int cpu = smp_processor_id();
120 struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
121
122
123
124
125
126 this_cpu_write(doublefault_stack.tss.sp,
127 (unsigned long)&cea->doublefault_stack.stack +
128 sizeof(doublefault_stack.stack));
129
130 set_df_gdt_entry(cpu);
131}
132