1
2
3
4
5#include <asm/fpu/internal.h>
6#include <asm/tlbflush.h>
7#include <asm/setup.h>
8
9#include <linux/sched.h>
10#include <linux/sched/task.h>
11#include <linux/init.h>
12
13
14
15
16static void fpu__init_cpu_generic(void)
17{
18 unsigned long cr0;
19 unsigned long cr4_mask = 0;
20
21 if (boot_cpu_has(X86_FEATURE_FXSR))
22 cr4_mask |= X86_CR4_OSFXSR;
23 if (boot_cpu_has(X86_FEATURE_XMM))
24 cr4_mask |= X86_CR4_OSXMMEXCPT;
25 if (cr4_mask)
26 cr4_set_bits(cr4_mask);
27
28 cr0 = read_cr0();
29 cr0 &= ~(X86_CR0_TS|X86_CR0_EM);
30 if (!boot_cpu_has(X86_FEATURE_FPU))
31 cr0 |= X86_CR0_EM;
32 write_cr0(cr0);
33
34
35#ifdef CONFIG_MATH_EMULATION
36 if (!boot_cpu_has(X86_FEATURE_FPU))
37 fpstate_init_soft(¤t->thread.fpu.state.soft);
38 else
39#endif
40 asm volatile ("fninit");
41}
42
43
44
45
46void fpu__init_cpu(void)
47{
48 fpu__init_cpu_generic();
49 fpu__init_cpu_xstate();
50}
51
52static bool fpu__probe_without_cpuid(void)
53{
54 unsigned long cr0;
55 u16 fsw, fcw;
56
57 fsw = fcw = 0xffff;
58
59 cr0 = read_cr0();
60 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
61 write_cr0(cr0);
62
63 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
64
65 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
66
67 return fsw == 0 && (fcw & 0x103f) == 0x003f;
68}
69
70static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
71{
72 if (!boot_cpu_has(X86_FEATURE_CPUID) &&
73 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
74 if (fpu__probe_without_cpuid())
75 setup_force_cpu_cap(X86_FEATURE_FPU);
76 else
77 setup_clear_cpu_cap(X86_FEATURE_FPU);
78 }
79
80#ifndef CONFIG_MATH_EMULATION
81 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
82 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
83 for (;;)
84 asm volatile("hlt");
85 }
86#endif
87}
88
89
90
91
92unsigned int mxcsr_feature_mask __ro_after_init = 0xffffffffu;
93EXPORT_SYMBOL_GPL(mxcsr_feature_mask);
94
95static void __init fpu__init_system_mxcsr(void)
96{
97 unsigned int mask = 0;
98
99 if (boot_cpu_has(X86_FEATURE_FXSR)) {
100
101 static struct fxregs_state fxregs __initdata;
102
103 asm volatile("fxsave %0" : "+m" (fxregs));
104
105 mask = fxregs.mxcsr_mask;
106
107
108
109
110
111
112 if (mask == 0)
113 mask = 0x0000ffbf;
114 }
115 mxcsr_feature_mask &= mask;
116}
117
118
119
120
121static void __init fpu__init_system_generic(void)
122{
123
124
125
126
127 fpstate_init(&init_fpstate);
128
129 fpu__init_system_mxcsr();
130}
131
132
133
134
135
136
137
138unsigned int fpu_kernel_xstate_size __ro_after_init;
139EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size);
140
141
142#define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
143
144
145
146
147
148
149
150#define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
151 BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
152 TYPE_ALIGN(TYPE)))
153
154
155
156
157static void __init fpu__init_task_struct_size(void)
158{
159 int task_size = sizeof(struct task_struct);
160
161
162
163
164
165 task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
166
167
168
169
170
171 task_size += fpu_kernel_xstate_size;
172
173
174
175
176
177
178
179
180 CHECK_MEMBER_AT_END_OF(struct fpu, state);
181 CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
182 CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
183
184 arch_task_struct_size = task_size;
185}
186
187
188
189
190
191
192
193static void __init fpu__init_system_xstate_size_legacy(void)
194{
195 static int on_boot_cpu __initdata = 1;
196
197 WARN_ON_FPU(!on_boot_cpu);
198 on_boot_cpu = 0;
199
200
201
202
203
204
205 if (!boot_cpu_has(X86_FEATURE_FPU)) {
206 fpu_kernel_xstate_size = sizeof(struct swregs_state);
207 } else {
208 if (boot_cpu_has(X86_FEATURE_FXSR))
209 fpu_kernel_xstate_size =
210 sizeof(struct fxregs_state);
211 else
212 fpu_kernel_xstate_size =
213 sizeof(struct fregs_state);
214 }
215
216 fpu_user_xstate_size = fpu_kernel_xstate_size;
217}
218
219
220static void __init fpu__init_system_ctx_switch(void)
221{
222 static bool on_boot_cpu __initdata = 1;
223
224 WARN_ON_FPU(!on_boot_cpu);
225 on_boot_cpu = 0;
226}
227
228
229
230
231
232void __init fpu__init_system(struct cpuinfo_x86 *c)
233{
234 fpu__init_system_early_generic(c);
235
236
237
238
239
240 fpu__init_cpu();
241
242 fpu__init_system_generic();
243 fpu__init_system_xstate_size_legacy();
244 fpu__init_system_xstate();
245 fpu__init_task_struct_size();
246
247 fpu__init_system_ctx_switch();
248}
249