1
2
3
4
5
6#include <linux/compat.h>
7#include <linux/cpu.h>
8#include <linux/pagemap.h>
9
10#include <asm/fpu/internal.h>
11#include <asm/fpu/signal.h>
12#include <asm/fpu/regset.h>
13#include <asm/fpu/xstate.h>
14
15#include <asm/sigframe.h>
16#include <asm/trace/fpu.h>
17
18static struct _fpx_sw_bytes fx_sw_reserved __ro_after_init;
19static struct _fpx_sw_bytes fx_sw_reserved_ia32 __ro_after_init;
20
21
22
23
24
25static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
26 struct _fpx_sw_bytes *fx_sw)
27{
28 int min_xstate_size = sizeof(struct fxregs_state) +
29 sizeof(struct xstate_header);
30 void __user *fpstate = fxbuf;
31 unsigned int magic2;
32
33 if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
34 return -EFAULT;
35
36
37 if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
38 fx_sw->xstate_size < min_xstate_size ||
39 fx_sw->xstate_size > fpu_user_xstate_size ||
40 fx_sw->xstate_size > fx_sw->extended_size)
41 goto setfx;
42
43
44
45
46
47
48
49 if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
50 return -EFAULT;
51
52 if (likely(magic2 == FP_XSTATE_MAGIC2))
53 return 0;
54setfx:
55 trace_x86_fpu_xstate_check_failed(¤t->thread.fpu);
56
57
58 fx_sw->magic1 = 0;
59 fx_sw->xstate_size = sizeof(struct fxregs_state);
60 fx_sw->xfeatures = XFEATURE_MASK_FPSSE;
61 return 0;
62}
63
64
65
66
67static inline int save_fsave_header(struct task_struct *tsk, void __user *buf)
68{
69 if (use_fxsr()) {
70 struct xregs_state *xsave = &tsk->thread.fpu.state.xsave;
71 struct user_i387_ia32_struct env;
72 struct _fpstate_32 __user *fp = buf;
73
74 fpregs_lock();
75 if (!test_thread_flag(TIF_NEED_FPU_LOAD))
76 fxsave(&tsk->thread.fpu.state.fxsave);
77 fpregs_unlock();
78
79 convert_from_fxsr(&env, tsk);
80
81 if (__copy_to_user(buf, &env, sizeof(env)) ||
82 __put_user(xsave->i387.swd, &fp->status) ||
83 __put_user(X86_FXSR_MAGIC, &fp->magic))
84 return -1;
85 } else {
86 struct fregs_state __user *fp = buf;
87 u32 swd;
88 if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
89 return -1;
90 }
91
92 return 0;
93}
94
95static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
96{
97 struct xregs_state __user *x = buf;
98 struct _fpx_sw_bytes *sw_bytes;
99 u32 xfeatures;
100 int err;
101
102
103 sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved;
104 err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
105
106 if (!use_xsave())
107 return err;
108
109 err |= __put_user(FP_XSTATE_MAGIC2,
110 (__u32 __user *)(buf + fpu_user_xstate_size));
111
112
113
114
115
116 err |= __get_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
117
118
119
120
121
122
123
124
125
126
127
128
129 xfeatures |= XFEATURE_MASK_FPSSE;
130
131 err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
132
133 return err;
134}
135
136static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)
137{
138 int err;
139
140 if (use_xsave())
141 err = xsave_to_user_sigframe(buf);
142 else if (use_fxsr())
143 err = fxsave_to_user_sigframe((struct fxregs_state __user *) buf);
144 else
145 err = fnsave_to_user_sigframe((struct fregs_state __user *) buf);
146
147 if (unlikely(err) && __clear_user(buf, fpu_user_xstate_size))
148 err = -EFAULT;
149 return err;
150}
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
174{
175 struct task_struct *tsk = current;
176 int ia32_fxstate = (buf != buf_fx);
177 int ret;
178
179 ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
180 IS_ENABLED(CONFIG_IA32_EMULATION));
181
182 if (!static_cpu_has(X86_FEATURE_FPU)) {
183 struct user_i387_ia32_struct fp;
184 fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
185 .left = sizeof(fp)});
186 return copy_to_user(buf, &fp, sizeof(fp)) ? -EFAULT : 0;
187 }
188
189 if (!access_ok(buf, size))
190 return -EACCES;
191retry:
192
193
194
195
196
197
198 fpregs_lock();
199 if (test_thread_flag(TIF_NEED_FPU_LOAD))
200 fpregs_restore_userregs();
201
202 pagefault_disable();
203 ret = copy_fpregs_to_sigframe(buf_fx);
204 pagefault_enable();
205 fpregs_unlock();
206
207 if (ret) {
208 if (!fault_in_pages_writeable(buf_fx, fpu_user_xstate_size))
209 goto retry;
210 return -EFAULT;
211 }
212
213
214 if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf))
215 return -1;
216
217 if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate))
218 return -1;
219
220 return 0;
221}
222
223static int __restore_fpregs_from_user(void __user *buf, u64 xrestore,
224 bool fx_only)
225{
226 if (use_xsave()) {
227 u64 init_bv = xfeatures_mask_uabi() & ~xrestore;
228 int ret;
229
230 if (likely(!fx_only))
231 ret = xrstor_from_user_sigframe(buf, xrestore);
232 else
233 ret = fxrstor_from_user_sigframe(buf);
234
235 if (!ret && unlikely(init_bv))
236 os_xrstor(&init_fpstate.xsave, init_bv);
237 return ret;
238 } else if (use_fxsr()) {
239 return fxrstor_from_user_sigframe(buf);
240 } else {
241 return frstor_from_user_sigframe(buf);
242 }
243}
244
245
246
247
248
249static int restore_fpregs_from_user(void __user *buf, u64 xrestore,
250 bool fx_only, unsigned int size)
251{
252 struct fpu *fpu = ¤t->thread.fpu;
253 int ret;
254
255retry:
256 fpregs_lock();
257 pagefault_disable();
258 ret = __restore_fpregs_from_user(buf, xrestore, fx_only);
259 pagefault_enable();
260
261 if (unlikely(ret)) {
262
263
264
265
266
267
268
269
270
271
272
273 if (test_thread_flag(TIF_NEED_FPU_LOAD))
274 __cpu_invalidate_fpregs_state();
275 fpregs_unlock();
276
277
278 if (ret != -EFAULT)
279 return -EINVAL;
280
281 ret = fault_in_pages_readable(buf, size);
282 if (!ret)
283 goto retry;
284 return ret;
285 }
286
287
288
289
290
291
292
293
294
295
296 if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
297 os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor());
298
299 fpregs_mark_activate();
300 fpregs_unlock();
301 return 0;
302}
303
304static int __fpu_restore_sig(void __user *buf, void __user *buf_fx,
305 bool ia32_fxstate)
306{
307 int state_size = fpu_kernel_xstate_size;
308 struct task_struct *tsk = current;
309 struct fpu *fpu = &tsk->thread.fpu;
310 struct user_i387_ia32_struct env;
311 u64 user_xfeatures = 0;
312 bool fx_only = false;
313 int ret;
314
315 if (use_xsave()) {
316 struct _fpx_sw_bytes fx_sw_user;
317
318 ret = check_xstate_in_sigframe(buf_fx, &fx_sw_user);
319 if (unlikely(ret))
320 return ret;
321
322 fx_only = !fx_sw_user.magic1;
323 state_size = fx_sw_user.xstate_size;
324 user_xfeatures = fx_sw_user.xfeatures;
325 } else {
326 user_xfeatures = XFEATURE_MASK_FPSSE;
327 }
328
329 if (likely(!ia32_fxstate)) {
330
331
332
333
334
335
336 return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only,
337 state_size);
338 }
339
340
341
342
343
344
345 ret = __copy_from_user(&env, buf, sizeof(env));
346 if (ret)
347 return ret;
348
349
350
351
352
353
354
355 fpregs_lock();
356 if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
357
358
359
360
361
362
363
364
365 if (xfeatures_mask_supervisor())
366 os_xsave(&fpu->state.xsave);
367 set_thread_flag(TIF_NEED_FPU_LOAD);
368 }
369 __fpu_invalidate_fpregs_state(fpu);
370 __cpu_invalidate_fpregs_state();
371 fpregs_unlock();
372
373 if (use_xsave() && !fx_only) {
374 ret = copy_sigframe_from_user_to_xstate(&fpu->state.xsave, buf_fx);
375 if (ret)
376 return ret;
377 } else {
378 if (__copy_from_user(&fpu->state.fxsave, buf_fx,
379 sizeof(fpu->state.fxsave)))
380 return -EFAULT;
381
382 if (IS_ENABLED(CONFIG_X86_64)) {
383
384 if (fpu->state.fxsave.mxcsr & ~mxcsr_feature_mask)
385 return -EINVAL;
386 } else {
387
388 fpu->state.fxsave.mxcsr &= mxcsr_feature_mask;
389 }
390
391
392 if (use_xsave())
393 fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
394 }
395
396
397 convert_to_fxsr(&fpu->state.fxsave, &env);
398
399 fpregs_lock();
400 if (use_xsave()) {
401
402
403
404
405
406
407
408
409
410 u64 mask = user_xfeatures | xfeatures_mask_supervisor();
411
412 fpu->state.xsave.header.xfeatures &= mask;
413 ret = os_xrstor_safe(&fpu->state.xsave, xfeatures_mask_all);
414 } else {
415 ret = fxrstor_safe(&fpu->state.fxsave);
416 }
417
418 if (likely(!ret))
419 fpregs_mark_activate();
420
421 fpregs_unlock();
422 return ret;
423}
424static inline int xstate_sigframe_size(void)
425{
426 return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE :
427 fpu_user_xstate_size;
428}
429
430
431
432
433int fpu__restore_sig(void __user *buf, int ia32_frame)
434{
435 unsigned int size = xstate_sigframe_size();
436 struct fpu *fpu = ¤t->thread.fpu;
437 void __user *buf_fx = buf;
438 bool ia32_fxstate = false;
439 int ret;
440
441 if (unlikely(!buf)) {
442 fpu__clear_user_states(fpu);
443 return 0;
444 }
445
446 ia32_frame &= (IS_ENABLED(CONFIG_X86_32) ||
447 IS_ENABLED(CONFIG_IA32_EMULATION));
448
449
450
451
452
453 if (ia32_frame && use_fxsr()) {
454 buf_fx = buf + sizeof(struct fregs_state);
455 size += sizeof(struct fregs_state);
456 ia32_fxstate = true;
457 }
458
459 if (!access_ok(buf, size)) {
460 ret = -EACCES;
461 goto out;
462 }
463
464 if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) {
465 ret = fpregs_soft_set(current, NULL, 0,
466 sizeof(struct user_i387_ia32_struct),
467 NULL, buf);
468 } else {
469 ret = __fpu_restore_sig(buf, buf_fx, ia32_fxstate);
470 }
471
472out:
473 if (unlikely(ret))
474 fpu__clear_user_states(fpu);
475 return ret;
476}
477
478unsigned long
479fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
480 unsigned long *buf_fx, unsigned long *size)
481{
482 unsigned long frame_size = xstate_sigframe_size();
483
484 *buf_fx = sp = round_down(sp - frame_size, 64);
485 if (ia32_frame && use_fxsr()) {
486 frame_size += sizeof(struct fregs_state);
487 sp -= sizeof(struct fregs_state);
488 }
489
490 *size = frame_size;
491
492 return sp;
493}
494
495unsigned long fpu__get_fpstate_size(void)
496{
497 unsigned long ret = xstate_sigframe_size();
498
499
500
501
502
503
504
505
506 if ((IS_ENABLED(CONFIG_IA32_EMULATION) ||
507 IS_ENABLED(CONFIG_X86_32)) && use_fxsr())
508 ret += sizeof(struct fregs_state);
509
510 return ret;
511}
512
513
514
515
516
517
518
519
520void fpu__init_prepare_fx_sw_frame(void)
521{
522 int size = fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE;
523
524 fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
525 fx_sw_reserved.extended_size = size;
526 fx_sw_reserved.xfeatures = xfeatures_mask_uabi();
527 fx_sw_reserved.xstate_size = fpu_user_xstate_size;
528
529 if (IS_ENABLED(CONFIG_IA32_EMULATION) ||
530 IS_ENABLED(CONFIG_X86_32)) {
531 int fsave_header_size = sizeof(struct fregs_state);
532
533 fx_sw_reserved_ia32 = fx_sw_reserved;
534 fx_sw_reserved_ia32.extended_size = size + fsave_header_size;
535 }
536}
537
538