1
2
3
4
5
6
7#include <linux/compat.h>
8#include <linux/cpu.h>
9#include <linux/mman.h>
10#include <linux/pkeys.h>
11#include <linux/seq_file.h>
12#include <linux/proc_fs.h>
13
14#include <asm/fpu/api.h>
15#include <asm/fpu/internal.h>
16#include <asm/fpu/signal.h>
17#include <asm/fpu/regset.h>
18#include <asm/fpu/xstate.h>
19
20#include <asm/tlbflush.h>
21#include <asm/cpufeature.h>
22
23
24
25
26
27
28static const char *xfeature_names[] =
29{
30 "x87 floating point registers" ,
31 "SSE registers" ,
32 "AVX registers" ,
33 "MPX bounds registers" ,
34 "MPX CSR" ,
35 "AVX-512 opmask" ,
36 "AVX-512 Hi256" ,
37 "AVX-512 ZMM_Hi256" ,
38 "Processor Trace (unused)" ,
39 "Protection Keys User registers",
40 "unknown xstate feature" ,
41};
42
43static short xsave_cpuid_features[] __initdata = {
44 X86_FEATURE_FPU,
45 X86_FEATURE_XMM,
46 X86_FEATURE_AVX,
47 X86_FEATURE_MPX,
48 X86_FEATURE_MPX,
49 X86_FEATURE_AVX512F,
50 X86_FEATURE_AVX512F,
51 X86_FEATURE_AVX512F,
52 X86_FEATURE_INTEL_PT,
53 X86_FEATURE_PKU,
54};
55
56
57
58
59u64 xfeatures_mask __read_mostly;
60
61static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
62static unsigned int xstate_sizes[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
63static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8];
64
65
66
67
68
69
70unsigned int fpu_user_xstate_size;
71
72
73
74
75
76
77int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
78{
79 u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask;
80
81 if (unlikely(feature_name)) {
82 long xfeature_idx, max_idx;
83 u64 xfeatures_print;
84
85
86
87
88
89
90
91 if (xfeatures_missing)
92 xfeatures_print = xfeatures_missing;
93 else
94 xfeatures_print = xfeatures_needed;
95
96 xfeature_idx = fls64(xfeatures_print)-1;
97 max_idx = ARRAY_SIZE(xfeature_names)-1;
98 xfeature_idx = min(xfeature_idx, max_idx);
99
100 *feature_name = xfeature_names[xfeature_idx];
101 }
102
103 if (xfeatures_missing)
104 return 0;
105
106 return 1;
107}
108EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
109
110static int xfeature_is_supervisor(int xfeature_nr)
111{
112
113
114
115
116
117
118
119
120 u32 eax, ebx, ecx, edx;
121
122 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
123 return !!(ecx & 1);
124}
125
126static int xfeature_is_user(int xfeature_nr)
127{
128 return !xfeature_is_supervisor(xfeature_nr);
129}
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146void fpstate_sanitize_xstate(struct fpu *fpu)
147{
148 struct fxregs_state *fx = &fpu->state.fxsave;
149 int feature_bit;
150 u64 xfeatures;
151
152 if (!use_xsaveopt())
153 return;
154
155 xfeatures = fpu->state.xsave.header.xfeatures;
156
157
158
159
160
161 if ((xfeatures & xfeatures_mask) == xfeatures_mask)
162 return;
163
164
165
166
167 if (!(xfeatures & XFEATURE_MASK_FP)) {
168 fx->cwd = 0x37f;
169 fx->swd = 0;
170 fx->twd = 0;
171 fx->fop = 0;
172 fx->rip = 0;
173 fx->rdp = 0;
174 memset(&fx->st_space[0], 0, 128);
175 }
176
177
178
179
180 if (!(xfeatures & XFEATURE_MASK_SSE))
181 memset(&fx->xmm_space[0], 0, 256);
182
183
184
185
186
187 feature_bit = 0x2;
188 xfeatures = (xfeatures_mask & ~xfeatures) >> 2;
189
190
191
192
193
194
195 while (xfeatures) {
196 if (xfeatures & 0x1) {
197 int offset = xstate_comp_offsets[feature_bit];
198 int size = xstate_sizes[feature_bit];
199
200 memcpy((void *)fx + offset,
201 (void *)&init_fpstate.xsave + offset,
202 size);
203 }
204
205 xfeatures >>= 1;
206 feature_bit++;
207 }
208}
209
210
211
212
213
214void fpu__init_cpu_xstate(void)
215{
216 if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask)
217 return;
218
219
220
221
222
223 WARN_ONCE((xfeatures_mask & XFEATURE_MASK_SUPERVISOR),
224 "x86/fpu: XSAVES supervisor states are not yet implemented.\n");
225
226 xfeatures_mask &= ~XFEATURE_MASK_SUPERVISOR;
227
228 cr4_set_bits(X86_CR4_OSXSAVE);
229 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
230}
231
232
233
234
235
236
237static int xfeature_enabled(enum xfeature xfeature)
238{
239 return !!(xfeatures_mask & (1UL << xfeature));
240}
241
242
243
244
245
246static void __init setup_xstate_features(void)
247{
248 u32 eax, ebx, ecx, edx, i;
249
250 unsigned int last_good_offset = offsetof(struct xregs_state,
251 extended_state_area);
252
253
254
255
256
257 xstate_offsets[0] = 0;
258 xstate_sizes[0] = offsetof(struct fxregs_state, xmm_space);
259 xstate_offsets[1] = xstate_sizes[0];
260 xstate_sizes[1] = FIELD_SIZEOF(struct fxregs_state, xmm_space);
261
262 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
263 if (!xfeature_enabled(i))
264 continue;
265
266 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
267
268
269
270
271
272 if (xfeature_is_user(i))
273 xstate_offsets[i] = ebx;
274
275 xstate_sizes[i] = eax;
276
277
278
279
280
281 WARN_ONCE(last_good_offset > xstate_offsets[i],
282 "x86/fpu: misordered xstate at %d\n", last_good_offset);
283 last_good_offset = xstate_offsets[i];
284 }
285}
286
287static void __init print_xstate_feature(u64 xstate_mask)
288{
289 const char *feature_name;
290
291 if (cpu_has_xfeatures(xstate_mask, &feature_name))
292 pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
293}
294
295
296
297
298static void __init print_xstate_features(void)
299{
300 print_xstate_feature(XFEATURE_MASK_FP);
301 print_xstate_feature(XFEATURE_MASK_SSE);
302 print_xstate_feature(XFEATURE_MASK_YMM);
303 print_xstate_feature(XFEATURE_MASK_BNDREGS);
304 print_xstate_feature(XFEATURE_MASK_BNDCSR);
305 print_xstate_feature(XFEATURE_MASK_OPMASK);
306 print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
307 print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
308 print_xstate_feature(XFEATURE_MASK_PKRU);
309}
310
311
312
313
314
315#define CHECK_XFEATURE(nr) do { \
316 WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \
317 WARN_ON(nr >= XFEATURE_MAX); \
318} while (0)
319
320
321
322
323
324static int xfeature_is_aligned(int xfeature_nr)
325{
326 u32 eax, ebx, ecx, edx;
327
328 CHECK_XFEATURE(xfeature_nr);
329 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
330
331
332
333
334
335 return !!(ecx & 2);
336}
337
338
339
340
341
342
343static void __init setup_xstate_comp(void)
344{
345 unsigned int xstate_comp_sizes[sizeof(xfeatures_mask)*8];
346 int i;
347
348
349
350
351
352
353 xstate_comp_offsets[0] = 0;
354 xstate_comp_offsets[1] = offsetof(struct fxregs_state, xmm_space);
355
356 if (!boot_cpu_has(X86_FEATURE_XSAVES)) {
357 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
358 if (xfeature_enabled(i)) {
359 xstate_comp_offsets[i] = xstate_offsets[i];
360 xstate_comp_sizes[i] = xstate_sizes[i];
361 }
362 }
363 return;
364 }
365
366 xstate_comp_offsets[FIRST_EXTENDED_XFEATURE] =
367 FXSAVE_SIZE + XSAVE_HDR_SIZE;
368
369 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
370 if (xfeature_enabled(i))
371 xstate_comp_sizes[i] = xstate_sizes[i];
372 else
373 xstate_comp_sizes[i] = 0;
374
375 if (i > FIRST_EXTENDED_XFEATURE) {
376 xstate_comp_offsets[i] = xstate_comp_offsets[i-1]
377 + xstate_comp_sizes[i-1];
378
379 if (xfeature_is_aligned(i))
380 xstate_comp_offsets[i] =
381 ALIGN(xstate_comp_offsets[i], 64);
382 }
383 }
384}
385
386
387
388
389static void __init print_xstate_offset_size(void)
390{
391 int i;
392
393 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
394 if (!xfeature_enabled(i))
395 continue;
396 pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
397 i, xstate_comp_offsets[i], i, xstate_sizes[i]);
398 }
399}
400
401
402
403
404static void __init setup_init_fpu_buf(void)
405{
406 static int on_boot_cpu __initdata = 1;
407
408 WARN_ON_FPU(!on_boot_cpu);
409 on_boot_cpu = 0;
410
411 if (!boot_cpu_has(X86_FEATURE_XSAVE))
412 return;
413
414 setup_xstate_features();
415 print_xstate_features();
416
417 if (boot_cpu_has(X86_FEATURE_XSAVES))
418 init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
419
420
421
422
423 copy_kernel_to_xregs_booting(&init_fpstate.xsave);
424
425
426
427
428
429 copy_xregs_to_kernel_booting(&init_fpstate.xsave);
430}
431
432static int xfeature_uncompacted_offset(int xfeature_nr)
433{
434 u32 eax, ebx, ecx, edx;
435
436
437
438
439
440
441 if (XFEATURE_MASK_SUPERVISOR & BIT_ULL(xfeature_nr)) {
442 WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
443 return -1;
444 }
445
446 CHECK_XFEATURE(xfeature_nr);
447 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
448 return ebx;
449}
450
451static int xfeature_size(int xfeature_nr)
452{
453 u32 eax, ebx, ecx, edx;
454
455 CHECK_XFEATURE(xfeature_nr);
456 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
457 return eax;
458}
459
460
461
462
463
464
465
466
467
468
469int using_compacted_format(void)
470{
471 return boot_cpu_has(X86_FEATURE_XSAVES);
472}
473
474
475int validate_xstate_header(const struct xstate_header *hdr)
476{
477
478 if (hdr->xfeatures & (~xfeatures_mask | XFEATURE_MASK_SUPERVISOR))
479 return -EINVAL;
480
481
482 if (hdr->xcomp_bv)
483 return -EINVAL;
484
485
486
487
488
489 BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
490
491
492 if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
493 return -EINVAL;
494
495 return 0;
496}
497
498static void __xstate_dump_leaves(void)
499{
500 int i;
501 u32 eax, ebx, ecx, edx;
502 static int should_dump = 1;
503
504 if (!should_dump)
505 return;
506 should_dump = 0;
507
508
509
510
511 for (i = 0; i < XFEATURE_MAX + 10; i++) {
512 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
513 pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
514 XSTATE_CPUID, i, eax, ebx, ecx, edx);
515 }
516}
517
518#define XSTATE_WARN_ON(x) do { \
519 if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \
520 __xstate_dump_leaves(); \
521 } \
522} while (0)
523
524#define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \
525 if ((nr == nr_macro) && \
526 WARN_ONCE(sz != sizeof(__struct), \
527 "%s: struct is %zu bytes, cpu state %d bytes\n", \
528 __stringify(nr_macro), sizeof(__struct), sz)) { \
529 __xstate_dump_leaves(); \
530 } \
531} while (0)
532
533
534
535
536
537
538static void check_xstate_against_struct(int nr)
539{
540
541
542
543 int sz = xfeature_size(nr);
544
545
546
547
548 XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct);
549 XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state);
550 XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state);
551 XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state);
552 XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
553 XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state);
554 XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state);
555
556
557
558
559
560
561 if ((nr < XFEATURE_YMM) ||
562 (nr >= XFEATURE_MAX) ||
563 (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR)) {
564 WARN_ONCE(1, "no structure for xstate: %d\n", nr);
565 XSTATE_WARN_ON(1);
566 }
567}
568
569
570
571
572
573
574static void do_extra_xstate_size_checks(void)
575{
576 int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
577 int i;
578
579 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
580 if (!xfeature_enabled(i))
581 continue;
582
583 check_xstate_against_struct(i);
584
585
586
587
588 if (!using_compacted_format())
589 XSTATE_WARN_ON(xfeature_is_supervisor(i));
590
591
592 if (xfeature_is_aligned(i))
593 paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64);
594
595
596
597
598
599
600 if (!using_compacted_format())
601 paranoid_xstate_size = xfeature_uncompacted_offset(i);
602
603
604
605
606 paranoid_xstate_size += xfeature_size(i);
607 }
608 XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size);
609}
610
611
612
613
614
615
616
617
618
619
620
621
622
623static unsigned int __init get_xsaves_size(void)
624{
625 unsigned int eax, ebx, ecx, edx;
626
627
628
629
630
631
632
633
634 cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
635 return ebx;
636}
637
638static unsigned int __init get_xsave_size(void)
639{
640 unsigned int eax, ebx, ecx, edx;
641
642
643
644
645
646
647
648 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
649 return ebx;
650}
651
652
653
654
655
656static bool is_supported_xstate_size(unsigned int test_xstate_size)
657{
658 if (test_xstate_size <= sizeof(union fpregs_state))
659 return true;
660
661 pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
662 sizeof(union fpregs_state), test_xstate_size);
663 return false;
664}
665
666static int __init init_xstate_size(void)
667{
668
669 unsigned int possible_xstate_size;
670 unsigned int xsave_size;
671
672 xsave_size = get_xsave_size();
673
674 if (boot_cpu_has(X86_FEATURE_XSAVES))
675 possible_xstate_size = get_xsaves_size();
676 else
677 possible_xstate_size = xsave_size;
678
679
680 if (!is_supported_xstate_size(possible_xstate_size))
681 return -EINVAL;
682
683
684
685
686
687 fpu_kernel_xstate_size = possible_xstate_size;
688 do_extra_xstate_size_checks();
689
690
691
692
693 fpu_user_xstate_size = xsave_size;
694 return 0;
695}
696
697
698
699
700
701static void fpu__init_disable_system_xstate(void)
702{
703 xfeatures_mask = 0;
704 cr4_clear_bits(X86_CR4_OSXSAVE);
705 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
706}
707
708
709
710
711
712void __init fpu__init_system_xstate(void)
713{
714 unsigned int eax, ebx, ecx, edx;
715 static int on_boot_cpu __initdata = 1;
716 int err;
717 int i;
718
719 WARN_ON_FPU(!on_boot_cpu);
720 on_boot_cpu = 0;
721
722 if (!boot_cpu_has(X86_FEATURE_FPU)) {
723 pr_info("x86/fpu: No FPU detected\n");
724 return;
725 }
726
727 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
728 pr_info("x86/fpu: x87 FPU will use %s\n",
729 boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
730 return;
731 }
732
733 if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
734 WARN_ON_FPU(1);
735 return;
736 }
737
738 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
739 xfeatures_mask = eax + ((u64)edx << 32);
740
741 if ((xfeatures_mask & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
742
743
744
745
746
747 pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask);
748 goto out_disable;
749 }
750
751
752
753
754 for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
755 if (!boot_cpu_has(xsave_cpuid_features[i]))
756 xfeatures_mask &= ~BIT(i);
757 }
758
759 xfeatures_mask &= fpu__get_supported_xfeatures_mask();
760
761
762 fpu__init_cpu_xstate();
763 err = init_xstate_size();
764 if (err)
765 goto out_disable;
766
767
768
769
770
771 update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask & ~XFEATURE_MASK_SUPERVISOR);
772
773 fpu__init_prepare_fx_sw_frame();
774 setup_init_fpu_buf();
775 setup_xstate_comp();
776 print_xstate_offset_size();
777
778 pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
779 xfeatures_mask,
780 fpu_kernel_xstate_size,
781 boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
782 return;
783
784out_disable:
785
786 fpu__init_disable_system_xstate();
787}
788
789
790
791
792void fpu__resume_cpu(void)
793{
794
795
796
797 if (boot_cpu_has(X86_FEATURE_XSAVE))
798 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
799}
800
801
802
803
804
805
806static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
807{
808 if (!xfeature_enabled(xfeature_nr)) {
809 WARN_ON_FPU(1);
810 return NULL;
811 }
812
813 return (void *)xsave + xstate_comp_offsets[xfeature_nr];
814}
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
834{
835
836
837
838 if (!boot_cpu_has(X86_FEATURE_XSAVE))
839 return NULL;
840
841
842
843
844
845
846 WARN_ONCE(!(xfeatures_mask & BIT_ULL(xfeature_nr)),
847 "get of unsupported state");
848
849
850
851
852
853
854
855
856
857
858
859 if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
860 return NULL;
861
862 return __raw_xsave_addr(xsave, xfeature_nr);
863}
864EXPORT_SYMBOL_GPL(get_xsave_addr);
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883const void *get_xsave_field_ptr(int xfeature_nr)
884{
885 struct fpu *fpu = ¤t->thread.fpu;
886
887
888
889
890
891 fpu__save(fpu);
892
893 return get_xsave_addr(&fpu->state.xsave, xfeature_nr);
894}
895
896#ifdef CONFIG_ARCH_HAS_PKEYS
897
898#define NR_VALID_PKRU_BITS (CONFIG_NR_PROTECTION_KEYS * 2)
899#define PKRU_VALID_MASK (NR_VALID_PKRU_BITS - 1)
900
901
902
903
904int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
905 unsigned long init_val)
906{
907 u32 old_pkru;
908 int pkey_shift = (pkey * PKRU_BITS_PER_PKEY);
909 u32 new_pkru_bits = 0;
910
911
912
913
914
915 if (!boot_cpu_has(X86_FEATURE_OSPKE))
916 return -EINVAL;
917
918
919 if (init_val & PKEY_DISABLE_ACCESS)
920 new_pkru_bits |= PKRU_AD_BIT;
921 if (init_val & PKEY_DISABLE_WRITE)
922 new_pkru_bits |= PKRU_WD_BIT;
923
924
925 new_pkru_bits <<= pkey_shift;
926
927
928 old_pkru = read_pkru();
929 old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
930
931
932 write_pkru(old_pkru | new_pkru_bits);
933
934 return 0;
935}
936#endif
937
938
939
940
941
942
943
944static inline bool xfeatures_mxcsr_quirk(u64 xfeatures)
945{
946 if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM)))
947 return false;
948
949 if (xfeatures & XFEATURE_MASK_FP)
950 return false;
951
952 return true;
953}
954
955
956
957
958
959static inline void
960__copy_xstate_to_kernel(void *kbuf, const void *data,
961 unsigned int offset, unsigned int size, unsigned int size_total)
962{
963 if (offset < size_total) {
964 unsigned int copy = min(size, size_total - offset);
965
966 memcpy(kbuf + offset, data, copy);
967 }
968}
969
970
971
972
973
974
975
976
977int copy_xstate_to_kernel(void *kbuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
978{
979 unsigned int offset, size;
980 struct xstate_header header;
981 int i;
982
983
984
985
986 if (unlikely(offset_start != 0))
987 return -EFAULT;
988
989
990
991
992 memset(&header, 0, sizeof(header));
993 header.xfeatures = xsave->header.xfeatures;
994 header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;
995
996
997
998
999 offset = offsetof(struct xregs_state, header);
1000 size = sizeof(header);
1001
1002 __copy_xstate_to_kernel(kbuf, &header, offset, size, size_total);
1003
1004 for (i = 0; i < XFEATURE_MAX; i++) {
1005
1006
1007
1008 if ((header.xfeatures >> i) & 1) {
1009 void *src = __raw_xsave_addr(xsave, i);
1010
1011 offset = xstate_offsets[i];
1012 size = xstate_sizes[i];
1013
1014
1015 if (offset + size > size_total)
1016 break;
1017
1018 __copy_xstate_to_kernel(kbuf, src, offset, size, size_total);
1019 }
1020
1021 }
1022
1023 if (xfeatures_mxcsr_quirk(header.xfeatures)) {
1024 offset = offsetof(struct fxregs_state, mxcsr);
1025 size = MXCSR_AND_FLAGS_SIZE;
1026 __copy_xstate_to_kernel(kbuf, &xsave->i387.mxcsr, offset, size, size_total);
1027 }
1028
1029
1030
1031
1032 offset = offsetof(struct fxregs_state, sw_reserved);
1033 size = sizeof(xstate_fx_sw_bytes);
1034
1035 __copy_xstate_to_kernel(kbuf, xstate_fx_sw_bytes, offset, size, size_total);
1036
1037 return 0;
1038}
1039
1040static inline int
1041__copy_xstate_to_user(void __user *ubuf, const void *data, unsigned int offset, unsigned int size, unsigned int size_total)
1042{
1043 if (!size)
1044 return 0;
1045
1046 if (offset < size_total) {
1047 unsigned int copy = min(size, size_total - offset);
1048
1049 if (__copy_to_user(ubuf + offset, data, copy))
1050 return -EFAULT;
1051 }
1052 return 0;
1053}
1054
1055
1056
1057
1058
1059
1060
1061int copy_xstate_to_user(void __user *ubuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
1062{
1063 unsigned int offset, size;
1064 int ret, i;
1065 struct xstate_header header;
1066
1067
1068
1069
1070 if (unlikely(offset_start != 0))
1071 return -EFAULT;
1072
1073
1074
1075
1076 memset(&header, 0, sizeof(header));
1077 header.xfeatures = xsave->header.xfeatures;
1078 header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;
1079
1080
1081
1082
1083 offset = offsetof(struct xregs_state, header);
1084 size = sizeof(header);
1085
1086 ret = __copy_xstate_to_user(ubuf, &header, offset, size, size_total);
1087 if (ret)
1088 return ret;
1089
1090 for (i = 0; i < XFEATURE_MAX; i++) {
1091
1092
1093
1094 if ((header.xfeatures >> i) & 1) {
1095 void *src = __raw_xsave_addr(xsave, i);
1096
1097 offset = xstate_offsets[i];
1098 size = xstate_sizes[i];
1099
1100
1101 if (offset + size > size_total)
1102 break;
1103
1104 ret = __copy_xstate_to_user(ubuf, src, offset, size, size_total);
1105 if (ret)
1106 return ret;
1107 }
1108
1109 }
1110
1111 if (xfeatures_mxcsr_quirk(header.xfeatures)) {
1112 offset = offsetof(struct fxregs_state, mxcsr);
1113 size = MXCSR_AND_FLAGS_SIZE;
1114 __copy_xstate_to_user(ubuf, &xsave->i387.mxcsr, offset, size, size_total);
1115 }
1116
1117
1118
1119
1120 offset = offsetof(struct fxregs_state, sw_reserved);
1121 size = sizeof(xstate_fx_sw_bytes);
1122
1123 ret = __copy_xstate_to_user(ubuf, xstate_fx_sw_bytes, offset, size, size_total);
1124 if (ret)
1125 return ret;
1126
1127 return 0;
1128}
1129
1130
1131
1132
1133
1134int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf)
1135{
1136 unsigned int offset, size;
1137 int i;
1138 struct xstate_header hdr;
1139
1140 offset = offsetof(struct xregs_state, header);
1141 size = sizeof(hdr);
1142
1143 memcpy(&hdr, kbuf + offset, size);
1144
1145 if (validate_xstate_header(&hdr))
1146 return -EINVAL;
1147
1148 for (i = 0; i < XFEATURE_MAX; i++) {
1149 u64 mask = ((u64)1 << i);
1150
1151 if (hdr.xfeatures & mask) {
1152 void *dst = __raw_xsave_addr(xsave, i);
1153
1154 offset = xstate_offsets[i];
1155 size = xstate_sizes[i];
1156
1157 memcpy(dst, kbuf + offset, size);
1158 }
1159 }
1160
1161 if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1162 offset = offsetof(struct fxregs_state, mxcsr);
1163 size = MXCSR_AND_FLAGS_SIZE;
1164 memcpy(&xsave->i387.mxcsr, kbuf + offset, size);
1165 }
1166
1167
1168
1169
1170
1171 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;
1172
1173
1174
1175
1176 xsave->header.xfeatures |= hdr.xfeatures;
1177
1178 return 0;
1179}
1180
1181
1182
1183
1184
1185
1186
1187int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf)
1188{
1189 unsigned int offset, size;
1190 int i;
1191 struct xstate_header hdr;
1192
1193 offset = offsetof(struct xregs_state, header);
1194 size = sizeof(hdr);
1195
1196 if (__copy_from_user(&hdr, ubuf + offset, size))
1197 return -EFAULT;
1198
1199 if (validate_xstate_header(&hdr))
1200 return -EINVAL;
1201
1202 for (i = 0; i < XFEATURE_MAX; i++) {
1203 u64 mask = ((u64)1 << i);
1204
1205 if (hdr.xfeatures & mask) {
1206 void *dst = __raw_xsave_addr(xsave, i);
1207
1208 offset = xstate_offsets[i];
1209 size = xstate_sizes[i];
1210
1211 if (__copy_from_user(dst, ubuf + offset, size))
1212 return -EFAULT;
1213 }
1214 }
1215
1216 if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1217 offset = offsetof(struct fxregs_state, mxcsr);
1218 size = MXCSR_AND_FLAGS_SIZE;
1219 if (__copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size))
1220 return -EFAULT;
1221 }
1222
1223
1224
1225
1226
1227 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;
1228
1229
1230
1231
1232 xsave->header.xfeatures |= hdr.xfeatures;
1233
1234 return 0;
1235}
1236
1237#ifdef CONFIG_PROC_PID_ARCH_STATUS
1238
1239
1240
1241
1242static void avx512_status(struct seq_file *m, struct task_struct *task)
1243{
1244 unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
1245 long delta;
1246
1247 if (!timestamp) {
1248
1249
1250
1251 delta = -1;
1252 } else {
1253 delta = (long)(jiffies - timestamp);
1254
1255
1256
1257 if (delta < 0)
1258 delta = LONG_MAX;
1259 delta = jiffies_to_msecs(delta);
1260 }
1261
1262 seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
1263 seq_putc(m, '\n');
1264}
1265
1266
1267
1268
1269int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
1270 struct pid *pid, struct task_struct *task)
1271{
1272
1273
1274
1275 if (cpu_feature_enabled(X86_FEATURE_AVX512F))
1276 avx512_status(m, task);
1277
1278 return 0;
1279}
1280#endif
1281