1
2
3
4
5
6
7
8
9#include <linux/bitmap.h>
10#include <linux/bitops.h>
11#include <linux/bottom_half.h>
12#include <linux/bug.h>
13#include <linux/cache.h>
14#include <linux/compat.h>
15#include <linux/compiler.h>
16#include <linux/cpu.h>
17#include <linux/cpu_pm.h>
18#include <linux/kernel.h>
19#include <linux/linkage.h>
20#include <linux/irqflags.h>
21#include <linux/init.h>
22#include <linux/percpu.h>
23#include <linux/prctl.h>
24#include <linux/preempt.h>
25#include <linux/ptrace.h>
26#include <linux/sched/signal.h>
27#include <linux/sched/task_stack.h>
28#include <linux/signal.h>
29#include <linux/slab.h>
30#include <linux/stddef.h>
31#include <linux/sysctl.h>
32#include <linux/swab.h>
33
34#include <asm/esr.h>
35#include <asm/fpsimd.h>
36#include <asm/cpufeature.h>
37#include <asm/cputype.h>
38#include <asm/processor.h>
39#include <asm/simd.h>
40#include <asm/sigcontext.h>
41#include <asm/sysreg.h>
42#include <asm/traps.h>
43#include <asm/virt.h>
44
45#define FPEXC_IOF (1 << 0)
46#define FPEXC_DZF (1 << 1)
47#define FPEXC_OFF (1 << 2)
48#define FPEXC_UFF (1 << 3)
49#define FPEXC_IXF (1 << 4)
50#define FPEXC_IDF (1 << 7)
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114struct fpsimd_last_state_struct {
115 struct user_fpsimd_state *st;
116 void *sve_state;
117 unsigned int sve_vl;
118};
119
120static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
121
122
123static int __sve_default_vl = -1;
124
125static int get_sve_default_vl(void)
126{
127 return READ_ONCE(__sve_default_vl);
128}
129
130#ifdef CONFIG_ARM64_SVE
131
132static void set_sve_default_vl(int val)
133{
134 WRITE_ONCE(__sve_default_vl, val);
135}
136
137
138int __ro_after_init sve_max_vl = SVE_VL_MIN;
139int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
140
141
142
143
144
145__ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
146
147static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
148
149static void __percpu *efi_sve_state;
150
151#else
152
153
154extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
155extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
156extern void __percpu *efi_sve_state;
157
158#endif
159
160DEFINE_PER_CPU(bool, fpsimd_context_busy);
161EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy);
162
163static void __get_cpu_fpsimd_context(void)
164{
165 bool busy = __this_cpu_xchg(fpsimd_context_busy, true);
166
167 WARN_ON(busy);
168}
169
170
171
172
173
174
175
176
177
178
179static void get_cpu_fpsimd_context(void)
180{
181 preempt_disable();
182 __get_cpu_fpsimd_context();
183}
184
185static void __put_cpu_fpsimd_context(void)
186{
187 bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
188
189 WARN_ON(!busy);
190}
191
192
193
194
195
196
197
198
199static void put_cpu_fpsimd_context(void)
200{
201 __put_cpu_fpsimd_context();
202 preempt_enable();
203}
204
205static bool have_cpu_fpsimd_context(void)
206{
207 return !preemptible() && __this_cpu_read(fpsimd_context_busy);
208}
209
210
211
212
213
214static void __sve_free(struct task_struct *task)
215{
216 kfree(task->thread.sve_state);
217 task->thread.sve_state = NULL;
218}
219
220static void sve_free(struct task_struct *task)
221{
222 WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
223
224 __sve_free(task);
225}
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281static void task_fpsimd_load(void)
282{
283 WARN_ON(!system_supports_fpsimd());
284 WARN_ON(!have_cpu_fpsimd_context());
285
286 if (system_supports_sve() && test_thread_flag(TIF_SVE))
287 sve_load_state(sve_pffr(¤t->thread),
288 ¤t->thread.uw.fpsimd_state.fpsr,
289 sve_vq_from_vl(current->thread.sve_vl) - 1);
290 else
291 fpsimd_load_state(¤t->thread.uw.fpsimd_state);
292}
293
294
295
296
297
298static void fpsimd_save(void)
299{
300 struct fpsimd_last_state_struct const *last =
301 this_cpu_ptr(&fpsimd_last_state);
302
303
304 WARN_ON(!system_supports_fpsimd());
305 WARN_ON(!have_cpu_fpsimd_context());
306
307 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
308 if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
309 if (WARN_ON(sve_get_vl() != last->sve_vl)) {
310
311
312
313
314
315 force_signal_inject(SIGKILL, SI_KERNEL, 0);
316 return;
317 }
318
319 sve_save_state((char *)last->sve_state +
320 sve_ffr_offset(last->sve_vl),
321 &last->st->fpsr);
322 } else
323 fpsimd_save_state(last->st);
324 }
325}
326
327
328
329
330
331
332
333static unsigned int find_supported_vector_length(unsigned int vl)
334{
335 int bit;
336 int max_vl = sve_max_vl;
337
338 if (WARN_ON(!sve_vl_valid(vl)))
339 vl = SVE_VL_MIN;
340
341 if (WARN_ON(!sve_vl_valid(max_vl)))
342 max_vl = SVE_VL_MIN;
343
344 if (vl > max_vl)
345 vl = max_vl;
346
347 bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
348 __vq_to_bit(sve_vq_from_vl(vl)));
349 return sve_vl_from_vq(__bit_to_vq(bit));
350}
351
352#if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
353
354static int sve_proc_do_default_vl(struct ctl_table *table, int write,
355 void *buffer, size_t *lenp, loff_t *ppos)
356{
357 int ret;
358 int vl = get_sve_default_vl();
359 struct ctl_table tmp_table = {
360 .data = &vl,
361 .maxlen = sizeof(vl),
362 };
363
364 ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
365 if (ret || !write)
366 return ret;
367
368
369 if (vl == -1)
370 vl = sve_max_vl;
371
372 if (!sve_vl_valid(vl))
373 return -EINVAL;
374
375 set_sve_default_vl(find_supported_vector_length(vl));
376 return 0;
377}
378
379static struct ctl_table sve_default_vl_table[] = {
380 {
381 .procname = "sve_default_vector_length",
382 .mode = 0644,
383 .proc_handler = sve_proc_do_default_vl,
384 },
385 { }
386};
387
388static int __init sve_sysctl_init(void)
389{
390 if (system_supports_sve())
391 if (!register_sysctl("abi", sve_default_vl_table))
392 return -EINVAL;
393
394 return 0;
395}
396
397#else
398static int __init sve_sysctl_init(void) { return 0; }
399#endif
400
401#define ZREG(sve_state, vq, n) ((char *)(sve_state) + \
402 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
403
404#ifdef CONFIG_CPU_BIG_ENDIAN
405static __uint128_t arm64_cpu_to_le128(__uint128_t x)
406{
407 u64 a = swab64(x);
408 u64 b = swab64(x >> 64);
409
410 return ((__uint128_t)a << 64) | b;
411}
412#else
413static __uint128_t arm64_cpu_to_le128(__uint128_t x)
414{
415 return x;
416}
417#endif
418
419#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
420
421static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
422 unsigned int vq)
423{
424 unsigned int i;
425 __uint128_t *p;
426
427 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
428 p = (__uint128_t *)ZREG(sst, vq, i);
429 *p = arm64_cpu_to_le128(fst->vregs[i]);
430 }
431}
432
433
434
435
436
437
438
439
440
441
442
443
444
445static void fpsimd_to_sve(struct task_struct *task)
446{
447 unsigned int vq;
448 void *sst = task->thread.sve_state;
449 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
450
451 if (!system_supports_sve())
452 return;
453
454 vq = sve_vq_from_vl(task->thread.sve_vl);
455 __fpsimd_to_sve(sst, fst, vq);
456}
457
458
459
460
461
462
463
464
465
466
467
468
469static void sve_to_fpsimd(struct task_struct *task)
470{
471 unsigned int vq;
472 void const *sst = task->thread.sve_state;
473 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
474 unsigned int i;
475 __uint128_t const *p;
476
477 if (!system_supports_sve())
478 return;
479
480 vq = sve_vq_from_vl(task->thread.sve_vl);
481 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
482 p = (__uint128_t const *)ZREG(sst, vq, i);
483 fst->vregs[i] = arm64_le128_to_cpu(*p);
484 }
485}
486
487#ifdef CONFIG_ARM64_SVE
488
489
490
491
492
493size_t sve_state_size(struct task_struct const *task)
494{
495 return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));
496}
497
498
499
500
501
502
503
504
505
506
507
508void sve_alloc(struct task_struct *task)
509{
510 if (task->thread.sve_state) {
511 memset(task->thread.sve_state, 0, sve_state_size(current));
512 return;
513 }
514
515
516 task->thread.sve_state =
517 kzalloc(sve_state_size(task), GFP_KERNEL);
518
519
520
521
522
523 BUG_ON(!task->thread.sve_state);
524}
525
526
527
528
529
530
531
532
533
534
535void fpsimd_sync_to_sve(struct task_struct *task)
536{
537 if (!test_tsk_thread_flag(task, TIF_SVE))
538 fpsimd_to_sve(task);
539}
540
541
542
543
544
545
546
547
548
549void sve_sync_to_fpsimd(struct task_struct *task)
550{
551 if (test_tsk_thread_flag(task, TIF_SVE))
552 sve_to_fpsimd(task);
553}
554
555
556
557
558
559
560
561
562
563
564
565
566
567void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
568{
569 unsigned int vq;
570 void *sst = task->thread.sve_state;
571 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
572
573 if (!test_tsk_thread_flag(task, TIF_SVE))
574 return;
575
576 vq = sve_vq_from_vl(task->thread.sve_vl);
577
578 memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
579 __fpsimd_to_sve(sst, fst, vq);
580}
581
582int sve_set_vector_length(struct task_struct *task,
583 unsigned long vl, unsigned long flags)
584{
585 if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
586 PR_SVE_SET_VL_ONEXEC))
587 return -EINVAL;
588
589 if (!sve_vl_valid(vl))
590 return -EINVAL;
591
592
593
594
595
596
597 if (vl > SVE_VL_ARCH_MAX)
598 vl = SVE_VL_ARCH_MAX;
599
600 vl = find_supported_vector_length(vl);
601
602 if (flags & (PR_SVE_VL_INHERIT |
603 PR_SVE_SET_VL_ONEXEC))
604 task->thread.sve_vl_onexec = vl;
605 else
606
607 task->thread.sve_vl_onexec = 0;
608
609
610 if (flags & PR_SVE_SET_VL_ONEXEC)
611 goto out;
612
613 if (vl == task->thread.sve_vl)
614 goto out;
615
616
617
618
619
620
621 if (task == current) {
622 get_cpu_fpsimd_context();
623
624 fpsimd_save();
625 }
626
627 fpsimd_flush_task_state(task);
628 if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
629 sve_to_fpsimd(task);
630
631 if (task == current)
632 put_cpu_fpsimd_context();
633
634
635
636
637
638 sve_free(task);
639
640 task->thread.sve_vl = vl;
641
642out:
643 update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
644 flags & PR_SVE_VL_INHERIT);
645
646 return 0;
647}
648
649
650
651
652
653
654
655static int sve_prctl_status(unsigned long flags)
656{
657 int ret;
658
659 if (flags & PR_SVE_SET_VL_ONEXEC)
660 ret = current->thread.sve_vl_onexec;
661 else
662 ret = current->thread.sve_vl;
663
664 if (test_thread_flag(TIF_SVE_VL_INHERIT))
665 ret |= PR_SVE_VL_INHERIT;
666
667 return ret;
668}
669
670
671int sve_set_current_vl(unsigned long arg)
672{
673 unsigned long vl, flags;
674 int ret;
675
676 vl = arg & PR_SVE_VL_LEN_MASK;
677 flags = arg & ~vl;
678
679 if (!system_supports_sve())
680 return -EINVAL;
681
682 ret = sve_set_vector_length(current, vl, flags);
683 if (ret)
684 return ret;
685
686 return sve_prctl_status(flags);
687}
688
689
690int sve_get_current_vl(void)
691{
692 if (!system_supports_sve())
693 return -EINVAL;
694
695 return sve_prctl_status(0);
696}
697
698static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
699{
700 unsigned int vq, vl;
701 unsigned long zcr;
702
703 bitmap_zero(map, SVE_VQ_MAX);
704
705 zcr = ZCR_ELx_LEN_MASK;
706 zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
707
708 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
709 write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1);
710 vl = sve_get_vl();
711 vq = sve_vq_from_vl(vl);
712 set_bit(__vq_to_bit(vq), map);
713 }
714}
715
716
717
718
719
720void __init sve_init_vq_map(void)
721{
722 sve_probe_vqs(sve_vq_map);
723 bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
724}
725
726
727
728
729
730
731void sve_update_vq_map(void)
732{
733 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
734
735 sve_probe_vqs(tmp_map);
736 bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
737 bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
738}
739
740
741
742
743
744int sve_verify_vq_map(void)
745{
746 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
747 unsigned long b;
748
749 sve_probe_vqs(tmp_map);
750
751 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
752 if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
753 pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
754 smp_processor_id());
755 return -EINVAL;
756 }
757
758 if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
759 return 0;
760
761
762
763
764
765
766
767
768 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
769
770 bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
771
772
773 b = find_last_bit(tmp_map, SVE_VQ_MAX);
774 if (b >= SVE_VQ_MAX)
775 return 0;
776
777
778
779
780
781 if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
782 pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
783 smp_processor_id());
784 return -EINVAL;
785 }
786
787 return 0;
788}
789
790static void __init sve_efi_setup(void)
791{
792 if (!IS_ENABLED(CONFIG_EFI))
793 return;
794
795
796
797
798
799
800 if (!sve_vl_valid(sve_max_vl))
801 goto fail;
802
803 efi_sve_state = __alloc_percpu(
804 SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
805 if (!efi_sve_state)
806 goto fail;
807
808 return;
809
810fail:
811 panic("Cannot allocate percpu memory for EFI SVE save/restore");
812}
813
814
815
816
817
818void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
819{
820 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
821 isb();
822}
823
824
825
826
827
828
829
830
831u64 read_zcr_features(void)
832{
833 u64 zcr;
834 unsigned int vq_max;
835
836
837
838
839
840 sve_kernel_enable(NULL);
841 write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
842
843 zcr = read_sysreg_s(SYS_ZCR_EL1);
844 zcr &= ~(u64)ZCR_ELx_LEN_MASK;
845 vq_max = sve_vq_from_vl(sve_get_vl());
846 zcr |= vq_max - 1;
847
848 return zcr;
849}
850
851void __init sve_setup(void)
852{
853 u64 zcr;
854 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
855 unsigned long b;
856
857 if (!system_supports_sve())
858 return;
859
860
861
862
863
864
865 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
866 set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
867
868 zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
869 sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
870
871
872
873
874
875 if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))
876 sve_max_vl = find_supported_vector_length(sve_max_vl);
877
878
879
880
881
882 set_sve_default_vl(find_supported_vector_length(64));
883
884 bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
885 SVE_VQ_MAX);
886
887 b = find_last_bit(tmp_map, SVE_VQ_MAX);
888 if (b >= SVE_VQ_MAX)
889
890 sve_max_virtualisable_vl = SVE_VQ_MAX;
891 else if (WARN_ON(b == SVE_VQ_MAX - 1))
892
893 sve_max_virtualisable_vl = SVE_VQ_MIN;
894 else
895 sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
896
897 if (sve_max_virtualisable_vl > sve_max_vl)
898 sve_max_virtualisable_vl = sve_max_vl;
899
900 pr_info("SVE: maximum available vector length %u bytes per vector\n",
901 sve_max_vl);
902 pr_info("SVE: default vector length %u bytes per vector\n",
903 get_sve_default_vl());
904
905
906 if (sve_max_virtualisable_vl < sve_max_vl)
907 pr_warn("SVE: unvirtualisable vector lengths present\n");
908
909 sve_efi_setup();
910}
911
912
913
914
915
916void fpsimd_release_task(struct task_struct *dead_task)
917{
918 __sve_free(dead_task);
919}
920
921#endif
922
923
924
925
926
927
928
929
930
931
932
933
934
935void do_sve_acc(unsigned int esr, struct pt_regs *regs)
936{
937
938 if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
939 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
940 return;
941 }
942
943 sve_alloc(current);
944
945 get_cpu_fpsimd_context();
946
947 fpsimd_save();
948
949
950 fpsimd_flush_task_state(current);
951
952 fpsimd_to_sve(current);
953 if (test_and_set_thread_flag(TIF_SVE))
954 WARN_ON(1);
955
956 put_cpu_fpsimd_context();
957}
958
959
960
961
962void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
963{
964
965 WARN_ON(1);
966}
967
968
969
970
971void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
972{
973 unsigned int si_code = FPE_FLTUNK;
974
975 if (esr & ESR_ELx_FP_EXC_TFV) {
976 if (esr & FPEXC_IOF)
977 si_code = FPE_FLTINV;
978 else if (esr & FPEXC_DZF)
979 si_code = FPE_FLTDIV;
980 else if (esr & FPEXC_OFF)
981 si_code = FPE_FLTOVF;
982 else if (esr & FPEXC_UFF)
983 si_code = FPE_FLTUND;
984 else if (esr & FPEXC_IXF)
985 si_code = FPE_FLTRES;
986 }
987
988 send_sig_fault(SIGFPE, si_code,
989 (void __user *)instruction_pointer(regs),
990 current);
991}
992
993void fpsimd_thread_switch(struct task_struct *next)
994{
995 bool wrong_task, wrong_cpu;
996
997 if (!system_supports_fpsimd())
998 return;
999
1000 __get_cpu_fpsimd_context();
1001
1002
1003 fpsimd_save();
1004
1005
1006
1007
1008
1009
1010 wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
1011 &next->thread.uw.fpsimd_state;
1012 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
1013
1014 update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
1015 wrong_task || wrong_cpu);
1016
1017 __put_cpu_fpsimd_context();
1018}
1019
1020void fpsimd_flush_thread(void)
1021{
1022 int vl, supported_vl;
1023
1024 if (!system_supports_fpsimd())
1025 return;
1026
1027 get_cpu_fpsimd_context();
1028
1029 fpsimd_flush_task_state(current);
1030 memset(¤t->thread.uw.fpsimd_state, 0,
1031 sizeof(current->thread.uw.fpsimd_state));
1032
1033 if (system_supports_sve()) {
1034 clear_thread_flag(TIF_SVE);
1035 sve_free(current);
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048 vl = current->thread.sve_vl_onexec ?
1049 current->thread.sve_vl_onexec : get_sve_default_vl();
1050
1051 if (WARN_ON(!sve_vl_valid(vl)))
1052 vl = SVE_VL_MIN;
1053
1054 supported_vl = find_supported_vector_length(vl);
1055 if (WARN_ON(supported_vl != vl))
1056 vl = supported_vl;
1057
1058 current->thread.sve_vl = vl;
1059
1060
1061
1062
1063
1064 if (!test_thread_flag(TIF_SVE_VL_INHERIT))
1065 current->thread.sve_vl_onexec = 0;
1066 }
1067
1068 put_cpu_fpsimd_context();
1069}
1070
1071
1072
1073
1074
1075void fpsimd_preserve_current_state(void)
1076{
1077 if (!system_supports_fpsimd())
1078 return;
1079
1080 get_cpu_fpsimd_context();
1081 fpsimd_save();
1082 put_cpu_fpsimd_context();
1083}
1084
1085
1086
1087
1088
1089
1090void fpsimd_signal_preserve_current_state(void)
1091{
1092 fpsimd_preserve_current_state();
1093 if (system_supports_sve() && test_thread_flag(TIF_SVE))
1094 sve_to_fpsimd(current);
1095}
1096
1097
1098
1099
1100
1101
1102void fpsimd_bind_task_to_cpu(void)
1103{
1104 struct fpsimd_last_state_struct *last =
1105 this_cpu_ptr(&fpsimd_last_state);
1106
1107 WARN_ON(!system_supports_fpsimd());
1108 last->st = ¤t->thread.uw.fpsimd_state;
1109 last->sve_state = current->thread.sve_state;
1110 last->sve_vl = current->thread.sve_vl;
1111 current->thread.fpsimd_cpu = smp_processor_id();
1112
1113 if (system_supports_sve()) {
1114
1115 if (test_thread_flag(TIF_SVE))
1116 sve_user_enable();
1117 else
1118 sve_user_disable();
1119
1120
1121 }
1122}
1123
1124void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1125 unsigned int sve_vl)
1126{
1127 struct fpsimd_last_state_struct *last =
1128 this_cpu_ptr(&fpsimd_last_state);
1129
1130 WARN_ON(!system_supports_fpsimd());
1131 WARN_ON(!in_softirq() && !irqs_disabled());
1132
1133 last->st = st;
1134 last->sve_state = sve_state;
1135 last->sve_vl = sve_vl;
1136}
1137
1138
1139
1140
1141
1142
1143void fpsimd_restore_current_state(void)
1144{
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154 if (!system_supports_fpsimd()) {
1155 clear_thread_flag(TIF_FOREIGN_FPSTATE);
1156 return;
1157 }
1158
1159 get_cpu_fpsimd_context();
1160
1161 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
1162 task_fpsimd_load();
1163 fpsimd_bind_task_to_cpu();
1164 }
1165
1166 put_cpu_fpsimd_context();
1167}
1168
1169
1170
1171
1172
1173
1174void fpsimd_update_current_state(struct user_fpsimd_state const *state)
1175{
1176 if (WARN_ON(!system_supports_fpsimd()))
1177 return;
1178
1179 get_cpu_fpsimd_context();
1180
1181 current->thread.uw.fpsimd_state = *state;
1182 if (system_supports_sve() && test_thread_flag(TIF_SVE))
1183 fpsimd_to_sve(current);
1184
1185 task_fpsimd_load();
1186 fpsimd_bind_task_to_cpu();
1187
1188 clear_thread_flag(TIF_FOREIGN_FPSTATE);
1189
1190 put_cpu_fpsimd_context();
1191}
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204void fpsimd_flush_task_state(struct task_struct *t)
1205{
1206 t->thread.fpsimd_cpu = NR_CPUS;
1207
1208
1209
1210
1211
1212 if (!system_supports_fpsimd())
1213 return;
1214 barrier();
1215 set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1216
1217 barrier();
1218}
1219
1220
1221
1222
1223
1224
1225static void fpsimd_flush_cpu_state(void)
1226{
1227 WARN_ON(!system_supports_fpsimd());
1228 __this_cpu_write(fpsimd_last_state.st, NULL);
1229 set_thread_flag(TIF_FOREIGN_FPSTATE);
1230}
1231
1232
1233
1234
1235
1236void fpsimd_save_and_flush_cpu_state(void)
1237{
1238 if (!system_supports_fpsimd())
1239 return;
1240 WARN_ON(preemptible());
1241 __get_cpu_fpsimd_context();
1242 fpsimd_save();
1243 fpsimd_flush_cpu_state();
1244 __put_cpu_fpsimd_context();
1245}
1246
1247#ifdef CONFIG_KERNEL_MODE_NEON
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266void kernel_neon_begin(void)
1267{
1268 if (WARN_ON(!system_supports_fpsimd()))
1269 return;
1270
1271 BUG_ON(!may_use_simd());
1272
1273 get_cpu_fpsimd_context();
1274
1275
1276 fpsimd_save();
1277
1278
1279 fpsimd_flush_cpu_state();
1280}
1281EXPORT_SYMBOL(kernel_neon_begin);
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292void kernel_neon_end(void)
1293{
1294 if (!system_supports_fpsimd())
1295 return;
1296
1297 put_cpu_fpsimd_context();
1298}
1299EXPORT_SYMBOL(kernel_neon_end);
1300
1301#ifdef CONFIG_EFI
1302
1303static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
1304static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
1305static DEFINE_PER_CPU(bool, efi_sve_state_used);
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324void __efi_fpsimd_begin(void)
1325{
1326 if (!system_supports_fpsimd())
1327 return;
1328
1329 WARN_ON(preemptible());
1330
1331 if (may_use_simd()) {
1332 kernel_neon_begin();
1333 } else {
1334
1335
1336
1337
1338 if (system_supports_sve() && likely(efi_sve_state)) {
1339 char *sve_state = this_cpu_ptr(efi_sve_state);
1340
1341 __this_cpu_write(efi_sve_state_used, true);
1342
1343 sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
1344 &this_cpu_ptr(&efi_fpsimd_state)->fpsr);
1345 } else {
1346 fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
1347 }
1348
1349 __this_cpu_write(efi_fpsimd_state_used, true);
1350 }
1351}
1352
1353
1354
1355
1356void __efi_fpsimd_end(void)
1357{
1358 if (!system_supports_fpsimd())
1359 return;
1360
1361 if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
1362 kernel_neon_end();
1363 } else {
1364 if (system_supports_sve() &&
1365 likely(__this_cpu_read(efi_sve_state_used))) {
1366 char const *sve_state = this_cpu_ptr(efi_sve_state);
1367
1368 sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
1369 &this_cpu_ptr(&efi_fpsimd_state)->fpsr,
1370 sve_vq_from_vl(sve_get_vl()) - 1);
1371
1372 __this_cpu_write(efi_sve_state_used, false);
1373 } else {
1374 fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
1375 }
1376 }
1377}
1378
1379#endif
1380
1381#endif
1382
1383#ifdef CONFIG_CPU_PM
1384static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
1385 unsigned long cmd, void *v)
1386{
1387 switch (cmd) {
1388 case CPU_PM_ENTER:
1389 fpsimd_save_and_flush_cpu_state();
1390 break;
1391 case CPU_PM_EXIT:
1392 break;
1393 case CPU_PM_ENTER_FAILED:
1394 default:
1395 return NOTIFY_DONE;
1396 }
1397 return NOTIFY_OK;
1398}
1399
1400static struct notifier_block fpsimd_cpu_pm_notifier_block = {
1401 .notifier_call = fpsimd_cpu_pm_notifier,
1402};
1403
1404static void __init fpsimd_pm_init(void)
1405{
1406 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
1407}
1408
1409#else
1410static inline void fpsimd_pm_init(void) { }
1411#endif
1412
1413#ifdef CONFIG_HOTPLUG_CPU
1414static int fpsimd_cpu_dead(unsigned int cpu)
1415{
1416 per_cpu(fpsimd_last_state.st, cpu) = NULL;
1417 return 0;
1418}
1419
1420static inline void fpsimd_hotplug_init(void)
1421{
1422 cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
1423 NULL, fpsimd_cpu_dead);
1424}
1425
1426#else
1427static inline void fpsimd_hotplug_init(void) { }
1428#endif
1429
1430
1431
1432
1433static int __init fpsimd_init(void)
1434{
1435 if (cpu_have_named_feature(FP)) {
1436 fpsimd_pm_init();
1437 fpsimd_hotplug_init();
1438 } else {
1439 pr_notice("Floating-point is not implemented\n");
1440 }
1441
1442 if (!cpu_have_named_feature(ASIMD))
1443 pr_notice("Advanced SIMD is not implemented\n");
1444
1445 return sve_sysctl_init();
1446}
1447core_initcall(fpsimd_init);
1448