1
2
3
4
5
6
7
8
9#include <linux/bitmap.h>
10#include <linux/bitops.h>
11#include <linux/bottom_half.h>
12#include <linux/bug.h>
13#include <linux/cache.h>
14#include <linux/compat.h>
15#include <linux/compiler.h>
16#include <linux/cpu.h>
17#include <linux/cpu_pm.h>
18#include <linux/kernel.h>
19#include <linux/linkage.h>
20#include <linux/irqflags.h>
21#include <linux/init.h>
22#include <linux/percpu.h>
23#include <linux/prctl.h>
24#include <linux/preempt.h>
25#include <linux/ptrace.h>
26#include <linux/sched/signal.h>
27#include <linux/sched/task_stack.h>
28#include <linux/signal.h>
29#include <linux/slab.h>
30#include <linux/stddef.h>
31#include <linux/sysctl.h>
32#include <linux/swab.h>
33
34#include <asm/esr.h>
35#include <asm/exception.h>
36#include <asm/fpsimd.h>
37#include <asm/cpufeature.h>
38#include <asm/cputype.h>
39#include <asm/neon.h>
40#include <asm/processor.h>
41#include <asm/simd.h>
42#include <asm/sigcontext.h>
43#include <asm/sysreg.h>
44#include <asm/traps.h>
45#include <asm/virt.h>
46
47#define FPEXC_IOF (1 << 0)
48#define FPEXC_DZF (1 << 1)
49#define FPEXC_OFF (1 << 2)
50#define FPEXC_UFF (1 << 3)
51#define FPEXC_IXF (1 << 4)
52#define FPEXC_IDF (1 << 7)
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116struct fpsimd_last_state_struct {
117 struct user_fpsimd_state *st;
118 void *sve_state;
119 unsigned int sve_vl;
120};
121
122static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
123
124
125static int __sve_default_vl = -1;
126
127static int get_sve_default_vl(void)
128{
129 return READ_ONCE(__sve_default_vl);
130}
131
132#ifdef CONFIG_ARM64_SVE
133
134static void set_sve_default_vl(int val)
135{
136 WRITE_ONCE(__sve_default_vl, val);
137}
138
139
140int __ro_after_init sve_max_vl = SVE_VL_MIN;
141int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
142
143
144
145
146
147__ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
148
149static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
150
151static void __percpu *efi_sve_state;
152
153#else
154
155
156extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
157extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
158extern void __percpu *efi_sve_state;
159
160#endif
161
162DEFINE_PER_CPU(bool, fpsimd_context_busy);
163EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy);
164
165static void fpsimd_bind_task_to_cpu(void);
166
167static void __get_cpu_fpsimd_context(void)
168{
169 bool busy = __this_cpu_xchg(fpsimd_context_busy, true);
170
171 WARN_ON(busy);
172}
173
174
175
176
177
178
179
180
181
182
183static void get_cpu_fpsimd_context(void)
184{
185 local_bh_disable();
186 __get_cpu_fpsimd_context();
187}
188
189static void __put_cpu_fpsimd_context(void)
190{
191 bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
192
193 WARN_ON(!busy);
194}
195
196
197
198
199
200
201
202
203static void put_cpu_fpsimd_context(void)
204{
205 __put_cpu_fpsimd_context();
206 local_bh_enable();
207}
208
209static bool have_cpu_fpsimd_context(void)
210{
211 return !preemptible() && __this_cpu_read(fpsimd_context_busy);
212}
213
214
215
216
217
218static void __sve_free(struct task_struct *task)
219{
220 kfree(task->thread.sve_state);
221 task->thread.sve_state = NULL;
222}
223
224static void sve_free(struct task_struct *task)
225{
226 WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
227
228 __sve_free(task);
229}
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285static void task_fpsimd_load(void)
286{
287 WARN_ON(!system_supports_fpsimd());
288 WARN_ON(!have_cpu_fpsimd_context());
289
290 if (IS_ENABLED(CONFIG_ARM64_SVE) && test_thread_flag(TIF_SVE))
291 sve_load_state(sve_pffr(¤t->thread),
292 ¤t->thread.uw.fpsimd_state.fpsr,
293 sve_vq_from_vl(current->thread.sve_vl) - 1);
294 else
295 fpsimd_load_state(¤t->thread.uw.fpsimd_state);
296}
297
298
299
300
301
302static void fpsimd_save(void)
303{
304 struct fpsimd_last_state_struct const *last =
305 this_cpu_ptr(&fpsimd_last_state);
306
307
308 WARN_ON(!system_supports_fpsimd());
309 WARN_ON(!have_cpu_fpsimd_context());
310
311 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
312 if (IS_ENABLED(CONFIG_ARM64_SVE) &&
313 test_thread_flag(TIF_SVE)) {
314 if (WARN_ON(sve_get_vl() != last->sve_vl)) {
315
316
317
318
319
320 force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
321 return;
322 }
323
324 sve_save_state((char *)last->sve_state +
325 sve_ffr_offset(last->sve_vl),
326 &last->st->fpsr);
327 } else
328 fpsimd_save_state(last->st);
329 }
330}
331
332
333
334
335
336
337
338static unsigned int find_supported_vector_length(unsigned int vl)
339{
340 int bit;
341 int max_vl = sve_max_vl;
342
343 if (WARN_ON(!sve_vl_valid(vl)))
344 vl = SVE_VL_MIN;
345
346 if (WARN_ON(!sve_vl_valid(max_vl)))
347 max_vl = SVE_VL_MIN;
348
349 if (vl > max_vl)
350 vl = max_vl;
351
352 bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
353 __vq_to_bit(sve_vq_from_vl(vl)));
354 return sve_vl_from_vq(__bit_to_vq(bit));
355}
356
357#if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
358
359static int sve_proc_do_default_vl(struct ctl_table *table, int write,
360 void *buffer, size_t *lenp, loff_t *ppos)
361{
362 int ret;
363 int vl = get_sve_default_vl();
364 struct ctl_table tmp_table = {
365 .data = &vl,
366 .maxlen = sizeof(vl),
367 };
368
369 ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
370 if (ret || !write)
371 return ret;
372
373
374 if (vl == -1)
375 vl = sve_max_vl;
376
377 if (!sve_vl_valid(vl))
378 return -EINVAL;
379
380 set_sve_default_vl(find_supported_vector_length(vl));
381 return 0;
382}
383
384static struct ctl_table sve_default_vl_table[] = {
385 {
386 .procname = "sve_default_vector_length",
387 .mode = 0644,
388 .proc_handler = sve_proc_do_default_vl,
389 },
390 { }
391};
392
393static int __init sve_sysctl_init(void)
394{
395 if (system_supports_sve())
396 if (!register_sysctl("abi", sve_default_vl_table))
397 return -EINVAL;
398
399 return 0;
400}
401
402#else
403static int __init sve_sysctl_init(void) { return 0; }
404#endif
405
406#define ZREG(sve_state, vq, n) ((char *)(sve_state) + \
407 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
408
409#ifdef CONFIG_CPU_BIG_ENDIAN
410static __uint128_t arm64_cpu_to_le128(__uint128_t x)
411{
412 u64 a = swab64(x);
413 u64 b = swab64(x >> 64);
414
415 return ((__uint128_t)a << 64) | b;
416}
417#else
418static __uint128_t arm64_cpu_to_le128(__uint128_t x)
419{
420 return x;
421}
422#endif
423
424#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
425
426static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
427 unsigned int vq)
428{
429 unsigned int i;
430 __uint128_t *p;
431
432 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
433 p = (__uint128_t *)ZREG(sst, vq, i);
434 *p = arm64_cpu_to_le128(fst->vregs[i]);
435 }
436}
437
438
439
440
441
442
443
444
445
446
447
448
449
450static void fpsimd_to_sve(struct task_struct *task)
451{
452 unsigned int vq;
453 void *sst = task->thread.sve_state;
454 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
455
456 if (!system_supports_sve())
457 return;
458
459 vq = sve_vq_from_vl(task->thread.sve_vl);
460 __fpsimd_to_sve(sst, fst, vq);
461}
462
463
464
465
466
467
468
469
470
471
472
473
474static void sve_to_fpsimd(struct task_struct *task)
475{
476 unsigned int vq;
477 void const *sst = task->thread.sve_state;
478 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
479 unsigned int i;
480 __uint128_t const *p;
481
482 if (!system_supports_sve())
483 return;
484
485 vq = sve_vq_from_vl(task->thread.sve_vl);
486 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
487 p = (__uint128_t const *)ZREG(sst, vq, i);
488 fst->vregs[i] = arm64_le128_to_cpu(*p);
489 }
490}
491
492#ifdef CONFIG_ARM64_SVE
493
494
495
496
497
498size_t sve_state_size(struct task_struct const *task)
499{
500 return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));
501}
502
503
504
505
506
507
508
509
510
511
512
513void sve_alloc(struct task_struct *task)
514{
515 if (task->thread.sve_state) {
516 memset(task->thread.sve_state, 0, sve_state_size(task));
517 return;
518 }
519
520
521 task->thread.sve_state =
522 kzalloc(sve_state_size(task), GFP_KERNEL);
523}
524
525
526
527
528
529
530
531
532
533
534void fpsimd_sync_to_sve(struct task_struct *task)
535{
536 if (!test_tsk_thread_flag(task, TIF_SVE))
537 fpsimd_to_sve(task);
538}
539
540
541
542
543
544
545
546
547
548void sve_sync_to_fpsimd(struct task_struct *task)
549{
550 if (test_tsk_thread_flag(task, TIF_SVE))
551 sve_to_fpsimd(task);
552}
553
554
555
556
557
558
559
560
561
562
563
564
565
566void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
567{
568 unsigned int vq;
569 void *sst = task->thread.sve_state;
570 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
571
572 if (!test_tsk_thread_flag(task, TIF_SVE))
573 return;
574
575 vq = sve_vq_from_vl(task->thread.sve_vl);
576
577 memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
578 __fpsimd_to_sve(sst, fst, vq);
579}
580
581int sve_set_vector_length(struct task_struct *task,
582 unsigned long vl, unsigned long flags)
583{
584 if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
585 PR_SVE_SET_VL_ONEXEC))
586 return -EINVAL;
587
588 if (!sve_vl_valid(vl))
589 return -EINVAL;
590
591
592
593
594
595
596 if (vl > SVE_VL_ARCH_MAX)
597 vl = SVE_VL_ARCH_MAX;
598
599 vl = find_supported_vector_length(vl);
600
601 if (flags & (PR_SVE_VL_INHERIT |
602 PR_SVE_SET_VL_ONEXEC))
603 task->thread.sve_vl_onexec = vl;
604 else
605
606 task->thread.sve_vl_onexec = 0;
607
608
609 if (flags & PR_SVE_SET_VL_ONEXEC)
610 goto out;
611
612 if (vl == task->thread.sve_vl)
613 goto out;
614
615
616
617
618
619
620 if (task == current) {
621 get_cpu_fpsimd_context();
622
623 fpsimd_save();
624 }
625
626 fpsimd_flush_task_state(task);
627 if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
628 sve_to_fpsimd(task);
629
630 if (task == current)
631 put_cpu_fpsimd_context();
632
633
634
635
636
637 sve_free(task);
638
639 task->thread.sve_vl = vl;
640
641out:
642 update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
643 flags & PR_SVE_VL_INHERIT);
644
645 return 0;
646}
647
648
649
650
651
652
653
654static int sve_prctl_status(unsigned long flags)
655{
656 int ret;
657
658 if (flags & PR_SVE_SET_VL_ONEXEC)
659 ret = current->thread.sve_vl_onexec;
660 else
661 ret = current->thread.sve_vl;
662
663 if (test_thread_flag(TIF_SVE_VL_INHERIT))
664 ret |= PR_SVE_VL_INHERIT;
665
666 return ret;
667}
668
669
670int sve_set_current_vl(unsigned long arg)
671{
672 unsigned long vl, flags;
673 int ret;
674
675 vl = arg & PR_SVE_VL_LEN_MASK;
676 flags = arg & ~vl;
677
678 if (!system_supports_sve() || is_compat_task())
679 return -EINVAL;
680
681 ret = sve_set_vector_length(current, vl, flags);
682 if (ret)
683 return ret;
684
685 return sve_prctl_status(flags);
686}
687
688
689int sve_get_current_vl(void)
690{
691 if (!system_supports_sve() || is_compat_task())
692 return -EINVAL;
693
694 return sve_prctl_status(0);
695}
696
697static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
698{
699 unsigned int vq, vl;
700 unsigned long zcr;
701
702 bitmap_zero(map, SVE_VQ_MAX);
703
704 zcr = ZCR_ELx_LEN_MASK;
705 zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
706
707 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
708 write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1);
709 vl = sve_get_vl();
710 vq = sve_vq_from_vl(vl);
711 set_bit(__vq_to_bit(vq), map);
712 }
713}
714
715
716
717
718
719void __init sve_init_vq_map(void)
720{
721 sve_probe_vqs(sve_vq_map);
722 bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
723}
724
725
726
727
728
729
730void sve_update_vq_map(void)
731{
732 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
733
734 sve_probe_vqs(tmp_map);
735 bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
736 bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
737}
738
739
740
741
742
743int sve_verify_vq_map(void)
744{
745 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
746 unsigned long b;
747
748 sve_probe_vqs(tmp_map);
749
750 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
751 if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
752 pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
753 smp_processor_id());
754 return -EINVAL;
755 }
756
757 if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
758 return 0;
759
760
761
762
763
764
765
766
767 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
768
769 bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
770
771
772 b = find_last_bit(tmp_map, SVE_VQ_MAX);
773 if (b >= SVE_VQ_MAX)
774 return 0;
775
776
777
778
779
780 if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
781 pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
782 smp_processor_id());
783 return -EINVAL;
784 }
785
786 return 0;
787}
788
789static void __init sve_efi_setup(void)
790{
791 if (!IS_ENABLED(CONFIG_EFI))
792 return;
793
794
795
796
797
798
799 if (!sve_vl_valid(sve_max_vl))
800 goto fail;
801
802 efi_sve_state = __alloc_percpu(
803 SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
804 if (!efi_sve_state)
805 goto fail;
806
807 return;
808
809fail:
810 panic("Cannot allocate percpu memory for EFI SVE save/restore");
811}
812
813
814
815
816
817void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
818{
819 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
820 isb();
821}
822
823
824
825
826
827
828
829
830u64 read_zcr_features(void)
831{
832 u64 zcr;
833 unsigned int vq_max;
834
835
836
837
838
839 sve_kernel_enable(NULL);
840 write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
841
842 zcr = read_sysreg_s(SYS_ZCR_EL1);
843 zcr &= ~(u64)ZCR_ELx_LEN_MASK;
844 vq_max = sve_vq_from_vl(sve_get_vl());
845 zcr |= vq_max - 1;
846
847 return zcr;
848}
849
850void __init sve_setup(void)
851{
852 u64 zcr;
853 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
854 unsigned long b;
855
856 if (!system_supports_sve())
857 return;
858
859
860
861
862
863
864 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
865 set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
866
867 zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
868 sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
869
870
871
872
873
874 if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))
875 sve_max_vl = find_supported_vector_length(sve_max_vl);
876
877
878
879
880
881 set_sve_default_vl(find_supported_vector_length(64));
882
883 bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
884 SVE_VQ_MAX);
885
886 b = find_last_bit(tmp_map, SVE_VQ_MAX);
887 if (b >= SVE_VQ_MAX)
888
889 sve_max_virtualisable_vl = SVE_VQ_MAX;
890 else if (WARN_ON(b == SVE_VQ_MAX - 1))
891
892 sve_max_virtualisable_vl = SVE_VQ_MIN;
893 else
894 sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
895
896 if (sve_max_virtualisable_vl > sve_max_vl)
897 sve_max_virtualisable_vl = sve_max_vl;
898
899 pr_info("SVE: maximum available vector length %u bytes per vector\n",
900 sve_max_vl);
901 pr_info("SVE: default vector length %u bytes per vector\n",
902 get_sve_default_vl());
903
904
905 if (sve_max_virtualisable_vl < sve_max_vl)
906 pr_warn("SVE: unvirtualisable vector lengths present\n");
907
908 sve_efi_setup();
909}
910
911
912
913
914
915void fpsimd_release_task(struct task_struct *dead_task)
916{
917 __sve_free(dead_task);
918}
919
920#endif
921
922
923
924
925
926
927
928
929
930
931
932
933void do_sve_acc(unsigned int esr, struct pt_regs *regs)
934{
935
936 if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
937 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
938 return;
939 }
940
941 sve_alloc(current);
942 if (!current->thread.sve_state) {
943 force_sig(SIGKILL);
944 return;
945 }
946
947 get_cpu_fpsimd_context();
948
949 if (test_and_set_thread_flag(TIF_SVE))
950 WARN_ON(1);
951
952
953
954
955
956
957
958
959 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
960 unsigned long vq_minus_one =
961 sve_vq_from_vl(current->thread.sve_vl) - 1;
962 sve_set_vq(vq_minus_one);
963 sve_flush_live(vq_minus_one);
964 fpsimd_bind_task_to_cpu();
965 } else {
966 fpsimd_to_sve(current);
967 }
968
969 put_cpu_fpsimd_context();
970}
971
972
973
974
975void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
976{
977
978 WARN_ON(1);
979}
980
981
982
983
984void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
985{
986 unsigned int si_code = FPE_FLTUNK;
987
988 if (esr & ESR_ELx_FP_EXC_TFV) {
989 if (esr & FPEXC_IOF)
990 si_code = FPE_FLTINV;
991 else if (esr & FPEXC_DZF)
992 si_code = FPE_FLTDIV;
993 else if (esr & FPEXC_OFF)
994 si_code = FPE_FLTOVF;
995 else if (esr & FPEXC_UFF)
996 si_code = FPE_FLTUND;
997 else if (esr & FPEXC_IXF)
998 si_code = FPE_FLTRES;
999 }
1000
1001 send_sig_fault(SIGFPE, si_code,
1002 (void __user *)instruction_pointer(regs),
1003 current);
1004}
1005
1006void fpsimd_thread_switch(struct task_struct *next)
1007{
1008 bool wrong_task, wrong_cpu;
1009
1010 if (!system_supports_fpsimd())
1011 return;
1012
1013 __get_cpu_fpsimd_context();
1014
1015
1016 fpsimd_save();
1017
1018
1019
1020
1021
1022
1023 wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
1024 &next->thread.uw.fpsimd_state;
1025 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
1026
1027 update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
1028 wrong_task || wrong_cpu);
1029
1030 __put_cpu_fpsimd_context();
1031}
1032
1033void fpsimd_flush_thread(void)
1034{
1035 int vl, supported_vl;
1036
1037 if (!system_supports_fpsimd())
1038 return;
1039
1040 get_cpu_fpsimd_context();
1041
1042 fpsimd_flush_task_state(current);
1043 memset(¤t->thread.uw.fpsimd_state, 0,
1044 sizeof(current->thread.uw.fpsimd_state));
1045
1046 if (system_supports_sve()) {
1047 clear_thread_flag(TIF_SVE);
1048 sve_free(current);
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061 vl = current->thread.sve_vl_onexec ?
1062 current->thread.sve_vl_onexec : get_sve_default_vl();
1063
1064 if (WARN_ON(!sve_vl_valid(vl)))
1065 vl = SVE_VL_MIN;
1066
1067 supported_vl = find_supported_vector_length(vl);
1068 if (WARN_ON(supported_vl != vl))
1069 vl = supported_vl;
1070
1071 current->thread.sve_vl = vl;
1072
1073
1074
1075
1076
1077 if (!test_thread_flag(TIF_SVE_VL_INHERIT))
1078 current->thread.sve_vl_onexec = 0;
1079 }
1080
1081 put_cpu_fpsimd_context();
1082}
1083
1084
1085
1086
1087
1088void fpsimd_preserve_current_state(void)
1089{
1090 if (!system_supports_fpsimd())
1091 return;
1092
1093 get_cpu_fpsimd_context();
1094 fpsimd_save();
1095 put_cpu_fpsimd_context();
1096}
1097
1098
1099
1100
1101
1102
1103void fpsimd_signal_preserve_current_state(void)
1104{
1105 fpsimd_preserve_current_state();
1106 if (test_thread_flag(TIF_SVE))
1107 sve_to_fpsimd(current);
1108}
1109
1110
1111
1112
1113
1114
1115static void fpsimd_bind_task_to_cpu(void)
1116{
1117 struct fpsimd_last_state_struct *last =
1118 this_cpu_ptr(&fpsimd_last_state);
1119
1120 WARN_ON(!system_supports_fpsimd());
1121 last->st = ¤t->thread.uw.fpsimd_state;
1122 last->sve_state = current->thread.sve_state;
1123 last->sve_vl = current->thread.sve_vl;
1124 current->thread.fpsimd_cpu = smp_processor_id();
1125
1126 if (system_supports_sve()) {
1127
1128 if (test_thread_flag(TIF_SVE))
1129 sve_user_enable();
1130 else
1131 sve_user_disable();
1132
1133
1134 }
1135}
1136
1137void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1138 unsigned int sve_vl)
1139{
1140 struct fpsimd_last_state_struct *last =
1141 this_cpu_ptr(&fpsimd_last_state);
1142
1143 WARN_ON(!system_supports_fpsimd());
1144 WARN_ON(!in_softirq() && !irqs_disabled());
1145
1146 last->st = st;
1147 last->sve_state = sve_state;
1148 last->sve_vl = sve_vl;
1149}
1150
1151
1152
1153
1154
1155
1156void fpsimd_restore_current_state(void)
1157{
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167 if (!system_supports_fpsimd()) {
1168 clear_thread_flag(TIF_FOREIGN_FPSTATE);
1169 return;
1170 }
1171
1172 get_cpu_fpsimd_context();
1173
1174 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
1175 task_fpsimd_load();
1176 fpsimd_bind_task_to_cpu();
1177 }
1178
1179 put_cpu_fpsimd_context();
1180}
1181
1182
1183
1184
1185
1186
1187void fpsimd_update_current_state(struct user_fpsimd_state const *state)
1188{
1189 if (WARN_ON(!system_supports_fpsimd()))
1190 return;
1191
1192 get_cpu_fpsimd_context();
1193
1194 current->thread.uw.fpsimd_state = *state;
1195 if (test_thread_flag(TIF_SVE))
1196 fpsimd_to_sve(current);
1197
1198 task_fpsimd_load();
1199 fpsimd_bind_task_to_cpu();
1200
1201 clear_thread_flag(TIF_FOREIGN_FPSTATE);
1202
1203 put_cpu_fpsimd_context();
1204}
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217void fpsimd_flush_task_state(struct task_struct *t)
1218{
1219 t->thread.fpsimd_cpu = NR_CPUS;
1220
1221
1222
1223
1224
1225 if (!system_supports_fpsimd())
1226 return;
1227 barrier();
1228 set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1229
1230 barrier();
1231}
1232
1233
1234
1235
1236
1237
1238static void fpsimd_flush_cpu_state(void)
1239{
1240 WARN_ON(!system_supports_fpsimd());
1241 __this_cpu_write(fpsimd_last_state.st, NULL);
1242 set_thread_flag(TIF_FOREIGN_FPSTATE);
1243}
1244
1245
1246
1247
1248
1249void fpsimd_save_and_flush_cpu_state(void)
1250{
1251 if (!system_supports_fpsimd())
1252 return;
1253 WARN_ON(preemptible());
1254 __get_cpu_fpsimd_context();
1255 fpsimd_save();
1256 fpsimd_flush_cpu_state();
1257 __put_cpu_fpsimd_context();
1258}
1259
1260#ifdef CONFIG_KERNEL_MODE_NEON
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279void kernel_neon_begin(void)
1280{
1281 if (WARN_ON(!system_supports_fpsimd()))
1282 return;
1283
1284 BUG_ON(!may_use_simd());
1285
1286 get_cpu_fpsimd_context();
1287
1288
1289 fpsimd_save();
1290
1291
1292 fpsimd_flush_cpu_state();
1293}
1294EXPORT_SYMBOL(kernel_neon_begin);
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305void kernel_neon_end(void)
1306{
1307 if (!system_supports_fpsimd())
1308 return;
1309
1310 put_cpu_fpsimd_context();
1311}
1312EXPORT_SYMBOL(kernel_neon_end);
1313
1314#ifdef CONFIG_EFI
1315
1316static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
1317static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
1318static DEFINE_PER_CPU(bool, efi_sve_state_used);
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337void __efi_fpsimd_begin(void)
1338{
1339 if (!system_supports_fpsimd())
1340 return;
1341
1342 WARN_ON(preemptible());
1343
1344 if (may_use_simd()) {
1345 kernel_neon_begin();
1346 } else {
1347
1348
1349
1350
1351 if (system_supports_sve() && likely(efi_sve_state)) {
1352 char *sve_state = this_cpu_ptr(efi_sve_state);
1353
1354 __this_cpu_write(efi_sve_state_used, true);
1355
1356 sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
1357 &this_cpu_ptr(&efi_fpsimd_state)->fpsr);
1358 } else {
1359 fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
1360 }
1361
1362 __this_cpu_write(efi_fpsimd_state_used, true);
1363 }
1364}
1365
1366
1367
1368
1369void __efi_fpsimd_end(void)
1370{
1371 if (!system_supports_fpsimd())
1372 return;
1373
1374 if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
1375 kernel_neon_end();
1376 } else {
1377 if (system_supports_sve() &&
1378 likely(__this_cpu_read(efi_sve_state_used))) {
1379 char const *sve_state = this_cpu_ptr(efi_sve_state);
1380
1381 sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
1382 &this_cpu_ptr(&efi_fpsimd_state)->fpsr,
1383 sve_vq_from_vl(sve_get_vl()) - 1);
1384
1385 __this_cpu_write(efi_sve_state_used, false);
1386 } else {
1387 fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
1388 }
1389 }
1390}
1391
1392#endif
1393
1394#endif
1395
1396#ifdef CONFIG_CPU_PM
1397static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
1398 unsigned long cmd, void *v)
1399{
1400 switch (cmd) {
1401 case CPU_PM_ENTER:
1402 fpsimd_save_and_flush_cpu_state();
1403 break;
1404 case CPU_PM_EXIT:
1405 break;
1406 case CPU_PM_ENTER_FAILED:
1407 default:
1408 return NOTIFY_DONE;
1409 }
1410 return NOTIFY_OK;
1411}
1412
1413static struct notifier_block fpsimd_cpu_pm_notifier_block = {
1414 .notifier_call = fpsimd_cpu_pm_notifier,
1415};
1416
1417static void __init fpsimd_pm_init(void)
1418{
1419 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
1420}
1421
1422#else
1423static inline void fpsimd_pm_init(void) { }
1424#endif
1425
1426#ifdef CONFIG_HOTPLUG_CPU
1427static int fpsimd_cpu_dead(unsigned int cpu)
1428{
1429 per_cpu(fpsimd_last_state.st, cpu) = NULL;
1430 return 0;
1431}
1432
1433static inline void fpsimd_hotplug_init(void)
1434{
1435 cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
1436 NULL, fpsimd_cpu_dead);
1437}
1438
1439#else
1440static inline void fpsimd_hotplug_init(void) { }
1441#endif
1442
1443
1444
1445
1446static int __init fpsimd_init(void)
1447{
1448 if (cpu_have_named_feature(FP)) {
1449 fpsimd_pm_init();
1450 fpsimd_hotplug_init();
1451 } else {
1452 pr_notice("Floating-point is not implemented\n");
1453 }
1454
1455 if (!cpu_have_named_feature(ASIMD))
1456 pr_notice("Advanced SIMD is not implemented\n");
1457
1458 return sve_sysctl_init();
1459}
1460core_initcall(fpsimd_init);
1461