1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/linkage.h>
17#include <asm/asm-offsets.h>
18#include <asm/processor.h>
19#include <asm/thread_info.h>
20#include <asm/uaccess.h>
21#include <asm/unistd.h>
22#include <asm/ptrace.h>
23#include <asm/current.h>
24#include <asm/pgtable.h>
25#include <asm/page.h>
26#include <asm/signal.h>
27#include <asm/tlbflush.h>
28
29
30
31#undef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION
32#undef KERNEL_STACK_OVERFLOW_CHECK
33#undef PREEMPTIBLE_KERNEL
34#undef ALLOCA_EXCEPTION_IN_IRAM
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49 .macro ffs_ws bit mask
50
51
52 nsau \bit, \mask
53 addi \bit, \bit, WSBITS - 32 + 1
54#else
55 movi \bit, WSBITS
56
57 _bltui \mask, 0x10000, 99f
58 addi \bit, \bit, -16
59 extui \mask, \mask, 16, 16
60#endif
61
6299: _bltui \mask, 0x100, 99f
63 addi \bit, \bit, -8
64 srli \mask, \mask, 8
65#endif
6699: _bltui \mask, 0x10, 99f
67 addi \bit, \bit, -4
68 srli \mask, \mask, 4
6999: _bltui \mask, 0x4, 99f
70 addi \bit, \bit, -2
71 srli \mask, \mask, 2
7299: _bltui \mask, 0x2, 99f
73 addi \bit, \bit, -1
7499:
75
76#endif
77 .endm
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110ENTRY(user_exception)
111
112
113
114 xsr a3, EXCSAVE_1
115 rsr a0, DEPC
116 s32i a1, a2, PT_AREG1
117 s32i a0, a2, PT_AREG2
118 s32i a3, a2, PT_AREG3
119 mov a1, a2
120
121 .globl _user_exception
122_user_exception:
123
124
125
126 movi a2, 0
127 rsr a3, SAR
128 xsr a2, ICOUNTLEVEL
129 s32i a3, a1, PT_SAR
130 s32i a2, a1, PT_ICOUNTLEVEL
131
132
133
134
135 rsr a2, WINDOWBASE
136 rsr a3, WINDOWSTART
137 ssr a2
138 s32i a2, a1, PT_WINDOWBASE
139 s32i a3, a1, PT_WINDOWSTART
140 slli a2, a3, 32-WSBITS
141 src a2, a3, a2
142 srli a2, a2, 32-WSBITS
143 s32i a2, a1, PT_WMASK
144
145
146
147 _bbsi.l a2, 1, 1f
148 s32i a4, a1, PT_AREG4
149 s32i a5, a1, PT_AREG5
150 s32i a6, a1, PT_AREG6
151 s32i a7, a1, PT_AREG7
152 _bbsi.l a2, 2, 1f
153 s32i a8, a1, PT_AREG8
154 s32i a9, a1, PT_AREG9
155 s32i a10, a1, PT_AREG10
156 s32i a11, a1, PT_AREG11
157 _bbsi.l a2, 3, 1f
158 s32i a12, a1, PT_AREG12
159 s32i a13, a1, PT_AREG13
160 s32i a14, a1, PT_AREG14
161 s32i a15, a1, PT_AREG15
162 _bnei a2, 1, 1f
163
164
165
166 j 2f
167
168
169
170
171
172
173
174
175
1761: addi a3, a2, -1
177 neg a3, a3
178 and a3, a3, a2
179
180
181
182 ffs_ws a0, a3
183
184
185
186
187
188
189 slli a3, a0, 4
190 extui a2, a2, 0, 4
191 or a2, a3, a2
192 s32i a2, a1, PT_WMASK
193
194
195
1961: rotw -1
197 s32i a0, a5, PT_AREG_END - 16
198 s32i a1, a5, PT_AREG_END - 12
199 s32i a2, a5, PT_AREG_END - 8
200 s32i a3, a5, PT_AREG_END - 4
201 addi a0, a4, -1
202 addi a1, a5, -16
203 _bnez a0, 1b
204
205
206
207 rsr a2, SAR
208 movi a3, 1
209 ssl a2
210 sll a3, a3
211 wsr a3, WINDOWSTART
212 wsr a2, WINDOWBASE
213 rsync
214
215
216
2172:
218
219
220
221
222
223
224 GET_CURRENT(a2,a1)
225 addi a2, a2, THREAD_CP_SAVE
226 xchal_extra_store_funcbody
227#endif
228
229
230
231 j common_exception
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262ENTRY(kernel_exception)
263
264
265
266 xsr a3, EXCSAVE_1
267 rsr a0, DEPC
268 s32i a1, a2, PT_AREG1
269 s32i a0, a2, PT_AREG2
270 s32i a3, a2, PT_AREG3
271 mov a1, a2
272
273 .globl _kernel_exception
274_kernel_exception:
275
276
277
278 movi a2, 0
279 rsr a3, SAR
280 xsr a2, ICOUNTLEVEL
281 s32i a3, a1, PT_SAR
282 s32i a2, a1, PT_ICOUNTLEVEL
283
284
285
286
287 rsr a2, WINDOWBASE
288 rsr a3, WINDOWSTART
289 ssr a2
290 slli a2, a3, 32-WSBITS
291 src a2, a3, a2
292 srli a2, a2, 32-WSBITS
293 s32i a2, a1, PT_WMASK
294
295
296
297 _bbsi.l a2, 1, 1f
298 s32i a4, a1, PT_AREG4
299 s32i a5, a1, PT_AREG5
300 s32i a6, a1, PT_AREG6
301 s32i a7, a1, PT_AREG7
302 _bbsi.l a2, 2, 1f
303 s32i a8, a1, PT_AREG8
304 s32i a9, a1, PT_AREG9
305 s32i a10, a1, PT_AREG10
306 s32i a11, a1, PT_AREG11
307 _bbsi.l a2, 3, 1f
308 s32i a12, a1, PT_AREG12
309 s32i a13, a1, PT_AREG13
310 s32i a14, a1, PT_AREG14
311 s32i a15, a1, PT_AREG15
312
3131:
314
315#ifdef KERNEL_STACK_OVERFLOW_CHECK
316
317
318 extui a2, a1, TASK_SIZE_BITS,XX
319 movi a3, SIZE??
320 _bge a2, a3, out_of_stack_panic
321
322#endif
323
324
325
326
327
328
329
330
331
332
333common_exception:
334
335
336
337 rsr a2, DEBUGCAUSE
338 rsr a3, EPC_1
339 s32i a2, a1, PT_DEBUGCAUSE
340 s32i a3, a1, PT_PC
341
342 movi a2, -1
343 rsr a3, EXCVADDR
344 s32i a2, a1, PT_SYSCALL
345 movi a2, 0
346 s32i a3, a1, PT_EXCVADDR
347 xsr a2, LCOUNT
348 s32i a2, a1, PT_LCOUNT
349
350
351
352 rsr a0, EXCCAUSE
353 movi a3, 0
354 rsr a2, EXCSAVE_1
355 s32i a0, a1, PT_EXCCAUSE
356 s32i a3, a2, EXC_TABLE_FIXUP
357
358
359
360
361
362
363
364
365
366 rsr a3, PS
367 addi a0, a0, -4
368 movi a2, 1
369 extui a3, a3, 0, 1
370 moveqz a3, a2, a0
371 movi a2, 1 << PS_WOE_BIT
372 or a3, a3, a2
373 rsr a0, EXCCAUSE
374 xsr a3, PS
375
376 s32i a3, a1, PT_PS
377
378
379
380 rsr a2, LBEG
381 rsr a3, LEND
382 s32i a2, a1, PT_LBEG
383 s32i a3, a1, PT_LEND
384
385
386
387
388
389 movi a4, exc_table
390 mov a6, a1
391 mov a7, a0
392 addx4 a4, a0, a4
393 l32i a4, a4, EXC_TABLE_DEFAULT
394
395
396
397 callx4 a4
398
399
400
401common_exception_return:
402
403
404
4051: l32i a3, a1, PT_PS
406 _bbsi.l a3, PS_UM_BIT, 2f
407 j kernel_exception_exit
408
409
410
411
412
413
414
4152: wsr a3, PS
416
417
418
419
420
421 GET_THREAD_INFO(a2,a1)
422 l32i a4, a2, TI_FLAGS
423
424
425
426
427
428
429 movi a3, 1 << PS_WOE_BIT
430
431 _bbsi.l a4, TIF_NEED_RESCHED, 3f
432 _bbci.l a4, TIF_SIGPENDING, 4f
433
434#ifndef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION
435 l32i a4, a1, PT_DEPC
436 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
437#endif
438
439
440
441 wsr a3, PS
442 movi a4, do_signal
443 mov a6, a1
444 movi a7, 0
445 callx4 a4
446 j 1b
447
4483:
449
450 wsr a3, PS
451 movi a4, schedule
452 callx4 a4
453 j 1b
454
455
456
4574:
458
459
460
461
462
463
464 addi a2, a2, THREAD_CP_SAVE
465 xchal_extra_load_funcbody
466
467
468
469
470
471
472#endif
473
474
475
476
477 l32i a2, a1, PT_WINDOWBASE
478 l32i a3, a1, PT_WINDOWSTART
479 wsr a1, DEPC
480 wsr a3, WINDOWSTART
481 ssr a2
482 wsr a2, WINDOWBASE
483 rsync
484 rsr a1, DEPC
485 l32i a2, a1, PT_WMASK
486 rotw -1
487 _bltui a6, 16, 1f
488
489
490
491
492
493
494
495
496 mov a2, a6
497 mov a3, a5
498
4992: rotw -1
500 addi a3, a7, -4*4
501 addi a2, a6, -16
502 l32i a4, a3, PT_AREG_END + 0
503 l32i a5, a3, PT_AREG_END + 4
504 l32i a6, a3, PT_AREG_END + 8
505 l32i a7, a3, PT_AREG_END + 12
506 _bgeui a2, 16, 2b
507
508
509
5101: rsr a0, WINDOWBASE
511 rsr a3, SAR
512 sub a3, a0, a3
513 beqz a3, 2f
514 extui a3, a3, 0, WBBITS
515
5161: rotw -1
517 addi a3, a7, -1
518 movi a4, 0
519 movi a5, 0
520 movi a6, 0
521 movi a7, 0
522 bgei a3, 1, 1b
523
524
525
526
527
528
529
5302: j common_exception_exit
531
532
533
534
535
536
537kernel_exception_exit:
538
539
540
541 wsr a3, PS
542
543#ifdef PREEMPTIBLE_KERNEL
544
545#ifdef CONFIG_PREEMPT
546
547
548
549
550
551
552
553
554 GET_THREAD_INFO(a2)
555 l32i a3, a2, TI_PREEMPT
556 bnez a3, 1f
557
558 l32i a2, a2, TI_FLAGS
559
5601:
561
562#endif
563
564#endif
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590 l32i a2, a1, PT_WMASK
591 _beqi a2, 1, common_exception_exit
592
593
594
595 rsr a3, WINDOWSTART
596 addi a0, a3, -1
597 and a3, a3, a0
598 _bnez a3, common_exception_exit
599
600
601
602 addi a0, a1, -16
603 l32i a3, a0, 0
604 l32i a4, a0, 4
605 s32i a3, a1, PT_SIZE+0
606 s32i a4, a1, PT_SIZE+4
607 l32i a3, a0, 8
608 l32i a4, a0, 12
609 s32i a3, a1, PT_SIZE+8
610 s32i a4, a1, PT_SIZE+12
611
612
613
614
615
616
617
618
619common_exception_exit:
620
621 _bbsi.l a2, 1, 1f
622 l32i a4, a1, PT_AREG4
623 l32i a5, a1, PT_AREG5
624 l32i a6, a1, PT_AREG6
625 l32i a7, a1, PT_AREG7
626 _bbsi.l a2, 2, 1f
627 l32i a8, a1, PT_AREG8
628 l32i a9, a1, PT_AREG9
629 l32i a10, a1, PT_AREG10
630 l32i a11, a1, PT_AREG11
631 _bbsi.l a2, 3, 1f
632 l32i a12, a1, PT_AREG12
633 l32i a13, a1, PT_AREG13
634 l32i a14, a1, PT_AREG14
635 l32i a15, a1, PT_AREG15
636
637
638
6391: l32i a2, a1, PT_PC
640 l32i a3, a1, PT_SAR
641 wsr a2, EPC_1
642 wsr a3, SAR
643
644
645
646 l32i a2, a1, PT_LBEG
647 l32i a3, a1, PT_LEND
648 wsr a2, LBEG
649 l32i a2, a1, PT_LCOUNT
650 wsr a3, LEND
651 wsr a2, LCOUNT
652
653
654
655 l32i a2, a1, PT_ICOUNTLEVEL
656 movi a3, -2
657 wsr a2, ICOUNTLEVEL
658 wsr a3, ICOUNT
659
660
661
662 l32i a0, a1, PT_DEPC
663 l32i a3, a1, PT_AREG3
664 l32i a2, a1, PT_AREG2
665 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
666
667
668
669 l32i a0, a1, PT_AREG0
670 l32i a1, a1, PT_AREG1
671 rfe
672
6731: wsr a0, DEPC
674 l32i a0, a1, PT_AREG0
675 l32i a1, a1, PT_AREG1
676 rfde
677
678
679
680
681
682
683
684
685
686ENTRY(debug_exception)
687
688 rsr a0, EPS + XCHAL_DEBUGLEVEL
689 bbsi.l a0, PS_EXCM_BIT, 1f
690
691
692
693 wsr a2, DEPC
694 rsr a2, EPC + XCHAL_DEBUGLEVEL
695 wsr a2, EPC_1
696
697 movi a2, EXCCAUSE_MAPPED_DEBUG
698 wsr a2, EXCCAUSE
699
700
701
702 movi a2, 1 << PS_EXCM_BIT
703 or a2, a0, a2
704 movi a0, debug_exception
705 wsr a2, PS
706 xsr a0, EXCSAVE + XCHAL_DEBUGLEVEL
707
708
709
710 bbsi.l a2, PS_UM_BIT, 2f
711
712 addi a2, a1, -16-PT_SIZE
713 s32i a0, a2, PT_AREG0
714 movi a0, 0
715 s32i a1, a2, PT_AREG1
716 s32i a0, a2, PT_DEPC
717 xsr a0, DEPC
718 s32i a3, a2, PT_AREG3
719 s32i a0, a2, PT_AREG2
720 mov a1, a2
721 j _kernel_exception
722
7232: rsr a2, EXCSAVE_1
724 l32i a2, a2, EXC_TABLE_KSTK
725 s32i a0, a2, PT_AREG0
726 movi a0, 0
727 s32i a1, a2, PT_AREG1
728 s32i a0, a2, PT_DEPC
729 xsr a0, DEPC
730 s32i a3, a2, PT_AREG3
731 s32i a0, a2, PT_AREG2
732 mov a1, a2
733 j _user_exception
734
735
7361: j 1b
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760 .align 4
761unrecoverable_text:
762 .ascii "Unrecoverable error in exception handler\0"
763
764ENTRY(unrecoverable_exception)
765
766 movi a0, 1
767 movi a1, 0
768
769 wsr a0, WINDOWSTART
770 wsr a1, WINDOWBASE
771 rsync
772
773 movi a1, (1 << PS_WOE_BIT) | 1
774 wsr a1, PS
775 rsync
776
777 movi a1, init_task
778 movi a0, 0
779 addi a1, a1, PT_REGS_OFFSET
780
781 movi a4, panic
782 movi a6, unrecoverable_text
783
784 callx4 a4
785
7861: j 1b
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 4, 4
817#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 0, 4
818#else
819#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 0, 4
820#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 4, 4
821#endif
822
823ENTRY(fast_alloca)
824
825
826
827 l32i a0, a2, PT_DEPC
828 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, .Lunhandled_double
829
830 rsr a0, DEPC
831 s32i a4, a2, PT_AREG4
832 s32i a0, a2, PT_AREG2
833
834
835
836 movi a0, 0
837 s32i a0, a3, EXC_TABLE_FIXUP
838
839
840
841 xsr a3, EXCSAVE_1
842 rsr a4, EPC_1
843 s32i a3, a2, PT_AREG3
844
845#ifdef ALLOCA_EXCEPTION_IN_IRAM
846
847#else
848
849 l8ui a0, a4, 1
850#endif
851 movi a3, .Lmovsp_src
852 _EXTUI_MOVSP_SRC(a0)
853 addx8 a3, a0, a3
854 jx a3
855
856.Lunhandled_double:
857 wsr a0, EXCSAVE_1
858 movi a0, unrecoverable_exception
859 callx0 a0
860
861 .align 8
862.Lmovsp_src:
863 l32i a3, a2, PT_AREG0; _j 1f; .align 8
864 mov a3, a1; _j 1f; .align 8
865 l32i a3, a2, PT_AREG2; _j 1f; .align 8
866 l32i a3, a2, PT_AREG3; _j 1f; .align 8
867 l32i a3, a2, PT_AREG4; _j 1f; .align 8
868 mov a3, a5; _j 1f; .align 8
869 mov a3, a6; _j 1f; .align 8
870 mov a3, a7; _j 1f; .align 8
871 mov a3, a8; _j 1f; .align 8
872 mov a3, a9; _j 1f; .align 8
873 mov a3, a10; _j 1f; .align 8
874 mov a3, a11; _j 1f; .align 8
875 mov a3, a12; _j 1f; .align 8
876 mov a3, a13; _j 1f; .align 8
877 mov a3, a14; _j 1f; .align 8
878 mov a3, a15; _j 1f; .align 8
879
8801:
881
882#ifdef ALLOCA_EXCEPTION_IN_IRAM
883
884#else
885 l8ui a0, a4, 0
886#endif
887 addi a4, a4, 3
888 _EXTUI_MOVSP_DST(a0)
889 wsr a4, EPC_1
890
891 _bnei a0, 1, 1f
892
893
894
895
896
897
898
899
900
901
902
903 l32e a0, a1, -16
904 l32e a4, a1, -12
905 s32e a0, a3, -16
906 s32e a4, a3, -12
907 l32e a0, a1, -8
908 l32e a4, a1, -4
909 s32e a0, a3, -8
910 s32e a4, a3, -4
911
912
913
914 mov a1, a3
915
916 l32i a4, a2, PT_AREG4
917 l32i a3, a2, PT_AREG3
918 l32i a0, a2, PT_AREG0
919 l32i a2, a2, PT_AREG2
920 rfe
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
9371: movi a4, .Lmovsp_dst
938 addx8 a4, a0, a4
939 jx a4
940
941 .align 8
942.Lmovsp_dst:
943 s32i a3, a2, PT_AREG0; _j 1f; .align 8
944 mov a1, a3; _j 1f; .align 8
945 s32i a3, a2, PT_AREG2; _j 1f; .align 8
946 s32i a3, a2, PT_AREG3; _j 1f; .align 8
947 s32i a3, a2, PT_AREG4; _j 1f; .align 8
948 mov a5, a3; _j 1f; .align 8
949 mov a6, a3; _j 1f; .align 8
950 mov a7, a3; _j 1f; .align 8
951 mov a8, a3; _j 1f; .align 8
952 mov a9, a3; _j 1f; .align 8
953 mov a10, a3; _j 1f; .align 8
954 mov a11, a3; _j 1f; .align 8
955 mov a12, a3; _j 1f; .align 8
956 mov a13, a3; _j 1f; .align 8
957 mov a14, a3; _j 1f; .align 8
958 mov a15, a3; _j 1f; .align 8
959
9601: l32i a4, a2, PT_AREG4
961 l32i a3, a2, PT_AREG3
962 l32i a0, a2, PT_AREG0
963 l32i a2, a2, PT_AREG2
964 rfe
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986ENTRY(fast_syscall_kernel)
987
988
989
990 rsr a0, EPC_1
991 addi a0, a0, 3
992 wsr a0, EPC_1
993
994 l32i a0, a2, PT_DEPC
995 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
996
997 rsr a0, DEPC
998 _beqz a0, fast_syscall_spill_registers
999 _beqi a0, __NR_xtensa, fast_syscall_xtensa
1000
1001 j kernel_exception
1002
1003ENTRY(fast_syscall_user)
1004
1005
1006
1007 rsr a0, EPC_1
1008 addi a0, a0, 3
1009 wsr a0, EPC_1
1010
1011 l32i a0, a2, PT_DEPC
1012 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
1013
1014 rsr a0, DEPC
1015 _beqz a0, fast_syscall_spill_registers
1016 _beqi a0, __NR_xtensa, fast_syscall_xtensa
1017
1018 j user_exception
1019
1020ENTRY(fast_syscall_unrecoverable)
1021
1022
1023
1024 l32i a0, a2, PT_AREG0
1025 xsr a2, DEPC
1026 rsr a3, EXCSAVE_1
1027
1028 wsr a0, EXCSAVE_1
1029 movi a0, unrecoverable_exception
1030 callx0 a0
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073#define TRY \
1074 .section __ex_table, "a"; \
1075 .word 66f, 67f; \
1076 .text; \
107766:
1078
1079#define CATCH \
108067:
1081
1082ENTRY(fast_syscall_xtensa)
1083
1084 xsr a3, EXCSAVE_1
1085
1086 s32i a7, a2, PT_AREG7
1087 movi a7, 4
1088 access_ok a3, a7, a0, a2, .Leac
1089
1090 addi a6, a6, -1
1091 _bgeui a6, SYS_XTENSA_COUNT - 1, .Lill
1092 _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP - 1, .Lnswp
1093
1094
1095
1096.Lswp:
1097
1098TRY l32i a0, a3, 0
1099 bne a0, a4, 1f
1100TRY s32i a5, a3, 0
1101 l32i a7, a2, PT_AREG7
1102 l32i a0, a2, PT_AREG0
1103 movi a2, 1
1104 addi a6, a6, 1
1105 rfe
1106
11071: l32i a7, a2, PT_AREG7
1108 l32i a0, a2, PT_AREG0
1109 movi a2, 0
1110 addi a6, a6, 1
1111 rfe
1112
1113.Lnswp:
1114
1115TRY l32i a7, a3, 0
1116 add a0, a4, a7
1117 moveqz a0, a4, a6
1118TRY s32i a0, a3, 0
1119
1120 mov a0, a2
1121 mov a2, a7
1122 l32i a7, a0, PT_AREG7
1123 l32i a0, a0, PT_AREG0
1124 addi a6, a6, 1
1125 rfe
1126
1127CATCH
1128.Leac: l32i a7, a2, PT_AREG7
1129 l32i a0, a2, PT_AREG0
1130 movi a2, -EFAULT
1131 rfe
1132
1133.Lill: l32i a7, a2, PT_AREG0
1134 l32i a0, a2, PT_AREG0
1135 movi a2, -EINVAL
1136 rfe
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156ENTRY(fast_syscall_spill_registers)
1157
1158
1159
1160 movi a0, fast_syscall_spill_registers_fixup
1161 s32i a0, a3, EXC_TABLE_FIXUP
1162 rsr a0, WINDOWBASE
1163 s32i a0, a3, EXC_TABLE_PARAM
1164
1165
1166
1167 rsr a0, SAR
1168 xsr a3, EXCSAVE_1
1169 s32i a0, a2, PT_AREG4
1170 s32i a3, a2, PT_AREG3
1171
1172
1173
1174 s32i a7, a2, PT_AREG5
1175 s32i a11, a2, PT_AREG6
1176 s32i a15, a2, PT_AREG7
1177
1178 call0 _spill_registers
1179
1180
1181
1182 l32i a3, a2, PT_AREG4
1183 l32i a0, a2, PT_AREG0
1184 wsr a3, SAR
1185 l32i a3, a2, PT_AREG3
1186
1187
1188
1189 l32i a7, a2, PT_AREG5
1190 l32i a11, a2, PT_AREG6
1191 l32i a15, a2, PT_AREG7
1192
1193 movi a2, 0
1194 rfe
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207fast_syscall_spill_registers_fixup:
1208
1209 rsr a2, WINDOWBASE
1210 xsr a0, DEPC
1211 ssl a2
1212
1213
1214
1215
1216
1217
1218 rsr a3, EXCSAVE_1
1219 slli a2, a3, 1
1220
1221 slli a3, a2, 32-WSBITS
1222 src a2, a2, a3
1223 wsr a2, WINDOWSTART
1224
1225 movi a3, exc_table
1226 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
1227 l32i a3, a3, EXC_TABLE_PARAM
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237 wsr a3, WINDOWBASE
1238 rsync
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250#ifdef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION
1251
1252
1253
1254
1255 l32i a7, a2, PT_AREG5
1256 l32i a11, a2, PT_AREG6
1257 l32i a15, a2, PT_AREG7
1258#endif
1259
1260
1261
1262 addi a2, a2, -PT_USER_SIZE
1263 s32i a0, a2, PT_AREG0
1264
1265
1266
1267 movi a3, fast_syscall_spill_registers_fixup_return
1268 s32i a3, a2, PT_DEPC
1269
1270
1271
1272 movi a3, exc_table
1273 rsr a0, EXCCAUSE
1274 addx4 a0, a0, a3
1275 l32i a0, a0, EXC_TABLE_FAST_USER
1276 jx a0
1277
1278fast_syscall_spill_registers_fixup_return:
1279
1280
1281
1282 wsr a2, DEPC
1283
1284
1285
1286 xsr a3, EXCSAVE_1
1287 movi a2, fast_syscall_spill_registers_fixup
1288 s32i a2, a3, EXC_TABLE_FIXUP
1289 rsr a2, WINDOWBASE
1290 s32i a2, a3, EXC_TABLE_PARAM
1291 l32i a2, a3, EXC_TABLE_KSTK
1292
1293#ifdef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION
1294
1295
1296 s32i a7, a2, PT_AREG5
1297 s32i a11, a2, PT_AREG6
1298 s32i a15, a2, PT_AREG7
1299#endif
1300
1301
1302
1303 rsr a3, SAR
1304 neg a3, a3
1305 wsr a3, WINDOWBASE
1306 rsync
1307
1308
1309
1310 movi a3, exc_table
1311 xsr a3, EXCSAVE_1
1312
1313 rfde
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329ENTRY(_spill_registers)
1330
1331
1332
1333
1334
1335
1336
1337 wsr a2, DEPC
1338 rsr a2, WINDOWBASE
1339 rsr a3, WINDOWSTART
1340 ssr a2
1341 slli a2, a3, WSBITS
1342 or a3, a3, a2
1343 srl a3, a3
1344
1345
1346
1347 extui a3, a3, 1, WSBITS-2
1348 movi a2, (1 << (WSBITS-1))
1349 _beqz a3, .Lnospill
1350
1351
1352
1353 or a3, a3, a2
1354
1355
1356
1357 wsr a3, WINDOWSTART
1358 neg a2, a3
1359 and a3, a2, a3
1360
1361 ffs_ws a2, a3
1362 movi a3, WSBITS
1363 sub a2, a3, a2
1364 ssr a2
1365
1366 rsr a3, WINDOWBASE
1367 add a3, a3, a2
1368 rsr a2, DEPC
1369 wsr a3, WINDOWBASE
1370 rsync
1371
1372 rsr a3, WINDOWSTART
1373 srl a3, a3
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383 _bbsi.l a3, 1, .Lc4
1384 _bbsi.l a3, 2, .Lc8
1385
1386
1387
1388 _bbci.l a3, 3, .Lc12
1389
1390 s32e a4, a1, -16
1391 l32e a4, a5, -12
1392 s32e a8, a4, -48
1393 mov a8, a4
1394 l32e a4, a1, -16
1395 j .Lc12c
1396
1397.Lloop: _bbsi.l a3, 1, .Lc4
1398 _bbci.l a3, 2, .Lc12
1399
1400.Lc8: s32e a4, a13, -16
1401 l32e a4, a5, -12
1402 s32e a8, a4, -32
1403 s32e a5, a13, -12
1404 s32e a6, a13, -8
1405 s32e a7, a13, -4
1406 s32e a9, a4, -28
1407 s32e a10, a4, -24
1408 s32e a11, a4, -20
1409
1410 srli a11, a3, 2
1411 rotw 2
1412 _bnei a3, 1, .Lloop
1413
1414.Lexit:
1415
1416 rotw 1
1417 rsr a3, WINDOWBASE
1418 ssl a3
1419 movi a3, 1
1420 sll a3, a3
1421 wsr a3, WINDOWSTART
1422
1423.Lnospill:
1424 jx a0
1425
1426.Lc4: s32e a4, a9, -16
1427 s32e a5, a9, -12
1428 s32e a6, a9, -8
1429 s32e a7, a9, -4
1430
1431 srli a7, a3, 1
1432 rotw 1
1433 _bnei a3, 1, .Lloop
1434 j .Lexit
1435
1436.Lc12: _bbci.l a3, 3, .Linvalid_mask
1437
1438
1439
1440 l32e a2, a5, -12
1441 s32e a8, a2, -48
1442 mov a8, a2
1443
1444.Lc12c: s32e a9, a8, -44
1445 s32e a10, a8, -40
1446 s32e a11, a8, -36
1447 s32e a12, a8, -32
1448 s32e a13, a8, -28
1449 s32e a14, a8, -24
1450 s32e a15, a8, -20
1451 srli a15, a3, 3
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463 rotw 1
1464 mov a5, a13
1465 rotw -1
1466
1467 s32e a4, a9, -16
1468 s32e a5, a9, -12
1469 s32e a6, a9, -8
1470 s32e a7, a9, -4
1471
1472 rotw 3
1473
1474 _beqi a3, 1, .Lexit
1475 j .Lloop
1476
1477.Linvalid_mask:
1478
1479
1480
1481
1482
1483
1484 rsr a0, PS
1485 _bbci.l a0, PS_UM_BIT, 1f
1486
1487
1488
1489
1490
1491 movi a0, 1
1492 movi a1, 0
1493
1494 wsr a0, WINDOWSTART
1495 wsr a1, WINDOWBASE
1496 rsync
1497
1498 movi a0, 0
1499
1500 movi a3, exc_table
1501 l32i a1, a3, EXC_TABLE_KSTK
1502 wsr a3, EXCSAVE_1
1503
1504 movi a4, (1 << PS_WOE_BIT) | 1
1505 wsr a4, PS
1506 rsync
1507
1508 movi a6, SIGSEGV
1509 movi a4, do_exit
1510 callx4 a4
1511
15121:
1513
1514 wsr a0, EXCSAVE_1
1515 movi a0, unrecoverable_exception
1516 callx0 a0
15171: j 1b
1518
1519
1520
1521
1522
1523ENTRY(fast_second_level_miss_double_kernel)
1524
15251: movi a0, unrecoverable_exception
1526 callx0 a0
15271: j 1b
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549ENTRY(fast_second_level_miss)
1550
1551
1552
1553 s32i a1, a2, PT_AREG1
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571 GET_CURRENT(a1,a2)
1572 l32i a0, a1, TASK_MM
1573 beqz a0, 9f
1574
1575
1576
1577
15788: rsr a3, EXCVADDR
1579 _PGD_OFFSET(a0, a3, a1)
1580 l32i a0, a0, 0
1581 beqz a0, 2f
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596 movi a1, -PAGE_OFFSET
1597 add a0, a0, a1
1598 extui a1, a0, 0, PAGE_SHIFT
1599 xor a0, a0, a1
1600
1601 movi a1, _PAGE_DIRECTORY
1602 or a0, a0, a1
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614 extui a3, a3, 28, 2
1615 rsr a1, PTEVADDR
1616 addx2 a3, a3, a3
1617 srli a1, a1, PAGE_SHIFT
1618 extui a3, a3, 2, 2
1619 slli a1, a1, PAGE_SHIFT
1620 addi a3, a3, DTLB_WAY_PGD
1621 add a1, a1, a3
1622
16233: wdtlb a0, a1
1624 dsync
1625
1626
1627
16284: movi a3, exc_table
1629 movi a0, 0
1630 s32i a0, a3, EXC_TABLE_FIXUP
1631
1632
1633
1634 l32i a0, a2, PT_AREG0
1635 l32i a1, a2, PT_AREG1
1636 l32i a2, a2, PT_DEPC
1637 xsr a3, EXCSAVE_1
1638
1639 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1640
1641
1642
1643 rsr a2, DEPC
1644 rfe
1645
1646
1647
16481: xsr a2, DEPC
1649 esync
1650 rfde
1651
16529: l32i a0, a1, TASK_ACTIVE_MM
1653 j 8b
1654
1655
1656
16572:
1658
1659
1660
1661
1662
1663
1664
1665
1666 l32i a0, a2, PT_DEPC
1667 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f
1668
1669
1670
1671 movi a0, __tlbtemp_mapping_start
1672 rsr a3, EPC_1
1673 bltu a3, a0, 2f
1674 movi a0, __tlbtemp_mapping_end
1675 bgeu a3, a0, 2f
1676
1677
1678
1679 movi a3, TLBTEMP_BASE_1
1680 rsr a0, EXCVADDR
1681 bltu a0, a3, 2f
1682
1683 addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT))
1684 bgeu a1, a3, 2f
1685
1686
1687
1688 movi a1, __tlbtemp_mapping_itlb
1689 rsr a3, EPC_1
1690 sub a3, a3, a1
1691
1692
1693
1694 movi a1, PAGE_MASK
1695 and a1, a1, a0
1696
1697
1698
1699 bgez a3, 1f
1700
1701
1702
1703 extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1
1704 add a1, a3, a1
1705
1706
1707
1708 mov a0, a6
1709 movnez a0, a7, a3
1710 j 3b
1711
1712
1713
17141: witlb a6, a1
1715 isync
1716 j 4b
1717
1718
1719#endif
1720
1721
17222:
1723
1724 movi a3, exc_table
1725 rsr a1, DEPC
1726 xsr a3, EXCSAVE_1
1727 s32i a1, a2, PT_AREG2
1728 s32i a3, a2, PT_AREG3
1729 mov a1, a2
1730
1731 rsr a2, PS
1732 bbsi.l a2, PS_UM_BIT, 1f
1733 j _kernel_exception
17341: j _user_exception
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755ENTRY(fast_store_prohibited)
1756
1757
1758
1759 s32i a1, a2, PT_AREG1
1760 s32i a4, a2, PT_AREG4
1761
1762 GET_CURRENT(a1,a2)
1763 l32i a0, a1, TASK_MM
1764 beqz a0, 9f
1765
17668: rsr a1, EXCVADDR
1767 _PGD_OFFSET(a0, a1, a4)
1768 l32i a0, a0, 0
1769 beqz a0, 2f
1770
1771
1772
1773 _PTE_OFFSET(a0, a1, a4)
1774 l32i a4, a0, 0
1775 bbci.l a4, _PAGE_WRITABLE_BIT, 2f
1776
1777 movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
1778 or a4, a4, a1
1779 rsr a1, EXCVADDR
1780 s32i a4, a0, 0
1781
1782
1783
1784 dhwb a0, 0
1785#endif
1786 pdtlb a0, a1
1787 wdtlb a4, a0
1788
1789
1790
1791 movi a0, 0
1792 s32i a0, a3, EXC_TABLE_FIXUP
1793
1794
1795
1796 l32i a4, a2, PT_AREG4
1797 l32i a1, a2, PT_AREG1
1798 l32i a0, a2, PT_AREG0
1799 l32i a2, a2, PT_DEPC
1800
1801
1802
1803 xsr a3, EXCSAVE_1
1804 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1805
1806 rsr a2, DEPC
1807 rfe
1808
1809
1810
18111: xsr a2, DEPC
1812 esync
1813 rfde
1814
18159: l32i a0, a1, TASK_ACTIVE_MM
1816 j 8b
1817
18182:
1819
1820 rsr a4, DEPC
1821 xsr a3, EXCSAVE_1
1822 s32i a4, a2, PT_AREG2
1823 s32i a3, a2, PT_AREG3
1824 l32i a4, a2, PT_AREG4
1825 mov a1, a2
1826
1827 rsr a2, PS
1828 bbsi.l a2, PS_UM_BIT, 1f
1829 j _kernel_exception
18301: j _user_exception
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851ENTRY(fast_coprocessor_double)
1852 wsr a0, EXCSAVE_1
1853 movi a0, unrecoverable_exception
1854 callx0 a0
1855
1856ENTRY(fast_coprocessor)
1857
1858
1859
1860 l32i a0, a2, PT_DEPC
1861 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_coprocessor_double
1862
1863
1864
1865 xsr a3, EXCSAVE_1
1866 s32i a3, a2, PT_AREG3
1867 rsr a3, SAR
1868 s32i a4, a2, PT_AREG4
1869 s32i a1, a2, PT_AREG1
1870 s32i a5, a1, PT_AREG5
1871 s32i a3, a2, PT_SAR
1872 mov a1, a2
1873
1874
1875
1876
1877
1878 s32i a6, a1, PT_AREG6
1879 s32i a7, a1, PT_AREG7
1880 s32i a8, a1, PT_AREG8
1881 s32i a9, a1, PT_AREG9
1882 s32i a10, a1, PT_AREG10
1883 s32i a11, a1, PT_AREG11
1884 s32i a12, a1, PT_AREG12
1885 s32i a13, a1, PT_AREG13
1886 s32i a14, a1, PT_AREG14
1887 s32i a15, a1, PT_AREG15
1888
1889
1890
1891 rsr a0, EXCCAUSE
1892 addi a3, a0, -XCHAL_EXCCAUSE_COPROCESSOR0_DISABLED
1893
1894
1895
1896 movi a4, 1
1897 ssl a3
1898 rsr a5, CPENABLE
1899 sll a4, a4
1900 or a4, a5, a4
1901 wsr a4, CPENABLE
1902 rsync
1903 movi a5, coprocessor_info
1904 addx8 a0, a4, a5
1905
1906 bne a4, a5, .Lload
1907
1908
1909
1910
1911
1912
1913 GET_CURRENT(a5,a1)
1914 l32i a4, a0, COPROCESSOR_INFO_OWNER
1915 beq a4, a5, .Ldone
1916
1917
1918
1919
1920
1921
1922
1923
1924 l32i a5, a0, COPROCESSOR_INFO_OFFSET
1925 addi a2, a4, THREAD_CP_SAVE
1926 add a2, a2, a5
1927
1928
1929
1930 xchal_cpi_store_funcbody
1931
1932
1933
1934
1935 movi a3, 32
1936 rsr a4, SAR
1937 sub a3, a3, a4
1938
1939.Lload:
1940
1941
1942
1943
1944
1945 GET_CURRENT(a4,a1)
1946 s32i a4, a0, 0
1947
1948
1949
1950 l32i a5, a0, COPROCESSOR_INFO_OFFSET
1951 addi a2, a4, THREAD_CP_SAVE
1952 add a2, a2, a4
1953
1954 xchal_cpi_load_funcbody
1955
1956
1957
1958
1959
1960.Ldone: l32i a15, a1, PT_AREG15
1961 l32i a14, a1, PT_AREG14
1962 l32i a13, a1, PT_AREG13
1963 l32i a12, a1, PT_AREG12
1964 l32i a11, a1, PT_AREG11
1965 l32i a10, a1, PT_AREG10
1966 l32i a9, a1, PT_AREG9
1967 l32i a8, a1, PT_AREG8
1968 l32i a7, a1, PT_AREG7
1969 l32i a6, a1, PT_AREG6
1970 l32i a5, a1, PT_AREG5
1971 l32i a4, a1, PT_AREG4
1972 l32i a3, a1, PT_AREG3
1973 l32i a2, a1, PT_AREG2
1974 l32i a0, a1, PT_AREG0
1975 l32i a1, a1, PT_AREG1
1976
1977 rfe
1978
1979#endif
1980
1981
1982
1983
1984
1985
1986
1987
1988ENTRY(system_call)
1989 entry a1, 32
1990
1991
1992
1993 l32i a3, a2, PT_AREG2
1994 mov a6, a2
1995 movi a4, do_syscall_trace_enter
1996 s32i a3, a2, PT_SYSCALL
1997 callx4 a4
1998
1999
2000
2001 movi a4, sys_call_table;
2002 movi a5, __NR_syscall_count
2003 movi a6, -ENOSYS
2004 bgeu a3, a5, 1f
2005
2006 addx4 a4, a3, a4
2007 l32i a4, a4, 0
2008 movi a5, sys_ni_syscall;
2009 beq a4, a5, 1f
2010
2011
2012
2013 l32i a6, a2, PT_AREG6
2014 l32i a7, a2, PT_AREG3
2015 l32i a8, a2, PT_AREG4
2016 l32i a9, a2, PT_AREG5
2017 l32i a10, a2, PT_AREG8
2018 l32i a11, a2, PT_AREG9
2019
2020
2021 s32i a2, a1, 0
2022
2023 callx4 a4
2024
20251:
2026
2027 s32i a6, a2, PT_AREG2
2028 movi a4, do_syscall_trace_leave
2029 mov a6, a2
2030 callx4 a4
2031 retw
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041ENTRY(kernel_thread)
2042 entry a1, 16
2043
2044 mov a5, a2
2045 mov a7, a3
2046
2047 movi a3, _CLONE_VM | _CLONE_UNTRACED
2048 movi a2, __NR_clone
2049 or a6, a4, a3
2050 mov a3, a1
2051 syscall
2052
2053 beq a3, a1, 1f
2054 mov a6, a7
2055 callx4 a5
2056
2057 movi a2, __NR_exit
2058 syscall
2059
20601: retw
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070ENTRY(kernel_execve)
2071 entry a1, 16
2072 mov a6, a2
2073 movi a2, __NR_execve
2074 syscall
2075
2076 retw
2077
2078
2079
2080
2081
2082
2083
2084
2085ENTRY(_switch_to)
2086
2087 entry a1, 16
2088
2089 mov a4, a3
2090
2091 s32i a0, a2, THREAD_RA
2092 s32i a1, a2, THREAD_SP
2093
2094
2095
2096 movi a5, (1 << PS_EXCM_BIT) | LOCKLEVEL
2097 xsr a5, PS
2098 rsr a3, EXCSAVE_1
2099 rsync
2100 s32i a3, a3, EXC_TABLE_FIXUP
2101
2102 call0 _spill_registers
2103
2104
2105
2106
2107
2108
2109
2110 l32i a0, a4, TASK_THREAD_INFO
2111 rsr a3, EXCSAVE_1
2112 movi a1, 0
2113 addi a0, a0, PT_REGS_OFFSET
2114 s32i a1, a3, EXC_TABLE_FIXUP
2115 s32i a0, a3, EXC_TABLE_KSTK
2116
2117
2118
2119 l32i a0, a4, THREAD_RA
2120 l32i a1, a4, THREAD_SP
2121
2122 wsr a5, PS
2123 rsync
2124
2125 retw
2126
2127
2128ENTRY(ret_from_fork)
2129
2130
2131
2132
2133 movi a4, schedule_tail
2134 callx4 a4
2135
2136 movi a4, do_syscall_trace_leave
2137 mov a6, a1
2138 callx4 a4
2139
2140 j common_exception_return
2141
2142