1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <asm/asm-offsets.h>
34#include <asm/unistd.h>
35#include <asm/errno.h>
36#include <asm/page.h>
37#include <asm/psw.h>
38#include <asm/thread_info.h>
39#include <asm/assembly.h>
40#include <asm/processor.h>
41#include <asm/cache.h>
42
43#include <linux/linkage.h>
44
45
46
47
48
49#define KILL_INSN break 0,0
50
51 .level LEVEL
52
53 .text
54
55 .import syscall_exit,code
56 .import syscall_exit_rfi,code
57
58
59
60
61
62
63
64
65
66 .align PAGE_SIZE
67ENTRY(linux_gateway_page)
68
69
70 .rept 44
71 KILL_INSN
72 .endr
73
74
75
76
77#define __NR_lws_entries (3)
78
79lws_entry:
80 gate lws_start, %r0
81 depi 3, 31, 2, %r31
82
83
84 .rept 10
85 KILL_INSN
86 .endr
87
88
89
90set_thread_pointer:
91 gate .+8, %r0
92 depi 3, 31, 2, %r31
93 be 0(%sr7,%r31)
94 mtctl %r26, %cr27
95
96
97
98 .rept 4
99 KILL_INSN
100 .endr
101
102
103 .align 256
104linux_gateway_entry:
105 gate .+8, %r0
106 mtsp %r0,%sr4
107 mtsp %r0,%sr5
108 mtsp %r0,%sr6
109 mfsp %sr7,%r1
110 mtsp %r1,%sr3
111
112#ifdef CONFIG_64BIT
113
114
115
116
117
118
119 ssm PSW_SM_W, %r1
120 extrd,u %r1,PSW_W_BIT,1,%r1
121
122
123 or,ev %r1,%r30,%r30
124 b,n 1f
125
126
127
128 depdi 0, 31, 32, %r26
129 depdi 0, 31, 32, %r25
130 depdi 0, 31, 32, %r24
131 depdi 0, 31, 32, %r23
132 depdi 0, 31, 32, %r22
133 depdi 0, 31, 32, %r21
1341:
135#endif
136 mfctl %cr30,%r1
137 xor %r1,%r30,%r30
138 xor %r1,%r30,%r1
139 xor %r1,%r30,%r30
140
141 ldo THREAD_SZ_ALGN+FRAME_SIZE(%r30),%r30
142
143
144
145
146
147
148
149 mtsp %r0,%sr7
150 STREGM %r1,FRAME_SIZE(%r30)
151 mfctl %cr30,%r1
152 LDREG TI_TASK(%r1),%r1
153
154
155
156
157
158
159 STREG %r0, TASK_PT_PSW(%r1)
160 STREG %r2, TASK_PT_GR2(%r1)
161 STREG %r19, TASK_PT_GR19(%r1)
162
163 LDREGM -FRAME_SIZE(%r30), %r2
164#ifdef CONFIG_64BIT
165 extrd,u %r2,63,1,%r19
166
167 xor %r19,%r2,%r2
168 depd,z %r19,1,1,%r19
169 std %r19,TASK_PT_PSW(%r1)
170#endif
171#endif
172 STREG %r2, TASK_PT_GR30(%r1)
173
174 STREG %r20, TASK_PT_GR20(%r1)
175 STREG %r21, TASK_PT_GR21(%r1)
176 STREG %r22, TASK_PT_GR22(%r1)
177 STREG %r23, TASK_PT_GR23(%r1)
178 STREG %r24, TASK_PT_GR24(%r1)
179 STREG %r25, TASK_PT_GR25(%r1)
180 STREG %r26, TASK_PT_GR26(%r1)
181 STREG %r27, TASK_PT_GR27(%r1)
182 STREG %r28, TASK_PT_GR28(%r1)
183 STREG %r0, TASK_PT_ORIG_R28(%r1)
184 STREG %r29, TASK_PT_GR29(%r1)
185 STREG %r31, TASK_PT_GR31(%r1)
186
187 ldo TASK_PT_FR0(%r1), %r27
188 save_fp %r27
189
190 mfctl %cr11, %r27
191 STREG %r27, TASK_PT_SAR(%r1)
192
193 loadgp
194
195#ifdef CONFIG_64BIT
196 ldo -16(%r30),%r29
197 copy %r19,%r2
198#else
199
200
201 stw %r22, -52(%r30)
202 stw %r21, -56(%r30)
203#endif
204
205
206 mfctl %cr30, %r1
207 LDREG TI_FLAGS(%r1),%r1
208 ldi _TIF_SYSCALL_TRACE_MASK, %r19
209 and,COND(=) %r1, %r19, %r0
210 b,n .Ltracesys
211
212
213
214
215#ifdef CONFIG_64BIT
216 ldil L%sys_call_table, %r1
217 or,= %r2,%r2,%r2
218 addil L%(sys_call_table64-sys_call_table), %r1
219 ldo R%sys_call_table(%r1), %r19
220 or,= %r2,%r2,%r2
221 ldo R%sys_call_table64(%r1), %r19
222#else
223 ldil L%sys_call_table, %r1
224 ldo R%sys_call_table(%r1), %r19
225#endif
226 comiclr,>> __NR_Linux_syscalls, %r20, %r0
227 b,n .Lsyscall_nosys
228
229 LDREGX %r20(%r19), %r19
230
231
232
233
234
235
236 ldi __NR_rt_sigreturn,%r2
237 comb,= %r2,%r20,.Lrt_sigreturn
238.Lin_syscall:
239 ldil L%syscall_exit,%r2
240 be 0(%sr7,%r19)
241 ldo R%syscall_exit(%r2),%r2
242.Lrt_sigreturn:
243 comib,<> 0,%r25,.Lin_syscall
244 ldil L%syscall_exit_rfi,%r2
245 be 0(%sr7,%r19)
246 ldo R%syscall_exit_rfi(%r2),%r2
247
248
249
250
251
252
253.Lsyscall_nosys:
254syscall_nosys:
255 ldil L%syscall_exit,%r1
256 be R%syscall_exit(%sr7,%r1)
257 ldo -ENOSYS(%r0),%r28
258
259
260
261
262.Ltracesys:
263tracesys:
264
265
266
267
268
269
270
271
272
273 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
274 LDREG TI_TASK(%r1), %r1
275 ssm 0,%r2
276 STREG %r2,TASK_PT_PSW(%r1)
277 mfsp %sr0,%r2
278 STREG %r2,TASK_PT_SR0(%r1)
279 mfsp %sr1,%r2
280 STREG %r2,TASK_PT_SR1(%r1)
281 mfsp %sr2,%r2
282 STREG %r2,TASK_PT_SR2(%r1)
283 mfsp %sr3,%r2
284 STREG %r2,TASK_PT_SR3(%r1)
285 STREG %r2,TASK_PT_SR4(%r1)
286 STREG %r2,TASK_PT_SR5(%r1)
287 STREG %r2,TASK_PT_SR6(%r1)
288 STREG %r2,TASK_PT_SR7(%r1)
289 STREG %r2,TASK_PT_IASQ0(%r1)
290 STREG %r2,TASK_PT_IASQ1(%r1)
291 LDREG TASK_PT_GR31(%r1),%r2
292 STREG %r2,TASK_PT_IAOQ0(%r1)
293 ldo 4(%r2),%r2
294 STREG %r2,TASK_PT_IAOQ1(%r1)
295 ldo TASK_REGS(%r1),%r2
296
297 STREG %r3,PT_GR3(%r2)
298 STREG %r4,PT_GR4(%r2)
299 STREG %r5,PT_GR5(%r2)
300 STREG %r6,PT_GR6(%r2)
301 STREG %r7,PT_GR7(%r2)
302 STREG %r8,PT_GR8(%r2)
303 STREG %r9,PT_GR9(%r2)
304 STREG %r10,PT_GR10(%r2)
305 STREG %r11,PT_GR11(%r2)
306 STREG %r12,PT_GR12(%r2)
307 STREG %r13,PT_GR13(%r2)
308 STREG %r14,PT_GR14(%r2)
309 STREG %r15,PT_GR15(%r2)
310 STREG %r16,PT_GR16(%r2)
311 STREG %r17,PT_GR17(%r2)
312 STREG %r18,PT_GR18(%r2)
313
314
315 copy %r2,%r26
316 ldil L%do_syscall_trace_enter,%r1
317 ldil L%tracesys_next,%r2
318 be R%do_syscall_trace_enter(%sr7,%r1)
319 ldo R%tracesys_next(%r2),%r2
320
321tracesys_next:
322
323
324
325
326 copy %ret0,%r20
327 ldil L%sys_call_table,%r1
328 ldo R%sys_call_table(%r1), %r19
329
330 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
331 LDREG TI_TASK(%r1), %r1
332 LDREG TASK_PT_GR26(%r1), %r26
333 LDREG TASK_PT_GR25(%r1), %r25
334 LDREG TASK_PT_GR24(%r1), %r24
335 LDREG TASK_PT_GR23(%r1), %r23
336 LDREG TASK_PT_GR22(%r1), %r22
337 LDREG TASK_PT_GR21(%r1), %r21
338#ifdef CONFIG_64BIT
339 ldo -16(%r30),%r29
340#else
341 stw %r22, -52(%r30)
342 stw %r21, -56(%r30)
343#endif
344
345 comiclr,>>= __NR_Linux_syscalls, %r20, %r0
346 b,n .Lsyscall_nosys
347
348 LDREGX %r20(%r19), %r19
349
350
351
352
353
354
355 ldi __NR_rt_sigreturn,%r2
356 comb,= %r2,%r20,.Ltrace_rt_sigreturn
357.Ltrace_in_syscall:
358 ldil L%tracesys_exit,%r2
359 be 0(%sr7,%r19)
360 ldo R%tracesys_exit(%r2),%r2
361
362
363
364
365tracesys_exit:
366 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
367 LDREG TI_TASK(%r1), %r1
368#ifdef CONFIG_64BIT
369 ldo -16(%r30),%r29
370#endif
371 ldo TASK_REGS(%r1),%r26
372 bl do_syscall_trace_exit,%r2
373 STREG %r28,TASK_PT_GR28(%r1)
374 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
375 LDREG TI_TASK(%r1), %r1
376 LDREG TASK_PT_GR28(%r1), %r28
377
378 ldil L%syscall_exit,%r1
379 be,n R%syscall_exit(%sr7,%r1)
380
381.Ltrace_rt_sigreturn:
382 comib,<> 0,%r25,.Ltrace_in_syscall
383 ldil L%tracesys_sigexit,%r2
384 be 0(%sr7,%r19)
385 ldo R%tracesys_sigexit(%r2),%r2
386
387tracesys_sigexit:
388 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1
389 LDREG TI_TASK(%r1), %r1
390#ifdef CONFIG_64BIT
391 ldo -16(%r30),%r29
392#endif
393 bl do_syscall_trace_exit,%r2
394 ldo TASK_REGS(%r1),%r26
395
396 ldil L%syscall_exit_rfi,%r1
397 be,n R%syscall_exit_rfi(%sr7,%r1)
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452lws_start:
453
454#ifdef CONFIG_64BIT
455
456
457
458 ssm PSW_SM_W, %r1
459 extrd,u %r1,PSW_W_BIT,1,%r1
460
461
462 or,ev %r1,%r30,%r30
463
464
465 depdi 0, 31, 32, %r20
466#endif
467
468
469 comiclr,>> __NR_lws_entries, %r20, %r0
470 b,n lws_exit_nosys
471
472
473 mfsp %sr7,%r1
474 mtsp %r1,%sr3
475 mtsp %r0,%sr2
476
477
478 ldil L%lws_table, %r1
479 ldo R%lws_table(%r1), %r28
480 LDREGX %r20(%sr2,r28), %r21
481
482
483 be,n 0(%sr2,%r21)
484
485lws_exit_nosys:
486 ldo -ENOSYS(%r0),%r21
487
488
489lws_exit:
490#ifdef CONFIG_64BIT
491
492
493
494
495 extrd,u,*<> %r30,63,1,%r1
496 rsm PSW_SM_W, %r0
497
498 xor %r30,%r1,%r30
499#endif
500 be,n 0(%sr7, %r31)
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532#define ENABLE_LWS_DEBUG 0
533
534
535lws_compare_and_swap64:
536#ifdef CONFIG_64BIT
537 b,n lws_compare_and_swap
538#else
539
540
541
542
543 b,n lws_exit_nosys
544#endif
545
546
547lws_compare_and_swap32:
548#ifdef CONFIG_64BIT
549
550 depdi 0, 31, 32, %r26
551 depdi 0, 31, 32, %r25
552 depdi 0, 31, 32, %r24
553#endif
554
555lws_compare_and_swap:
556
557 ldil L%lws_lock_start, %r20
558 ldo R%lws_lock_start(%r20), %r28
559
560
561 extru %r26, 27, 4, %r20
562
563
564
565
566 shlw %r20, 4, %r20
567 add %r20, %r28, %r20
568
569
570
571
572
573
574
575
576
577 ldw 4(%sr2,%r20), %r28
578
579 mfctl %cr27, %r21
580 cmpb,<>,n %r21, %r28, cas_lock
581 b lws_exit
582 ldo -EDEADLOCK(%r0), %r21
583cas_lock:
584 cmpb,=,n %r0, %r28, cas_nocontend
585 ldo 1(%r0), %r28
586 b lws_exit
587 ldo -EAGAIN(%r0), %r21
588cas_nocontend:
589# endif
590
591
592 rsm PSW_SM_I, %r0
593
594 LDCW 0(%sr2,%r20), %r28
595 cmpb,<>,n %r0, %r28, cas_action
596cas_wouldblock:
597 ldo 2(%r0), %r28
598 ssm PSW_SM_I, %r0
599 b lws_exit
600 ldo -EAGAIN(%r0), %r21
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618cas_action:
619
620
621 mfctl %cr27, %r1
622 stw %r1, 4(%sr2,%r20)
623#endif
624
6251: ldw,ma 0(%sr3,%r26), %r28
626 sub,<> %r28, %r25, %r0
6272: stw,ma %r24, 0(%sr3,%r26)
628
629 stw,ma %r20, 0(%sr2,%r20)
630
631
632 stw %r0, 4(%sr2,%r20)
633#endif
634
635 ssm PSW_SM_I, %r0
636
637 b lws_exit
638 copy %r0, %r21
639
6403:
641
642
643 stw %r20, 0(%sr2,%r20)
644
645 stw %r0, 4(%sr2,%r20)
646#endif
647 ssm PSW_SM_I, %r0
648 b lws_exit
649 ldo -EFAULT(%r0),%r21
650 nop
651 nop
652 nop
653 nop
654
655
656
657
658 ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page)
659 ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page)
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684lws_compare_and_swap_2:
685#ifdef CONFIG_64BIT
686
687 depdi 0, 31, 32, %r26
688 depdi 0, 31, 32, %r25
689 depdi 0, 31, 32, %r24
690 depdi 0, 31, 32, %r23
691#endif
692
693
694 subi,>>= 4, %r23, %r0
695 b,n lws_exit_nosys
696
697
698
699 shlw %r23, 2, %r29
700 blr %r29, %r0
701 nop
702
703
7044: ldb 0(%sr3,%r25), %r25
705 b cas2_lock_start
7065: ldb 0(%sr3,%r24), %r24
707 nop
708 nop
709 nop
710 nop
711 nop
712
713
7146: ldh 0(%sr3,%r25), %r25
715 b cas2_lock_start
7167: ldh 0(%sr3,%r24), %r24
717 nop
718 nop
719 nop
720 nop
721 nop
722
723
7248: ldw 0(%sr3,%r25), %r25
725 b cas2_lock_start
7269: ldw 0(%sr3,%r24), %r24
727 nop
728 nop
729 nop
730 nop
731 nop
732
733
734#ifdef CONFIG_64BIT
73510: ldd 0(%sr3,%r25), %r25
73611: ldd 0(%sr3,%r24), %r24
737#else
738
73910: ldw 0(%sr3,%r25), %r22
74011: ldw 4(%sr3,%r25), %r23
741
74212: flddx 0(%sr3,%r24), %fr4
743#endif
744
745cas2_lock_start:
746
747 ldil L%lws_lock_start, %r20
748 ldo R%lws_lock_start(%r20), %r28
749
750
751 extru %r26, 27, 4, %r20
752
753
754
755
756 shlw %r20, 4, %r20
757 add %r20, %r28, %r20
758
759 rsm PSW_SM_I, %r0
760
761 LDCW 0(%sr2,%r20), %r28
762 cmpb,<>,n %r0, %r28, cas2_action
763cas2_wouldblock:
764 ldo 2(%r0), %r28
765 ssm PSW_SM_I, %r0
766 b lws_exit
767 ldo -EAGAIN(%r0), %r21
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785cas2_action:
786
787 blr %r29, %r0
788
789 ldo 1(%r0),%r28
790
791
79213: ldb,ma 0(%sr3,%r26), %r29
793 sub,= %r29, %r25, %r0
794 b,n cas2_end
79514: stb,ma %r24, 0(%sr3,%r26)
796 b cas2_end
797 copy %r0, %r28
798 nop
799 nop
800
801
80215: ldh,ma 0(%sr3,%r26), %r29
803 sub,= %r29, %r25, %r0
804 b,n cas2_end
80516: sth,ma %r24, 0(%sr3,%r26)
806 b cas2_end
807 copy %r0, %r28
808 nop
809 nop
810
811
81217: ldw,ma 0(%sr3,%r26), %r29
813 sub,= %r29, %r25, %r0
814 b,n cas2_end
81518: stw,ma %r24, 0(%sr3,%r26)
816 b cas2_end
817 copy %r0, %r28
818 nop
819 nop
820
821
822#ifdef CONFIG_64BIT
82319: ldd,ma 0(%sr3,%r26), %r29
824 sub,= %r29, %r25, %r0
825 b,n cas2_end
82620: std,ma %r24, 0(%sr3,%r26)
827 copy %r0, %r28
828#else
829
83019: ldw,ma 0(%sr3,%r26), %r29
831 sub,= %r29, %r22, %r0
832 b,n cas2_end
833
83420: ldw,ma 4(%sr3,%r26), %r29
835 sub,= %r29, %r23, %r0
836 b,n cas2_end
837
83821: fstdx %fr4, 0(%sr3,%r26)
839 copy %r0, %r28
840#endif
841
842cas2_end:
843
844 stw,ma %r20, 0(%sr2,%r20)
845
846 ssm PSW_SM_I, %r0
847
848 b lws_exit
849 copy %r0, %r21
850
85122:
852
853
854 stw %r20, 0(%sr2,%r20)
855 ssm PSW_SM_I, %r0
856 ldo 1(%r0),%r28
857 b lws_exit
858 ldo -EFAULT(%r0),%r21
859 nop
860 nop
861 nop
862
863
864
865 ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page)
866 ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page)
867 ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page)
868 ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page)
869 ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page)
870 ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page)
871 ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page)
872 ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page)
873 ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page)
874 ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page)
875 ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page)
876 ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page)
877 ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page)
878 ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page)
879 ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page)
880 ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page)
881#ifndef CONFIG_64BIT
882 ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page)
883 ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page)
884#endif
885
886
887 .align PAGE_SIZE
888END(linux_gateway_page)
889ENTRY(end_linux_gateway_page)
890
891
892
893
894#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_
895
896 .section .rodata,"a"
897
898 .align 8
899
900
901ENTRY(lws_table)
902 LWS_ENTRY(compare_and_swap32)
903 LWS_ENTRY(compare_and_swap64)
904 LWS_ENTRY(compare_and_swap_2)
905END(lws_table)
906
907
908 .align 8
909ENTRY(sys_call_table)
910#include "syscall_table.S"
911END(sys_call_table)
912
913#ifdef CONFIG_64BIT
914 .align 8
915ENTRY(sys_call_table64)
916#define SYSCALL_TABLE_64BIT
917#include "syscall_table.S"
918END(sys_call_table64)
919#endif
920
921
922
923
924
925
926
927
928
929 .section .data
930 .align L1_CACHE_BYTES
931ENTRY(lws_lock_start)
932
933 .rept 16
934
935 .word 1
936 .word 0
937 .word 0
938 .word 0
939 .endr
940END(lws_lock_start)
941 .previous
942
943.end
944
945
946