1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <asm/asm.h>
17#include <asm/asm-offsets.h>
18#include <asm/regdef.h>
19
20#define dst a0
21#define src a1
22#define len a2
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73#define EXC(inst_reg,addr,handler) \
749: inst_reg, addr; \
75 .section __ex_table,"a"; \
76 PTR 9b, handler; \
77 .previous
78
79
80
81
82
83#define LOAD ld
84#define LOADL ldl
85#define LOADR ldr
86#define STOREL sdl
87#define STORER sdr
88#define STORE sd
89#define ADD daddu
90#define SUB dsubu
91#define SRL dsrl
92#define SRA dsra
93#define SLL dsll
94#define SLLV dsllv
95#define SRLV dsrlv
96#define NBYTES 8
97#define LOG_NBYTES 3
98
99
100
101
102
103
104#undef t0
105#undef t1
106#undef t2
107#undef t3
108#define t0 $8
109#define t1 $9
110#define t2 $10
111#define t3 $11
112#define t4 $12
113#define t5 $13
114#define t6 $14
115#define t7 $15
116
117#ifdef CONFIG_CPU_LITTLE_ENDIAN
118#define LDFIRST LOADR
119#define LDREST LOADL
120#define STFIRST STORER
121#define STREST STOREL
122#define SHIFT_DISCARD SLLV
123#else
124#define LDFIRST LOADL
125#define LDREST LOADR
126#define STFIRST STOREL
127#define STREST STORER
128#define SHIFT_DISCARD SRLV
129#endif
130
131#define FIRST(unit) ((unit)*NBYTES)
132#define REST(unit) (FIRST(unit)+NBYTES-1)
133#define UNIT(unit) FIRST(unit)
134
135#define ADDRMASK (NBYTES-1)
136
137 .text
138 .set noreorder
139 .set noat
140
141
142
143
144LEAF(__copy_user_inatomic)
145 b __copy_user_common
146 li t7, 1
147 END(__copy_user_inatomic)
148
149
150
151
152
153
154
155 .align 5
156LEAF(memcpy)
157 move v0, dst
158__memcpy:
159FEXPORT(__copy_user)
160 li t7, 0
161__copy_user_common:
162
163
164
165
166
167
168
169
170 pref 0, 0(src)
171 sltu t0, len, NBYTES
172 bnez t0, copy_bytes_checklen
173 and t0, src, ADDRMASK
174 bnez t0, src_unaligned
175 sltu t0, len, 4*NBYTES
176 bnez t0, less_than_4units
177 sltu t0, len, 8*NBYTES
178 bnez t0, less_than_8units
179 sltu t0, len, 16*NBYTES
180 bnez t0, cleanup_both_aligned
181 sltu t0, len, 128+1
182 bnez t0, 1f
183 sltu t0, len, 256+1
184 bnez t0, 1f
185 pref 0, 128(src)
186
187
1882: pref 0, 256(src)
189
190
1911:
192EXC( LOAD t0, UNIT(0)(src), l_exc)
193EXC( LOAD t1, UNIT(1)(src), l_exc_copy)
194EXC( LOAD t2, UNIT(2)(src), l_exc_copy)
195EXC( LOAD t3, UNIT(3)(src), l_exc_copy)
196 SUB len, len, 16*NBYTES
197EXC( STORE t0, UNIT(0)(dst), s_exc_p16u)
198EXC( STORE t1, UNIT(1)(dst), s_exc_p15u)
199EXC( STORE t2, UNIT(2)(dst), s_exc_p14u)
200EXC( STORE t3, UNIT(3)(dst), s_exc_p13u)
201EXC( LOAD t0, UNIT(4)(src), l_exc_copy)
202EXC( LOAD t1, UNIT(5)(src), l_exc_copy)
203EXC( LOAD t2, UNIT(6)(src), l_exc_copy)
204EXC( LOAD t3, UNIT(7)(src), l_exc_copy)
205EXC( STORE t0, UNIT(4)(dst), s_exc_p12u)
206EXC( STORE t1, UNIT(5)(dst), s_exc_p11u)
207EXC( STORE t2, UNIT(6)(dst), s_exc_p10u)
208 ADD src, src, 16*NBYTES
209EXC( STORE t3, UNIT(7)(dst), s_exc_p9u)
210 ADD dst, dst, 16*NBYTES
211EXC( LOAD t0, UNIT(-8)(src), l_exc_copy)
212EXC( LOAD t1, UNIT(-7)(src), l_exc_copy)
213EXC( LOAD t2, UNIT(-6)(src), l_exc_copy)
214EXC( LOAD t3, UNIT(-5)(src), l_exc_copy)
215EXC( STORE t0, UNIT(-8)(dst), s_exc_p8u)
216EXC( STORE t1, UNIT(-7)(dst), s_exc_p7u)
217EXC( STORE t2, UNIT(-6)(dst), s_exc_p6u)
218EXC( STORE t3, UNIT(-5)(dst), s_exc_p5u)
219EXC( LOAD t0, UNIT(-4)(src), l_exc_copy)
220EXC( LOAD t1, UNIT(-3)(src), l_exc_copy)
221EXC( LOAD t2, UNIT(-2)(src), l_exc_copy)
222EXC( LOAD t3, UNIT(-1)(src), l_exc_copy)
223EXC( STORE t0, UNIT(-4)(dst), s_exc_p4u)
224EXC( STORE t1, UNIT(-3)(dst), s_exc_p3u)
225EXC( STORE t2, UNIT(-2)(dst), s_exc_p2u)
226EXC( STORE t3, UNIT(-1)(dst), s_exc_p1u)
227 sltu t0, len, 256+1
228 beqz t0, 2b
229 sltu t0, len, 128
230 beqz t0, 1b
231 nop
232
233
234
235cleanup_both_aligned:
236 beqz len, done
237 sltu t0, len, 8*NBYTES
238 bnez t0, less_than_8units
239 nop
240EXC( LOAD t0, UNIT(0)(src), l_exc)
241EXC( LOAD t1, UNIT(1)(src), l_exc_copy)
242EXC( LOAD t2, UNIT(2)(src), l_exc_copy)
243EXC( LOAD t3, UNIT(3)(src), l_exc_copy)
244 SUB len, len, 8*NBYTES
245EXC( STORE t0, UNIT(0)(dst), s_exc_p8u)
246EXC( STORE t1, UNIT(1)(dst), s_exc_p7u)
247EXC( STORE t2, UNIT(2)(dst), s_exc_p6u)
248EXC( STORE t3, UNIT(3)(dst), s_exc_p5u)
249EXC( LOAD t0, UNIT(4)(src), l_exc_copy)
250EXC( LOAD t1, UNIT(5)(src), l_exc_copy)
251EXC( LOAD t2, UNIT(6)(src), l_exc_copy)
252EXC( LOAD t3, UNIT(7)(src), l_exc_copy)
253EXC( STORE t0, UNIT(4)(dst), s_exc_p4u)
254EXC( STORE t1, UNIT(5)(dst), s_exc_p3u)
255EXC( STORE t2, UNIT(6)(dst), s_exc_p2u)
256EXC( STORE t3, UNIT(7)(dst), s_exc_p1u)
257 ADD src, src, 8*NBYTES
258 beqz len, done
259 ADD dst, dst, 8*NBYTES
260
261
262
263less_than_8units:
264 sltu t0, len, 4*NBYTES
265 bnez t0, less_than_4units
266 nop
267EXC( LOAD t0, UNIT(0)(src), l_exc)
268EXC( LOAD t1, UNIT(1)(src), l_exc_copy)
269EXC( LOAD t2, UNIT(2)(src), l_exc_copy)
270EXC( LOAD t3, UNIT(3)(src), l_exc_copy)
271 SUB len, len, 4*NBYTES
272EXC( STORE t0, UNIT(0)(dst), s_exc_p4u)
273EXC( STORE t1, UNIT(1)(dst), s_exc_p3u)
274EXC( STORE t2, UNIT(2)(dst), s_exc_p2u)
275EXC( STORE t3, UNIT(3)(dst), s_exc_p1u)
276 ADD src, src, 4*NBYTES
277 beqz len, done
278 ADD dst, dst, 4*NBYTES
279
280
281
282
283less_than_4units:
284 sltu t0, len, 1*NBYTES
285 bnez t0, copy_bytes_checklen
286 nop
287
288
289
290EXC( LOAD t0, 0(src), l_exc)
291 SUB len, len, NBYTES
292 sltu t1, len, 8
293EXC( STORE t0, 0(dst), s_exc_p1u)
294 ADD src, src, NBYTES
295 bnez t1, copy_bytes_checklen
296 ADD dst, dst, NBYTES
297
298
299
300EXC( LOAD t0, 0(src), l_exc)
301 SUB len, len, NBYTES
302 sltu t1, len, 8
303EXC( STORE t0, 0(dst), s_exc_p1u)
304 ADD src, src, NBYTES
305 bnez t1, copy_bytes_checklen
306 ADD dst, dst, NBYTES
307
308
309
310EXC( LOAD t0, 0(src), l_exc)
311 SUB len, len, NBYTES
312 ADD src, src, NBYTES
313 ADD dst, dst, NBYTES
314 b copy_bytes_checklen
315EXC( STORE t0, -8(dst), s_exc_p1u)
316
317src_unaligned:
318#define rem t8
319 SRL t0, len, LOG_NBYTES+2
320 beqz t0, cleanup_src_unaligned
321 and rem, len, (4*NBYTES-1)
3221:
323
324
325
326
327
328
329EXC( LDFIRST t0, FIRST(0)(src), l_exc)
330EXC( LDFIRST t1, FIRST(1)(src), l_exc_copy)
331 SUB len, len, 4*NBYTES
332EXC( LDREST t0, REST(0)(src), l_exc_copy)
333EXC( LDREST t1, REST(1)(src), l_exc_copy)
334EXC( LDFIRST t2, FIRST(2)(src), l_exc_copy)
335EXC( LDFIRST t3, FIRST(3)(src), l_exc_copy)
336EXC( LDREST t2, REST(2)(src), l_exc_copy)
337EXC( LDREST t3, REST(3)(src), l_exc_copy)
338 ADD src, src, 4*NBYTES
339EXC( STORE t0, UNIT(0)(dst), s_exc_p4u)
340EXC( STORE t1, UNIT(1)(dst), s_exc_p3u)
341EXC( STORE t2, UNIT(2)(dst), s_exc_p2u)
342EXC( STORE t3, UNIT(3)(dst), s_exc_p1u)
343 bne len, rem, 1b
344 ADD dst, dst, 4*NBYTES
345
346cleanup_src_unaligned:
347 beqz len, done
348 and rem, len, NBYTES-1
349 beq rem, len, copy_bytes
350 nop
3511:
352EXC( LDFIRST t0, FIRST(0)(src), l_exc)
353EXC( LDREST t0, REST(0)(src), l_exc_copy)
354 SUB len, len, NBYTES
355EXC( STORE t0, 0(dst), s_exc_p1u)
356 ADD src, src, NBYTES
357 bne len, rem, 1b
358 ADD dst, dst, NBYTES
359
360copy_bytes_checklen:
361 beqz len, done
362 nop
363copy_bytes:
364
365#define COPY_BYTE(N) \
366EXC( lb t0, N(src), l_exc); \
367 SUB len, len, 1; \
368 beqz len, done; \
369EXC( sb t0, N(dst), s_exc_p1)
370
371 COPY_BYTE(0)
372 COPY_BYTE(1)
373 COPY_BYTE(2)
374 COPY_BYTE(3)
375 COPY_BYTE(4)
376 COPY_BYTE(5)
377EXC( lb t0, NBYTES-2(src), l_exc)
378 SUB len, len, 1
379 jr ra
380EXC( sb t0, NBYTES-2(dst), s_exc_p1)
381done:
382 jr ra
383 nop
384 END(memcpy)
385
386l_exc_copy:
387
388
389
390
391
392
393
394
395
396
397 LOAD t0, TI_TASK($28)
398 LOAD t0, THREAD_BUADDR(t0)
3991:
400EXC( lb t1, 0(src), l_exc)
401 ADD src, src, 1
402 sb t1, 0(dst)
403 bne src, t0, 1b
404 ADD dst, dst, 1
405l_exc:
406 LOAD t0, TI_TASK($28)
407 LOAD t0, THREAD_BUADDR(t0)
408 SUB len, AT, t0
409 bnez t7, 2f
410
411
412
413
414
415 ADD dst, t0
416 SUB dst, src
417
418
419
420
421 beqz len, done
422 SUB src, len, 1
4231: sb zero, 0(dst)
424 ADD dst, dst, 1
425 bnez src, 1b
426 SUB src, src, 1
4272: jr ra
428 nop
429
430
431#define SEXC(n) \
432s_exc_p
433 jr ra; \
434 ADD len, len, n*NBYTES
435
436SEXC(16)
437SEXC(15)
438SEXC(14)
439SEXC(13)
440SEXC(12)
441SEXC(11)
442SEXC(10)
443SEXC(9)
444SEXC(8)
445SEXC(7)
446SEXC(6)
447SEXC(5)
448SEXC(4)
449SEXC(3)
450SEXC(2)
451SEXC(1)
452
453s_exc_p1:
454 jr ra
455 ADD len, len, 1
456s_exc:
457 jr ra
458 nop
459
460 .align 5
461LEAF(memmove)
462 ADD t0, a0, a2
463 ADD t1, a1, a2
464 sltu t0, a1, t0
465 sltu t1, a0, t1
466 and t0, t1
467 beqz t0, __memcpy
468 move v0, a0
469 beqz a2, r_out
470 END(memmove)
471
472
473LEAF(__rmemcpy)
474 sltu t0, a1, a0
475 beqz t0, r_end_bytes_up
476 nop
477 ADD a0, a2
478 ADD a1, a2
479
480r_end_bytes:
481 lb t0, -1(a1)
482 SUB a2, a2, 0x1
483 sb t0, -1(a0)
484 SUB a1, a1, 0x1
485 bnez a2, r_end_bytes
486 SUB a0, a0, 0x1
487
488r_out:
489 jr ra
490 move a2, zero
491
492r_end_bytes_up:
493 lb t0, (a1)
494 SUB a2, a2, 0x1
495 sb t0, (a0)
496 ADD a1, a1, 0x1
497 bnez a2, r_end_bytes_up
498 ADD a0, a0, 0x1
499
500 jr ra
501 move a2, zero
502 END(__rmemcpy)
503