1
2
3
4
5
6
7
8
9
10
11#include <asm/uaccess.h>
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29unsigned long
30__copy_user (void __user *pdst, const void *psrc, unsigned long pn)
31{
32
33
34
35
36
37
38
39
40 register char *dst __asm__ ("r13") = pdst;
41 register const char *src __asm__ ("r11") = psrc;
42 register int n __asm__ ("r12") = pn;
43 register int retn __asm__ ("r10") = 0;
44
45
46
47
48
49 if (((unsigned long) dst & 3) != 0
50
51
52 && n >= 3)
53 {
54 if ((unsigned long) dst & 1)
55 {
56 __asm_copy_to_user_1 (dst, src, retn);
57 n--;
58 }
59
60 if ((unsigned long) dst & 2)
61 {
62 __asm_copy_to_user_2 (dst, src, retn);
63 n -= 2;
64 }
65 }
66
67
68
69 if (n >= 44)
70 {
71
72
73
74
75
76
77 __asm__ volatile ("\
78 ;; Check that the register asm declaration got right. \n\
79 ;; The GCC manual explicitly says TRT will happen. \n\
80 .ifnc %0%1%2%3,$r13$r11$r12$r10 \n\
81 .err \n\
82 .endif \n\
83 \n\
84 ;; Save the registers we'll use in the movem process \n\
85 ;; on the stack. \n\
86 subq 11*4,$sp \n\
87 movem $r10,[$sp] \n\
88 \n\
89 ;; Now we've got this: \n\
90 ;; r11 - src \n\
91 ;; r13 - dst \n\
92 ;; r12 - n \n\
93 \n\
94 ;; Update n for the first loop \n\
95 subq 44,$r12 \n\
960: \n\
97 movem [$r11+],$r10 \n\
98 subq 44,$r12 \n\
991: bge 0b \n\
100 movem $r10,[$r13+] \n\
1013: \n\
102 addq 44,$r12 ;; compensate for last loop underflowing n \n\
103 \n\
104 ;; Restore registers from stack \n\
105 movem [$sp+],$r10 \n\
1062: \n\
107 .section .fixup,\"ax\" \n\
1084: \n\
109; When failing on any of the 1..44 bytes in a chunk, we adjust back the \n\
110; source pointer and just drop through to the by-16 and by-4 loops to \n\
111; get the correct number of failing bytes. This necessarily means a \n\
112; few extra exceptions, but invalid user pointers shouldn't happen in \n\
113; time-critical code anyway. \n\
114 jump 3b \n\
115 subq 44,$r11 \n\
116 \n\
117 .previous \n\
118 .section __ex_table,\"a\" \n\
119 .dword 1b,4b \n\
120 .previous"
121
122 : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
123 : "0" (dst), "1" (src), "2" (n), "3" (retn));
124
125 }
126
127 while (n >= 16)
128 {
129 __asm_copy_to_user_16 (dst, src, retn);
130 n -= 16;
131 }
132
133
134
135 while (n >= 4)
136 {
137 __asm_copy_to_user_4 (dst, src, retn);
138 n -= 4;
139 }
140
141 switch (n)
142 {
143 case 0:
144 break;
145 case 1:
146 __asm_copy_to_user_1 (dst, src, retn);
147 break;
148 case 2:
149 __asm_copy_to_user_2 (dst, src, retn);
150 break;
151 case 3:
152 __asm_copy_to_user_3 (dst, src, retn);
153 break;
154 }
155
156 return retn;
157}
158
159
160
161
162
163unsigned long
164__copy_user_zeroing(void *pdst, const void __user *psrc, unsigned long pn)
165{
166
167
168
169
170
171
172
173
174 register char *dst __asm__ ("r13") = pdst;
175 register const char *src __asm__ ("r11") = psrc;
176 register int n __asm__ ("r12") = pn;
177 register int retn __asm__ ("r10") = 0;
178
179
180
181
182 if (((unsigned long) src & 3) != 0)
183 {
184 if (((unsigned long) src & 1) && n != 0)
185 {
186 __asm_copy_from_user_1 (dst, src, retn);
187 n--;
188 }
189
190 if (((unsigned long) src & 2) && n >= 2)
191 {
192 __asm_copy_from_user_2 (dst, src, retn);
193 n -= 2;
194 }
195
196
197
198
199 if (retn != 0)
200 goto copy_exception_bytes;
201 }
202
203
204
205 if (n >= 44)
206 {
207
208
209
210
211 __asm__ volatile ("\
212 .ifnc %0%1%2%3,$r13$r11$r12$r10 \n\
213 .err \n\
214 .endif \n\
215 \n\
216 ;; Save the registers we'll use in the movem process \n\
217 ;; on the stack. \n\
218 subq 11*4,$sp \n\
219 movem $r10,[$sp] \n\
220 \n\
221 ;; Now we've got this: \n\
222 ;; r11 - src \n\
223 ;; r13 - dst \n\
224 ;; r12 - n \n\
225 \n\
226 ;; Update n for the first loop \n\
227 subq 44,$r12 \n\
2280: \n\
229 movem [$r11+],$r10 \n\
230 \n\
231 subq 44,$r12 \n\
232 bge 0b \n\
233 movem $r10,[$r13+] \n\
234 \n\
2354: \n\
236 addq 44,$r12 ;; compensate for last loop underflowing n \n\
237 \n\
238 ;; Restore registers from stack \n\
239 movem [$sp+],$r10 \n\
240 .section .fixup,\"ax\" \n\
241 \n\
242;; Do not jump back into the loop if we fail. For some uses, we get a \n\
243;; page fault somewhere on the line. Without checking for page limits, \n\
244;; we don't know where, but we need to copy accurately and keep an \n\
245;; accurate count; not just clear the whole line. To do that, we fall \n\
246;; down in the code below, proceeding with smaller amounts. It should \n\
247;; be kept in mind that we have to cater to code like what at one time \n\
248;; was in fs/super.c: \n\
249;; i = size - copy_from_user((void *)page, data, size); \n\
250;; which would cause repeated faults while clearing the remainder of \n\
251;; the SIZE bytes at PAGE after the first fault. \n\
252;; A caveat here is that we must not fall through from a failing page \n\
253;; to a valid page. \n\
254 \n\
2553: \n\
256 jump 4b ;; Fall through, pretending the fault didn't happen. \n\
257 nop \n\
258 \n\
259 .previous \n\
260 .section __ex_table,\"a\" \n\
261 .dword 0b,3b \n\
262 .previous"
263
264 : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
265 : "0" (dst), "1" (src), "2" (n), "3" (retn));
266 }
267
268
269
270
271
272
273
274
275
276
277 while (n >= 4)
278 {
279 __asm_copy_from_user_4 (dst, src, retn);
280 n -= 4;
281
282 if (retn)
283 goto copy_exception_bytes;
284 }
285
286
287 switch (n)
288 {
289
290
291
292 case 0:
293
294
295 break;
296 case 1:
297 __asm_copy_from_user_1 (dst, src, retn);
298 break;
299 case 2:
300 __asm_copy_from_user_2 (dst, src, retn);
301 break;
302 case 3:
303 __asm_copy_from_user_3 (dst, src, retn);
304 break;
305 }
306
307
308
309 return retn;
310
311copy_exception_bytes:
312
313
314
315
316 {
317 char *endp;
318 for (endp = dst + n; dst < endp; dst++)
319 *dst = 0;
320 }
321
322 return retn + n;
323}
324
325
326
327unsigned long
328__do_clear_user (void __user *pto, unsigned long pn)
329{
330
331
332
333
334
335
336
337
338 register char *dst __asm__ ("r13") = pto;
339 register int n __asm__ ("r12") = pn;
340 register int retn __asm__ ("r10") = 0;
341
342
343 if (((unsigned long) dst & 3) != 0
344
345 && n >= 3)
346 {
347 if ((unsigned long) dst & 1)
348 {
349 __asm_clear_1 (dst, retn);
350 n--;
351 }
352
353 if ((unsigned long) dst & 2)
354 {
355 __asm_clear_2 (dst, retn);
356 n -= 2;
357 }
358 }
359
360
361
362 if (n >= 48)
363 {
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380 __asm__ volatile ("\
381 .ifnc %0%1%2,$r13$r12$r10 \n\
382 .err \n\
383 .endif \n\
384 \n\
385 ;; Save the registers we'll clobber in the movem process \n\
386 ;; on the stack. Don't mention them to gcc, it will only be \n\
387 ;; upset. \n\
388 subq 11*4,$sp \n\
389 movem $r10,[$sp] \n\
390 \n\
391 clear.d $r0 \n\
392 clear.d $r1 \n\
393 clear.d $r2 \n\
394 clear.d $r3 \n\
395 clear.d $r4 \n\
396 clear.d $r5 \n\
397 clear.d $r6 \n\
398 clear.d $r7 \n\
399 clear.d $r8 \n\
400 clear.d $r9 \n\
401 clear.d $r10 \n\
402 clear.d $r11 \n\
403 \n\
404 ;; Now we've got this: \n\
405 ;; r13 - dst \n\
406 ;; r12 - n \n\
407 \n\
408 ;; Update n for the first loop \n\
409 subq 12*4,$r12 \n\
4100: \n\
411 subq 12*4,$r12 \n\
4121: \n\
413 bge 0b \n\
414 movem $r11,[$r13+] \n\
415 \n\
416 addq 12*4,$r12 ;; compensate for last loop underflowing n \n\
417 \n\
418 ;; Restore registers from stack \n\
419 movem [$sp+],$r10 \n\
4202: \n\
421 .section .fixup,\"ax\" \n\
4223: \n\
423 movem [$sp],$r10 \n\
424 addq 12*4,$r10 \n\
425 addq 12*4,$r13 \n\
426 movem $r10,[$sp] \n\
427 jump 0b \n\
428 clear.d $r10 \n\
429 \n\
430 .previous \n\
431 .section __ex_table,\"a\" \n\
432 .dword 1b,3b \n\
433 .previous"
434
435 : "=r" (dst), "=r" (n), "=r" (retn)
436 : "0" (dst), "1" (n), "2" (retn)
437 : "r11");
438 }
439
440 while (n >= 16)
441 {
442 __asm_clear_16 (dst, retn);
443 n -= 16;
444 }
445
446
447
448 while (n >= 4)
449 {
450 __asm_clear_4 (dst, retn);
451 n -= 4;
452 }
453
454 switch (n)
455 {
456 case 0:
457 break;
458 case 1:
459 __asm_clear_1 (dst, retn);
460 break;
461 case 2:
462 __asm_clear_2 (dst, retn);
463 break;
464 case 3:
465 __asm_clear_3 (dst, retn);
466 break;
467 }
468
469 return retn;
470}
471