1
2
3
4
5
6
7
8
9#include <linux/prefetch.h>
10#include <linux/string.h>
11#include <linux/thread_info.h>
12#include <asm/uaccess.h>
13
14unsigned long
15__generic_copy_to_user(void __user *to, const void *from, unsigned long n)
16{
17 prefetch(from);
18 if (access_ok(VERIFY_WRITE, to, n))
19 __copy_user(to,from,n);
20 return n;
21}
22
23unsigned long
24__generic_copy_from_user(void *to, const void __user *from, unsigned long n)
25{
26 prefetchw(to);
27 if (access_ok(VERIFY_READ, from, n))
28 __copy_user_zeroing(to,from,n);
29 else
30 memset(to, 0, n);
31 return n;
32}
33
34
35
36
37
38
39#ifdef CONFIG_ISA_DUAL_ISSUE
40
41#define __do_strncpy_from_user(dst,src,count,res) \
42do { \
43 int __d0, __d1, __d2; \
44 __asm__ __volatile__( \
45 " beqz %1, 2f\n" \
46 " .fillinsn\n" \
47 "0: ldb r14, @%3 || addi %3, #1\n" \
48 " stb r14, @%4 || addi %4, #1\n" \
49 " beqz r14, 1f\n" \
50 " addi %1, #-1\n" \
51 " bnez %1, 0b\n" \
52 " .fillinsn\n" \
53 "1: sub %0, %1\n" \
54 " .fillinsn\n" \
55 "2:\n" \
56 ".section .fixup,\"ax\"\n" \
57 " .balign 4\n" \
58 "3: seth r14, #high(2b)\n" \
59 " or3 r14, r14, #low(2b)\n" \
60 " jmp r14 || ldi %0, #%5\n" \
61 ".previous\n" \
62 ".section __ex_table,\"a\"\n" \
63 " .balign 4\n" \
64 " .long 0b,3b\n" \
65 ".previous" \
66 : "=&r"(res), "=&r"(count), "=&r" (__d0), "=&r" (__d1), \
67 "=&r" (__d2) \
68 : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), \
69 "4"(dst) \
70 : "r14", "cbit", "memory"); \
71} while (0)
72
73#else
74
75#define __do_strncpy_from_user(dst,src,count,res) \
76do { \
77 int __d0, __d1, __d2; \
78 __asm__ __volatile__( \
79 " beqz %1, 2f\n" \
80 " .fillinsn\n" \
81 "0: ldb r14, @%3\n" \
82 " stb r14, @%4\n" \
83 " addi %3, #1\n" \
84 " addi %4, #1\n" \
85 " beqz r14, 1f\n" \
86 " addi %1, #-1\n" \
87 " bnez %1, 0b\n" \
88 " .fillinsn\n" \
89 "1: sub %0, %1\n" \
90 " .fillinsn\n" \
91 "2:\n" \
92 ".section .fixup,\"ax\"\n" \
93 " .balign 4\n" \
94 "3: ldi %0, #%5\n" \
95 " seth r14, #high(2b)\n" \
96 " or3 r14, r14, #low(2b)\n" \
97 " jmp r14\n" \
98 ".previous\n" \
99 ".section __ex_table,\"a\"\n" \
100 " .balign 4\n" \
101 " .long 0b,3b\n" \
102 ".previous" \
103 : "=&r"(res), "=&r"(count), "=&r" (__d0), "=&r" (__d1), \
104 "=&r" (__d2) \
105 : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), \
106 "4"(dst) \
107 : "r14", "cbit", "memory"); \
108} while (0)
109
110#endif
111
112long
113__strncpy_from_user(char *dst, const char __user *src, long count)
114{
115 long res;
116 __do_strncpy_from_user(dst, src, count, res);
117 return res;
118}
119
120long
121strncpy_from_user(char *dst, const char __user *src, long count)
122{
123 long res = -EFAULT;
124 if (access_ok(VERIFY_READ, src, 1))
125 __do_strncpy_from_user(dst, src, count, res);
126 return res;
127}
128
129
130
131
132
133
134#ifdef CONFIG_ISA_DUAL_ISSUE
135
136#define __do_clear_user(addr,size) \
137do { \
138 int __dst, __c; \
139 __asm__ __volatile__( \
140 " beqz %1, 9f\n" \
141 " and3 r14, %0, #3\n" \
142 " bnez r14, 2f\n" \
143 " and3 r14, %1, #3\n" \
144 " bnez r14, 2f\n" \
145 " and3 %1, %1, #3\n" \
146 " beqz %2, 2f\n" \
147 " addi %0, #-4\n" \
148 " .fillinsn\n" \
149 "0: ; word clear \n" \
150 " st %6, @+%0 || addi %2, #-1\n" \
151 " bnez %2, 0b\n" \
152 " beqz %1, 9f\n" \
153 " .fillinsn\n" \
154 "2: ; byte clear \n" \
155 " stb %6, @%0 || addi %1, #-1\n" \
156 " addi %0, #1\n" \
157 " bnez %1, 2b\n" \
158 " .fillinsn\n" \
159 "9:\n" \
160 ".section .fixup,\"ax\"\n" \
161 " .balign 4\n" \
162 "4: slli %2, #2\n" \
163 " seth r14, #high(9b)\n" \
164 " or3 r14, r14, #low(9b)\n" \
165 " jmp r14 || add %1, %2\n" \
166 ".previous\n" \
167 ".section __ex_table,\"a\"\n" \
168 " .balign 4\n" \
169 " .long 0b,4b\n" \
170 " .long 2b,9b\n" \
171 ".previous\n" \
172 : "=&r"(__dst), "=&r"(size), "=&r"(__c) \
173 : "0"(addr), "1"(size), "2"(size / 4), "r"(0) \
174 : "r14", "cbit", "memory"); \
175} while (0)
176
177#else
178
179#define __do_clear_user(addr,size) \
180do { \
181 int __dst, __c; \
182 __asm__ __volatile__( \
183 " beqz %1, 9f\n" \
184 " and3 r14, %0, #3\n" \
185 " bnez r14, 2f\n" \
186 " and3 r14, %1, #3\n" \
187 " bnez r14, 2f\n" \
188 " and3 %1, %1, #3\n" \
189 " beqz %2, 2f\n" \
190 " addi %0, #-4\n" \
191 " .fillinsn\n" \
192 "0: st %6, @+%0 ; word clear \n" \
193 " addi %2, #-1\n" \
194 " bnez %2, 0b\n" \
195 " beqz %1, 9f\n" \
196 " .fillinsn\n" \
197 "2: stb %6, @%0 ; byte clear \n" \
198 " addi %1, #-1\n" \
199 " addi %0, #1\n" \
200 " bnez %1, 2b\n" \
201 " .fillinsn\n" \
202 "9:\n" \
203 ".section .fixup,\"ax\"\n" \
204 " .balign 4\n" \
205 "4: slli %2, #2\n" \
206 " add %1, %2\n" \
207 " seth r14, #high(9b)\n" \
208 " or3 r14, r14, #low(9b)\n" \
209 " jmp r14\n" \
210 ".previous\n" \
211 ".section __ex_table,\"a\"\n" \
212 " .balign 4\n" \
213 " .long 0b,4b\n" \
214 " .long 2b,9b\n" \
215 ".previous\n" \
216 : "=&r"(__dst), "=&r"(size), "=&r"(__c) \
217 : "0"(addr), "1"(size), "2"(size / 4), "r"(0) \
218 : "r14", "cbit", "memory"); \
219} while (0)
220
221#endif
222
223unsigned long
224clear_user(void __user *to, unsigned long n)
225{
226 if (access_ok(VERIFY_WRITE, to, n))
227 __do_clear_user(to, n);
228 return n;
229}
230
231unsigned long
232__clear_user(void __user *to, unsigned long n)
233{
234 __do_clear_user(to, n);
235 return n;
236}
237
238
239
240
241
242
243
244#ifdef CONFIG_ISA_DUAL_ISSUE
245
246long strnlen_user(const char __user *s, long n)
247{
248 unsigned long mask = -__addr_ok(s);
249 unsigned long res;
250
251 __asm__ __volatile__(
252 " and %0, %5 || mv r1, %1\n"
253 " beqz %0, strnlen_exit\n"
254 " and3 r0, %1, #3\n"
255 " bnez r0, strnlen_byte_loop\n"
256 " cmpui %0, #4\n"
257 " bc strnlen_byte_loop\n"
258 "strnlen_word_loop:\n"
259 "0: ld r0, @%1+\n"
260 " pcmpbz r0\n"
261 " bc strnlen_last_bytes_fixup\n"
262 " addi %0, #-4\n"
263 " beqz %0, strnlen_exit\n"
264 " bgtz %0, strnlen_word_loop\n"
265 "strnlen_last_bytes:\n"
266 " mv %0, %4\n"
267 "strnlen_last_bytes_fixup:\n"
268 " addi %1, #-4\n"
269 "strnlen_byte_loop:\n"
270 "1: ldb r0, @%1 || addi %0, #-1\n"
271 " beqz r0, strnlen_exit\n"
272 " addi %1, #1\n"
273 " bnez %0, strnlen_byte_loop\n"
274 "strnlen_exit:\n"
275 " sub %1, r1\n"
276 " add3 %0, %1, #1\n"
277 " .fillinsn\n"
278 "9:\n"
279 ".section .fixup,\"ax\"\n"
280 " .balign 4\n"
281 "4: addi %1, #-4\n"
282 " .fillinsn\n"
283 "5: seth r1, #high(9b)\n"
284 " or3 r1, r1, #low(9b)\n"
285 " jmp r1 || ldi %0, #0\n"
286 ".previous\n"
287 ".section __ex_table,\"a\"\n"
288 " .balign 4\n"
289 " .long 0b,4b\n"
290 " .long 1b,5b\n"
291 ".previous"
292 : "=&r" (res), "=r" (s)
293 : "0" (n), "1" (s), "r" (n & 3), "r" (mask), "r"(0x01010101)
294 : "r0", "r1", "cbit");
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310 return res & mask;
311}
312
313#else
314
315long strnlen_user(const char __user *s, long n)
316{
317 unsigned long mask = -__addr_ok(s);
318 unsigned long res;
319
320 __asm__ __volatile__(
321 " and %0, %5\n"
322 " mv r1, %1\n"
323 " beqz %0, strnlen_exit\n"
324 " and3 r0, %1, #3\n"
325 " bnez r0, strnlen_byte_loop\n"
326 " cmpui %0, #4\n"
327 " bc strnlen_byte_loop\n"
328 " sll3 r3, %6, #7\n"
329 "strnlen_word_loop:\n"
330 "0: ld r0, @%1+\n"
331 " not r2, r0\n"
332 " sub r0, %6\n"
333 " and r2, r3\n"
334 " and r2, r0\n"
335 " bnez r2, strnlen_last_bytes_fixup\n"
336 " addi %0, #-4\n"
337 " beqz %0, strnlen_exit\n"
338 " bgtz %0, strnlen_word_loop\n"
339 "strnlen_last_bytes:\n"
340 " mv %0, %4\n"
341 "strnlen_last_bytes_fixup:\n"
342 " addi %1, #-4\n"
343 "strnlen_byte_loop:\n"
344 "1: ldb r0, @%1\n"
345 " addi %0, #-1\n"
346 " beqz r0, strnlen_exit\n"
347 " addi %1, #1\n"
348 " bnez %0, strnlen_byte_loop\n"
349 "strnlen_exit:\n"
350 " sub %1, r1\n"
351 " add3 %0, %1, #1\n"
352 " .fillinsn\n"
353 "9:\n"
354 ".section .fixup,\"ax\"\n"
355 " .balign 4\n"
356 "4: addi %1, #-4\n"
357 " .fillinsn\n"
358 "5: ldi %0, #0\n"
359 " seth r1, #high(9b)\n"
360 " or3 r1, r1, #low(9b)\n"
361 " jmp r1\n"
362 ".previous\n"
363 ".section __ex_table,\"a\"\n"
364 " .balign 4\n"
365 " .long 0b,4b\n"
366 " .long 1b,5b\n"
367 ".previous"
368 : "=&r" (res), "=r" (s)
369 : "0" (n), "1" (s), "r" (n & 3), "r" (mask), "r"(0x01010101)
370 : "r0", "r1", "r2", "r3", "cbit");
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386 return res & mask;
387}
388
389#endif
390
391
