1
2
3
4
5#include <linux/export.h>
6#include <linux/mm.h>
7#include <linux/uaccess.h>
8
9bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
10 size_t size)
11{
12 return true;
13}
14
15#ifdef HAVE_GET_KERNEL_NOFAULT
16
17#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \
18 while (len >= sizeof(type)) { \
19 __get_kernel_nofault(dst, src, type, err_label); \
20 dst += sizeof(type); \
21 src += sizeof(type); \
22 len -= sizeof(type); \
23 }
24
25long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
26{
27 unsigned long align = 0;
28
29 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
30 align = (unsigned long)dst | (unsigned long)src;
31
32 if (!copy_from_kernel_nofault_allowed(src, size))
33 return -ERANGE;
34
35 pagefault_disable();
36 if (!(align & 7))
37 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
38 if (!(align & 3))
39 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
40 if (!(align & 1))
41 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
42 copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
43 pagefault_enable();
44 return 0;
45Efault:
46 pagefault_enable();
47 return -EFAULT;
48}
49EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
50
51#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \
52 while (len >= sizeof(type)) { \
53 __put_kernel_nofault(dst, src, type, err_label); \
54 dst += sizeof(type); \
55 src += sizeof(type); \
56 len -= sizeof(type); \
57 }
58
59long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
60{
61 unsigned long align = 0;
62
63 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
64 align = (unsigned long)dst | (unsigned long)src;
65
66 pagefault_disable();
67 if (!(align & 7))
68 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
69 if (!(align & 3))
70 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
71 if (!(align & 1))
72 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
73 copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
74 pagefault_enable();
75 return 0;
76Efault:
77 pagefault_enable();
78 return -EFAULT;
79}
80
81long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
82{
83 const void *src = unsafe_addr;
84
85 if (unlikely(count <= 0))
86 return 0;
87 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
88 return -ERANGE;
89
90 pagefault_disable();
91 do {
92 __get_kernel_nofault(dst, src, u8, Efault);
93 dst++;
94 src++;
95 } while (dst[-1] && src - unsafe_addr < count);
96 pagefault_enable();
97
98 dst[-1] = '\0';
99 return src - unsafe_addr;
100Efault:
101 pagefault_enable();
102 dst[-1] = '\0';
103 return -EFAULT;
104}
105#else
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
122{
123 long ret;
124 mm_segment_t old_fs = get_fs();
125
126 if (!copy_from_kernel_nofault_allowed(src, size))
127 return -ERANGE;
128
129 set_fs(KERNEL_DS);
130 pagefault_disable();
131 ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
132 size);
133 pagefault_enable();
134 set_fs(old_fs);
135
136 if (ret)
137 return -EFAULT;
138 return 0;
139}
140EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
141
142
143
144
145
146
147
148
149
150
151long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
152{
153 long ret;
154 mm_segment_t old_fs = get_fs();
155
156 set_fs(KERNEL_DS);
157 pagefault_disable();
158 ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
159 pagefault_enable();
160 set_fs(old_fs);
161
162 if (ret)
163 return -EFAULT;
164 return 0;
165}
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
187{
188 mm_segment_t old_fs = get_fs();
189 const void *src = unsafe_addr;
190 long ret;
191
192 if (unlikely(count <= 0))
193 return 0;
194 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
195 return -ERANGE;
196
197 set_fs(KERNEL_DS);
198 pagefault_disable();
199
200 do {
201 ret = __get_user(*dst++, (const char __user __force *)src++);
202 } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
203
204 dst[-1] = '\0';
205 pagefault_enable();
206 set_fs(old_fs);
207
208 return ret ? -EFAULT : src - unsafe_addr;
209}
210#endif
211
212
213
214
215
216
217
218
219
220
221long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
222{
223 long ret = -EFAULT;
224 mm_segment_t old_fs = force_uaccess_begin();
225
226 if (access_ok(src, size)) {
227 pagefault_disable();
228 ret = __copy_from_user_inatomic(dst, src, size);
229 pagefault_enable();
230 }
231 force_uaccess_end(old_fs);
232
233 if (ret)
234 return -EFAULT;
235 return 0;
236}
237EXPORT_SYMBOL_GPL(copy_from_user_nofault);
238
239
240
241
242
243
244
245
246
247
248long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
249{
250 long ret = -EFAULT;
251 mm_segment_t old_fs = force_uaccess_begin();
252
253 if (access_ok(dst, size)) {
254 pagefault_disable();
255 ret = __copy_to_user_inatomic(dst, src, size);
256 pagefault_enable();
257 }
258 force_uaccess_end(old_fs);
259
260 if (ret)
261 return -EFAULT;
262 return 0;
263}
264EXPORT_SYMBOL_GPL(copy_to_user_nofault);
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
285 long count)
286{
287 mm_segment_t old_fs;
288 long ret;
289
290 if (unlikely(count <= 0))
291 return 0;
292
293 old_fs = force_uaccess_begin();
294 pagefault_disable();
295 ret = strncpy_from_user(dst, unsafe_addr, count);
296 pagefault_enable();
297 force_uaccess_end(old_fs);
298
299 if (ret >= count) {
300 ret = count;
301 dst[ret - 1] = '\0';
302 } else if (ret > 0) {
303 ret++;
304 }
305
306 return ret;
307}
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325long strnlen_user_nofault(const void __user *unsafe_addr, long count)
326{
327 mm_segment_t old_fs;
328 int ret;
329
330 old_fs = force_uaccess_begin();
331 pagefault_disable();
332 ret = strnlen_user(unsafe_addr, count);
333 pagefault_enable();
334 force_uaccess_end(old_fs);
335
336 return ret;
337}
338