1
2
3
4
5
6
7
8#include <linux/export.h>
9#include <linux/uaccess.h>
10#include <linux/highmem.h>
11
12
13
14
15
16unsigned long __clear_user(void __user *addr, unsigned long size)
17{
18 long __d0;
19 might_fault();
20
21
22 stac();
23 asm volatile(
24 " testq %[size8],%[size8]\n"
25 " jz 4f\n"
26 "0: movq $0,(%[dst])\n"
27 " addq $8,%[dst]\n"
28 " decl %%ecx ; jnz 0b\n"
29 "4: movq %[size1],%%rcx\n"
30 " testl %%ecx,%%ecx\n"
31 " jz 2f\n"
32 "1: movb $0,(%[dst])\n"
33 " incq %[dst]\n"
34 " decl %%ecx ; jnz 1b\n"
35 "2:\n"
36 ".section .fixup,\"ax\"\n"
37 "3: lea 0(%[size1],%[size8],8),%[size8]\n"
38 " jmp 2b\n"
39 ".previous\n"
40 _ASM_EXTABLE_UA(0b, 3b)
41 _ASM_EXTABLE_UA(1b, 2b)
42 : [size8] "=&c"(size), [dst] "=&D" (__d0)
43 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
44 clac();
45 return size;
46}
47EXPORT_SYMBOL(__clear_user);
48
49unsigned long clear_user(void __user *to, unsigned long n)
50{
51 if (access_ok(VERIFY_WRITE, to, n))
52 return __clear_user(to, n);
53 return n;
54}
55EXPORT_SYMBOL(clear_user);
56
57
58
59
60
61
62__visible unsigned long
63copy_user_handle_tail(char *to, char *from, unsigned len)
64{
65 for (; len; --len, to++) {
66 char c;
67
68 if (__get_user_nocheck(c, from++, sizeof(char)))
69 break;
70 if (__put_user_nocheck(c, to, sizeof(char)))
71 break;
72 }
73 clac();
74 return len;
75}
76
77
78
79
80
81
82__visible unsigned long
83mcsafe_handle_tail(char *to, char *from, unsigned len)
84{
85 for (; len; --len, to++, from++) {
86
87
88
89
90 unsigned long rem = __memcpy_mcsafe(to, from, 1);
91
92 if (rem)
93 break;
94 }
95 return len;
96}
97
98#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
99
100
101
102
103
104
105
106
107
108static void clean_cache_range(void *addr, size_t size)
109{
110 u16 x86_clflush_size = boot_cpu_data.x86_clflush_size;
111 unsigned long clflush_mask = x86_clflush_size - 1;
112 void *vend = addr + size;
113 void *p;
114
115 for (p = (void *)((unsigned long)addr & ~clflush_mask);
116 p < vend; p += x86_clflush_size)
117 clwb(p);
118}
119
120void arch_wb_cache_pmem(void *addr, size_t size)
121{
122 clean_cache_range(addr, size);
123}
124EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
125
126long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
127{
128 unsigned long flushed, dest = (unsigned long) dst;
129 long rc = __copy_user_nocache(dst, src, size, 0);
130
131
132
133
134
135
136
137
138
139 if (size < 8) {
140 if (!IS_ALIGNED(dest, 4) || size != 4)
141 clean_cache_range(dst, 1);
142 } else {
143 if (!IS_ALIGNED(dest, 8)) {
144 dest = ALIGN(dest, boot_cpu_data.x86_clflush_size);
145 clean_cache_range(dst, 1);
146 }
147
148 flushed = dest - (unsigned long) dst;
149 if (size > flushed && !IS_ALIGNED(size - flushed, 8))
150 clean_cache_range(dst + size - 1, 1);
151 }
152
153 return rc;
154}
155
156void __memcpy_flushcache(void *_dst, const void *_src, size_t size)
157{
158 unsigned long dest = (unsigned long) _dst;
159 unsigned long source = (unsigned long) _src;
160
161
162 if (!IS_ALIGNED(dest, 8)) {
163 unsigned len = min_t(unsigned, size, ALIGN(dest, 8) - dest);
164
165 memcpy((void *) dest, (void *) source, len);
166 clean_cache_range((void *) dest, len);
167 dest += len;
168 source += len;
169 size -= len;
170 if (!size)
171 return;
172 }
173
174
175 while (size >= 32) {
176 asm("movq (%0), %%r8\n"
177 "movq 8(%0), %%r9\n"
178 "movq 16(%0), %%r10\n"
179 "movq 24(%0), %%r11\n"
180 "movnti %%r8, (%1)\n"
181 "movnti %%r9, 8(%1)\n"
182 "movnti %%r10, 16(%1)\n"
183 "movnti %%r11, 24(%1)\n"
184 :: "r" (source), "r" (dest)
185 : "memory", "r8", "r9", "r10", "r11");
186 dest += 32;
187 source += 32;
188 size -= 32;
189 }
190
191
192 while (size >= 8) {
193 asm("movq (%0), %%r8\n"
194 "movnti %%r8, (%1)\n"
195 :: "r" (source), "r" (dest)
196 : "memory", "r8");
197 dest += 8;
198 source += 8;
199 size -= 8;
200 }
201
202
203 while (size >= 4) {
204 asm("movl (%0), %%r8d\n"
205 "movnti %%r8d, (%1)\n"
206 :: "r" (source), "r" (dest)
207 : "memory", "r8");
208 dest += 4;
209 source += 4;
210 size -= 4;
211 }
212
213
214 if (size) {
215 memcpy((void *) dest, (void *) source, size);
216 clean_cache_range((void *) dest, size);
217 }
218}
219EXPORT_SYMBOL_GPL(__memcpy_flushcache);
220
221void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
222 size_t len)
223{
224 char *from = kmap_atomic(page);
225
226 memcpy_flushcache(to, from + offset, len);
227 kunmap_atomic(from);
228}
229#endif
230