1
2
3#include <linux/highmem.h>
4#include <linux/kprobes.h>
5
6
7
8
9
10static inline bool flush_coherent_icache(void)
11{
12
13
14
15
16
17
18 if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
19 mb();
20 icbi((void *)PAGE_OFFSET);
21 mb();
22 isync();
23 return true;
24 }
25
26 return false;
27}
28
29
30
31
32
33
34static void invalidate_icache_range(unsigned long start, unsigned long stop)
35{
36 unsigned long shift = l1_icache_shift();
37 unsigned long bytes = l1_icache_bytes();
38 char *addr = (char *)(start & ~(bytes - 1));
39 unsigned long size = stop - (unsigned long)addr + (bytes - 1);
40 unsigned long i;
41
42 for (i = 0; i < size >> shift; i++, addr += bytes)
43 icbi(addr);
44
45 mb();
46 isync();
47}
48
49
50
51
52
53
54
55
56
57
58void flush_icache_range(unsigned long start, unsigned long stop)
59{
60 if (flush_coherent_icache())
61 return;
62
63 clean_dcache_range(start, stop);
64
65 if (IS_ENABLED(CONFIG_44x)) {
66
67
68
69
70
71 iccci((void *)start);
72 mb();
73 isync();
74 } else
75 invalidate_icache_range(start, stop);
76}
77EXPORT_SYMBOL(flush_icache_range);
78
79#ifdef CONFIG_HIGHMEM
80
81
82
83
84static void flush_dcache_icache_phys(unsigned long physaddr)
85{
86 unsigned long bytes = l1_dcache_bytes();
87 unsigned long nb = PAGE_SIZE / bytes;
88 unsigned long addr = physaddr & PAGE_MASK;
89 unsigned long msr, msr0;
90 unsigned long loop1 = addr, loop2 = addr;
91
92 msr0 = mfmsr();
93 msr = msr0 & ~MSR_DR;
94
95
96
97
98 asm volatile(
99 " mtctr %2;\n"
100 " mtmsr %3;\n"
101 " isync;\n"
102 "0: dcbst 0, %0;\n"
103 " addi %0, %0, %4;\n"
104 " bdnz 0b;\n"
105 " sync;\n"
106 " mtctr %2;\n"
107 "1: icbi 0, %1;\n"
108 " addi %1, %1, %4;\n"
109 " bdnz 1b;\n"
110 " sync;\n"
111 " mtmsr %5;\n"
112 " isync;\n"
113 : "+&r" (loop1), "+&r" (loop2)
114 : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0)
115 : "ctr", "memory");
116}
117NOKPROBE_SYMBOL(flush_dcache_icache_phys)
118#else
119static void flush_dcache_icache_phys(unsigned long physaddr)
120{
121}
122#endif
123
124
125
126
127
128
129
130
131static void __flush_dcache_icache(void *p)
132{
133 unsigned long addr = (unsigned long)p & PAGE_MASK;
134
135 clean_dcache_range(addr, addr + PAGE_SIZE);
136
137
138
139
140
141
142
143
144
145 if (mmu_has_feature(MMU_FTR_TYPE_44x))
146 return;
147
148 invalidate_icache_range(addr, addr + PAGE_SIZE);
149}
150
151static void flush_dcache_icache_hugepage(struct page *page)
152{
153 int i;
154 int nr = compound_nr(page);
155
156 if (!PageHighMem(page)) {
157 for (i = 0; i < nr; i++)
158 __flush_dcache_icache(lowmem_page_address(page + i));
159 } else {
160 for (i = 0; i < nr; i++) {
161 void *start = kmap_local_page(page + i);
162
163 __flush_dcache_icache(start);
164 kunmap_local(start);
165 }
166 }
167}
168
169void flush_dcache_icache_page(struct page *page)
170{
171 if (flush_coherent_icache())
172 return;
173
174 if (PageCompound(page))
175 return flush_dcache_icache_hugepage(page);
176
177 if (!PageHighMem(page)) {
178 __flush_dcache_icache(lowmem_page_address(page));
179 } else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) {
180 void *start = kmap_local_page(page);
181
182 __flush_dcache_icache(start);
183 kunmap_local(start);
184 } else {
185 flush_dcache_icache_phys(page_to_phys(page));
186 }
187}
188EXPORT_SYMBOL(flush_dcache_icache_page);
189
190void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
191{
192 clear_page(page);
193
194
195
196
197
198
199 flush_dcache_page(pg);
200}
201EXPORT_SYMBOL(clear_user_page);
202
203void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
204 struct page *pg)
205{
206 copy_page(vto, vfrom);
207
208
209
210
211
212
213
214
215
216
217
218#if 0
219 if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
220 return;
221#endif
222
223 flush_dcache_page(pg);
224}
225
226void flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
227 unsigned long addr, int len)
228{
229 void *maddr;
230
231 maddr = kmap_local_page(page) + (addr & ~PAGE_MASK);
232 flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len);
233 kunmap_local(maddr);
234}
235