1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/kernel.h>
18#include <linux/mm.h>
19#include <linux/spinlock.h>
20#include <linux/list.h>
21#include <linux/bootmem.h>
22#include <linux/rmap.h>
23#include <linux/pagemap.h>
24#include <linux/mutex.h>
25#include <linux/interrupt.h>
26#include <linux/sysctl.h>
27#include <linux/pagevec.h>
28#include <linux/ptrace.h>
29#include <linux/timex.h>
30#include <linux/cache.h>
31#include <linux/smp.h>
32#include <linux/module.h>
33
34#include <asm/page.h>
35#include <asm/sections.h>
36#include <asm/tlbflush.h>
37#include <asm/pgalloc.h>
38#include <asm/homecache.h>
39
40#include <arch/sim.h>
41
42#include "migrate.h"
43
44
45#if CHIP_HAS_COHERENT_LOCAL_CACHE()
46
47
48
49
50
51
52static int __write_once noallocl2;
53static int __init set_noallocl2(char *str)
54{
55 noallocl2 = 1;
56 return 0;
57}
58early_param("noallocl2", set_noallocl2);
59
60#else
61
62#define noallocl2 0
63
64#endif
65
66
67#define mark_caches_evicted_start() 0
68#define mark_caches_evicted_finish(mask, timestamp) do {} while (0)
69
70
71
72
73
74
75
76static void hv_flush_update(const struct cpumask *cache_cpumask,
77 struct cpumask *tlb_cpumask,
78 unsigned long tlb_va, unsigned long tlb_length,
79 HV_Remote_ASID *asids, int asidcount)
80{
81 struct cpumask mask;
82 int i, cpu;
83
84 cpumask_clear(&mask);
85 if (cache_cpumask)
86 cpumask_or(&mask, &mask, cache_cpumask);
87 if (tlb_cpumask && tlb_length) {
88 cpumask_or(&mask, &mask, tlb_cpumask);
89 }
90
91 for (i = 0; i < asidcount; ++i)
92 cpumask_set_cpu(asids[i].y * smp_width + asids[i].x, &mask);
93
94
95
96
97
98 for_each_cpu(cpu, &mask)
99 ++per_cpu(irq_stat, cpu).irq_hv_flush_count;
100}
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120void flush_remote(unsigned long cache_pfn, unsigned long cache_control,
121 const struct cpumask *cache_cpumask_orig,
122 HV_VirtAddr tlb_va, unsigned long tlb_length,
123 unsigned long tlb_pgsize,
124 const struct cpumask *tlb_cpumask_orig,
125 HV_Remote_ASID *asids, int asidcount)
126{
127 int rc;
128 int timestamp = 0;
129 struct cpumask cache_cpumask_copy, tlb_cpumask_copy;
130 struct cpumask *cache_cpumask, *tlb_cpumask;
131 HV_PhysAddr cache_pa;
132 char cache_buf[NR_CPUS*5], tlb_buf[NR_CPUS*5];
133
134 mb();
135
136
137
138
139 if (cache_cpumask_orig && cache_control) {
140 cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig);
141 cache_cpumask = &cache_cpumask_copy;
142 } else {
143 cpumask_clear(&cache_cpumask_copy);
144 cache_cpumask = NULL;
145 }
146 if (cache_cpumask == NULL)
147 cache_control = 0;
148 if (tlb_cpumask_orig && tlb_length) {
149 cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig);
150 tlb_cpumask = &tlb_cpumask_copy;
151 } else {
152 cpumask_clear(&tlb_cpumask_copy);
153 tlb_cpumask = NULL;
154 }
155
156 hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length,
157 asids, asidcount);
158 cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT;
159 if (cache_control & HV_FLUSH_EVICT_L2)
160 timestamp = mark_caches_evicted_start();
161 rc = hv_flush_remote(cache_pa, cache_control,
162 cpumask_bits(cache_cpumask),
163 tlb_va, tlb_length, tlb_pgsize,
164 cpumask_bits(tlb_cpumask),
165 asids, asidcount);
166 if (cache_control & HV_FLUSH_EVICT_L2)
167 mark_caches_evicted_finish(cache_cpumask, timestamp);
168 if (rc == 0)
169 return;
170 cpumask_scnprintf(cache_buf, sizeof(cache_buf), &cache_cpumask_copy);
171 cpumask_scnprintf(tlb_buf, sizeof(tlb_buf), &tlb_cpumask_copy);
172
173 pr_err("hv_flush_remote(%#llx, %#lx, %p [%s],"
174 " %#lx, %#lx, %#lx, %p [%s], %p, %d) = %d\n",
175 cache_pa, cache_control, cache_cpumask, cache_buf,
176 (unsigned long)tlb_va, tlb_length, tlb_pgsize,
177 tlb_cpumask, tlb_buf,
178 asids, asidcount, rc);
179 panic("Unsafe to continue.");
180}
181
182void homecache_evict(const struct cpumask *mask)
183{
184 flush_remote(0, HV_FLUSH_EVICT_L2, mask, 0, 0, 0, NULL, NULL, 0);
185}
186
187
188static void homecache_mask(struct page *page, int pages,
189 struct cpumask *home_mask)
190{
191 int i;
192 cpumask_clear(home_mask);
193 for (i = 0; i < pages; ++i) {
194 int home = page_home(&page[i]);
195 if (home == PAGE_HOME_IMMUTABLE ||
196 home == PAGE_HOME_INCOHERENT) {
197 cpumask_copy(home_mask, cpu_possible_mask);
198 return;
199 }
200#if CHIP_HAS_CBOX_HOME_MAP()
201 if (home == PAGE_HOME_HASH) {
202 cpumask_or(home_mask, home_mask, &hash_for_home_map);
203 continue;
204 }
205#endif
206 if (home == PAGE_HOME_UNCACHED)
207 continue;
208 BUG_ON(home < 0 || home >= NR_CPUS);
209 cpumask_set_cpu(home, home_mask);
210 }
211}
212
213
214
215
216
217static unsigned long cache_flush_length(unsigned long length)
218{
219 return (length >= CHIP_L2_CACHE_SIZE()) ? HV_FLUSH_EVICT_L2 : length;
220}
221
222
223void homecache_flush_cache(struct page *page, int order)
224{
225 int pages = 1 << order;
226 int length = cache_flush_length(pages * PAGE_SIZE);
227 unsigned long pfn = page_to_pfn(page);
228 struct cpumask home_mask;
229
230 homecache_mask(page, pages, &home_mask);
231 flush_remote(pfn, length, &home_mask, 0, 0, 0, NULL, NULL, 0);
232 sim_validate_lines_evicted(PFN_PHYS(pfn), pages * PAGE_SIZE);
233}
234
235
236
237static int pte_to_home(pte_t pte)
238{
239 if (hv_pte_get_nc(pte))
240 return PAGE_HOME_IMMUTABLE;
241 switch (hv_pte_get_mode(pte)) {
242 case HV_PTE_MODE_CACHE_TILE_L3:
243 return get_remote_cache_cpu(pte);
244 case HV_PTE_MODE_CACHE_NO_L3:
245 return PAGE_HOME_INCOHERENT;
246 case HV_PTE_MODE_UNCACHED:
247 return PAGE_HOME_UNCACHED;
248#if CHIP_HAS_CBOX_HOME_MAP()
249 case HV_PTE_MODE_CACHE_HASH_L3:
250 return PAGE_HOME_HASH;
251#endif
252 }
253 panic("Bad PTE %#llx\n", pte.val);
254}
255
256
257pte_t pte_set_home(pte_t pte, int home)
258{
259
260 if (pte_file(pte))
261 return pte;
262
263#if CHIP_HAS_MMIO()
264
265 if (hv_pte_get_mode(pte) == HV_PTE_MODE_MMIO)
266 return pte;
267#endif
268
269
270
271
272
273
274
275
276
277
278 if (hv_pte_get_nc(pte) && home != PAGE_HOME_IMMUTABLE) {
279 pte = hv_pte_clear_nc(pte);
280 pr_err("non-immutable page incoherently referenced: %#llx\n",
281 pte.val);
282 }
283
284 switch (home) {
285
286 case PAGE_HOME_UNCACHED:
287 pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
288 break;
289
290 case PAGE_HOME_INCOHERENT:
291 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
292 break;
293
294 case PAGE_HOME_IMMUTABLE:
295
296
297
298
299 BUG_ON(hv_pte_get_writable(pte));
300 if (pte_get_forcecache(pte)) {
301
302 if (hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_TILE_L3
303 && pte_get_anyhome(pte)) {
304 pte = hv_pte_set_mode(pte,
305 HV_PTE_MODE_CACHE_NO_L3);
306 }
307 } else
308#if CHIP_HAS_CBOX_HOME_MAP()
309 if (hash_default)
310 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
311 else
312#endif
313 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
314 pte = hv_pte_set_nc(pte);
315 break;
316
317#if CHIP_HAS_CBOX_HOME_MAP()
318 case PAGE_HOME_HASH:
319 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
320 break;
321#endif
322
323 default:
324 BUG_ON(home < 0 || home >= NR_CPUS ||
325 !cpu_is_valid_lotar(home));
326 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_TILE_L3);
327 pte = set_remote_cache_cpu(pte, home);
328 break;
329 }
330
331#if CHIP_HAS_NC_AND_NOALLOC_BITS()
332 if (noallocl2)
333 pte = hv_pte_set_no_alloc_l2(pte);
334
335
336 if (hv_pte_get_no_alloc_l2(pte) && hv_pte_get_no_alloc_l1(pte) &&
337 hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_NO_L3) {
338 pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
339 }
340#endif
341
342
343 BUG_ON(hv_pte_get_mode(pte) == 0);
344
345 return pte;
346}
347EXPORT_SYMBOL(pte_set_home);
348
349
350
351
352
353
354
355
356#if CHIP_HAS_CBOX_HOME_MAP()
357static inline int initial_page_home(void) { return PAGE_HOME_HASH; }
358#else
359static inline int initial_page_home(void) { return 0; }
360#endif
361
362int page_home(struct page *page)
363{
364 if (PageHighMem(page)) {
365 return initial_page_home();
366 } else {
367 unsigned long kva = (unsigned long)page_address(page);
368 return pte_to_home(*virt_to_pte(NULL, kva));
369 }
370}
371
372void homecache_change_page_home(struct page *page, int order, int home)
373{
374 int i, pages = (1 << order);
375 unsigned long kva;
376
377 BUG_ON(PageHighMem(page));
378 BUG_ON(page_count(page) > 1);
379 BUG_ON(page_mapcount(page) != 0);
380 kva = (unsigned long) page_address(page);
381 flush_remote(0, HV_FLUSH_EVICT_L2, &cpu_cacheable_map,
382 kva, pages * PAGE_SIZE, PAGE_SIZE, cpu_online_mask,
383 NULL, 0);
384
385 for (i = 0; i < pages; ++i, kva += PAGE_SIZE) {
386 pte_t *ptep = virt_to_pte(NULL, kva);
387 pte_t pteval = *ptep;
388 BUG_ON(!pte_present(pteval) || pte_huge(pteval));
389 *ptep = pte_set_home(pteval, home);
390 }
391}
392
393struct page *homecache_alloc_pages(gfp_t gfp_mask,
394 unsigned int order, int home)
395{
396 struct page *page;
397 BUG_ON(gfp_mask & __GFP_HIGHMEM);
398 page = alloc_pages(gfp_mask, order);
399 if (page)
400 homecache_change_page_home(page, order, home);
401 return page;
402}
403EXPORT_SYMBOL(homecache_alloc_pages);
404
405struct page *homecache_alloc_pages_node(int nid, gfp_t gfp_mask,
406 unsigned int order, int home)
407{
408 struct page *page;
409 BUG_ON(gfp_mask & __GFP_HIGHMEM);
410 page = alloc_pages_node(nid, gfp_mask, order);
411 if (page)
412 homecache_change_page_home(page, order, home);
413 return page;
414}
415
416void homecache_free_pages(unsigned long addr, unsigned int order)
417{
418 struct page *page;
419
420 if (addr == 0)
421 return;
422
423 VM_BUG_ON(!virt_addr_valid((void *)addr));
424 page = virt_to_page((void *)addr);
425 if (put_page_testzero(page)) {
426 int pages = (1 << order);
427 homecache_change_page_home(page, order, initial_page_home());
428 while (pages--)
429 __free_page(page++);
430 }
431}
432