1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/kernel.h>
18#include <linux/mm.h>
19#include <linux/spinlock.h>
20#include <linux/list.h>
21#include <linux/bootmem.h>
22#include <linux/rmap.h>
23#include <linux/pagemap.h>
24#include <linux/mutex.h>
25#include <linux/interrupt.h>
26#include <linux/sysctl.h>
27#include <linux/pagevec.h>
28#include <linux/ptrace.h>
29#include <linux/timex.h>
30#include <linux/cache.h>
31#include <linux/smp.h>
32#include <linux/module.h>
33#include <linux/hugetlb.h>
34
35#include <asm/page.h>
36#include <asm/sections.h>
37#include <asm/tlbflush.h>
38#include <asm/pgalloc.h>
39#include <asm/homecache.h>
40
41#include <arch/sim.h>
42
43#include "migrate.h"
44
45
46#if CHIP_HAS_COHERENT_LOCAL_CACHE()
47
48
49
50
51
52
53static int __write_once noallocl2;
54static int __init set_noallocl2(char *str)
55{
56 noallocl2 = 1;
57 return 0;
58}
59early_param("noallocl2", set_noallocl2);
60
61#else
62
63#define noallocl2 0
64
65#endif
66
67
68
69
70
71
72
73static void hv_flush_update(const struct cpumask *cache_cpumask,
74 struct cpumask *tlb_cpumask,
75 unsigned long tlb_va, unsigned long tlb_length,
76 HV_Remote_ASID *asids, int asidcount)
77{
78 struct cpumask mask;
79 int i, cpu;
80
81 cpumask_clear(&mask);
82 if (cache_cpumask)
83 cpumask_or(&mask, &mask, cache_cpumask);
84 if (tlb_cpumask && tlb_length) {
85 cpumask_or(&mask, &mask, tlb_cpumask);
86 }
87
88 for (i = 0; i < asidcount; ++i)
89 cpumask_set_cpu(asids[i].y * smp_width + asids[i].x, &mask);
90
91
92
93
94
95 for_each_cpu(cpu, &mask)
96 ++per_cpu(irq_stat, cpu).irq_hv_flush_count;
97}
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116void flush_remote(unsigned long cache_pfn, unsigned long cache_control,
117 const struct cpumask *cache_cpumask_orig,
118 HV_VirtAddr tlb_va, unsigned long tlb_length,
119 unsigned long tlb_pgsize,
120 const struct cpumask *tlb_cpumask_orig,
121 HV_Remote_ASID *asids, int asidcount)
122{
123 int rc;
124 struct cpumask cache_cpumask_copy, tlb_cpumask_copy;
125 struct cpumask *cache_cpumask, *tlb_cpumask;
126 HV_PhysAddr cache_pa;
127 char cache_buf[NR_CPUS*5], tlb_buf[NR_CPUS*5];
128
129 mb();
130
131
132
133
134 if (cache_cpumask_orig && cache_control) {
135 cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig);
136 cache_cpumask = &cache_cpumask_copy;
137 } else {
138 cpumask_clear(&cache_cpumask_copy);
139 cache_cpumask = NULL;
140 }
141 if (cache_cpumask == NULL)
142 cache_control = 0;
143 if (tlb_cpumask_orig && tlb_length) {
144 cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig);
145 tlb_cpumask = &tlb_cpumask_copy;
146 } else {
147 cpumask_clear(&tlb_cpumask_copy);
148 tlb_cpumask = NULL;
149 }
150
151 hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length,
152 asids, asidcount);
153 cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT;
154 rc = hv_flush_remote(cache_pa, cache_control,
155 cpumask_bits(cache_cpumask),
156 tlb_va, tlb_length, tlb_pgsize,
157 cpumask_bits(tlb_cpumask),
158 asids, asidcount);
159 if (rc == 0)
160 return;
161 cpumask_scnprintf(cache_buf, sizeof(cache_buf), &cache_cpumask_copy);
162 cpumask_scnprintf(tlb_buf, sizeof(tlb_buf), &tlb_cpumask_copy);
163
164 pr_err("hv_flush_remote(%#llx, %#lx, %p [%s],"
165 " %#lx, %#lx, %#lx, %p [%s], %p, %d) = %d\n",
166 cache_pa, cache_control, cache_cpumask, cache_buf,
167 (unsigned long)tlb_va, tlb_length, tlb_pgsize,
168 tlb_cpumask, tlb_buf,
169 asids, asidcount, rc);
170 panic("Unsafe to continue.");
171}
172
173static void homecache_finv_page_va(void* va, int home)
174{
175 if (home == smp_processor_id()) {
176 finv_buffer_local(va, PAGE_SIZE);
177 } else if (home == PAGE_HOME_HASH) {
178 finv_buffer_remote(va, PAGE_SIZE, 1);
179 } else {
180 BUG_ON(home < 0 || home >= NR_CPUS);
181 finv_buffer_remote(va, PAGE_SIZE, 0);
182 }
183}
184
185void homecache_finv_map_page(struct page *page, int home)
186{
187 unsigned long flags;
188 unsigned long va;
189 pte_t *ptep;
190 pte_t pte;
191
192 if (home == PAGE_HOME_UNCACHED)
193 return;
194 local_irq_save(flags);
195#ifdef CONFIG_HIGHMEM
196 va = __fix_to_virt(FIX_KMAP_BEGIN + kmap_atomic_idx_push() +
197 (KM_TYPE_NR * smp_processor_id()));
198#else
199 va = __fix_to_virt(FIX_HOMECACHE_BEGIN + smp_processor_id());
200#endif
201 ptep = virt_to_pte(NULL, (unsigned long)va);
202 pte = pfn_pte(page_to_pfn(page), PAGE_KERNEL);
203 __set_pte(ptep, pte_set_home(pte, home));
204 homecache_finv_page_va((void *)va, home);
205 __pte_clear(ptep);
206 hv_flush_page(va, PAGE_SIZE);
207#ifdef CONFIG_HIGHMEM
208 kmap_atomic_idx_pop();
209#endif
210 local_irq_restore(flags);
211}
212
213static void homecache_finv_page_home(struct page *page, int home)
214{
215 if (!PageHighMem(page) && home == page_home(page))
216 homecache_finv_page_va(page_address(page), home);
217 else
218 homecache_finv_map_page(page, home);
219}
220
221static inline bool incoherent_home(int home)
222{
223 return home == PAGE_HOME_IMMUTABLE || home == PAGE_HOME_INCOHERENT;
224}
225
226static void homecache_finv_page_internal(struct page *page, int force_map)
227{
228 int home = page_home(page);
229 if (home == PAGE_HOME_UNCACHED)
230 return;
231 if (incoherent_home(home)) {
232 int cpu;
233 for_each_cpu(cpu, &cpu_cacheable_map)
234 homecache_finv_map_page(page, cpu);
235 } else if (force_map) {
236
237 homecache_finv_map_page(page, home);
238 } else {
239 homecache_finv_page_home(page, home);
240 }
241 sim_validate_lines_evicted(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
242}
243
244void homecache_finv_page(struct page *page)
245{
246 homecache_finv_page_internal(page, 0);
247}
248
249void homecache_evict(const struct cpumask *mask)
250{
251 flush_remote(0, HV_FLUSH_EVICT_L2, mask, 0, 0, 0, NULL, NULL, 0);
252}
253
254
255static int pte_to_home(pte_t pte)
256{
257 if (hv_pte_get_nc(pte))
258 return PAGE_HOME_IMMUTABLE;
259 switch (hv_pte_get_mode(pte)) {
260 case HV_PTE_MODE_CACHE_TILE_L3:
261 return get_remote_cache_cpu(pte);
262 case HV_PTE_MODE_CACHE_NO_L3:
263 return PAGE_HOME_INCOHERENT;
264 case HV_PTE_MODE_UNCACHED:
265 return PAGE_HOME_UNCACHED;
266#if CHIP_HAS_CBOX_HOME_MAP()
267 case HV_PTE_MODE_CACHE_HASH_L3:
268 return PAGE_HOME_HASH;
269#endif
270 }
271 panic("Bad PTE %#llx\n", pte.val);
272}
273
274
275pte_t pte_set_home(pte_t pte, int home)
276{
277
278 if (pte_file(pte))
279 return pte;
280
281#if CHIP_HAS_MMIO()
282
283 if (hv_pte_get_mode(pte) == HV_PTE_MODE_MMIO)
284 return pte;
285#endif
286
287
288
289
290
291
292
293
294
295
296 if (hv_pte_get_nc(pte) && home != PAGE_HOME_IMMUTABLE) {
297 pte = hv_pte_clear_nc(pte);
298 pr_err("non-immutable page incoherently referenced: %#llx\n",
299 pte.val);
300 }
301
302 switch (home) {
303
304 case PAGE_HOME_UNCACHED:
305 pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
306 break;
307
308 case PAGE_HOME_INCOHERENT:
309 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
310 break;
311
312 case PAGE_HOME_IMMUTABLE:
313
314
315
316
317 BUG_ON(hv_pte_get_writable(pte));
318 if (pte_get_forcecache(pte)) {
319
320 if (hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_TILE_L3
321 && pte_get_anyhome(pte)) {
322 pte = hv_pte_set_mode(pte,
323 HV_PTE_MODE_CACHE_NO_L3);
324 }
325 } else
326#if CHIP_HAS_CBOX_HOME_MAP()
327 if (hash_default)
328 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
329 else
330#endif
331 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
332 pte = hv_pte_set_nc(pte);
333 break;
334
335#if CHIP_HAS_CBOX_HOME_MAP()
336 case PAGE_HOME_HASH:
337 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
338 break;
339#endif
340
341 default:
342 BUG_ON(home < 0 || home >= NR_CPUS ||
343 !cpu_is_valid_lotar(home));
344 pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_TILE_L3);
345 pte = set_remote_cache_cpu(pte, home);
346 break;
347 }
348
349#if CHIP_HAS_NC_AND_NOALLOC_BITS()
350 if (noallocl2)
351 pte = hv_pte_set_no_alloc_l2(pte);
352
353
354 if (hv_pte_get_no_alloc_l2(pte) && hv_pte_get_no_alloc_l1(pte) &&
355 hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_NO_L3) {
356 pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
357 }
358#endif
359
360
361 BUG_ON(hv_pte_get_mode(pte) == 0);
362
363 return pte;
364}
365EXPORT_SYMBOL(pte_set_home);
366
367
368
369
370
371
372
373
374#if CHIP_HAS_CBOX_HOME_MAP()
375static inline int initial_page_home(void) { return PAGE_HOME_HASH; }
376#else
377static inline int initial_page_home(void) { return 0; }
378#endif
379
380int page_home(struct page *page)
381{
382 if (PageHighMem(page)) {
383 return initial_page_home();
384 } else {
385 unsigned long kva = (unsigned long)page_address(page);
386 return pte_to_home(*virt_to_pte(NULL, kva));
387 }
388}
389EXPORT_SYMBOL(page_home);
390
391void homecache_change_page_home(struct page *page, int order, int home)
392{
393 int i, pages = (1 << order);
394 unsigned long kva;
395
396 BUG_ON(PageHighMem(page));
397 BUG_ON(page_count(page) > 1);
398 BUG_ON(page_mapcount(page) != 0);
399 kva = (unsigned long) page_address(page);
400 flush_remote(0, HV_FLUSH_EVICT_L2, &cpu_cacheable_map,
401 kva, pages * PAGE_SIZE, PAGE_SIZE, cpu_online_mask,
402 NULL, 0);
403
404 for (i = 0; i < pages; ++i, kva += PAGE_SIZE) {
405 pte_t *ptep = virt_to_pte(NULL, kva);
406 pte_t pteval = *ptep;
407 BUG_ON(!pte_present(pteval) || pte_huge(pteval));
408 __set_pte(ptep, pte_set_home(pteval, home));
409 }
410}
411EXPORT_SYMBOL(homecache_change_page_home);
412
413struct page *homecache_alloc_pages(gfp_t gfp_mask,
414 unsigned int order, int home)
415{
416 struct page *page;
417 BUG_ON(gfp_mask & __GFP_HIGHMEM);
418 page = alloc_pages(gfp_mask, order);
419 if (page)
420 homecache_change_page_home(page, order, home);
421 return page;
422}
423EXPORT_SYMBOL(homecache_alloc_pages);
424
425struct page *homecache_alloc_pages_node(int nid, gfp_t gfp_mask,
426 unsigned int order, int home)
427{
428 struct page *page;
429 BUG_ON(gfp_mask & __GFP_HIGHMEM);
430 page = alloc_pages_node(nid, gfp_mask, order);
431 if (page)
432 homecache_change_page_home(page, order, home);
433 return page;
434}
435
436void __homecache_free_pages(struct page *page, unsigned int order)
437{
438 if (put_page_testzero(page)) {
439 homecache_change_page_home(page, order, initial_page_home());
440 if (order == 0) {
441 free_hot_cold_page(page, 0);
442 } else {
443 init_page_count(page);
444 __free_pages(page, order);
445 }
446 }
447}
448EXPORT_SYMBOL(__homecache_free_pages);
449
450void homecache_free_pages(unsigned long addr, unsigned int order)
451{
452 if (addr != 0) {
453 VM_BUG_ON(!virt_addr_valid((void *)addr));
454 __homecache_free_pages(virt_to_page((void *)addr), order);
455 }
456}
457EXPORT_SYMBOL(homecache_free_pages);
458