1
2
3
4
5
6
7
8
9
10
11
12
13#undef DEBUG_LOW
14
15#include <linux/spinlock.h>
16#include <linux/bitops.h>
17#include <linux/of.h>
18#include <linux/threads.h>
19#include <linux/smp.h>
20
21#include <asm/machdep.h>
22#include <asm/mmu.h>
23#include <asm/mmu_context.h>
24#include <asm/pgtable.h>
25#include <asm/tlbflush.h>
26#include <asm/tlb.h>
27#include <asm/cputable.h>
28#include <asm/udbg.h>
29#include <asm/kexec.h>
30#include <asm/ppc-opcode.h>
31
32#ifdef DEBUG_LOW
33#define DBG_LOW(fmt...) udbg_printf(fmt)
34#else
35#define DBG_LOW(fmt...)
36#endif
37
38#ifdef __BIG_ENDIAN__
39#define HPTE_LOCK_BIT 3
40#else
41#define HPTE_LOCK_BIT (56+3)
42#endif
43
44DEFINE_RAW_SPINLOCK(native_tlbie_lock);
45
46static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
47{
48 unsigned long va;
49 unsigned int penc;
50 unsigned long sllp;
51
52
53
54
55
56
57
58
59 va = vpn << VPN_SHIFT;
60
61
62
63
64
65 va &= ~(0xffffULL << 48);
66
67 switch (psize) {
68 case MMU_PAGE_4K:
69
70 va &= ~((1ul << (64 - 52)) - 1);
71 va |= ssize << 8;
72 sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) |
73 ((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4);
74 va |= sllp << 5;
75 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
76 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
77 : "memory");
78 break;
79 default:
80
81 penc = mmu_psize_defs[psize].penc[apsize];
82 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
83 va |= penc << 12;
84 va |= ssize << 8;
85
86
87
88
89
90
91
92 va |= (vpn & 0xfe);
93 va |= 1;
94 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
95 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
96 : "memory");
97 break;
98 }
99}
100
101static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
102{
103 unsigned long va;
104 unsigned int penc;
105 unsigned long sllp;
106
107
108 va = vpn << VPN_SHIFT;
109
110
111
112
113
114 va &= ~(0xffffULL << 48);
115
116 switch (psize) {
117 case MMU_PAGE_4K:
118
119 va &= ~((1ul << (64 - 52)) - 1);
120 va |= ssize << 8;
121 sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) |
122 ((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4);
123 va |= sllp << 5;
124 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
125 : : "r"(va) : "memory");
126 break;
127 default:
128
129 penc = mmu_psize_defs[psize].penc[apsize];
130 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
131 va |= penc << 12;
132 va |= ssize << 8;
133
134
135
136
137
138
139
140 va |= (vpn & 0xfe);
141 va |= 1;
142 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
143 : : "r"(va) : "memory");
144 break;
145 }
146
147}
148
149static inline void tlbie(unsigned long vpn, int psize, int apsize,
150 int ssize, int local)
151{
152 unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL);
153 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
154
155 if (use_local)
156 use_local = mmu_psize_defs[psize].tlbiel;
157 if (lock_tlbie && !use_local)
158 raw_spin_lock(&native_tlbie_lock);
159 asm volatile("ptesync": : :"memory");
160 if (use_local) {
161 __tlbiel(vpn, psize, apsize, ssize);
162 asm volatile("ptesync": : :"memory");
163 } else {
164 __tlbie(vpn, psize, apsize, ssize);
165 asm volatile("eieio; tlbsync; ptesync": : :"memory");
166 }
167 if (lock_tlbie && !use_local)
168 raw_spin_unlock(&native_tlbie_lock);
169}
170
171static inline void native_lock_hpte(struct hash_pte *hptep)
172{
173 unsigned long *word = (unsigned long *)&hptep->v;
174
175 while (1) {
176 if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
177 break;
178 while(test_bit(HPTE_LOCK_BIT, word))
179 cpu_relax();
180 }
181}
182
183static inline void native_unlock_hpte(struct hash_pte *hptep)
184{
185 unsigned long *word = (unsigned long *)&hptep->v;
186
187 clear_bit_unlock(HPTE_LOCK_BIT, word);
188}
189
190static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
191 unsigned long pa, unsigned long rflags,
192 unsigned long vflags, int psize, int apsize, int ssize)
193{
194 struct hash_pte *hptep = htab_address + hpte_group;
195 unsigned long hpte_v, hpte_r;
196 int i;
197
198 if (!(vflags & HPTE_V_BOLTED)) {
199 DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx,"
200 " rflags=%lx, vflags=%lx, psize=%d)\n",
201 hpte_group, vpn, pa, rflags, vflags, psize);
202 }
203
204 for (i = 0; i < HPTES_PER_GROUP; i++) {
205 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
206
207 native_lock_hpte(hptep);
208 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID))
209 break;
210 native_unlock_hpte(hptep);
211 }
212
213 hptep++;
214 }
215
216 if (i == HPTES_PER_GROUP)
217 return -1;
218
219 hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
220 hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
221
222 if (!(vflags & HPTE_V_BOLTED)) {
223 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
224 i, hpte_v, hpte_r);
225 }
226
227 hptep->r = cpu_to_be64(hpte_r);
228
229 eieio();
230
231
232
233
234 hptep->v = cpu_to_be64(hpte_v);
235
236 __asm__ __volatile__ ("ptesync" : : : "memory");
237
238 return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
239}
240
241static long native_hpte_remove(unsigned long hpte_group)
242{
243 struct hash_pte *hptep;
244 int i;
245 int slot_offset;
246 unsigned long hpte_v;
247
248 DBG_LOW(" remove(group=%lx)\n", hpte_group);
249
250
251 slot_offset = mftb() & 0x7;
252
253 for (i = 0; i < HPTES_PER_GROUP; i++) {
254 hptep = htab_address + hpte_group + slot_offset;
255 hpte_v = be64_to_cpu(hptep->v);
256
257 if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
258
259 native_lock_hpte(hptep);
260 hpte_v = be64_to_cpu(hptep->v);
261 if ((hpte_v & HPTE_V_VALID)
262 && !(hpte_v & HPTE_V_BOLTED))
263 break;
264 native_unlock_hpte(hptep);
265 }
266
267 slot_offset++;
268 slot_offset &= 0x7;
269 }
270
271 if (i == HPTES_PER_GROUP)
272 return -1;
273
274
275 hptep->v = 0;
276
277 return i;
278}
279
280static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
281 unsigned long vpn, int bpsize,
282 int apsize, int ssize, int local)
283{
284 struct hash_pte *hptep = htab_address + slot;
285 unsigned long hpte_v, want_v;
286 int ret = 0;
287
288 want_v = hpte_encode_avpn(vpn, bpsize, ssize);
289
290 DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
291 vpn, want_v & HPTE_V_AVPN, slot, newpp);
292
293 native_lock_hpte(hptep);
294
295 hpte_v = be64_to_cpu(hptep->v);
296
297
298
299
300
301
302
303 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
304 DBG_LOW(" -> miss\n");
305 ret = -1;
306 } else {
307 DBG_LOW(" -> hit\n");
308
309 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & ~(HPTE_R_PP | HPTE_R_N)) |
310 (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C)));
311 }
312 native_unlock_hpte(hptep);
313
314
315 tlbie(vpn, bpsize, apsize, ssize, local);
316
317 return ret;
318}
319
320static long native_hpte_find(unsigned long vpn, int psize, int ssize)
321{
322 struct hash_pte *hptep;
323 unsigned long hash;
324 unsigned long i;
325 long slot;
326 unsigned long want_v, hpte_v;
327
328 hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
329 want_v = hpte_encode_avpn(vpn, psize, ssize);
330
331
332 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
333 for (i = 0; i < HPTES_PER_GROUP; i++) {
334 hptep = htab_address + slot;
335 hpte_v = be64_to_cpu(hptep->v);
336
337 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
338
339 return slot;
340 ++slot;
341 }
342
343 return -1;
344}
345
346
347
348
349
350
351
352
353static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
354 int psize, int ssize)
355{
356 unsigned long vpn;
357 unsigned long vsid;
358 long slot;
359 struct hash_pte *hptep;
360
361 vsid = get_kernel_vsid(ea, ssize);
362 vpn = hpt_vpn(ea, vsid, ssize);
363
364 slot = native_hpte_find(vpn, psize, ssize);
365 if (slot == -1)
366 panic("could not find page to bolt\n");
367 hptep = htab_address + slot;
368
369
370 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
371 ~(HPTE_R_PP | HPTE_R_N)) |
372 (newpp & (HPTE_R_PP | HPTE_R_N)));
373
374
375
376
377 tlbie(vpn, psize, psize, ssize, 0);
378}
379
380static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
381 int bpsize, int apsize, int ssize, int local)
382{
383 struct hash_pte *hptep = htab_address + slot;
384 unsigned long hpte_v;
385 unsigned long want_v;
386 unsigned long flags;
387
388 local_irq_save(flags);
389
390 DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
391
392 want_v = hpte_encode_avpn(vpn, bpsize, ssize);
393 native_lock_hpte(hptep);
394 hpte_v = be64_to_cpu(hptep->v);
395
396
397
398
399
400
401
402
403 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
404 native_unlock_hpte(hptep);
405 else
406
407 hptep->v = 0;
408
409
410 tlbie(vpn, bpsize, apsize, ssize, local);
411
412 local_irq_restore(flags);
413}
414
415static void native_hugepage_invalidate(struct mm_struct *mm,
416 unsigned char *hpte_slot_array,
417 unsigned long addr, int psize)
418{
419 int ssize = 0, i;
420 int lock_tlbie;
421 struct hash_pte *hptep;
422 int actual_psize = MMU_PAGE_16M;
423 unsigned int max_hpte_count, valid;
424 unsigned long flags, s_addr = addr;
425 unsigned long hpte_v, want_v, shift;
426 unsigned long hidx, vpn = 0, vsid, hash, slot;
427
428 shift = mmu_psize_defs[psize].shift;
429 max_hpte_count = 1U << (PMD_SHIFT - shift);
430
431 local_irq_save(flags);
432 for (i = 0; i < max_hpte_count; i++) {
433 valid = hpte_valid(hpte_slot_array, i);
434 if (!valid)
435 continue;
436 hidx = hpte_hash_index(hpte_slot_array, i);
437
438
439 addr = s_addr + (i * (1ul << shift));
440 if (!is_kernel_addr(addr)) {
441 ssize = user_segment_size(addr);
442 vsid = get_vsid(mm->context.id, addr, ssize);
443 WARN_ON(vsid == 0);
444 } else {
445 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
446 ssize = mmu_kernel_ssize;
447 }
448
449 vpn = hpt_vpn(addr, vsid, ssize);
450 hash = hpt_hash(vpn, shift, ssize);
451 if (hidx & _PTEIDX_SECONDARY)
452 hash = ~hash;
453
454 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
455 slot += hidx & _PTEIDX_GROUP_IX;
456
457 hptep = htab_address + slot;
458 want_v = hpte_encode_avpn(vpn, psize, ssize);
459 native_lock_hpte(hptep);
460 hpte_v = be64_to_cpu(hptep->v);
461
462
463 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
464 native_unlock_hpte(hptep);
465 else
466
467 hptep->v = 0;
468 }
469
470
471
472
473 lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
474 if (lock_tlbie)
475 raw_spin_lock(&native_tlbie_lock);
476
477 asm volatile("ptesync":::"memory");
478 __tlbie(vpn, psize, actual_psize, ssize);
479 asm volatile("eieio; tlbsync; ptesync":::"memory");
480
481 if (lock_tlbie)
482 raw_spin_unlock(&native_tlbie_lock);
483
484 local_irq_restore(flags);
485}
486
487static inline int __hpte_actual_psize(unsigned int lp, int psize)
488{
489 int i, shift;
490 unsigned int mask;
491
492
493 for (i = 1; i < MMU_PAGE_COUNT; i++) {
494
495
496 if (mmu_psize_defs[psize].penc[i] == -1)
497 continue;
498
499
500
501
502
503
504
505
506
507 shift = mmu_psize_defs[i].shift - LP_SHIFT;
508 if (shift > LP_BITS)
509 shift = LP_BITS;
510 mask = (1 << shift) - 1;
511 if ((lp & mask) == mmu_psize_defs[psize].penc[i])
512 return i;
513 }
514 return -1;
515}
516
517static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
518 int *psize, int *apsize, int *ssize, unsigned long *vpn)
519{
520 unsigned long avpn, pteg, vpi;
521 unsigned long hpte_v = be64_to_cpu(hpte->v);
522 unsigned long hpte_r = be64_to_cpu(hpte->r);
523 unsigned long vsid, seg_off;
524 int size, a_size, shift;
525
526 unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
527
528 if (!(hpte_v & HPTE_V_LARGE)) {
529 size = MMU_PAGE_4K;
530 a_size = MMU_PAGE_4K;
531 } else {
532 for (size = 0; size < MMU_PAGE_COUNT; size++) {
533
534
535 if (!mmu_psize_defs[size].shift)
536 continue;
537
538 a_size = __hpte_actual_psize(lp, size);
539 if (a_size != -1)
540 break;
541 }
542 }
543
544 *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
545 shift = mmu_psize_defs[size].shift;
546
547 avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
548 pteg = slot / HPTES_PER_GROUP;
549 if (hpte_v & HPTE_V_SECONDARY)
550 pteg = ~pteg;
551
552 switch (*ssize) {
553 case MMU_SEGSIZE_256M:
554
555 seg_off = (avpn & 0x1f) << 23;
556 vsid = avpn >> 5;
557
558 if (shift < 23) {
559 vpi = (vsid ^ pteg) & htab_hash_mask;
560 seg_off |= vpi << shift;
561 }
562 *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT;
563 break;
564 case MMU_SEGSIZE_1T:
565
566 seg_off = (avpn & 0x1ffff) << 23;
567 vsid = avpn >> 17;
568 if (shift < 23) {
569 vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
570 seg_off |= vpi << shift;
571 }
572 *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT;
573 break;
574 default:
575 *vpn = size = 0;
576 }
577 *psize = size;
578 *apsize = a_size;
579}
580
581
582
583
584
585
586
587
588
589static void native_hpte_clear(void)
590{
591 unsigned long vpn = 0;
592 unsigned long slot, slots, flags;
593 struct hash_pte *hptep = htab_address;
594 unsigned long hpte_v;
595 unsigned long pteg_count;
596 int psize, apsize, ssize;
597
598 pteg_count = htab_hash_mask + 1;
599
600 local_irq_save(flags);
601
602
603
604
605 raw_spin_lock(&native_tlbie_lock);
606
607 slots = pteg_count * HPTES_PER_GROUP;
608
609 for (slot = 0; slot < slots; slot++, hptep++) {
610
611
612
613
614
615 hpte_v = be64_to_cpu(hptep->v);
616
617
618
619
620
621 if (hpte_v & HPTE_V_VALID) {
622 hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
623 hptep->v = 0;
624 __tlbie(vpn, psize, apsize, ssize);
625 }
626 }
627
628 asm volatile("eieio; tlbsync; ptesync":::"memory");
629 raw_spin_unlock(&native_tlbie_lock);
630 local_irq_restore(flags);
631}
632
633
634
635
636
637static void native_flush_hash_range(unsigned long number, int local)
638{
639 unsigned long vpn;
640 unsigned long hash, index, hidx, shift, slot;
641 struct hash_pte *hptep;
642 unsigned long hpte_v;
643 unsigned long want_v;
644 unsigned long flags;
645 real_pte_t pte;
646 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
647 unsigned long psize = batch->psize;
648 int ssize = batch->ssize;
649 int i;
650
651 local_irq_save(flags);
652
653 for (i = 0; i < number; i++) {
654 vpn = batch->vpn[i];
655 pte = batch->pte[i];
656
657 pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
658 hash = hpt_hash(vpn, shift, ssize);
659 hidx = __rpte_to_hidx(pte, index);
660 if (hidx & _PTEIDX_SECONDARY)
661 hash = ~hash;
662 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
663 slot += hidx & _PTEIDX_GROUP_IX;
664 hptep = htab_address + slot;
665 want_v = hpte_encode_avpn(vpn, psize, ssize);
666 native_lock_hpte(hptep);
667 hpte_v = be64_to_cpu(hptep->v);
668 if (!HPTE_V_COMPARE(hpte_v, want_v) ||
669 !(hpte_v & HPTE_V_VALID))
670 native_unlock_hpte(hptep);
671 else
672 hptep->v = 0;
673 } pte_iterate_hashed_end();
674 }
675
676 if (mmu_has_feature(MMU_FTR_TLBIEL) &&
677 mmu_psize_defs[psize].tlbiel && local) {
678 asm volatile("ptesync":::"memory");
679 for (i = 0; i < number; i++) {
680 vpn = batch->vpn[i];
681 pte = batch->pte[i];
682
683 pte_iterate_hashed_subpages(pte, psize,
684 vpn, index, shift) {
685 __tlbiel(vpn, psize, psize, ssize);
686 } pte_iterate_hashed_end();
687 }
688 asm volatile("ptesync":::"memory");
689 } else {
690 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
691
692 if (lock_tlbie)
693 raw_spin_lock(&native_tlbie_lock);
694
695 asm volatile("ptesync":::"memory");
696 for (i = 0; i < number; i++) {
697 vpn = batch->vpn[i];
698 pte = batch->pte[i];
699
700 pte_iterate_hashed_subpages(pte, psize,
701 vpn, index, shift) {
702 __tlbie(vpn, psize, psize, ssize);
703 } pte_iterate_hashed_end();
704 }
705 asm volatile("eieio; tlbsync; ptesync":::"memory");
706
707 if (lock_tlbie)
708 raw_spin_unlock(&native_tlbie_lock);
709 }
710
711 local_irq_restore(flags);
712}
713
714void __init hpte_init_native(void)
715{
716 ppc_md.hpte_invalidate = native_hpte_invalidate;
717 ppc_md.hpte_updatepp = native_hpte_updatepp;
718 ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp;
719 ppc_md.hpte_insert = native_hpte_insert;
720 ppc_md.hpte_remove = native_hpte_remove;
721 ppc_md.hpte_clear_all = native_hpte_clear;
722 ppc_md.flush_hash_range = native_flush_hash_range;
723 ppc_md.hugepage_invalidate = native_hugepage_invalidate;
724}
725