1
2
3
4
5
6
7
8
9#undef DEBUG_LOW
10
11#include <linux/spinlock.h>
12#include <linux/bitops.h>
13#include <linux/of.h>
14#include <linux/processor.h>
15#include <linux/threads.h>
16#include <linux/smp.h>
17
18#include <asm/machdep.h>
19#include <asm/mmu.h>
20#include <asm/mmu_context.h>
21#include <asm/pgtable.h>
22#include <asm/trace.h>
23#include <asm/tlb.h>
24#include <asm/cputable.h>
25#include <asm/udbg.h>
26#include <asm/kexec.h>
27#include <asm/ppc-opcode.h>
28#include <asm/feature-fixups.h>
29
30#include <misc/cxl-base.h>
31
32#ifdef DEBUG_LOW
33#define DBG_LOW(fmt...) udbg_printf(fmt)
34#else
35#define DBG_LOW(fmt...)
36#endif
37
38#ifdef __BIG_ENDIAN__
39#define HPTE_LOCK_BIT 3
40#else
41#define HPTE_LOCK_BIT (56+3)
42#endif
43
44static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
45
46static inline void tlbiel_hash_set_isa206(unsigned int set, unsigned int is)
47{
48 unsigned long rb;
49
50 rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
51
52 asm volatile("tlbiel %0" : : "r" (rb));
53}
54
55
56
57
58
59static __always_inline void tlbiel_hash_set_isa300(unsigned int set, unsigned int is,
60 unsigned int pid,
61 unsigned int ric, unsigned int prs)
62{
63 unsigned long rb;
64 unsigned long rs;
65 unsigned int r = 0;
66
67 rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
68 rs = ((unsigned long)pid << PPC_BITLSHIFT(31));
69
70 asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4)
71 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r)
72 : "memory");
73}
74
75
76static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is)
77{
78 unsigned int set;
79
80 asm volatile("ptesync": : :"memory");
81
82 for (set = 0; set < num_sets; set++)
83 tlbiel_hash_set_isa206(set, is);
84
85 asm volatile("ptesync": : :"memory");
86}
87
88static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)
89{
90 unsigned int set;
91
92 asm volatile("ptesync": : :"memory");
93
94
95
96
97
98
99 tlbiel_hash_set_isa300(0, is, 0, 2, 0);
100 for (set = 1; set < num_sets; set++)
101 tlbiel_hash_set_isa300(set, is, 0, 0, 0);
102
103
104
105
106
107
108
109
110
111 tlbiel_hash_set_isa300(0, is, 0, 2, 1);
112
113 asm volatile("ptesync": : :"memory");
114
115 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory");
116}
117
118void hash__tlbiel_all(unsigned int action)
119{
120 unsigned int is;
121
122 switch (action) {
123 case TLB_INVAL_SCOPE_GLOBAL:
124 is = 3;
125 break;
126 case TLB_INVAL_SCOPE_LPID:
127 is = 2;
128 break;
129 default:
130 BUG();
131 }
132
133 if (early_cpu_has_feature(CPU_FTR_ARCH_300))
134 tlbiel_all_isa300(POWER9_TLB_SETS_HASH, is);
135 else if (early_cpu_has_feature(CPU_FTR_ARCH_207S))
136 tlbiel_all_isa206(POWER8_TLB_SETS, is);
137 else if (early_cpu_has_feature(CPU_FTR_ARCH_206))
138 tlbiel_all_isa206(POWER7_TLB_SETS, is);
139 else
140 WARN(1, "%s called on pre-POWER7 CPU\n", __func__);
141}
142
143static inline unsigned long ___tlbie(unsigned long vpn, int psize,
144 int apsize, int ssize)
145{
146 unsigned long va;
147 unsigned int penc;
148 unsigned long sllp;
149
150
151
152
153
154
155
156
157 va = vpn << VPN_SHIFT;
158
159
160
161
162
163 if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
164 va &= ~(0xffffULL << 48);
165
166 switch (psize) {
167 case MMU_PAGE_4K:
168
169 va &= ~((1ul << (64 - 52)) - 1);
170 va |= ssize << 8;
171 sllp = get_sllp_encoding(apsize);
172 va |= sllp << 5;
173 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
174 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
175 : "memory");
176 break;
177 default:
178
179 penc = mmu_psize_defs[psize].penc[apsize];
180 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
181 va |= penc << 12;
182 va |= ssize << 8;
183
184
185
186
187
188
189
190 va |= (vpn & 0xfe);
191 va |= 1;
192 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
193 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
194 : "memory");
195 break;
196 }
197 return va;
198}
199
200static inline void fixup_tlbie_vpn(unsigned long vpn, int psize,
201 int apsize, int ssize)
202{
203 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
204
205
206 unsigned long rb,rs,prs,r,ric;
207
208 rb = PPC_BIT(52);
209 rs = 0;
210 prs = 0;
211 r = 1;
212 ric = 0;
213
214
215
216
217
218 asm volatile("ptesync": : :"memory");
219 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
220 : : "r"(rb), "i"(r), "i"(prs),
221 "i"(ric), "r"(rs) : "memory");
222 }
223
224
225 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
226
227 asm volatile("ptesync": : :"memory");
228 ___tlbie(vpn, psize, apsize, ssize);
229 }
230}
231
232static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
233{
234 unsigned long rb;
235
236 rb = ___tlbie(vpn, psize, apsize, ssize);
237 trace_tlbie(0, 0, rb, 0, 0, 0, 0);
238}
239
240static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
241{
242 unsigned long va;
243 unsigned int penc;
244 unsigned long sllp;
245
246
247 va = vpn << VPN_SHIFT;
248
249
250
251
252
253 if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
254 va &= ~(0xffffULL << 48);
255
256 switch (psize) {
257 case MMU_PAGE_4K:
258
259 va &= ~((1ul << (64 - 52)) - 1);
260 va |= ssize << 8;
261 sllp = get_sllp_encoding(apsize);
262 va |= sllp << 5;
263 asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,0", %1)
264 : : "r" (va), "i" (CPU_FTR_ARCH_206)
265 : "memory");
266 break;
267 default:
268
269 penc = mmu_psize_defs[psize].penc[apsize];
270 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
271 va |= penc << 12;
272 va |= ssize << 8;
273
274
275
276
277
278
279
280 va |= (vpn & 0xfe);
281 va |= 1;
282 asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,1", %1)
283 : : "r" (va), "i" (CPU_FTR_ARCH_206)
284 : "memory");
285 break;
286 }
287 trace_tlbie(0, 1, va, 0, 0, 0, 0);
288
289}
290
291static inline void tlbie(unsigned long vpn, int psize, int apsize,
292 int ssize, int local)
293{
294 unsigned int use_local;
295 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
296
297 use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && !cxl_ctx_in_use();
298
299 if (use_local)
300 use_local = mmu_psize_defs[psize].tlbiel;
301 if (lock_tlbie && !use_local)
302 raw_spin_lock(&native_tlbie_lock);
303 asm volatile("ptesync": : :"memory");
304 if (use_local) {
305 __tlbiel(vpn, psize, apsize, ssize);
306 asm volatile("ptesync": : :"memory");
307 } else {
308 __tlbie(vpn, psize, apsize, ssize);
309 fixup_tlbie_vpn(vpn, psize, apsize, ssize);
310 asm volatile("eieio; tlbsync; ptesync": : :"memory");
311 }
312 if (lock_tlbie && !use_local)
313 raw_spin_unlock(&native_tlbie_lock);
314}
315
316static inline void native_lock_hpte(struct hash_pte *hptep)
317{
318 unsigned long *word = (unsigned long *)&hptep->v;
319
320 while (1) {
321 if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
322 break;
323 spin_begin();
324 while(test_bit(HPTE_LOCK_BIT, word))
325 spin_cpu_relax();
326 spin_end();
327 }
328}
329
330static inline void native_unlock_hpte(struct hash_pte *hptep)
331{
332 unsigned long *word = (unsigned long *)&hptep->v;
333
334 clear_bit_unlock(HPTE_LOCK_BIT, word);
335}
336
337static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
338 unsigned long pa, unsigned long rflags,
339 unsigned long vflags, int psize, int apsize, int ssize)
340{
341 struct hash_pte *hptep = htab_address + hpte_group;
342 unsigned long hpte_v, hpte_r;
343 int i;
344
345 if (!(vflags & HPTE_V_BOLTED)) {
346 DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx,"
347 " rflags=%lx, vflags=%lx, psize=%d)\n",
348 hpte_group, vpn, pa, rflags, vflags, psize);
349 }
350
351 for (i = 0; i < HPTES_PER_GROUP; i++) {
352 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
353
354 native_lock_hpte(hptep);
355 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID))
356 break;
357 native_unlock_hpte(hptep);
358 }
359
360 hptep++;
361 }
362
363 if (i == HPTES_PER_GROUP)
364 return -1;
365
366 hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
367 hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
368
369 if (!(vflags & HPTE_V_BOLTED)) {
370 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
371 i, hpte_v, hpte_r);
372 }
373
374 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
375 hpte_r = hpte_old_to_new_r(hpte_v, hpte_r);
376 hpte_v = hpte_old_to_new_v(hpte_v);
377 }
378
379 hptep->r = cpu_to_be64(hpte_r);
380
381 eieio();
382
383
384
385
386 hptep->v = cpu_to_be64(hpte_v);
387
388 __asm__ __volatile__ ("ptesync" : : : "memory");
389
390 return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
391}
392
393static long native_hpte_remove(unsigned long hpte_group)
394{
395 struct hash_pte *hptep;
396 int i;
397 int slot_offset;
398 unsigned long hpte_v;
399
400 DBG_LOW(" remove(group=%lx)\n", hpte_group);
401
402
403 slot_offset = mftb() & 0x7;
404
405 for (i = 0; i < HPTES_PER_GROUP; i++) {
406 hptep = htab_address + hpte_group + slot_offset;
407 hpte_v = be64_to_cpu(hptep->v);
408
409 if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
410
411 native_lock_hpte(hptep);
412 hpte_v = be64_to_cpu(hptep->v);
413 if ((hpte_v & HPTE_V_VALID)
414 && !(hpte_v & HPTE_V_BOLTED))
415 break;
416 native_unlock_hpte(hptep);
417 }
418
419 slot_offset++;
420 slot_offset &= 0x7;
421 }
422
423 if (i == HPTES_PER_GROUP)
424 return -1;
425
426
427 hptep->v = 0;
428
429 return i;
430}
431
432static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
433 unsigned long vpn, int bpsize,
434 int apsize, int ssize, unsigned long flags)
435{
436 struct hash_pte *hptep = htab_address + slot;
437 unsigned long hpte_v, want_v;
438 int ret = 0, local = 0;
439
440 want_v = hpte_encode_avpn(vpn, bpsize, ssize);
441
442 DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
443 vpn, want_v & HPTE_V_AVPN, slot, newpp);
444
445 hpte_v = hpte_get_old_v(hptep);
446
447
448
449
450
451
452
453 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
454 DBG_LOW(" -> miss\n");
455 ret = -1;
456 } else {
457 native_lock_hpte(hptep);
458
459 hpte_v = hpte_get_old_v(hptep);
460 if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) ||
461 !(hpte_v & HPTE_V_VALID))) {
462 ret = -1;
463 } else {
464 DBG_LOW(" -> hit\n");
465
466 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
467 ~(HPTE_R_PPP | HPTE_R_N)) |
468 (newpp & (HPTE_R_PPP | HPTE_R_N |
469 HPTE_R_C)));
470 }
471 native_unlock_hpte(hptep);
472 }
473
474 if (flags & HPTE_LOCAL_UPDATE)
475 local = 1;
476
477
478
479 if (!(flags & HPTE_NOHPTE_UPDATE))
480 tlbie(vpn, bpsize, apsize, ssize, local);
481
482 return ret;
483}
484
485static long __native_hpte_find(unsigned long want_v, unsigned long slot)
486{
487 struct hash_pte *hptep;
488 unsigned long hpte_v;
489 unsigned long i;
490
491 for (i = 0; i < HPTES_PER_GROUP; i++) {
492
493 hptep = htab_address + slot;
494 hpte_v = hpte_get_old_v(hptep);
495 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
496
497 return slot;
498 ++slot;
499 }
500
501 return -1;
502}
503
504static long native_hpte_find(unsigned long vpn, int psize, int ssize)
505{
506 unsigned long hpte_group;
507 unsigned long want_v;
508 unsigned long hash;
509 long slot;
510
511 hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
512 want_v = hpte_encode_avpn(vpn, psize, ssize);
513
514
515
516
517
518 hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
519 slot = __native_hpte_find(want_v, hpte_group);
520 if (slot < 0) {
521
522 hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
523 slot = __native_hpte_find(want_v, hpte_group);
524 if (slot < 0)
525 return -1;
526 }
527
528 return slot;
529}
530
531
532
533
534
535
536
537
538static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
539 int psize, int ssize)
540{
541 unsigned long vpn;
542 unsigned long vsid;
543 long slot;
544 struct hash_pte *hptep;
545
546 vsid = get_kernel_vsid(ea, ssize);
547 vpn = hpt_vpn(ea, vsid, ssize);
548
549 slot = native_hpte_find(vpn, psize, ssize);
550 if (slot == -1)
551 panic("could not find page to bolt\n");
552 hptep = htab_address + slot;
553
554
555 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
556 ~(HPTE_R_PPP | HPTE_R_N)) |
557 (newpp & (HPTE_R_PPP | HPTE_R_N)));
558
559
560
561
562 tlbie(vpn, psize, psize, ssize, 0);
563}
564
565
566
567
568
569
570static int native_hpte_removebolted(unsigned long ea, int psize, int ssize)
571{
572 unsigned long vpn;
573 unsigned long vsid;
574 long slot;
575 struct hash_pte *hptep;
576
577 vsid = get_kernel_vsid(ea, ssize);
578 vpn = hpt_vpn(ea, vsid, ssize);
579
580 slot = native_hpte_find(vpn, psize, ssize);
581 if (slot == -1)
582 return -ENOENT;
583
584 hptep = htab_address + slot;
585
586 VM_WARN_ON(!(be64_to_cpu(hptep->v) & HPTE_V_BOLTED));
587
588
589 hptep->v = 0;
590
591
592 tlbie(vpn, psize, psize, ssize, 0);
593 return 0;
594}
595
596
597static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
598 int bpsize, int apsize, int ssize, int local)
599{
600 struct hash_pte *hptep = htab_address + slot;
601 unsigned long hpte_v;
602 unsigned long want_v;
603 unsigned long flags;
604
605 local_irq_save(flags);
606
607 DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
608
609 want_v = hpte_encode_avpn(vpn, bpsize, ssize);
610 hpte_v = hpte_get_old_v(hptep);
611
612 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
613 native_lock_hpte(hptep);
614
615 hpte_v = hpte_get_old_v(hptep);
616
617 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
618
619 hptep->v = 0;
620 else
621 native_unlock_hpte(hptep);
622 }
623
624
625
626
627
628
629
630 tlbie(vpn, bpsize, apsize, ssize, local);
631
632 local_irq_restore(flags);
633}
634
635#ifdef CONFIG_TRANSPARENT_HUGEPAGE
636static void native_hugepage_invalidate(unsigned long vsid,
637 unsigned long addr,
638 unsigned char *hpte_slot_array,
639 int psize, int ssize, int local)
640{
641 int i;
642 struct hash_pte *hptep;
643 int actual_psize = MMU_PAGE_16M;
644 unsigned int max_hpte_count, valid;
645 unsigned long flags, s_addr = addr;
646 unsigned long hpte_v, want_v, shift;
647 unsigned long hidx, vpn = 0, hash, slot;
648
649 shift = mmu_psize_defs[psize].shift;
650 max_hpte_count = 1U << (PMD_SHIFT - shift);
651
652 local_irq_save(flags);
653 for (i = 0; i < max_hpte_count; i++) {
654 valid = hpte_valid(hpte_slot_array, i);
655 if (!valid)
656 continue;
657 hidx = hpte_hash_index(hpte_slot_array, i);
658
659
660 addr = s_addr + (i * (1ul << shift));
661 vpn = hpt_vpn(addr, vsid, ssize);
662 hash = hpt_hash(vpn, shift, ssize);
663 if (hidx & _PTEIDX_SECONDARY)
664 hash = ~hash;
665
666 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
667 slot += hidx & _PTEIDX_GROUP_IX;
668
669 hptep = htab_address + slot;
670 want_v = hpte_encode_avpn(vpn, psize, ssize);
671 hpte_v = hpte_get_old_v(hptep);
672
673
674 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
675
676 native_lock_hpte(hptep);
677 hpte_v = hpte_get_old_v(hptep);
678
679 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
680
681
682
683
684 hptep->v = 0;
685 } else
686 native_unlock_hpte(hptep);
687 }
688
689
690
691
692
693 tlbie(vpn, psize, actual_psize, ssize, local);
694 }
695 local_irq_restore(flags);
696}
697#else
698static void native_hugepage_invalidate(unsigned long vsid,
699 unsigned long addr,
700 unsigned char *hpte_slot_array,
701 int psize, int ssize, int local)
702{
703 WARN(1, "%s called without THP support\n", __func__);
704}
705#endif
706
707static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
708 int *psize, int *apsize, int *ssize, unsigned long *vpn)
709{
710 unsigned long avpn, pteg, vpi;
711 unsigned long hpte_v = be64_to_cpu(hpte->v);
712 unsigned long hpte_r = be64_to_cpu(hpte->r);
713 unsigned long vsid, seg_off;
714 int size, a_size, shift;
715
716 unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
717
718 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
719 hpte_v = hpte_new_to_old_v(hpte_v, hpte_r);
720 hpte_r = hpte_new_to_old_r(hpte_r);
721 }
722 if (!(hpte_v & HPTE_V_LARGE)) {
723 size = MMU_PAGE_4K;
724 a_size = MMU_PAGE_4K;
725 } else {
726 size = hpte_page_sizes[lp] & 0xf;
727 a_size = hpte_page_sizes[lp] >> 4;
728 }
729
730 *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
731 shift = mmu_psize_defs[size].shift;
732
733 avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
734 pteg = slot / HPTES_PER_GROUP;
735 if (hpte_v & HPTE_V_SECONDARY)
736 pteg = ~pteg;
737
738 switch (*ssize) {
739 case MMU_SEGSIZE_256M:
740
741 seg_off = (avpn & 0x1f) << 23;
742 vsid = avpn >> 5;
743
744 if (shift < 23) {
745 vpi = (vsid ^ pteg) & htab_hash_mask;
746 seg_off |= vpi << shift;
747 }
748 *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT;
749 break;
750 case MMU_SEGSIZE_1T:
751
752 seg_off = (avpn & 0x1ffff) << 23;
753 vsid = avpn >> 17;
754 if (shift < 23) {
755 vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
756 seg_off |= vpi << shift;
757 }
758 *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT;
759 break;
760 default:
761 *vpn = size = 0;
762 }
763 *psize = size;
764 *apsize = a_size;
765}
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783static void native_hpte_clear(void)
784{
785 unsigned long vpn = 0;
786 unsigned long slot, slots;
787 struct hash_pte *hptep = htab_address;
788 unsigned long hpte_v;
789 unsigned long pteg_count;
790 int psize, apsize, ssize;
791
792 pteg_count = htab_hash_mask + 1;
793
794 slots = pteg_count * HPTES_PER_GROUP;
795
796 for (slot = 0; slot < slots; slot++, hptep++) {
797
798
799
800
801
802 hpte_v = be64_to_cpu(hptep->v);
803
804
805
806
807
808 if (hpte_v & HPTE_V_VALID) {
809 hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
810 hptep->v = 0;
811 ___tlbie(vpn, psize, apsize, ssize);
812 }
813 }
814
815 asm volatile("eieio; tlbsync; ptesync":::"memory");
816}
817
818
819
820
821
822static void native_flush_hash_range(unsigned long number, int local)
823{
824 unsigned long vpn = 0;
825 unsigned long hash, index, hidx, shift, slot;
826 struct hash_pte *hptep;
827 unsigned long hpte_v;
828 unsigned long want_v;
829 unsigned long flags;
830 real_pte_t pte;
831 struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
832 unsigned long psize = batch->psize;
833 int ssize = batch->ssize;
834 int i;
835 unsigned int use_local;
836
837 use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) &&
838 mmu_psize_defs[psize].tlbiel && !cxl_ctx_in_use();
839
840 local_irq_save(flags);
841
842 for (i = 0; i < number; i++) {
843 vpn = batch->vpn[i];
844 pte = batch->pte[i];
845
846 pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
847 hash = hpt_hash(vpn, shift, ssize);
848 hidx = __rpte_to_hidx(pte, index);
849 if (hidx & _PTEIDX_SECONDARY)
850 hash = ~hash;
851 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
852 slot += hidx & _PTEIDX_GROUP_IX;
853 hptep = htab_address + slot;
854 want_v = hpte_encode_avpn(vpn, psize, ssize);
855 hpte_v = hpte_get_old_v(hptep);
856
857 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
858 continue;
859
860 native_lock_hpte(hptep);
861 hpte_v = hpte_get_old_v(hptep);
862
863 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
864 native_unlock_hpte(hptep);
865 else
866 hptep->v = 0;
867
868 } pte_iterate_hashed_end();
869 }
870
871 if (use_local) {
872 asm volatile("ptesync":::"memory");
873 for (i = 0; i < number; i++) {
874 vpn = batch->vpn[i];
875 pte = batch->pte[i];
876
877 pte_iterate_hashed_subpages(pte, psize,
878 vpn, index, shift) {
879 __tlbiel(vpn, psize, psize, ssize);
880 } pte_iterate_hashed_end();
881 }
882 asm volatile("ptesync":::"memory");
883 } else {
884 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
885
886 if (lock_tlbie)
887 raw_spin_lock(&native_tlbie_lock);
888
889 asm volatile("ptesync":::"memory");
890 for (i = 0; i < number; i++) {
891 vpn = batch->vpn[i];
892 pte = batch->pte[i];
893
894 pte_iterate_hashed_subpages(pte, psize,
895 vpn, index, shift) {
896 __tlbie(vpn, psize, psize, ssize);
897 } pte_iterate_hashed_end();
898 }
899
900
901
902 fixup_tlbie_vpn(vpn, psize, psize, ssize);
903 asm volatile("eieio; tlbsync; ptesync":::"memory");
904
905 if (lock_tlbie)
906 raw_spin_unlock(&native_tlbie_lock);
907 }
908
909 local_irq_restore(flags);
910}
911
912void __init hpte_init_native(void)
913{
914 mmu_hash_ops.hpte_invalidate = native_hpte_invalidate;
915 mmu_hash_ops.hpte_updatepp = native_hpte_updatepp;
916 mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
917 mmu_hash_ops.hpte_removebolted = native_hpte_removebolted;
918 mmu_hash_ops.hpte_insert = native_hpte_insert;
919 mmu_hash_ops.hpte_remove = native_hpte_remove;
920 mmu_hash_ops.hpte_clear_all = native_hpte_clear;
921 mmu_hash_ops.flush_hash_range = native_flush_hash_range;
922 mmu_hash_ops.hugepage_invalidate = native_hugepage_invalidate;
923}
924