1
2
3
4
5
6
7
8#include <linux/mm.h>
9#include <linux/hugetlb.h>
10#include <linux/memblock.h>
11#include <linux/mmu_context.h>
12#include <linux/sched/mm.h>
13
14#include <asm/ppc-opcode.h>
15#include <asm/tlb.h>
16#include <asm/tlbflush.h>
17#include <asm/trace.h>
18#include <asm/cputhreads.h>
19#include <asm/plpar_wrappers.h>
20
21#define RIC_FLUSH_TLB 0
22#define RIC_FLUSH_PWC 1
23#define RIC_FLUSH_ALL 2
24
25
26
27
28
29static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is,
30 unsigned int pid,
31 unsigned int ric, unsigned int prs)
32{
33 unsigned long rb;
34 unsigned long rs;
35
36 rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
37 rs = ((unsigned long)pid << PPC_BITLSHIFT(31));
38
39 asm volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
40 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
41 : "memory");
42}
43
44static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)
45{
46 unsigned int set;
47
48 asm volatile("ptesync": : :"memory");
49
50
51
52
53
54
55
56 if (early_cpu_has_feature(CPU_FTR_HVMODE)) {
57
58 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
59 for (set = 1; set < num_sets; set++)
60 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
61 }
62
63
64 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
65 for (set = 1; set < num_sets; set++)
66 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
67
68 ppc_after_tlbiel_barrier();
69}
70
71void radix__tlbiel_all(unsigned int action)
72{
73 unsigned int is;
74
75 switch (action) {
76 case TLB_INVAL_SCOPE_GLOBAL:
77 is = 3;
78 break;
79 case TLB_INVAL_SCOPE_LPID:
80 is = 2;
81 break;
82 default:
83 BUG();
84 }
85
86 if (early_cpu_has_feature(CPU_FTR_ARCH_300))
87 tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is);
88 else
89 WARN(1, "%s called on pre-POWER9 CPU\n", __func__);
90
91 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory");
92}
93
94static __always_inline void __tlbiel_pid(unsigned long pid, int set,
95 unsigned long ric)
96{
97 unsigned long rb,rs,prs,r;
98
99 rb = PPC_BIT(53);
100 rb |= set << PPC_BITLSHIFT(51);
101 rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
102 prs = 1;
103 r = 1;
104
105 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
106 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
107 trace_tlbie(0, 1, rb, rs, ric, prs, r);
108}
109
110static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric)
111{
112 unsigned long rb,rs,prs,r;
113
114 rb = PPC_BIT(53);
115 rs = pid << PPC_BITLSHIFT(31);
116 prs = 1;
117 r = 1;
118
119 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
120 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
121 trace_tlbie(0, 0, rb, rs, ric, prs, r);
122}
123
124static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric)
125{
126 unsigned long rb,rs,prs,r;
127
128 rb = PPC_BIT(52);
129 rs = lpid;
130 prs = 0;
131 r = 1;
132
133 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
134 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
135 trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
136}
137
138static __always_inline void __tlbie_lpid_guest(unsigned long lpid, unsigned long ric)
139{
140 unsigned long rb,rs,prs,r;
141
142 rb = PPC_BIT(52);
143 rs = lpid;
144 prs = 1;
145 r = 1;
146
147 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
148 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
149 trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
150}
151
152static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid,
153 unsigned long ap, unsigned long ric)
154{
155 unsigned long rb,rs,prs,r;
156
157 rb = va & ~(PPC_BITMASK(52, 63));
158 rb |= ap << PPC_BITLSHIFT(58);
159 rs = pid << PPC_BITLSHIFT(31);
160 prs = 1;
161 r = 1;
162
163 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
164 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
165 trace_tlbie(0, 1, rb, rs, ric, prs, r);
166}
167
168static __always_inline void __tlbie_va(unsigned long va, unsigned long pid,
169 unsigned long ap, unsigned long ric)
170{
171 unsigned long rb,rs,prs,r;
172
173 rb = va & ~(PPC_BITMASK(52, 63));
174 rb |= ap << PPC_BITLSHIFT(58);
175 rs = pid << PPC_BITLSHIFT(31);
176 prs = 1;
177 r = 1;
178
179 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
180 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
181 trace_tlbie(0, 0, rb, rs, ric, prs, r);
182}
183
184static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid,
185 unsigned long ap, unsigned long ric)
186{
187 unsigned long rb,rs,prs,r;
188
189 rb = va & ~(PPC_BITMASK(52, 63));
190 rb |= ap << PPC_BITLSHIFT(58);
191 rs = lpid;
192 prs = 0;
193 r = 1;
194
195 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
196 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
197 trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
198}
199
200
201static inline void fixup_tlbie_va(unsigned long va, unsigned long pid,
202 unsigned long ap)
203{
204 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
205 asm volatile("ptesync": : :"memory");
206 __tlbie_va(va, 0, ap, RIC_FLUSH_TLB);
207 }
208
209 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
210 asm volatile("ptesync": : :"memory");
211 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
212 }
213}
214
215static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid,
216 unsigned long ap)
217{
218 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
219 asm volatile("ptesync": : :"memory");
220 __tlbie_pid(0, RIC_FLUSH_TLB);
221 }
222
223 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
224 asm volatile("ptesync": : :"memory");
225 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
226 }
227}
228
229static inline void fixup_tlbie_pid(unsigned long pid)
230{
231
232
233
234
235 unsigned long va = ((1UL << 52) - 1);
236
237 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
238 asm volatile("ptesync": : :"memory");
239 __tlbie_pid(0, RIC_FLUSH_TLB);
240 }
241
242 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
243 asm volatile("ptesync": : :"memory");
244 __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
245 }
246}
247
248
249static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid,
250 unsigned long ap)
251{
252 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
253 asm volatile("ptesync": : :"memory");
254 __tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB);
255 }
256
257 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
258 asm volatile("ptesync": : :"memory");
259 __tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB);
260 }
261}
262
263static inline void fixup_tlbie_lpid(unsigned long lpid)
264{
265
266
267
268
269 unsigned long va = ((1UL << 52) - 1);
270
271 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
272 asm volatile("ptesync": : :"memory");
273 __tlbie_lpid(0, RIC_FLUSH_TLB);
274 }
275
276 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
277 asm volatile("ptesync": : :"memory");
278 __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
279 }
280}
281
282
283
284
285static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
286{
287 int set;
288
289 asm volatile("ptesync": : :"memory");
290
291
292
293
294
295 __tlbiel_pid(pid, 0, ric);
296
297
298 if (ric == RIC_FLUSH_PWC) {
299 ppc_after_tlbiel_barrier();
300 return;
301 }
302
303
304 for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
305 __tlbiel_pid(pid, set, RIC_FLUSH_TLB);
306
307 ppc_after_tlbiel_barrier();
308 asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory");
309}
310
311static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
312{
313 asm volatile("ptesync": : :"memory");
314
315
316
317
318
319
320 switch (ric) {
321 case RIC_FLUSH_TLB:
322 __tlbie_pid(pid, RIC_FLUSH_TLB);
323 fixup_tlbie_pid(pid);
324 break;
325 case RIC_FLUSH_PWC:
326 __tlbie_pid(pid, RIC_FLUSH_PWC);
327 break;
328 case RIC_FLUSH_ALL:
329 default:
330 __tlbie_pid(pid, RIC_FLUSH_ALL);
331 fixup_tlbie_pid(pid);
332 }
333 asm volatile("eieio; tlbsync; ptesync": : :"memory");
334}
335
336struct tlbiel_pid {
337 unsigned long pid;
338 unsigned long ric;
339};
340
341static void do_tlbiel_pid(void *info)
342{
343 struct tlbiel_pid *t = info;
344
345 if (t->ric == RIC_FLUSH_TLB)
346 _tlbiel_pid(t->pid, RIC_FLUSH_TLB);
347 else if (t->ric == RIC_FLUSH_PWC)
348 _tlbiel_pid(t->pid, RIC_FLUSH_PWC);
349 else
350 _tlbiel_pid(t->pid, RIC_FLUSH_ALL);
351}
352
353static inline void _tlbiel_pid_multicast(struct mm_struct *mm,
354 unsigned long pid, unsigned long ric)
355{
356 struct cpumask *cpus = mm_cpumask(mm);
357 struct tlbiel_pid t = { .pid = pid, .ric = ric };
358
359 on_each_cpu_mask(cpus, do_tlbiel_pid, &t, 1);
360
361
362
363
364
365 if (atomic_read(&mm->context.copros) > 0)
366 _tlbie_pid(pid, RIC_FLUSH_ALL);
367}
368
369static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric)
370{
371 asm volatile("ptesync": : :"memory");
372
373
374
375
376
377
378 switch (ric) {
379 case RIC_FLUSH_TLB:
380 __tlbie_lpid(lpid, RIC_FLUSH_TLB);
381 fixup_tlbie_lpid(lpid);
382 break;
383 case RIC_FLUSH_PWC:
384 __tlbie_lpid(lpid, RIC_FLUSH_PWC);
385 break;
386 case RIC_FLUSH_ALL:
387 default:
388 __tlbie_lpid(lpid, RIC_FLUSH_ALL);
389 fixup_tlbie_lpid(lpid);
390 }
391 asm volatile("eieio; tlbsync; ptesync": : :"memory");
392}
393
394static __always_inline void _tlbie_lpid_guest(unsigned long lpid, unsigned long ric)
395{
396
397
398
399
400
401 switch (ric) {
402 case RIC_FLUSH_TLB:
403 __tlbie_lpid_guest(lpid, RIC_FLUSH_TLB);
404 break;
405 case RIC_FLUSH_PWC:
406 __tlbie_lpid_guest(lpid, RIC_FLUSH_PWC);
407 break;
408 case RIC_FLUSH_ALL:
409 default:
410 __tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
411 }
412 fixup_tlbie_lpid(lpid);
413 asm volatile("eieio; tlbsync; ptesync": : :"memory");
414}
415
416static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
417 unsigned long pid, unsigned long page_size,
418 unsigned long psize)
419{
420 unsigned long addr;
421 unsigned long ap = mmu_get_ap(psize);
422
423 for (addr = start; addr < end; addr += page_size)
424 __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
425}
426
427static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid,
428 unsigned long psize, unsigned long ric)
429{
430 unsigned long ap = mmu_get_ap(psize);
431
432 asm volatile("ptesync": : :"memory");
433 __tlbiel_va(va, pid, ap, ric);
434 ppc_after_tlbiel_barrier();
435}
436
437static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
438 unsigned long pid, unsigned long page_size,
439 unsigned long psize, bool also_pwc)
440{
441 asm volatile("ptesync": : :"memory");
442 if (also_pwc)
443 __tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
444 __tlbiel_va_range(start, end, pid, page_size, psize);
445 ppc_after_tlbiel_barrier();
446}
447
448static inline void __tlbie_va_range(unsigned long start, unsigned long end,
449 unsigned long pid, unsigned long page_size,
450 unsigned long psize)
451{
452 unsigned long addr;
453 unsigned long ap = mmu_get_ap(psize);
454
455 for (addr = start; addr < end; addr += page_size)
456 __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
457
458 fixup_tlbie_va_range(addr - page_size, pid, ap);
459}
460
461static __always_inline void _tlbie_va(unsigned long va, unsigned long pid,
462 unsigned long psize, unsigned long ric)
463{
464 unsigned long ap = mmu_get_ap(psize);
465
466 asm volatile("ptesync": : :"memory");
467 __tlbie_va(va, pid, ap, ric);
468 fixup_tlbie_va(va, pid, ap);
469 asm volatile("eieio; tlbsync; ptesync": : :"memory");
470}
471
472struct tlbiel_va {
473 unsigned long pid;
474 unsigned long va;
475 unsigned long psize;
476 unsigned long ric;
477};
478
479static void do_tlbiel_va(void *info)
480{
481 struct tlbiel_va *t = info;
482
483 if (t->ric == RIC_FLUSH_TLB)
484 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB);
485 else if (t->ric == RIC_FLUSH_PWC)
486 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC);
487 else
488 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL);
489}
490
491static inline void _tlbiel_va_multicast(struct mm_struct *mm,
492 unsigned long va, unsigned long pid,
493 unsigned long psize, unsigned long ric)
494{
495 struct cpumask *cpus = mm_cpumask(mm);
496 struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric };
497 on_each_cpu_mask(cpus, do_tlbiel_va, &t, 1);
498 if (atomic_read(&mm->context.copros) > 0)
499 _tlbie_va(va, pid, psize, RIC_FLUSH_TLB);
500}
501
502struct tlbiel_va_range {
503 unsigned long pid;
504 unsigned long start;
505 unsigned long end;
506 unsigned long page_size;
507 unsigned long psize;
508 bool also_pwc;
509};
510
511static void do_tlbiel_va_range(void *info)
512{
513 struct tlbiel_va_range *t = info;
514
515 _tlbiel_va_range(t->start, t->end, t->pid, t->page_size,
516 t->psize, t->also_pwc);
517}
518
519static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid,
520 unsigned long psize, unsigned long ric)
521{
522 unsigned long ap = mmu_get_ap(psize);
523
524 asm volatile("ptesync": : :"memory");
525 __tlbie_lpid_va(va, lpid, ap, ric);
526 fixup_tlbie_lpid_va(va, lpid, ap);
527 asm volatile("eieio; tlbsync; ptesync": : :"memory");
528}
529
530static inline void _tlbie_va_range(unsigned long start, unsigned long end,
531 unsigned long pid, unsigned long page_size,
532 unsigned long psize, bool also_pwc)
533{
534 asm volatile("ptesync": : :"memory");
535 if (also_pwc)
536 __tlbie_pid(pid, RIC_FLUSH_PWC);
537 __tlbie_va_range(start, end, pid, page_size, psize);
538 asm volatile("eieio; tlbsync; ptesync": : :"memory");
539}
540
541static inline void _tlbiel_va_range_multicast(struct mm_struct *mm,
542 unsigned long start, unsigned long end,
543 unsigned long pid, unsigned long page_size,
544 unsigned long psize, bool also_pwc)
545{
546 struct cpumask *cpus = mm_cpumask(mm);
547 struct tlbiel_va_range t = { .start = start, .end = end,
548 .pid = pid, .page_size = page_size,
549 .psize = psize, .also_pwc = also_pwc };
550
551 on_each_cpu_mask(cpus, do_tlbiel_va_range, &t, 1);
552 if (atomic_read(&mm->context.copros) > 0)
553 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
554}
555
556
557
558
559
560
561
562
563
564
565
566
567void radix__local_flush_tlb_mm(struct mm_struct *mm)
568{
569 unsigned long pid;
570
571 preempt_disable();
572 pid = mm->context.id;
573 if (pid != MMU_NO_CONTEXT)
574 _tlbiel_pid(pid, RIC_FLUSH_TLB);
575 preempt_enable();
576}
577EXPORT_SYMBOL(radix__local_flush_tlb_mm);
578
579#ifndef CONFIG_SMP
580void radix__local_flush_all_mm(struct mm_struct *mm)
581{
582 unsigned long pid;
583
584 preempt_disable();
585 pid = mm->context.id;
586 if (pid != MMU_NO_CONTEXT)
587 _tlbiel_pid(pid, RIC_FLUSH_ALL);
588 preempt_enable();
589}
590EXPORT_SYMBOL(radix__local_flush_all_mm);
591
592static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
593{
594 radix__local_flush_all_mm(mm);
595}
596#endif
597
598void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
599 int psize)
600{
601 unsigned long pid;
602
603 preempt_disable();
604 pid = mm->context.id;
605 if (pid != MMU_NO_CONTEXT)
606 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
607 preempt_enable();
608}
609
610void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
611{
612#ifdef CONFIG_HUGETLB_PAGE
613
614 if (is_vm_hugetlb_page(vma))
615 return radix__local_flush_hugetlb_page(vma, vmaddr);
616#endif
617 radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
618}
619EXPORT_SYMBOL(radix__local_flush_tlb_page);
620
621static bool mm_is_singlethreaded(struct mm_struct *mm)
622{
623 if (atomic_read(&mm->context.copros) > 0)
624 return false;
625 if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm)
626 return true;
627 return false;
628}
629
630static bool mm_needs_flush_escalation(struct mm_struct *mm)
631{
632
633
634
635
636
637 if (atomic_read(&mm->context.copros) > 0)
638 return true;
639 return false;
640}
641
642#ifdef CONFIG_SMP
643static void do_exit_flush_lazy_tlb(void *arg)
644{
645 struct mm_struct *mm = arg;
646 unsigned long pid = mm->context.id;
647
648
649
650
651
652
653
654
655 if (current->mm == mm)
656 goto out_flush;
657
658 if (current->active_mm == mm) {
659 WARN_ON_ONCE(current->mm != NULL);
660
661 mmgrab(&init_mm);
662 current->active_mm = &init_mm;
663 switch_mm_irqs_off(mm, &init_mm, current);
664 mmdrop(mm);
665 }
666
667 atomic_dec(&mm->context.active_cpus);
668 cpumask_clear_cpu(smp_processor_id(), mm_cpumask(mm));
669
670out_flush:
671 _tlbiel_pid(pid, RIC_FLUSH_ALL);
672}
673
674static void exit_flush_lazy_tlbs(struct mm_struct *mm)
675{
676
677
678
679
680
681
682
683 smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb,
684 (void *)mm, 1);
685}
686
687void radix__flush_tlb_mm(struct mm_struct *mm)
688{
689 unsigned long pid;
690
691 pid = mm->context.id;
692 if (unlikely(pid == MMU_NO_CONTEXT))
693 return;
694
695 preempt_disable();
696
697
698
699
700 smp_mb();
701 if (!mm_is_thread_local(mm)) {
702 if (unlikely(mm_is_singlethreaded(mm))) {
703 exit_flush_lazy_tlbs(mm);
704 goto local;
705 }
706
707 if (!mmu_has_feature(MMU_FTR_GTSE)) {
708 unsigned long tgt = H_RPTI_TARGET_CMMU;
709
710 if (atomic_read(&mm->context.copros) > 0)
711 tgt |= H_RPTI_TARGET_NMMU;
712 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
713 H_RPTI_PAGE_ALL, 0, -1UL);
714 } else if (cputlb_use_tlbie()) {
715 if (mm_needs_flush_escalation(mm))
716 _tlbie_pid(pid, RIC_FLUSH_ALL);
717 else
718 _tlbie_pid(pid, RIC_FLUSH_TLB);
719 } else {
720 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
721 }
722 } else {
723local:
724 _tlbiel_pid(pid, RIC_FLUSH_TLB);
725 }
726 preempt_enable();
727}
728EXPORT_SYMBOL(radix__flush_tlb_mm);
729
730static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
731{
732 unsigned long pid;
733
734 pid = mm->context.id;
735 if (unlikely(pid == MMU_NO_CONTEXT))
736 return;
737
738 preempt_disable();
739 smp_mb();
740 if (!mm_is_thread_local(mm)) {
741 if (unlikely(mm_is_singlethreaded(mm))) {
742 if (!fullmm) {
743 exit_flush_lazy_tlbs(mm);
744 goto local;
745 }
746 }
747 if (!mmu_has_feature(MMU_FTR_GTSE)) {
748 unsigned long tgt = H_RPTI_TARGET_CMMU;
749 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
750 H_RPTI_TYPE_PRT;
751
752 if (atomic_read(&mm->context.copros) > 0)
753 tgt |= H_RPTI_TARGET_NMMU;
754 pseries_rpt_invalidate(pid, tgt, type,
755 H_RPTI_PAGE_ALL, 0, -1UL);
756 } else if (cputlb_use_tlbie())
757 _tlbie_pid(pid, RIC_FLUSH_ALL);
758 else
759 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
760 } else {
761local:
762 _tlbiel_pid(pid, RIC_FLUSH_ALL);
763 }
764 preempt_enable();
765}
766
767void radix__flush_all_mm(struct mm_struct *mm)
768{
769 __flush_all_mm(mm, false);
770}
771EXPORT_SYMBOL(radix__flush_all_mm);
772
773void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
774 int psize)
775{
776 unsigned long pid;
777
778 pid = mm->context.id;
779 if (unlikely(pid == MMU_NO_CONTEXT))
780 return;
781
782 preempt_disable();
783 smp_mb();
784 if (!mm_is_thread_local(mm)) {
785 if (unlikely(mm_is_singlethreaded(mm))) {
786 exit_flush_lazy_tlbs(mm);
787 goto local;
788 }
789 if (!mmu_has_feature(MMU_FTR_GTSE)) {
790 unsigned long tgt, pg_sizes, size;
791
792 tgt = H_RPTI_TARGET_CMMU;
793 pg_sizes = psize_to_rpti_pgsize(psize);
794 size = 1UL << mmu_psize_to_shift(psize);
795
796 if (atomic_read(&mm->context.copros) > 0)
797 tgt |= H_RPTI_TARGET_NMMU;
798 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
799 pg_sizes, vmaddr,
800 vmaddr + size);
801 } else if (cputlb_use_tlbie())
802 _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
803 else
804 _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
805 } else {
806local:
807 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
808 }
809 preempt_enable();
810}
811
812void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
813{
814#ifdef CONFIG_HUGETLB_PAGE
815 if (is_vm_hugetlb_page(vma))
816 return radix__flush_hugetlb_page(vma, vmaddr);
817#endif
818 radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
819}
820EXPORT_SYMBOL(radix__flush_tlb_page);
821
822#else
823static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
824#endif
825
826static void do_tlbiel_kernel(void *info)
827{
828 _tlbiel_pid(0, RIC_FLUSH_ALL);
829}
830
831static inline void _tlbiel_kernel_broadcast(void)
832{
833 on_each_cpu(do_tlbiel_kernel, NULL, 1);
834 if (tlbie_capable) {
835
836
837
838
839
840 _tlbie_pid(0, RIC_FLUSH_ALL);
841 }
842}
843
844
845
846
847
848
849void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
850{
851 if (!mmu_has_feature(MMU_FTR_GTSE)) {
852 unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU;
853 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
854 H_RPTI_TYPE_PRT;
855
856 pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL,
857 start, end);
858 } else if (cputlb_use_tlbie())
859 _tlbie_pid(0, RIC_FLUSH_ALL);
860 else
861 _tlbiel_kernel_broadcast();
862}
863EXPORT_SYMBOL(radix__flush_tlb_kernel_range);
864
865#define TLB_FLUSH_ALL -1UL
866
867
868
869
870
871
872
873
874
875
876static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
877static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2;
878
879static inline void __radix__flush_tlb_range(struct mm_struct *mm,
880 unsigned long start, unsigned long end)
881
882{
883 unsigned long pid;
884 unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift;
885 unsigned long page_size = 1UL << page_shift;
886 unsigned long nr_pages = (end - start) >> page_shift;
887 bool local, full;
888
889 pid = mm->context.id;
890 if (unlikely(pid == MMU_NO_CONTEXT))
891 return;
892
893 preempt_disable();
894 smp_mb();
895 if (!mm_is_thread_local(mm)) {
896 if (unlikely(mm_is_singlethreaded(mm))) {
897 if (end != TLB_FLUSH_ALL) {
898 exit_flush_lazy_tlbs(mm);
899 goto is_local;
900 }
901 }
902 local = false;
903 full = (end == TLB_FLUSH_ALL ||
904 nr_pages > tlb_single_page_flush_ceiling);
905 } else {
906is_local:
907 local = true;
908 full = (end == TLB_FLUSH_ALL ||
909 nr_pages > tlb_local_single_page_flush_ceiling);
910 }
911
912 if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
913 unsigned long tgt = H_RPTI_TARGET_CMMU;
914 unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);
915
916 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
917 pg_sizes |= psize_to_rpti_pgsize(MMU_PAGE_2M);
918 if (atomic_read(&mm->context.copros) > 0)
919 tgt |= H_RPTI_TARGET_NMMU;
920 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes,
921 start, end);
922 } else if (full) {
923 if (local) {
924 _tlbiel_pid(pid, RIC_FLUSH_TLB);
925 } else {
926 if (cputlb_use_tlbie()) {
927 if (mm_needs_flush_escalation(mm))
928 _tlbie_pid(pid, RIC_FLUSH_ALL);
929 else
930 _tlbie_pid(pid, RIC_FLUSH_TLB);
931 } else {
932 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
933 }
934 }
935 } else {
936 bool hflush = false;
937 unsigned long hstart, hend;
938
939 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
940 hstart = (start + PMD_SIZE - 1) & PMD_MASK;
941 hend = end & PMD_MASK;
942 if (hstart < hend)
943 hflush = true;
944 }
945
946 if (local) {
947 asm volatile("ptesync": : :"memory");
948 __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize);
949 if (hflush)
950 __tlbiel_va_range(hstart, hend, pid,
951 PMD_SIZE, MMU_PAGE_2M);
952 ppc_after_tlbiel_barrier();
953 } else if (cputlb_use_tlbie()) {
954 asm volatile("ptesync": : :"memory");
955 __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize);
956 if (hflush)
957 __tlbie_va_range(hstart, hend, pid,
958 PMD_SIZE, MMU_PAGE_2M);
959 asm volatile("eieio; tlbsync; ptesync": : :"memory");
960 } else {
961 _tlbiel_va_range_multicast(mm,
962 start, end, pid, page_size, mmu_virtual_psize, false);
963 if (hflush)
964 _tlbiel_va_range_multicast(mm,
965 hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false);
966 }
967 }
968 preempt_enable();
969}
970
971void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
972 unsigned long end)
973
974{
975#ifdef CONFIG_HUGETLB_PAGE
976 if (is_vm_hugetlb_page(vma))
977 return radix__flush_hugetlb_tlb_range(vma, start, end);
978#endif
979
980 __radix__flush_tlb_range(vma->vm_mm, start, end);
981}
982EXPORT_SYMBOL(radix__flush_tlb_range);
983
984static int radix_get_mmu_psize(int page_size)
985{
986 int psize;
987
988 if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift))
989 psize = mmu_virtual_psize;
990 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift))
991 psize = MMU_PAGE_2M;
992 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift))
993 psize = MMU_PAGE_1G;
994 else
995 return -1;
996 return psize;
997}
998
999
1000
1001
1002void radix__flush_tlb_lpid_page(unsigned int lpid,
1003 unsigned long addr,
1004 unsigned long page_size)
1005{
1006 int psize = radix_get_mmu_psize(page_size);
1007
1008 _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB);
1009}
1010EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page);
1011
1012
1013
1014
1015void radix__flush_pwc_lpid(unsigned int lpid)
1016{
1017 _tlbie_lpid(lpid, RIC_FLUSH_PWC);
1018}
1019EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid);
1020
1021
1022
1023
1024void radix__flush_all_lpid(unsigned int lpid)
1025{
1026 _tlbie_lpid(lpid, RIC_FLUSH_ALL);
1027}
1028EXPORT_SYMBOL_GPL(radix__flush_all_lpid);
1029
1030
1031
1032
1033void radix__flush_all_lpid_guest(unsigned int lpid)
1034{
1035 _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
1036}
1037
1038static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
1039 unsigned long end, int psize);
1040
1041void radix__tlb_flush(struct mmu_gather *tlb)
1042{
1043 int psize = 0;
1044 struct mm_struct *mm = tlb->mm;
1045 int page_size = tlb->page_size;
1046 unsigned long start = tlb->start;
1047 unsigned long end = tlb->end;
1048
1049
1050
1051
1052
1053
1054
1055
1056 if (tlb->fullmm || tlb->need_flush_all) {
1057 __flush_all_mm(mm, true);
1058 } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) {
1059 if (!tlb->freed_tables)
1060 radix__flush_tlb_mm(mm);
1061 else
1062 radix__flush_all_mm(mm);
1063 } else {
1064 if (!tlb->freed_tables)
1065 radix__flush_tlb_range_psize(mm, start, end, psize);
1066 else
1067 radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
1068 }
1069}
1070
1071static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
1072 unsigned long start, unsigned long end,
1073 int psize, bool also_pwc)
1074{
1075 unsigned long pid;
1076 unsigned int page_shift = mmu_psize_defs[psize].shift;
1077 unsigned long page_size = 1UL << page_shift;
1078 unsigned long nr_pages = (end - start) >> page_shift;
1079 bool local, full;
1080
1081 pid = mm->context.id;
1082 if (unlikely(pid == MMU_NO_CONTEXT))
1083 return;
1084
1085 preempt_disable();
1086 smp_mb();
1087 if (!mm_is_thread_local(mm)) {
1088 if (unlikely(mm_is_singlethreaded(mm))) {
1089 if (end != TLB_FLUSH_ALL) {
1090 exit_flush_lazy_tlbs(mm);
1091 goto is_local;
1092 }
1093 }
1094 local = false;
1095 full = (end == TLB_FLUSH_ALL ||
1096 nr_pages > tlb_single_page_flush_ceiling);
1097 } else {
1098is_local:
1099 local = true;
1100 full = (end == TLB_FLUSH_ALL ||
1101 nr_pages > tlb_local_single_page_flush_ceiling);
1102 }
1103
1104 if (!mmu_has_feature(MMU_FTR_GTSE) && !local) {
1105 unsigned long tgt = H_RPTI_TARGET_CMMU;
1106 unsigned long type = H_RPTI_TYPE_TLB;
1107 unsigned long pg_sizes = psize_to_rpti_pgsize(psize);
1108
1109 if (also_pwc)
1110 type |= H_RPTI_TYPE_PWC;
1111 if (atomic_read(&mm->context.copros) > 0)
1112 tgt |= H_RPTI_TARGET_NMMU;
1113 pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
1114 } else if (full) {
1115 if (local) {
1116 _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1117 } else {
1118 if (cputlb_use_tlbie()) {
1119 if (mm_needs_flush_escalation(mm))
1120 also_pwc = true;
1121
1122 _tlbie_pid(pid,
1123 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1124 } else {
1125 _tlbiel_pid_multicast(mm, pid,
1126 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1127 }
1128
1129 }
1130 } else {
1131 if (local)
1132 _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc);
1133 else if (cputlb_use_tlbie())
1134 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
1135 else
1136 _tlbiel_va_range_multicast(mm,
1137 start, end, pid, page_size, psize, also_pwc);
1138 }
1139 preempt_enable();
1140}
1141
1142void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
1143 unsigned long end, int psize)
1144{
1145 return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
1146}
1147
1148static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
1149 unsigned long end, int psize)
1150{
1151 __radix__flush_tlb_range_psize(mm, start, end, psize, true);
1152}
1153
1154#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1155void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
1156{
1157 unsigned long pid, end;
1158
1159 pid = mm->context.id;
1160 if (unlikely(pid == MMU_NO_CONTEXT))
1161 return;
1162
1163
1164 if (PAGE_SIZE == 0x1000) {
1165 radix__flush_all_mm(mm);
1166 return;
1167 }
1168
1169 end = addr + HPAGE_PMD_SIZE;
1170
1171
1172 preempt_disable();
1173 smp_mb();
1174 if (!mm_is_thread_local(mm)) {
1175 if (unlikely(mm_is_singlethreaded(mm))) {
1176 exit_flush_lazy_tlbs(mm);
1177 goto local;
1178 }
1179 if (!mmu_has_feature(MMU_FTR_GTSE)) {
1180 unsigned long tgt, type, pg_sizes;
1181
1182 tgt = H_RPTI_TARGET_CMMU;
1183 type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
1184 H_RPTI_TYPE_PRT;
1185 pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);
1186
1187 if (atomic_read(&mm->context.copros) > 0)
1188 tgt |= H_RPTI_TARGET_NMMU;
1189 pseries_rpt_invalidate(pid, tgt, type, pg_sizes,
1190 addr, end);
1191 } else if (cputlb_use_tlbie())
1192 _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1193 else
1194 _tlbiel_va_range_multicast(mm,
1195 addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1196 } else {
1197local:
1198 _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1199 }
1200
1201 preempt_enable();
1202}
1203#endif
1204
1205void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
1206 unsigned long start, unsigned long end)
1207{
1208 radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
1209}
1210EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
1211
1212void radix__flush_tlb_all(void)
1213{
1214 unsigned long rb,prs,r,rs;
1215 unsigned long ric = RIC_FLUSH_ALL;
1216
1217 rb = 0x3 << PPC_BITLSHIFT(53);
1218 prs = 0;
1219 r = 1;
1220 rs = 1 & ((1UL << 32) - 1);
1221
1222 asm volatile("ptesync": : :"memory");
1223
1224
1225
1226 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
1227 : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");
1228
1229
1230
1231 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
1232 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
1233 asm volatile("eieio; tlbsync; ptesync": : :"memory");
1234}
1235
1236#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1237extern void radix_kvm_prefetch_workaround(struct mm_struct *mm)
1238{
1239 unsigned long pid = mm->context.id;
1240
1241 if (unlikely(pid == MMU_NO_CONTEXT))
1242 return;
1243
1244 if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
1245 return;
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262 if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) {
1263 int cpu = smp_processor_id();
1264 int sib = cpu_first_thread_sibling(cpu);
1265 bool flush = false;
1266
1267 for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
1268 if (sib == cpu)
1269 continue;
1270 if (!cpu_possible(sib))
1271 continue;
1272 if (paca_ptrs[sib]->kvm_hstate.kvm_vcpu)
1273 flush = true;
1274 }
1275 if (flush)
1276 _tlbiel_pid(pid, RIC_FLUSH_ALL);
1277 }
1278}
1279EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround);
1280#endif
1281