1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/kernel.h>
25#include <linux/mm.h>
26#include <linux/init.h>
27#include <linux/percpu.h>
28#include <linux/hardirq.h>
29#include <asm/pgalloc.h>
30#include <asm/tlbflush.h>
31#include <asm/tlb.h>
32#include <asm/bug.h>
33
34DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
35
36
37
38
39
40
41
42void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
43 pte_t *ptep, unsigned long pte, int huge)
44{
45 unsigned long vpn;
46 struct ppc64_tlb_batch *batch = &get_cpu_var(ppc64_tlb_batch);
47 unsigned long vsid;
48 unsigned int psize;
49 int ssize;
50 real_pte_t rpte;
51 int i;
52
53 i = batch->index;
54
55
56
57
58
59
60
61
62 if (huge) {
63#ifdef CONFIG_HUGETLB_PAGE
64 psize = get_slice_psize(mm, addr);
65
66 addr &= ~((1UL << mmu_psize_defs[psize].shift) - 1);
67#else
68 BUG();
69 psize = pte_pagesize_index(mm, addr, pte);
70#endif
71 } else {
72 psize = pte_pagesize_index(mm, addr, pte);
73
74
75
76
77 addr &= PAGE_MASK;
78 }
79
80
81
82 if (!is_kernel_addr(addr)) {
83 ssize = user_segment_size(addr);
84 vsid = get_vsid(mm->context.id, addr, ssize);
85 } else {
86 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
87 ssize = mmu_kernel_ssize;
88 }
89 WARN_ON(vsid == 0);
90 vpn = hpt_vpn(addr, vsid, ssize);
91 rpte = __real_pte(__pte(pte), ptep);
92
93
94
95
96
97
98
99 if (!batch->active) {
100 flush_hash_page(vpn, rpte, psize, ssize, 0);
101 put_cpu_var(ppc64_tlb_batch);
102 return;
103 }
104
105
106
107
108
109
110
111
112
113
114
115 if (i != 0 && (mm != batch->mm || batch->psize != psize ||
116 batch->ssize != ssize)) {
117 __flush_tlb_pending(batch);
118 i = 0;
119 }
120 if (i == 0) {
121 batch->mm = mm;
122 batch->psize = psize;
123 batch->ssize = ssize;
124 }
125 batch->pte[i] = rpte;
126 batch->vpn[i] = vpn;
127 batch->index = ++i;
128 if (i >= PPC64_TLB_BATCH_NR)
129 __flush_tlb_pending(batch);
130 put_cpu_var(ppc64_tlb_batch);
131}
132
133
134
135
136
137
138
139
140void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
141{
142 const struct cpumask *tmp;
143 int i, local = 0;
144
145 i = batch->index;
146 tmp = cpumask_of(smp_processor_id());
147 if (cpumask_equal(mm_cpumask(batch->mm), tmp))
148 local = 1;
149 if (i == 1)
150 flush_hash_page(batch->vpn[0], batch->pte[0],
151 batch->psize, batch->ssize, local);
152 else
153 flush_hash_range(i, local);
154 batch->index = 0;
155}
156
157void tlb_flush(struct mmu_gather *tlb)
158{
159 struct ppc64_tlb_batch *tlbbatch = &get_cpu_var(ppc64_tlb_batch);
160
161
162
163
164
165 if (tlbbatch->index)
166 __flush_tlb_pending(tlbbatch);
167
168 put_cpu_var(ppc64_tlb_batch);
169}
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,
190 unsigned long end)
191{
192 int hugepage_shift;
193 unsigned long flags;
194
195 start = _ALIGN_DOWN(start, PAGE_SIZE);
196 end = _ALIGN_UP(end, PAGE_SIZE);
197
198 BUG_ON(!mm->pgd);
199
200
201
202
203
204
205
206
207 local_irq_save(flags);
208 arch_enter_lazy_mmu_mode();
209 for (; start < end; start += PAGE_SIZE) {
210 pte_t *ptep = find_linux_pte_or_hugepte(mm->pgd, start,
211 &hugepage_shift);
212 unsigned long pte;
213
214 if (ptep == NULL)
215 continue;
216 pte = pte_val(*ptep);
217 if (!(pte & _PAGE_HASHPTE))
218 continue;
219 if (unlikely(hugepage_shift && pmd_trans_huge(*(pmd_t *)pte)))
220 hpte_do_hugepage_flush(mm, start, (pmd_t *)pte);
221 else
222 hpte_need_flush(mm, start, ptep, pte, 0);
223 }
224 arch_leave_lazy_mmu_mode();
225 local_irq_restore(flags);
226}
227
228void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr)
229{
230 pte_t *pte;
231 pte_t *start_pte;
232 unsigned long flags;
233
234 addr = _ALIGN_DOWN(addr, PMD_SIZE);
235
236
237
238
239
240
241
242 local_irq_save(flags);
243 arch_enter_lazy_mmu_mode();
244 start_pte = pte_offset_map(pmd, addr);
245 for (pte = start_pte; pte < start_pte + PTRS_PER_PTE; pte++) {
246 unsigned long pteval = pte_val(*pte);
247 if (pteval & _PAGE_HASHPTE)
248 hpte_need_flush(mm, addr, pte, pteval, 0);
249 addr += PAGE_SIZE;
250 }
251 arch_leave_lazy_mmu_mode();
252 local_irq_restore(flags);
253}
254