1#include "qemu/osdep.h"
2#include "qemu/cutils.h"
3#include "qemu/memalign.h"
4#include "qemu/error-report.h"
5#include "cpu.h"
6#include "helper_regs.h"
7#include "hw/ppc/spapr.h"
8#include "mmu-hash64.h"
9#include "mmu-book3s-v3.h"
10
11
12static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
13{
14
15
16
17 if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
18 return false;
19 }
20 return true;
21}
22
23static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
24 target_ulong opcode, target_ulong *args)
25{
26 target_ulong flags = args[0];
27 target_ulong ptex = args[1];
28 target_ulong pteh = args[2];
29 target_ulong ptel = args[3];
30 unsigned apshift;
31 target_ulong raddr;
32 target_ulong slot;
33 const ppc_hash_pte64_t *hptes;
34
35 apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
36 if (!apshift) {
37
38 return H_PARAMETER;
39 }
40
41 raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
42
43 if (is_ram_address(spapr, raddr)) {
44
45 if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
46 return H_PARAMETER;
47 }
48 } else {
49 target_ulong wimg_flags;
50
51
52
53
54 wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
55
56 if (wimg_flags != HPTE64_R_I &&
57 wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
58 return H_PARAMETER;
59 }
60 }
61
62 pteh &= ~0x60ULL;
63
64 if (!valid_ptex(cpu, ptex)) {
65 return H_PARAMETER;
66 }
67
68 slot = ptex & 7ULL;
69 ptex = ptex & ~7ULL;
70
71 if (likely((flags & H_EXACT) == 0)) {
72 hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
73 for (slot = 0; slot < 8; slot++) {
74 if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
75 break;
76 }
77 }
78 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
79 if (slot == 8) {
80 return H_PTEG_FULL;
81 }
82 } else {
83 hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
84 if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
85 ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
86 return H_PTEG_FULL;
87 }
88 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
89 }
90
91 spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
92
93 args[0] = ptex + slot;
94 return H_SUCCESS;
95}
96
97typedef enum {
98 REMOVE_SUCCESS = 0,
99 REMOVE_NOT_FOUND = 1,
100 REMOVE_PARM = 2,
101 REMOVE_HW = 3,
102} RemoveResult;
103
104static RemoveResult remove_hpte(PowerPCCPU *cpu
105 , target_ulong ptex,
106 target_ulong avpn,
107 target_ulong flags,
108 target_ulong *vp, target_ulong *rp)
109{
110 const ppc_hash_pte64_t *hptes;
111 target_ulong v, r;
112
113 if (!valid_ptex(cpu, ptex)) {
114 return REMOVE_PARM;
115 }
116
117 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
118 v = ppc_hash64_hpte0(cpu, hptes, 0);
119 r = ppc_hash64_hpte1(cpu, hptes, 0);
120 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
121
122 if ((v & HPTE64_V_VALID) == 0 ||
123 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
124 ((flags & H_ANDCOND) && (v & avpn) != 0)) {
125 return REMOVE_NOT_FOUND;
126 }
127 *vp = v;
128 *rp = r;
129 spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
130 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
131 return REMOVE_SUCCESS;
132}
133
134static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
135 target_ulong opcode, target_ulong *args)
136{
137 CPUPPCState *env = &cpu->env;
138 target_ulong flags = args[0];
139 target_ulong ptex = args[1];
140 target_ulong avpn = args[2];
141 RemoveResult ret;
142
143 ret = remove_hpte(cpu, ptex, avpn, flags,
144 &args[0], &args[1]);
145
146 switch (ret) {
147 case REMOVE_SUCCESS:
148 check_tlb_flush(env, true);
149 return H_SUCCESS;
150
151 case REMOVE_NOT_FOUND:
152 return H_NOT_FOUND;
153
154 case REMOVE_PARM:
155 return H_PARAMETER;
156
157 case REMOVE_HW:
158 return H_HARDWARE;
159 }
160
161 g_assert_not_reached();
162}
163
164#define H_BULK_REMOVE_TYPE 0xc000000000000000ULL
165#define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL
166#define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL
167#define H_BULK_REMOVE_END 0xc000000000000000ULL
168#define H_BULK_REMOVE_CODE 0x3000000000000000ULL
169#define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL
170#define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL
171#define H_BULK_REMOVE_PARM 0x2000000000000000ULL
172#define H_BULK_REMOVE_HW 0x3000000000000000ULL
173#define H_BULK_REMOVE_RC 0x0c00000000000000ULL
174#define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL
175#define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL
176#define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL
177#define H_BULK_REMOVE_AVPN 0x0200000000000000ULL
178#define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL
179
180#define H_BULK_REMOVE_MAX_BATCH 4
181
182static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
183 target_ulong opcode, target_ulong *args)
184{
185 CPUPPCState *env = &cpu->env;
186 int i;
187 target_ulong rc = H_SUCCESS;
188
189 for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
190 target_ulong *tsh = &args[i*2];
191 target_ulong tsl = args[i*2 + 1];
192 target_ulong v, r, ret;
193
194 if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
195 break;
196 } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
197 return H_PARAMETER;
198 }
199
200 *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
201 *tsh |= H_BULK_REMOVE_RESPONSE;
202
203 if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
204 *tsh |= H_BULK_REMOVE_PARM;
205 return H_PARAMETER;
206 }
207
208 ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
209 (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
210 &v, &r);
211
212 *tsh |= ret << 60;
213
214 switch (ret) {
215 case REMOVE_SUCCESS:
216 *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
217 break;
218
219 case REMOVE_PARM:
220 rc = H_PARAMETER;
221 goto exit;
222
223 case REMOVE_HW:
224 rc = H_HARDWARE;
225 goto exit;
226 }
227 }
228 exit:
229 check_tlb_flush(env, true);
230
231 return rc;
232}
233
234static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
235 target_ulong opcode, target_ulong *args)
236{
237 CPUPPCState *env = &cpu->env;
238 target_ulong flags = args[0];
239 target_ulong ptex = args[1];
240 target_ulong avpn = args[2];
241 const ppc_hash_pte64_t *hptes;
242 target_ulong v, r;
243
244 if (!valid_ptex(cpu, ptex)) {
245 return H_PARAMETER;
246 }
247
248 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
249 v = ppc_hash64_hpte0(cpu, hptes, 0);
250 r = ppc_hash64_hpte1(cpu, hptes, 0);
251 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
252
253 if ((v & HPTE64_V_VALID) == 0 ||
254 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
255 return H_NOT_FOUND;
256 }
257
258 r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
259 HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
260 r |= (flags << 55) & HPTE64_R_PP0;
261 r |= (flags << 48) & HPTE64_R_KEY_HI;
262 r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
263 spapr_store_hpte(cpu, ptex,
264 (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
265 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
266
267 check_tlb_flush(env, true);
268
269 spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
270 return H_SUCCESS;
271}
272
273static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
274 target_ulong opcode, target_ulong *args)
275{
276 target_ulong flags = args[0];
277 target_ulong ptex = args[1];
278 int i, ridx, n_entries = 1;
279 const ppc_hash_pte64_t *hptes;
280
281 if (!valid_ptex(cpu, ptex)) {
282 return H_PARAMETER;
283 }
284
285 if (flags & H_READ_4) {
286
287 ptex &= ~(3ULL);
288 n_entries = 4;
289 }
290
291 hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
292 for (i = 0, ridx = 0; i < n_entries; i++) {
293 args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
294 args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
295 }
296 ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
297
298 return H_SUCCESS;
299}
300
301struct SpaprPendingHpt {
302
303 int shift;
304 QemuThread thread;
305
306
307 bool complete;
308
309
310
311 int ret;
312 void *hpt;
313};
314
315static void free_pending_hpt(SpaprPendingHpt *pending)
316{
317 if (pending->hpt) {
318 qemu_vfree(pending->hpt);
319 }
320
321 g_free(pending);
322}
323
324static void *hpt_prepare_thread(void *opaque)
325{
326 SpaprPendingHpt *pending = opaque;
327 size_t size = 1ULL << pending->shift;
328
329 pending->hpt = qemu_try_memalign(size, size);
330 if (pending->hpt) {
331 memset(pending->hpt, 0, size);
332 pending->ret = H_SUCCESS;
333 } else {
334 pending->ret = H_NO_MEM;
335 }
336
337 qemu_mutex_lock_iothread();
338
339 if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
340
341 pending->complete = true;
342 } else {
343
344 free_pending_hpt(pending);
345 }
346
347 qemu_mutex_unlock_iothread();
348 return NULL;
349}
350
351
352static void cancel_hpt_prepare(SpaprMachineState *spapr)
353{
354 SpaprPendingHpt *pending = spapr->pending_hpt;
355
356
357 spapr->pending_hpt = NULL;
358
359 if (!pending) {
360
361 return;
362 }
363
364 if (!pending->complete) {
365
366 return;
367 }
368
369 free_pending_hpt(pending);
370}
371
372target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
373 SpaprMachineState *spapr,
374 target_ulong shift)
375{
376 SpaprPendingHpt *pending = spapr->pending_hpt;
377
378 if (pending) {
379
380 if (pending->shift == shift) {
381
382 if (pending->complete) {
383 return pending->ret;
384 } else {
385 return H_LONG_BUSY_ORDER_100_MSEC;
386 }
387 }
388
389
390 cancel_hpt_prepare(spapr);
391 }
392
393 if (!shift) {
394
395 return H_SUCCESS;
396 }
397
398
399
400 pending = g_new0(SpaprPendingHpt, 1);
401 pending->shift = shift;
402 pending->ret = H_HARDWARE;
403
404 qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
405 hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
406
407 spapr->pending_hpt = pending;
408
409
410
411 return H_LONG_BUSY_ORDER_100_MSEC;
412}
413
414static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
415{
416 uint8_t *addr = htab;
417
418 addr += pteg * HASH_PTEG_SIZE_64;
419 addr += slot * HASH_PTE_SIZE_64;
420 return ldq_p(addr);
421}
422
423static void new_hpte_store(void *htab, uint64_t pteg, int slot,
424 uint64_t pte0, uint64_t pte1)
425{
426 uint8_t *addr = htab;
427
428 addr += pteg * HASH_PTEG_SIZE_64;
429 addr += slot * HASH_PTE_SIZE_64;
430
431 stq_p(addr, pte0);
432 stq_p(addr + HPTE64_DW1, pte1);
433}
434
435static int rehash_hpte(PowerPCCPU *cpu,
436 const ppc_hash_pte64_t *hptes,
437 void *old_hpt, uint64_t oldsize,
438 void *new_hpt, uint64_t newsize,
439 uint64_t pteg, int slot)
440{
441 uint64_t old_hash_mask = (oldsize >> 7) - 1;
442 uint64_t new_hash_mask = (newsize >> 7) - 1;
443 target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
444 target_ulong pte1;
445 uint64_t avpn;
446 unsigned base_pg_shift;
447 uint64_t hash, new_pteg, replace_pte0;
448
449 if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
450 return H_SUCCESS;
451 }
452
453 pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
454
455 base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
456 assert(base_pg_shift);
457 avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
458
459 if (pte0 & HPTE64_V_SECONDARY) {
460 pteg = ~pteg;
461 }
462
463 if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
464 uint64_t offset, vsid;
465
466
467 offset = (avpn & 0x1f) << 23;
468 vsid = avpn >> 5;
469
470 if (base_pg_shift < 23) {
471 offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
472 }
473
474 hash = vsid ^ (offset >> base_pg_shift);
475 } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
476 uint64_t offset, vsid;
477
478
479 offset = (avpn & 0x1ffff) << 23;
480 vsid = avpn >> 17;
481 if (base_pg_shift < 23) {
482 offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
483 << base_pg_shift;
484 }
485
486 hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
487 } else {
488 error_report("rehash_pte: Bad segment size in HPTE");
489 return H_HARDWARE;
490 }
491
492 new_pteg = hash & new_hash_mask;
493 if (pte0 & HPTE64_V_SECONDARY) {
494 assert(~pteg == (hash & old_hash_mask));
495 new_pteg = ~new_pteg;
496 } else {
497 assert(pteg == (hash & old_hash_mask));
498 }
499 assert((oldsize != newsize) || (pteg == new_pteg));
500 replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
501
502
503
504
505
506
507 if (replace_pte0 & HPTE64_V_VALID) {
508 assert(newsize < oldsize);
509 if (replace_pte0 & HPTE64_V_BOLTED) {
510 if (pte0 & HPTE64_V_BOLTED) {
511
512 return H_PTEG_FULL;
513 } else {
514
515 return H_SUCCESS;
516 }
517 }
518 }
519
520 new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
521 return H_SUCCESS;
522}
523
524static int rehash_hpt(PowerPCCPU *cpu,
525 void *old_hpt, uint64_t oldsize,
526 void *new_hpt, uint64_t newsize)
527{
528 uint64_t n_ptegs = oldsize >> 7;
529 uint64_t pteg;
530 int slot;
531 int rc;
532
533 for (pteg = 0; pteg < n_ptegs; pteg++) {
534 hwaddr ptex = pteg * HPTES_PER_GROUP;
535 const ppc_hash_pte64_t *hptes
536 = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
537
538 if (!hptes) {
539 return H_HARDWARE;
540 }
541
542 for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
543 rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
544 pteg, slot);
545 if (rc != H_SUCCESS) {
546 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
547 return rc;
548 }
549 }
550 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
551 }
552
553 return H_SUCCESS;
554}
555
556target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
557 SpaprMachineState *spapr,
558 target_ulong flags,
559 target_ulong shift)
560{
561 SpaprPendingHpt *pending = spapr->pending_hpt;
562 int rc;
563 size_t newsize;
564
565 if (flags != 0) {
566 return H_PARAMETER;
567 }
568
569 if (!pending || (pending->shift != shift)) {
570
571 return H_CLOSED;
572 }
573
574 if (!pending->complete) {
575
576 return H_BUSY;
577 }
578
579
580 g_assert(spapr->htab_shift);
581
582 newsize = 1ULL << pending->shift;
583 rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
584 pending->hpt, newsize);
585 if (rc == H_SUCCESS) {
586 qemu_vfree(spapr->htab);
587 spapr->htab = pending->hpt;
588 spapr->htab_shift = pending->shift;
589
590 push_sregs_to_kvm_pr(spapr);
591
592 pending->hpt = NULL;
593 }
594
595
596 spapr->pending_hpt = NULL;
597 free_pending_hpt(pending);
598
599 return rc;
600}
601
602static void hypercall_register_types(void)
603{
604
605 spapr_register_hypercall(H_ENTER, h_enter);
606 spapr_register_hypercall(H_REMOVE, h_remove);
607 spapr_register_hypercall(H_PROTECT, h_protect);
608 spapr_register_hypercall(H_READ, h_read);
609
610
611 spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
612
613}
614
615type_init(hypercall_register_types)
616