1#include "qemu/osdep.h"
2#include "qemu/cutils.h"
3#include "qapi/error.h"
4#include "sysemu/hw_accel.h"
5#include "sysemu/runstate.h"
6#include "qemu/log.h"
7#include "qemu/main-loop.h"
8#include "qemu/module.h"
9#include "qemu/error-report.h"
10#include "cpu.h"
11#include "exec/exec-all.h"
12#include "helper_regs.h"
13#include "hw/ppc/spapr.h"
14#include "hw/ppc/spapr_cpu_core.h"
15#include "mmu-hash64.h"
16#include "cpu-models.h"
17#include "trace.h"
18#include "kvm_ppc.h"
19#include "hw/ppc/fdt.h"
20#include "hw/ppc/spapr_ovec.h"
21#include "mmu-book3s-v3.h"
22#include "hw/mem/memory-device.h"
23
24static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
25{
26
27
28
29 if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
30 return false;
31 }
32 return true;
33}
34
35static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
36 target_ulong opcode, target_ulong *args)
37{
38 target_ulong flags = args[0];
39 target_ulong ptex = args[1];
40 target_ulong pteh = args[2];
41 target_ulong ptel = args[3];
42 unsigned apshift;
43 target_ulong raddr;
44 target_ulong slot;
45 const ppc_hash_pte64_t *hptes;
46
47 apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
48 if (!apshift) {
49
50 return H_PARAMETER;
51 }
52
53 raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
54
55 if (is_ram_address(spapr, raddr)) {
56
57 if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
58 return H_PARAMETER;
59 }
60 } else {
61 target_ulong wimg_flags;
62
63
64
65
66 wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
67
68 if (wimg_flags != HPTE64_R_I &&
69 wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
70 return H_PARAMETER;
71 }
72 }
73
74 pteh &= ~0x60ULL;
75
76 if (!valid_ptex(cpu, ptex)) {
77 return H_PARAMETER;
78 }
79
80 slot = ptex & 7ULL;
81 ptex = ptex & ~7ULL;
82
83 if (likely((flags & H_EXACT) == 0)) {
84 hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
85 for (slot = 0; slot < 8; slot++) {
86 if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
87 break;
88 }
89 }
90 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
91 if (slot == 8) {
92 return H_PTEG_FULL;
93 }
94 } else {
95 hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
96 if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
97 ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
98 return H_PTEG_FULL;
99 }
100 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
101 }
102
103 spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
104
105 args[0] = ptex + slot;
106 return H_SUCCESS;
107}
108
109typedef enum {
110 REMOVE_SUCCESS = 0,
111 REMOVE_NOT_FOUND = 1,
112 REMOVE_PARM = 2,
113 REMOVE_HW = 3,
114} RemoveResult;
115
116static RemoveResult remove_hpte(PowerPCCPU *cpu
117 , target_ulong ptex,
118 target_ulong avpn,
119 target_ulong flags,
120 target_ulong *vp, target_ulong *rp)
121{
122 const ppc_hash_pte64_t *hptes;
123 target_ulong v, r;
124
125 if (!valid_ptex(cpu, ptex)) {
126 return REMOVE_PARM;
127 }
128
129 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
130 v = ppc_hash64_hpte0(cpu, hptes, 0);
131 r = ppc_hash64_hpte1(cpu, hptes, 0);
132 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
133
134 if ((v & HPTE64_V_VALID) == 0 ||
135 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
136 ((flags & H_ANDCOND) && (v & avpn) != 0)) {
137 return REMOVE_NOT_FOUND;
138 }
139 *vp = v;
140 *rp = r;
141 spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
142 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
143 return REMOVE_SUCCESS;
144}
145
146static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
147 target_ulong opcode, target_ulong *args)
148{
149 CPUPPCState *env = &cpu->env;
150 target_ulong flags = args[0];
151 target_ulong ptex = args[1];
152 target_ulong avpn = args[2];
153 RemoveResult ret;
154
155 ret = remove_hpte(cpu, ptex, avpn, flags,
156 &args[0], &args[1]);
157
158 switch (ret) {
159 case REMOVE_SUCCESS:
160 check_tlb_flush(env, true);
161 return H_SUCCESS;
162
163 case REMOVE_NOT_FOUND:
164 return H_NOT_FOUND;
165
166 case REMOVE_PARM:
167 return H_PARAMETER;
168
169 case REMOVE_HW:
170 return H_HARDWARE;
171 }
172
173 g_assert_not_reached();
174}
175
176#define H_BULK_REMOVE_TYPE 0xc000000000000000ULL
177#define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL
178#define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL
179#define H_BULK_REMOVE_END 0xc000000000000000ULL
180#define H_BULK_REMOVE_CODE 0x3000000000000000ULL
181#define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL
182#define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL
183#define H_BULK_REMOVE_PARM 0x2000000000000000ULL
184#define H_BULK_REMOVE_HW 0x3000000000000000ULL
185#define H_BULK_REMOVE_RC 0x0c00000000000000ULL
186#define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL
187#define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL
188#define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL
189#define H_BULK_REMOVE_AVPN 0x0200000000000000ULL
190#define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL
191
192#define H_BULK_REMOVE_MAX_BATCH 4
193
194static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
195 target_ulong opcode, target_ulong *args)
196{
197 CPUPPCState *env = &cpu->env;
198 int i;
199 target_ulong rc = H_SUCCESS;
200
201 for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
202 target_ulong *tsh = &args[i*2];
203 target_ulong tsl = args[i*2 + 1];
204 target_ulong v, r, ret;
205
206 if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
207 break;
208 } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
209 return H_PARAMETER;
210 }
211
212 *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
213 *tsh |= H_BULK_REMOVE_RESPONSE;
214
215 if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
216 *tsh |= H_BULK_REMOVE_PARM;
217 return H_PARAMETER;
218 }
219
220 ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
221 (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
222 &v, &r);
223
224 *tsh |= ret << 60;
225
226 switch (ret) {
227 case REMOVE_SUCCESS:
228 *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
229 break;
230
231 case REMOVE_PARM:
232 rc = H_PARAMETER;
233 goto exit;
234
235 case REMOVE_HW:
236 rc = H_HARDWARE;
237 goto exit;
238 }
239 }
240 exit:
241 check_tlb_flush(env, true);
242
243 return rc;
244}
245
246static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
247 target_ulong opcode, target_ulong *args)
248{
249 CPUPPCState *env = &cpu->env;
250 target_ulong flags = args[0];
251 target_ulong ptex = args[1];
252 target_ulong avpn = args[2];
253 const ppc_hash_pte64_t *hptes;
254 target_ulong v, r;
255
256 if (!valid_ptex(cpu, ptex)) {
257 return H_PARAMETER;
258 }
259
260 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
261 v = ppc_hash64_hpte0(cpu, hptes, 0);
262 r = ppc_hash64_hpte1(cpu, hptes, 0);
263 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
264
265 if ((v & HPTE64_V_VALID) == 0 ||
266 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
267 return H_NOT_FOUND;
268 }
269
270 r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
271 HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
272 r |= (flags << 55) & HPTE64_R_PP0;
273 r |= (flags << 48) & HPTE64_R_KEY_HI;
274 r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
275 spapr_store_hpte(cpu, ptex,
276 (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
277 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
278
279 check_tlb_flush(env, true);
280
281 spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
282 return H_SUCCESS;
283}
284
285static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
286 target_ulong opcode, target_ulong *args)
287{
288 target_ulong flags = args[0];
289 target_ulong ptex = args[1];
290 int i, ridx, n_entries = 1;
291 const ppc_hash_pte64_t *hptes;
292
293 if (!valid_ptex(cpu, ptex)) {
294 return H_PARAMETER;
295 }
296
297 if (flags & H_READ_4) {
298
299 ptex &= ~(3ULL);
300 n_entries = 4;
301 }
302
303 hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
304 for (i = 0, ridx = 0; i < n_entries; i++) {
305 args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
306 args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
307 }
308 ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
309
310 return H_SUCCESS;
311}
312
313struct SpaprPendingHpt {
314
315 int shift;
316 QemuThread thread;
317
318
319 bool complete;
320
321
322
323 int ret;
324 void *hpt;
325};
326
327static void free_pending_hpt(SpaprPendingHpt *pending)
328{
329 if (pending->hpt) {
330 qemu_vfree(pending->hpt);
331 }
332
333 g_free(pending);
334}
335
336static void *hpt_prepare_thread(void *opaque)
337{
338 SpaprPendingHpt *pending = opaque;
339 size_t size = 1ULL << pending->shift;
340
341 pending->hpt = qemu_try_memalign(size, size);
342 if (pending->hpt) {
343 memset(pending->hpt, 0, size);
344 pending->ret = H_SUCCESS;
345 } else {
346 pending->ret = H_NO_MEM;
347 }
348
349 qemu_mutex_lock_iothread();
350
351 if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
352
353 pending->complete = true;
354 } else {
355
356 free_pending_hpt(pending);
357 }
358
359 qemu_mutex_unlock_iothread();
360 return NULL;
361}
362
363
364static void cancel_hpt_prepare(SpaprMachineState *spapr)
365{
366 SpaprPendingHpt *pending = spapr->pending_hpt;
367
368
369 spapr->pending_hpt = NULL;
370
371 if (!pending) {
372
373 return;
374 }
375
376 if (!pending->complete) {
377
378 return;
379 }
380
381 free_pending_hpt(pending);
382}
383
384target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
385 SpaprMachineState *spapr,
386 target_ulong shift)
387{
388 SpaprPendingHpt *pending = spapr->pending_hpt;
389
390 if (pending) {
391
392 if (pending->shift == shift) {
393
394 if (pending->complete) {
395 return pending->ret;
396 } else {
397 return H_LONG_BUSY_ORDER_100_MSEC;
398 }
399 }
400
401
402 cancel_hpt_prepare(spapr);
403 }
404
405 if (!shift) {
406
407 return H_SUCCESS;
408 }
409
410
411
412 pending = g_new0(SpaprPendingHpt, 1);
413 pending->shift = shift;
414 pending->ret = H_HARDWARE;
415
416 qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
417 hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
418
419 spapr->pending_hpt = pending;
420
421
422
423 return H_LONG_BUSY_ORDER_100_MSEC;
424}
425
426static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
427{
428 uint8_t *addr = htab;
429
430 addr += pteg * HASH_PTEG_SIZE_64;
431 addr += slot * HASH_PTE_SIZE_64;
432 return ldq_p(addr);
433}
434
435static void new_hpte_store(void *htab, uint64_t pteg, int slot,
436 uint64_t pte0, uint64_t pte1)
437{
438 uint8_t *addr = htab;
439
440 addr += pteg * HASH_PTEG_SIZE_64;
441 addr += slot * HASH_PTE_SIZE_64;
442
443 stq_p(addr, pte0);
444 stq_p(addr + HASH_PTE_SIZE_64 / 2, pte1);
445}
446
447static int rehash_hpte(PowerPCCPU *cpu,
448 const ppc_hash_pte64_t *hptes,
449 void *old_hpt, uint64_t oldsize,
450 void *new_hpt, uint64_t newsize,
451 uint64_t pteg, int slot)
452{
453 uint64_t old_hash_mask = (oldsize >> 7) - 1;
454 uint64_t new_hash_mask = (newsize >> 7) - 1;
455 target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
456 target_ulong pte1;
457 uint64_t avpn;
458 unsigned base_pg_shift;
459 uint64_t hash, new_pteg, replace_pte0;
460
461 if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
462 return H_SUCCESS;
463 }
464
465 pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
466
467 base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
468 assert(base_pg_shift);
469 avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
470
471 if (pte0 & HPTE64_V_SECONDARY) {
472 pteg = ~pteg;
473 }
474
475 if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
476 uint64_t offset, vsid;
477
478
479 offset = (avpn & 0x1f) << 23;
480 vsid = avpn >> 5;
481
482 if (base_pg_shift < 23) {
483 offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
484 }
485
486 hash = vsid ^ (offset >> base_pg_shift);
487 } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
488 uint64_t offset, vsid;
489
490
491 offset = (avpn & 0x1ffff) << 23;
492 vsid = avpn >> 17;
493 if (base_pg_shift < 23) {
494 offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
495 << base_pg_shift;
496 }
497
498 hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
499 } else {
500 error_report("rehash_pte: Bad segment size in HPTE");
501 return H_HARDWARE;
502 }
503
504 new_pteg = hash & new_hash_mask;
505 if (pte0 & HPTE64_V_SECONDARY) {
506 assert(~pteg == (hash & old_hash_mask));
507 new_pteg = ~new_pteg;
508 } else {
509 assert(pteg == (hash & old_hash_mask));
510 }
511 assert((oldsize != newsize) || (pteg == new_pteg));
512 replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
513
514
515
516
517
518
519 if (replace_pte0 & HPTE64_V_VALID) {
520 assert(newsize < oldsize);
521 if (replace_pte0 & HPTE64_V_BOLTED) {
522 if (pte0 & HPTE64_V_BOLTED) {
523
524 return H_PTEG_FULL;
525 } else {
526
527 return H_SUCCESS;
528 }
529 }
530 }
531
532 new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
533 return H_SUCCESS;
534}
535
536static int rehash_hpt(PowerPCCPU *cpu,
537 void *old_hpt, uint64_t oldsize,
538 void *new_hpt, uint64_t newsize)
539{
540 uint64_t n_ptegs = oldsize >> 7;
541 uint64_t pteg;
542 int slot;
543 int rc;
544
545 for (pteg = 0; pteg < n_ptegs; pteg++) {
546 hwaddr ptex = pteg * HPTES_PER_GROUP;
547 const ppc_hash_pte64_t *hptes
548 = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
549
550 if (!hptes) {
551 return H_HARDWARE;
552 }
553
554 for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
555 rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
556 pteg, slot);
557 if (rc != H_SUCCESS) {
558 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
559 return rc;
560 }
561 }
562 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
563 }
564
565 return H_SUCCESS;
566}
567
568target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
569 SpaprMachineState *spapr,
570 target_ulong flags,
571 target_ulong shift)
572{
573 SpaprPendingHpt *pending = spapr->pending_hpt;
574 int rc;
575 size_t newsize;
576
577 if (flags != 0) {
578 return H_PARAMETER;
579 }
580
581 if (!pending || (pending->shift != shift)) {
582
583 return H_CLOSED;
584 }
585
586 if (!pending->complete) {
587
588 return H_BUSY;
589 }
590
591
592 g_assert(spapr->htab_shift);
593
594 newsize = 1ULL << pending->shift;
595 rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
596 pending->hpt, newsize);
597 if (rc == H_SUCCESS) {
598 qemu_vfree(spapr->htab);
599 spapr->htab = pending->hpt;
600 spapr->htab_shift = pending->shift;
601
602 push_sregs_to_kvm_pr(spapr);
603
604 pending->hpt = NULL;
605 }
606
607
608 spapr->pending_hpt = NULL;
609 free_pending_hpt(pending);
610
611 return rc;
612}
613
614static void hypercall_register_types(void)
615{
616
617 spapr_register_hypercall(H_ENTER, h_enter);
618 spapr_register_hypercall(H_REMOVE, h_remove);
619 spapr_register_hypercall(H_PROTECT, h_protect);
620 spapr_register_hypercall(H_READ, h_read);
621
622
623 spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
624
625}
626
627type_init(hypercall_register_types)
628