1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include "qemu/osdep.h"
20#include "qemu/bitops.h"
21#include "hw/irq.h"
22#include "hw/sysbus.h"
23#include "migration/vmstate.h"
24#include "hw/qdev-core.h"
25#include "hw/pci/pci.h"
26#include "cpu.h"
27#include "trace.h"
28#include "qemu/log.h"
29#include "qemu/error-report.h"
30#include "qapi/error.h"
31
32#include "hw/arm/smmuv3.h"
33#include "smmuv3-internal.h"
34#include "smmu-internal.h"
35
36
37
38
39
40
41
42
43static void smmuv3_trigger_irq(SMMUv3State *s, SMMUIrq irq,
44 uint32_t gerror_mask)
45{
46
47 bool pulse = false;
48
49 switch (irq) {
50 case SMMU_IRQ_EVTQ:
51 pulse = smmuv3_eventq_irq_enabled(s);
52 break;
53 case SMMU_IRQ_PRIQ:
54 qemu_log_mask(LOG_UNIMP, "PRI not yet supported\n");
55 break;
56 case SMMU_IRQ_CMD_SYNC:
57 pulse = true;
58 break;
59 case SMMU_IRQ_GERROR:
60 {
61 uint32_t pending = s->gerror ^ s->gerrorn;
62 uint32_t new_gerrors = ~pending & gerror_mask;
63
64 if (!new_gerrors) {
65
66 return;
67 }
68 s->gerror ^= new_gerrors;
69 trace_smmuv3_write_gerror(new_gerrors, s->gerror);
70
71 pulse = smmuv3_gerror_irq_enabled(s);
72 break;
73 }
74 }
75 if (pulse) {
76 trace_smmuv3_trigger_irq(irq);
77 qemu_irq_pulse(s->irq[irq]);
78 }
79}
80
81static void smmuv3_write_gerrorn(SMMUv3State *s, uint32_t new_gerrorn)
82{
83 uint32_t pending = s->gerror ^ s->gerrorn;
84 uint32_t toggled = s->gerrorn ^ new_gerrorn;
85
86 if (toggled & ~pending) {
87 qemu_log_mask(LOG_GUEST_ERROR,
88 "guest toggles non pending errors = 0x%x\n",
89 toggled & ~pending);
90 }
91
92
93
94
95
96 s->gerrorn = new_gerrorn;
97
98 trace_smmuv3_write_gerrorn(toggled & pending, s->gerrorn);
99}
100
101static inline MemTxResult queue_read(SMMUQueue *q, void *data)
102{
103 dma_addr_t addr = Q_CONS_ENTRY(q);
104
105 return dma_memory_read(&address_space_memory, addr, data, q->entry_size,
106 MEMTXATTRS_UNSPECIFIED);
107}
108
109static MemTxResult queue_write(SMMUQueue *q, void *data)
110{
111 dma_addr_t addr = Q_PROD_ENTRY(q);
112 MemTxResult ret;
113
114 ret = dma_memory_write(&address_space_memory, addr, data, q->entry_size,
115 MEMTXATTRS_UNSPECIFIED);
116 if (ret != MEMTX_OK) {
117 return ret;
118 }
119
120 queue_prod_incr(q);
121 return MEMTX_OK;
122}
123
124static MemTxResult smmuv3_write_eventq(SMMUv3State *s, Evt *evt)
125{
126 SMMUQueue *q = &s->eventq;
127 MemTxResult r;
128
129 if (!smmuv3_eventq_enabled(s)) {
130 return MEMTX_ERROR;
131 }
132
133 if (smmuv3_q_full(q)) {
134 return MEMTX_ERROR;
135 }
136
137 r = queue_write(q, evt);
138 if (r != MEMTX_OK) {
139 return r;
140 }
141
142 if (!smmuv3_q_empty(q)) {
143 smmuv3_trigger_irq(s, SMMU_IRQ_EVTQ, 0);
144 }
145 return MEMTX_OK;
146}
147
148void smmuv3_record_event(SMMUv3State *s, SMMUEventInfo *info)
149{
150 Evt evt = {};
151 MemTxResult r;
152
153 if (!smmuv3_eventq_enabled(s)) {
154 return;
155 }
156
157 EVT_SET_TYPE(&evt, info->type);
158 EVT_SET_SID(&evt, info->sid);
159
160 switch (info->type) {
161 case SMMU_EVT_NONE:
162 return;
163 case SMMU_EVT_F_UUT:
164 EVT_SET_SSID(&evt, info->u.f_uut.ssid);
165 EVT_SET_SSV(&evt, info->u.f_uut.ssv);
166 EVT_SET_ADDR(&evt, info->u.f_uut.addr);
167 EVT_SET_RNW(&evt, info->u.f_uut.rnw);
168 EVT_SET_PNU(&evt, info->u.f_uut.pnu);
169 EVT_SET_IND(&evt, info->u.f_uut.ind);
170 break;
171 case SMMU_EVT_C_BAD_STREAMID:
172 EVT_SET_SSID(&evt, info->u.c_bad_streamid.ssid);
173 EVT_SET_SSV(&evt, info->u.c_bad_streamid.ssv);
174 break;
175 case SMMU_EVT_F_STE_FETCH:
176 EVT_SET_SSID(&evt, info->u.f_ste_fetch.ssid);
177 EVT_SET_SSV(&evt, info->u.f_ste_fetch.ssv);
178 EVT_SET_ADDR2(&evt, info->u.f_ste_fetch.addr);
179 break;
180 case SMMU_EVT_C_BAD_STE:
181 EVT_SET_SSID(&evt, info->u.c_bad_ste.ssid);
182 EVT_SET_SSV(&evt, info->u.c_bad_ste.ssv);
183 break;
184 case SMMU_EVT_F_STREAM_DISABLED:
185 break;
186 case SMMU_EVT_F_TRANS_FORBIDDEN:
187 EVT_SET_ADDR(&evt, info->u.f_transl_forbidden.addr);
188 EVT_SET_RNW(&evt, info->u.f_transl_forbidden.rnw);
189 break;
190 case SMMU_EVT_C_BAD_SUBSTREAMID:
191 EVT_SET_SSID(&evt, info->u.c_bad_substream.ssid);
192 break;
193 case SMMU_EVT_F_CD_FETCH:
194 EVT_SET_SSID(&evt, info->u.f_cd_fetch.ssid);
195 EVT_SET_SSV(&evt, info->u.f_cd_fetch.ssv);
196 EVT_SET_ADDR(&evt, info->u.f_cd_fetch.addr);
197 break;
198 case SMMU_EVT_C_BAD_CD:
199 EVT_SET_SSID(&evt, info->u.c_bad_cd.ssid);
200 EVT_SET_SSV(&evt, info->u.c_bad_cd.ssv);
201 break;
202 case SMMU_EVT_F_WALK_EABT:
203 case SMMU_EVT_F_TRANSLATION:
204 case SMMU_EVT_F_ADDR_SIZE:
205 case SMMU_EVT_F_ACCESS:
206 case SMMU_EVT_F_PERMISSION:
207 EVT_SET_STALL(&evt, info->u.f_walk_eabt.stall);
208 EVT_SET_STAG(&evt, info->u.f_walk_eabt.stag);
209 EVT_SET_SSID(&evt, info->u.f_walk_eabt.ssid);
210 EVT_SET_SSV(&evt, info->u.f_walk_eabt.ssv);
211 EVT_SET_S2(&evt, info->u.f_walk_eabt.s2);
212 EVT_SET_ADDR(&evt, info->u.f_walk_eabt.addr);
213 EVT_SET_RNW(&evt, info->u.f_walk_eabt.rnw);
214 EVT_SET_PNU(&evt, info->u.f_walk_eabt.pnu);
215 EVT_SET_IND(&evt, info->u.f_walk_eabt.ind);
216 EVT_SET_CLASS(&evt, info->u.f_walk_eabt.class);
217 EVT_SET_ADDR2(&evt, info->u.f_walk_eabt.addr2);
218 break;
219 case SMMU_EVT_F_CFG_CONFLICT:
220 EVT_SET_SSID(&evt, info->u.f_cfg_conflict.ssid);
221 EVT_SET_SSV(&evt, info->u.f_cfg_conflict.ssv);
222 break;
223
224 case SMMU_EVT_F_BAD_ATS_TREQ:
225 case SMMU_EVT_F_TLB_CONFLICT:
226 case SMMU_EVT_E_PAGE_REQ:
227 default:
228 g_assert_not_reached();
229 }
230
231 trace_smmuv3_record_event(smmu_event_string(info->type), info->sid);
232 r = smmuv3_write_eventq(s, &evt);
233 if (r != MEMTX_OK) {
234 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_EVENTQ_ABT_ERR_MASK);
235 }
236 info->recorded = true;
237}
238
239static void smmuv3_init_regs(SMMUv3State *s)
240{
241
242
243
244
245 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1);
246 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTF, 2);
247 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, COHACC, 1);
248 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ASID16, 1);
249 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTENDIAN, 2);
250 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STALL_MODEL, 1);
251
252 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TERM_MODEL, 1);
253
254 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STLEVEL, 1);
255
256 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SIDSIZE, SMMU_IDR1_SIDSIZE);
257 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, EVENTQS, SMMU_EVENTQS);
258 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, CMDQS, SMMU_CMDQS);
259
260 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1);
261 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1);
262
263
264 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1);
265 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN16K, 1);
266 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1);
267 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS);
268
269 s->cmdq.base = deposit64(s->cmdq.base, 0, 5, SMMU_CMDQS);
270 s->cmdq.prod = 0;
271 s->cmdq.cons = 0;
272 s->cmdq.entry_size = sizeof(struct Cmd);
273 s->eventq.base = deposit64(s->eventq.base, 0, 5, SMMU_EVENTQS);
274 s->eventq.prod = 0;
275 s->eventq.cons = 0;
276 s->eventq.entry_size = sizeof(struct Evt);
277
278 s->features = 0;
279 s->sid_split = 0;
280 s->aidr = 0x1;
281 s->cr[0] = 0;
282 s->cr0ack = 0;
283 s->irq_ctrl = 0;
284 s->gerror = 0;
285 s->gerrorn = 0;
286 s->statusr = 0;
287}
288
289static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
290 SMMUEventInfo *event)
291{
292 int ret;
293
294 trace_smmuv3_get_ste(addr);
295
296 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
297 MEMTXATTRS_UNSPECIFIED);
298 if (ret != MEMTX_OK) {
299 qemu_log_mask(LOG_GUEST_ERROR,
300 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
301 event->type = SMMU_EVT_F_STE_FETCH;
302 event->u.f_ste_fetch.addr = addr;
303 return -EINVAL;
304 }
305 return 0;
306
307}
308
309
310static int smmu_get_cd(SMMUv3State *s, STE *ste, uint32_t ssid,
311 CD *buf, SMMUEventInfo *event)
312{
313 dma_addr_t addr = STE_CTXPTR(ste);
314 int ret;
315
316 trace_smmuv3_get_cd(addr);
317
318 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
319 MEMTXATTRS_UNSPECIFIED);
320 if (ret != MEMTX_OK) {
321 qemu_log_mask(LOG_GUEST_ERROR,
322 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
323 event->type = SMMU_EVT_F_CD_FETCH;
324 event->u.f_ste_fetch.addr = addr;
325 return -EINVAL;
326 }
327 return 0;
328}
329
330
331static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg,
332 STE *ste, SMMUEventInfo *event)
333{
334 uint32_t config;
335
336 if (!STE_VALID(ste)) {
337 if (!event->inval_ste_allowed) {
338 qemu_log_mask(LOG_GUEST_ERROR, "invalid STE\n");
339 }
340 goto bad_ste;
341 }
342
343 config = STE_CONFIG(ste);
344
345 if (STE_CFG_ABORT(config)) {
346 cfg->aborted = true;
347 return 0;
348 }
349
350 if (STE_CFG_BYPASS(config)) {
351 cfg->bypassed = true;
352 return 0;
353 }
354
355 if (STE_CFG_S2_ENABLED(config)) {
356 qemu_log_mask(LOG_UNIMP, "SMMUv3 does not support stage 2 yet\n");
357 goto bad_ste;
358 }
359
360 if (STE_S1CDMAX(ste) != 0) {
361 qemu_log_mask(LOG_UNIMP,
362 "SMMUv3 does not support multiple context descriptors yet\n");
363 goto bad_ste;
364 }
365
366 if (STE_S1STALLD(ste)) {
367 qemu_log_mask(LOG_UNIMP,
368 "SMMUv3 S1 stalling fault model not allowed yet\n");
369 goto bad_ste;
370 }
371 return 0;
372
373bad_ste:
374 event->type = SMMU_EVT_C_BAD_STE;
375 return -EINVAL;
376}
377
378
379
380
381
382
383
384
385
386
387
388
389
390static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste,
391 SMMUEventInfo *event)
392{
393 dma_addr_t addr, strtab_base;
394 uint32_t log2size;
395 int strtab_size_shift;
396 int ret;
397
398 trace_smmuv3_find_ste(sid, s->features, s->sid_split);
399 log2size = FIELD_EX32(s->strtab_base_cfg, STRTAB_BASE_CFG, LOG2SIZE);
400
401
402
403 if (sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE))) {
404 event->type = SMMU_EVT_C_BAD_STREAMID;
405 return -EINVAL;
406 }
407 if (s->features & SMMU_FEATURE_2LVL_STE) {
408 int l1_ste_offset, l2_ste_offset, max_l2_ste, span;
409 dma_addr_t l1ptr, l2ptr;
410 STEDesc l1std;
411
412
413
414
415
416 strtab_size_shift = MAX(5, (int)log2size - s->sid_split - 1 + 3);
417 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
418 ~MAKE_64BIT_MASK(0, strtab_size_shift);
419 l1_ste_offset = sid >> s->sid_split;
420 l2_ste_offset = sid & ((1 << s->sid_split) - 1);
421 l1ptr = (dma_addr_t)(strtab_base + l1_ste_offset * sizeof(l1std));
422
423 ret = dma_memory_read(&address_space_memory, l1ptr, &l1std,
424 sizeof(l1std), MEMTXATTRS_UNSPECIFIED);
425 if (ret != MEMTX_OK) {
426 qemu_log_mask(LOG_GUEST_ERROR,
427 "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr);
428 event->type = SMMU_EVT_F_STE_FETCH;
429 event->u.f_ste_fetch.addr = l1ptr;
430 return -EINVAL;
431 }
432
433 span = L1STD_SPAN(&l1std);
434
435 if (!span) {
436
437 if (!event->inval_ste_allowed) {
438 qemu_log_mask(LOG_GUEST_ERROR,
439 "invalid sid=%d (L1STD span=0)\n", sid);
440 }
441 event->type = SMMU_EVT_C_BAD_STREAMID;
442 return -EINVAL;
443 }
444 max_l2_ste = (1 << span) - 1;
445 l2ptr = l1std_l2ptr(&l1std);
446 trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset,
447 l2ptr, l2_ste_offset, max_l2_ste);
448 if (l2_ste_offset > max_l2_ste) {
449 qemu_log_mask(LOG_GUEST_ERROR,
450 "l2_ste_offset=%d > max_l2_ste=%d\n",
451 l2_ste_offset, max_l2_ste);
452 event->type = SMMU_EVT_C_BAD_STE;
453 return -EINVAL;
454 }
455 addr = l2ptr + l2_ste_offset * sizeof(*ste);
456 } else {
457 strtab_size_shift = log2size + 5;
458 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
459 ~MAKE_64BIT_MASK(0, strtab_size_shift);
460 addr = strtab_base + sid * sizeof(*ste);
461 }
462
463 if (smmu_get_ste(s, addr, ste, event)) {
464 return -EINVAL;
465 }
466
467 return 0;
468}
469
470static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event)
471{
472 int ret = -EINVAL;
473 int i;
474
475 if (!CD_VALID(cd) || !CD_AARCH64(cd)) {
476 goto bad_cd;
477 }
478 if (!CD_A(cd)) {
479 goto bad_cd;
480 }
481 if (CD_S(cd)) {
482 goto bad_cd;
483 }
484 if (CD_HA(cd) || CD_HD(cd)) {
485 goto bad_cd;
486 }
487
488
489 cfg->aa64 = true;
490 cfg->stage = 1;
491
492 cfg->oas = oas2bits(CD_IPS(cd));
493 cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas);
494 cfg->tbi = CD_TBI(cd);
495 cfg->asid = CD_ASID(cd);
496
497 trace_smmuv3_decode_cd(cfg->oas);
498
499
500 for (i = 0; i <= 1; i++) {
501 int tg, tsz;
502 SMMUTransTableInfo *tt = &cfg->tt[i];
503
504 cfg->tt[i].disabled = CD_EPD(cd, i);
505 if (cfg->tt[i].disabled) {
506 continue;
507 }
508
509 tsz = CD_TSZ(cd, i);
510 if (tsz < 16 || tsz > 39) {
511 goto bad_cd;
512 }
513
514 tg = CD_TG(cd, i);
515 tt->granule_sz = tg2granule(tg, i);
516 if ((tt->granule_sz != 12 && tt->granule_sz != 14 &&
517 tt->granule_sz != 16) || CD_ENDI(cd)) {
518 goto bad_cd;
519 }
520
521 tt->tsz = tsz;
522 tt->ttb = CD_TTB(cd, i);
523 if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) {
524 goto bad_cd;
525 }
526 tt->had = CD_HAD(cd, i);
527 trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had);
528 }
529
530 event->record_trans_faults = CD_R(cd);
531
532 return 0;
533
534bad_cd:
535 event->type = SMMU_EVT_C_BAD_CD;
536 return ret;
537}
538
539
540
541
542
543
544
545
546
547
548
549
550static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg,
551 SMMUEventInfo *event)
552{
553 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
554 uint32_t sid = smmu_get_sid(sdev);
555 SMMUv3State *s = sdev->smmu;
556 int ret;
557 STE ste;
558 CD cd;
559
560 ret = smmu_find_ste(s, sid, &ste, event);
561 if (ret) {
562 return ret;
563 }
564
565 ret = decode_ste(s, cfg, &ste, event);
566 if (ret) {
567 return ret;
568 }
569
570 if (cfg->aborted || cfg->bypassed) {
571 return 0;
572 }
573
574 ret = smmu_get_cd(s, &ste, 0 , &cd, event);
575 if (ret) {
576 return ret;
577 }
578
579 return decode_cd(cfg, &cd, event);
580}
581
582
583
584
585
586
587
588
589
590
591
592
593
594static SMMUTransCfg *smmuv3_get_config(SMMUDevice *sdev, SMMUEventInfo *event)
595{
596 SMMUv3State *s = sdev->smmu;
597 SMMUState *bc = &s->smmu_state;
598 SMMUTransCfg *cfg;
599
600 cfg = g_hash_table_lookup(bc->configs, sdev);
601 if (cfg) {
602 sdev->cfg_cache_hits++;
603 trace_smmuv3_config_cache_hit(smmu_get_sid(sdev),
604 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
605 100 * sdev->cfg_cache_hits /
606 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
607 } else {
608 sdev->cfg_cache_misses++;
609 trace_smmuv3_config_cache_miss(smmu_get_sid(sdev),
610 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
611 100 * sdev->cfg_cache_hits /
612 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
613 cfg = g_new0(SMMUTransCfg, 1);
614
615 if (!smmuv3_decode_config(&sdev->iommu, cfg, event)) {
616 g_hash_table_insert(bc->configs, sdev, cfg);
617 } else {
618 g_free(cfg);
619 cfg = NULL;
620 }
621 }
622 return cfg;
623}
624
625static void smmuv3_flush_config(SMMUDevice *sdev)
626{
627 SMMUv3State *s = sdev->smmu;
628 SMMUState *bc = &s->smmu_state;
629
630 trace_smmuv3_config_cache_inv(smmu_get_sid(sdev));
631 g_hash_table_remove(bc->configs, sdev);
632}
633
634static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
635 IOMMUAccessFlags flag, int iommu_idx)
636{
637 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
638 SMMUv3State *s = sdev->smmu;
639 uint32_t sid = smmu_get_sid(sdev);
640 SMMUEventInfo event = {.type = SMMU_EVT_NONE,
641 .sid = sid,
642 .inval_ste_allowed = false};
643 SMMUPTWEventInfo ptw_info = {};
644 SMMUTranslationStatus status;
645 SMMUState *bs = ARM_SMMU(s);
646 uint64_t page_mask, aligned_addr;
647 SMMUTLBEntry *cached_entry = NULL;
648 SMMUTransTableInfo *tt;
649 SMMUTransCfg *cfg = NULL;
650 IOMMUTLBEntry entry = {
651 .target_as = &address_space_memory,
652 .iova = addr,
653 .translated_addr = addr,
654 .addr_mask = ~(hwaddr)0,
655 .perm = IOMMU_NONE,
656 };
657
658 qemu_mutex_lock(&s->mutex);
659
660 if (!smmu_enabled(s)) {
661 status = SMMU_TRANS_DISABLE;
662 goto epilogue;
663 }
664
665 cfg = smmuv3_get_config(sdev, &event);
666 if (!cfg) {
667 status = SMMU_TRANS_ERROR;
668 goto epilogue;
669 }
670
671 if (cfg->aborted) {
672 status = SMMU_TRANS_ABORT;
673 goto epilogue;
674 }
675
676 if (cfg->bypassed) {
677 status = SMMU_TRANS_BYPASS;
678 goto epilogue;
679 }
680
681 tt = select_tt(cfg, addr);
682 if (!tt) {
683 if (event.record_trans_faults) {
684 event.type = SMMU_EVT_F_TRANSLATION;
685 event.u.f_translation.addr = addr;
686 event.u.f_translation.rnw = flag & 0x1;
687 }
688 status = SMMU_TRANS_ERROR;
689 goto epilogue;
690 }
691
692 page_mask = (1ULL << (tt->granule_sz)) - 1;
693 aligned_addr = addr & ~page_mask;
694
695 cached_entry = smmu_iotlb_lookup(bs, cfg, tt, aligned_addr);
696 if (cached_entry) {
697 if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) {
698 status = SMMU_TRANS_ERROR;
699 if (event.record_trans_faults) {
700 event.type = SMMU_EVT_F_PERMISSION;
701 event.u.f_permission.addr = addr;
702 event.u.f_permission.rnw = flag & 0x1;
703 }
704 } else {
705 status = SMMU_TRANS_SUCCESS;
706 }
707 goto epilogue;
708 }
709
710 cached_entry = g_new0(SMMUTLBEntry, 1);
711
712 if (smmu_ptw(cfg, aligned_addr, flag, cached_entry, &ptw_info)) {
713 g_free(cached_entry);
714 switch (ptw_info.type) {
715 case SMMU_PTW_ERR_WALK_EABT:
716 event.type = SMMU_EVT_F_WALK_EABT;
717 event.u.f_walk_eabt.addr = addr;
718 event.u.f_walk_eabt.rnw = flag & 0x1;
719 event.u.f_walk_eabt.class = 0x1;
720 event.u.f_walk_eabt.addr2 = ptw_info.addr;
721 break;
722 case SMMU_PTW_ERR_TRANSLATION:
723 if (event.record_trans_faults) {
724 event.type = SMMU_EVT_F_TRANSLATION;
725 event.u.f_translation.addr = addr;
726 event.u.f_translation.rnw = flag & 0x1;
727 }
728 break;
729 case SMMU_PTW_ERR_ADDR_SIZE:
730 if (event.record_trans_faults) {
731 event.type = SMMU_EVT_F_ADDR_SIZE;
732 event.u.f_addr_size.addr = addr;
733 event.u.f_addr_size.rnw = flag & 0x1;
734 }
735 break;
736 case SMMU_PTW_ERR_ACCESS:
737 if (event.record_trans_faults) {
738 event.type = SMMU_EVT_F_ACCESS;
739 event.u.f_access.addr = addr;
740 event.u.f_access.rnw = flag & 0x1;
741 }
742 break;
743 case SMMU_PTW_ERR_PERMISSION:
744 if (event.record_trans_faults) {
745 event.type = SMMU_EVT_F_PERMISSION;
746 event.u.f_permission.addr = addr;
747 event.u.f_permission.rnw = flag & 0x1;
748 }
749 break;
750 default:
751 g_assert_not_reached();
752 }
753 status = SMMU_TRANS_ERROR;
754 } else {
755 smmu_iotlb_insert(bs, cfg, cached_entry);
756 status = SMMU_TRANS_SUCCESS;
757 }
758
759epilogue:
760 qemu_mutex_unlock(&s->mutex);
761 switch (status) {
762 case SMMU_TRANS_SUCCESS:
763 entry.perm = flag;
764 entry.translated_addr = cached_entry->entry.translated_addr +
765 (addr & cached_entry->entry.addr_mask);
766 entry.addr_mask = cached_entry->entry.addr_mask;
767 trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
768 entry.translated_addr, entry.perm);
769 break;
770 case SMMU_TRANS_DISABLE:
771 entry.perm = flag;
772 entry.addr_mask = ~TARGET_PAGE_MASK;
773 trace_smmuv3_translate_disable(mr->parent_obj.name, sid, addr,
774 entry.perm);
775 break;
776 case SMMU_TRANS_BYPASS:
777 entry.perm = flag;
778 entry.addr_mask = ~TARGET_PAGE_MASK;
779 trace_smmuv3_translate_bypass(mr->parent_obj.name, sid, addr,
780 entry.perm);
781 break;
782 case SMMU_TRANS_ABORT:
783
784 trace_smmuv3_translate_abort(mr->parent_obj.name, sid, addr,
785 entry.perm);
786 break;
787 case SMMU_TRANS_ERROR:
788 qemu_log_mask(LOG_GUEST_ERROR,
789 "%s translation failed for iova=0x%"PRIx64"(%s)\n",
790 mr->parent_obj.name, addr, smmu_event_string(event.type));
791 smmuv3_record_event(s, &event);
792 break;
793 }
794
795 return entry;
796}
797
798
799
800
801
802
803
804
805
806
807
808
809static void smmuv3_notify_iova(IOMMUMemoryRegion *mr,
810 IOMMUNotifier *n,
811 int asid, dma_addr_t iova,
812 uint8_t tg, uint64_t num_pages)
813{
814 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
815 IOMMUTLBEvent event;
816 uint8_t granule;
817
818 if (!tg) {
819 SMMUEventInfo event = {.inval_ste_allowed = true};
820 SMMUTransCfg *cfg = smmuv3_get_config(sdev, &event);
821 SMMUTransTableInfo *tt;
822
823 if (!cfg) {
824 return;
825 }
826
827 if (asid >= 0 && cfg->asid != asid) {
828 return;
829 }
830
831 tt = select_tt(cfg, iova);
832 if (!tt) {
833 return;
834 }
835 granule = tt->granule_sz;
836 } else {
837 granule = tg * 2 + 10;
838 }
839
840 event.type = IOMMU_NOTIFIER_UNMAP;
841 event.entry.target_as = &address_space_memory;
842 event.entry.iova = iova;
843 event.entry.addr_mask = num_pages * (1 << granule) - 1;
844 event.entry.perm = IOMMU_NONE;
845
846 memory_region_notify_iommu_one(n, &event);
847}
848
849
850static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, dma_addr_t iova,
851 uint8_t tg, uint64_t num_pages)
852{
853 SMMUDevice *sdev;
854
855 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
856 IOMMUMemoryRegion *mr = &sdev->iommu;
857 IOMMUNotifier *n;
858
859 trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, iova,
860 tg, num_pages);
861
862 IOMMU_NOTIFIER_FOREACH(n, mr) {
863 smmuv3_notify_iova(mr, n, asid, iova, tg, num_pages);
864 }
865 }
866}
867
868static void smmuv3_s1_range_inval(SMMUState *s, Cmd *cmd)
869{
870 dma_addr_t end, addr = CMD_ADDR(cmd);
871 uint8_t type = CMD_TYPE(cmd);
872 uint16_t vmid = CMD_VMID(cmd);
873 uint8_t scale = CMD_SCALE(cmd);
874 uint8_t num = CMD_NUM(cmd);
875 uint8_t ttl = CMD_TTL(cmd);
876 bool leaf = CMD_LEAF(cmd);
877 uint8_t tg = CMD_TG(cmd);
878 uint64_t num_pages;
879 uint8_t granule;
880 int asid = -1;
881
882 if (type == SMMU_CMD_TLBI_NH_VA) {
883 asid = CMD_ASID(cmd);
884 }
885
886 if (!tg) {
887 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, 1, ttl, leaf);
888 smmuv3_inv_notifiers_iova(s, asid, addr, tg, 1);
889 smmu_iotlb_inv_iova(s, asid, addr, tg, 1, ttl);
890 return;
891 }
892
893
894
895 num_pages = (num + 1) * BIT_ULL(scale);
896 granule = tg * 2 + 10;
897
898
899 end = addr + (num_pages << granule) - 1;
900
901 while (addr != end + 1) {
902 uint64_t mask = dma_aligned_pow2_mask(addr, end, 64);
903
904 num_pages = (mask + 1) >> granule;
905 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf);
906 smmuv3_inv_notifiers_iova(s, asid, addr, tg, num_pages);
907 smmu_iotlb_inv_iova(s, asid, addr, tg, num_pages, ttl);
908 addr += mask + 1;
909 }
910}
911
912static gboolean
913smmuv3_invalidate_ste(gpointer key, gpointer value, gpointer user_data)
914{
915 SMMUDevice *sdev = (SMMUDevice *)key;
916 uint32_t sid = smmu_get_sid(sdev);
917 SMMUSIDRange *sid_range = (SMMUSIDRange *)user_data;
918
919 if (sid < sid_range->start || sid > sid_range->end) {
920 return false;
921 }
922 trace_smmuv3_config_cache_inv(sid);
923 return true;
924}
925
926static int smmuv3_cmdq_consume(SMMUv3State *s)
927{
928 SMMUState *bs = ARM_SMMU(s);
929 SMMUCmdError cmd_error = SMMU_CERROR_NONE;
930 SMMUQueue *q = &s->cmdq;
931 SMMUCommandType type = 0;
932
933 if (!smmuv3_cmdq_enabled(s)) {
934 return 0;
935 }
936
937
938
939
940
941
942
943 while (!smmuv3_q_empty(q)) {
944 uint32_t pending = s->gerror ^ s->gerrorn;
945 Cmd cmd;
946
947 trace_smmuv3_cmdq_consume(Q_PROD(q), Q_CONS(q),
948 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
949
950 if (FIELD_EX32(pending, GERROR, CMDQ_ERR)) {
951 break;
952 }
953
954 if (queue_read(q, &cmd) != MEMTX_OK) {
955 cmd_error = SMMU_CERROR_ABT;
956 break;
957 }
958
959 type = CMD_TYPE(&cmd);
960
961 trace_smmuv3_cmdq_opcode(smmu_cmd_string(type));
962
963 qemu_mutex_lock(&s->mutex);
964 switch (type) {
965 case SMMU_CMD_SYNC:
966 if (CMD_SYNC_CS(&cmd) & CMD_SYNC_SIG_IRQ) {
967 smmuv3_trigger_irq(s, SMMU_IRQ_CMD_SYNC, 0);
968 }
969 break;
970 case SMMU_CMD_PREFETCH_CONFIG:
971 case SMMU_CMD_PREFETCH_ADDR:
972 break;
973 case SMMU_CMD_CFGI_STE:
974 {
975 uint32_t sid = CMD_SID(&cmd);
976 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
977 SMMUDevice *sdev;
978
979 if (CMD_SSEC(&cmd)) {
980 cmd_error = SMMU_CERROR_ILL;
981 break;
982 }
983
984 if (!mr) {
985 break;
986 }
987
988 trace_smmuv3_cmdq_cfgi_ste(sid);
989 sdev = container_of(mr, SMMUDevice, iommu);
990 smmuv3_flush_config(sdev);
991
992 break;
993 }
994 case SMMU_CMD_CFGI_STE_RANGE:
995 {
996 uint32_t sid = CMD_SID(&cmd), mask;
997 uint8_t range = CMD_STE_RANGE(&cmd);
998 SMMUSIDRange sid_range;
999
1000 if (CMD_SSEC(&cmd)) {
1001 cmd_error = SMMU_CERROR_ILL;
1002 break;
1003 }
1004
1005 mask = (1ULL << (range + 1)) - 1;
1006 sid_range.start = sid & ~mask;
1007 sid_range.end = sid_range.start + mask;
1008
1009 trace_smmuv3_cmdq_cfgi_ste_range(sid_range.start, sid_range.end);
1010 g_hash_table_foreach_remove(bs->configs, smmuv3_invalidate_ste,
1011 &sid_range);
1012 break;
1013 }
1014 case SMMU_CMD_CFGI_CD:
1015 case SMMU_CMD_CFGI_CD_ALL:
1016 {
1017 uint32_t sid = CMD_SID(&cmd);
1018 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
1019 SMMUDevice *sdev;
1020
1021 if (CMD_SSEC(&cmd)) {
1022 cmd_error = SMMU_CERROR_ILL;
1023 break;
1024 }
1025
1026 if (!mr) {
1027 break;
1028 }
1029
1030 trace_smmuv3_cmdq_cfgi_cd(sid);
1031 sdev = container_of(mr, SMMUDevice, iommu);
1032 smmuv3_flush_config(sdev);
1033 break;
1034 }
1035 case SMMU_CMD_TLBI_NH_ASID:
1036 {
1037 uint16_t asid = CMD_ASID(&cmd);
1038
1039 trace_smmuv3_cmdq_tlbi_nh_asid(asid);
1040 smmu_inv_notifiers_all(&s->smmu_state);
1041 smmu_iotlb_inv_asid(bs, asid);
1042 break;
1043 }
1044 case SMMU_CMD_TLBI_NH_ALL:
1045 case SMMU_CMD_TLBI_NSNH_ALL:
1046 trace_smmuv3_cmdq_tlbi_nh();
1047 smmu_inv_notifiers_all(&s->smmu_state);
1048 smmu_iotlb_inv_all(bs);
1049 break;
1050 case SMMU_CMD_TLBI_NH_VAA:
1051 case SMMU_CMD_TLBI_NH_VA:
1052 smmuv3_s1_range_inval(bs, &cmd);
1053 break;
1054 case SMMU_CMD_TLBI_EL3_ALL:
1055 case SMMU_CMD_TLBI_EL3_VA:
1056 case SMMU_CMD_TLBI_EL2_ALL:
1057 case SMMU_CMD_TLBI_EL2_ASID:
1058 case SMMU_CMD_TLBI_EL2_VA:
1059 case SMMU_CMD_TLBI_EL2_VAA:
1060 case SMMU_CMD_TLBI_S12_VMALL:
1061 case SMMU_CMD_TLBI_S2_IPA:
1062 case SMMU_CMD_ATC_INV:
1063 case SMMU_CMD_PRI_RESP:
1064 case SMMU_CMD_RESUME:
1065 case SMMU_CMD_STALL_TERM:
1066 trace_smmuv3_unhandled_cmd(type);
1067 break;
1068 default:
1069 cmd_error = SMMU_CERROR_ILL;
1070 qemu_log_mask(LOG_GUEST_ERROR,
1071 "Illegal command type: %d\n", CMD_TYPE(&cmd));
1072 break;
1073 }
1074 qemu_mutex_unlock(&s->mutex);
1075 if (cmd_error) {
1076 break;
1077 }
1078
1079
1080
1081
1082
1083 queue_cons_incr(q);
1084 }
1085
1086 if (cmd_error) {
1087 trace_smmuv3_cmdq_consume_error(smmu_cmd_string(type), cmd_error);
1088 smmu_write_cmdq_err(s, cmd_error);
1089 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_CMDQ_ERR_MASK);
1090 }
1091
1092 trace_smmuv3_cmdq_consume_out(Q_PROD(q), Q_CONS(q),
1093 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1094
1095 return 0;
1096}
1097
1098static MemTxResult smmu_writell(SMMUv3State *s, hwaddr offset,
1099 uint64_t data, MemTxAttrs attrs)
1100{
1101 switch (offset) {
1102 case A_GERROR_IRQ_CFG0:
1103 s->gerror_irq_cfg0 = data;
1104 return MEMTX_OK;
1105 case A_STRTAB_BASE:
1106 s->strtab_base = data;
1107 return MEMTX_OK;
1108 case A_CMDQ_BASE:
1109 s->cmdq.base = data;
1110 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1111 if (s->cmdq.log2size > SMMU_CMDQS) {
1112 s->cmdq.log2size = SMMU_CMDQS;
1113 }
1114 return MEMTX_OK;
1115 case A_EVENTQ_BASE:
1116 s->eventq.base = data;
1117 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1118 if (s->eventq.log2size > SMMU_EVENTQS) {
1119 s->eventq.log2size = SMMU_EVENTQS;
1120 }
1121 return MEMTX_OK;
1122 case A_EVENTQ_IRQ_CFG0:
1123 s->eventq_irq_cfg0 = data;
1124 return MEMTX_OK;
1125 default:
1126 qemu_log_mask(LOG_UNIMP,
1127 "%s Unexpected 64-bit access to 0x%"PRIx64" (WI)\n",
1128 __func__, offset);
1129 return MEMTX_OK;
1130 }
1131}
1132
1133static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
1134 uint64_t data, MemTxAttrs attrs)
1135{
1136 switch (offset) {
1137 case A_CR0:
1138 s->cr[0] = data;
1139 s->cr0ack = data & ~SMMU_CR0_RESERVED;
1140
1141 smmuv3_cmdq_consume(s);
1142 return MEMTX_OK;
1143 case A_CR1:
1144 s->cr[1] = data;
1145 return MEMTX_OK;
1146 case A_CR2:
1147 s->cr[2] = data;
1148 return MEMTX_OK;
1149 case A_IRQ_CTRL:
1150 s->irq_ctrl = data;
1151 return MEMTX_OK;
1152 case A_GERRORN:
1153 smmuv3_write_gerrorn(s, data);
1154
1155
1156
1157
1158 smmuv3_cmdq_consume(s);
1159 return MEMTX_OK;
1160 case A_GERROR_IRQ_CFG0:
1161 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 0, 32, data);
1162 return MEMTX_OK;
1163 case A_GERROR_IRQ_CFG0 + 4:
1164 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 32, 32, data);
1165 return MEMTX_OK;
1166 case A_GERROR_IRQ_CFG1:
1167 s->gerror_irq_cfg1 = data;
1168 return MEMTX_OK;
1169 case A_GERROR_IRQ_CFG2:
1170 s->gerror_irq_cfg2 = data;
1171 return MEMTX_OK;
1172 case A_STRTAB_BASE:
1173 s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
1174 return MEMTX_OK;
1175 case A_STRTAB_BASE + 4:
1176 s->strtab_base = deposit64(s->strtab_base, 32, 32, data);
1177 return MEMTX_OK;
1178 case A_STRTAB_BASE_CFG:
1179 s->strtab_base_cfg = data;
1180 if (FIELD_EX32(data, STRTAB_BASE_CFG, FMT) == 1) {
1181 s->sid_split = FIELD_EX32(data, STRTAB_BASE_CFG, SPLIT);
1182 s->features |= SMMU_FEATURE_2LVL_STE;
1183 }
1184 return MEMTX_OK;
1185 case A_CMDQ_BASE:
1186 s->cmdq.base = deposit64(s->cmdq.base, 0, 32, data);
1187 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1188 if (s->cmdq.log2size > SMMU_CMDQS) {
1189 s->cmdq.log2size = SMMU_CMDQS;
1190 }
1191 return MEMTX_OK;
1192 case A_CMDQ_BASE + 4:
1193 s->cmdq.base = deposit64(s->cmdq.base, 32, 32, data);
1194 return MEMTX_OK;
1195 case A_CMDQ_PROD:
1196 s->cmdq.prod = data;
1197 smmuv3_cmdq_consume(s);
1198 return MEMTX_OK;
1199 case A_CMDQ_CONS:
1200 s->cmdq.cons = data;
1201 return MEMTX_OK;
1202 case A_EVENTQ_BASE:
1203 s->eventq.base = deposit64(s->eventq.base, 0, 32, data);
1204 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1205 if (s->eventq.log2size > SMMU_EVENTQS) {
1206 s->eventq.log2size = SMMU_EVENTQS;
1207 }
1208 return MEMTX_OK;
1209 case A_EVENTQ_BASE + 4:
1210 s->eventq.base = deposit64(s->eventq.base, 32, 32, data);
1211 return MEMTX_OK;
1212 case A_EVENTQ_PROD:
1213 s->eventq.prod = data;
1214 return MEMTX_OK;
1215 case A_EVENTQ_CONS:
1216 s->eventq.cons = data;
1217 return MEMTX_OK;
1218 case A_EVENTQ_IRQ_CFG0:
1219 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 0, 32, data);
1220 return MEMTX_OK;
1221 case A_EVENTQ_IRQ_CFG0 + 4:
1222 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 32, 32, data);
1223 return MEMTX_OK;
1224 case A_EVENTQ_IRQ_CFG1:
1225 s->eventq_irq_cfg1 = data;
1226 return MEMTX_OK;
1227 case A_EVENTQ_IRQ_CFG2:
1228 s->eventq_irq_cfg2 = data;
1229 return MEMTX_OK;
1230 default:
1231 qemu_log_mask(LOG_UNIMP,
1232 "%s Unexpected 32-bit access to 0x%"PRIx64" (WI)\n",
1233 __func__, offset);
1234 return MEMTX_OK;
1235 }
1236}
1237
1238static MemTxResult smmu_write_mmio(void *opaque, hwaddr offset, uint64_t data,
1239 unsigned size, MemTxAttrs attrs)
1240{
1241 SMMUState *sys = opaque;
1242 SMMUv3State *s = ARM_SMMUV3(sys);
1243 MemTxResult r;
1244
1245
1246 offset &= ~0x10000;
1247
1248 switch (size) {
1249 case 8:
1250 r = smmu_writell(s, offset, data, attrs);
1251 break;
1252 case 4:
1253 r = smmu_writel(s, offset, data, attrs);
1254 break;
1255 default:
1256 r = MEMTX_ERROR;
1257 break;
1258 }
1259
1260 trace_smmuv3_write_mmio(offset, data, size, r);
1261 return r;
1262}
1263
1264static MemTxResult smmu_readll(SMMUv3State *s, hwaddr offset,
1265 uint64_t *data, MemTxAttrs attrs)
1266{
1267 switch (offset) {
1268 case A_GERROR_IRQ_CFG0:
1269 *data = s->gerror_irq_cfg0;
1270 return MEMTX_OK;
1271 case A_STRTAB_BASE:
1272 *data = s->strtab_base;
1273 return MEMTX_OK;
1274 case A_CMDQ_BASE:
1275 *data = s->cmdq.base;
1276 return MEMTX_OK;
1277 case A_EVENTQ_BASE:
1278 *data = s->eventq.base;
1279 return MEMTX_OK;
1280 default:
1281 *data = 0;
1282 qemu_log_mask(LOG_UNIMP,
1283 "%s Unexpected 64-bit access to 0x%"PRIx64" (RAZ)\n",
1284 __func__, offset);
1285 return MEMTX_OK;
1286 }
1287}
1288
1289static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
1290 uint64_t *data, MemTxAttrs attrs)
1291{
1292 switch (offset) {
1293 case A_IDREGS ... A_IDREGS + 0x2f:
1294 *data = smmuv3_idreg(offset - A_IDREGS);
1295 return MEMTX_OK;
1296 case A_IDR0 ... A_IDR5:
1297 *data = s->idr[(offset - A_IDR0) / 4];
1298 return MEMTX_OK;
1299 case A_IIDR:
1300 *data = s->iidr;
1301 return MEMTX_OK;
1302 case A_AIDR:
1303 *data = s->aidr;
1304 return MEMTX_OK;
1305 case A_CR0:
1306 *data = s->cr[0];
1307 return MEMTX_OK;
1308 case A_CR0ACK:
1309 *data = s->cr0ack;
1310 return MEMTX_OK;
1311 case A_CR1:
1312 *data = s->cr[1];
1313 return MEMTX_OK;
1314 case A_CR2:
1315 *data = s->cr[2];
1316 return MEMTX_OK;
1317 case A_STATUSR:
1318 *data = s->statusr;
1319 return MEMTX_OK;
1320 case A_IRQ_CTRL:
1321 case A_IRQ_CTRL_ACK:
1322 *data = s->irq_ctrl;
1323 return MEMTX_OK;
1324 case A_GERROR:
1325 *data = s->gerror;
1326 return MEMTX_OK;
1327 case A_GERRORN:
1328 *data = s->gerrorn;
1329 return MEMTX_OK;
1330 case A_GERROR_IRQ_CFG0:
1331 *data = extract64(s->gerror_irq_cfg0, 0, 32);
1332 return MEMTX_OK;
1333 case A_GERROR_IRQ_CFG0 + 4:
1334 *data = extract64(s->gerror_irq_cfg0, 32, 32);
1335 return MEMTX_OK;
1336 case A_GERROR_IRQ_CFG1:
1337 *data = s->gerror_irq_cfg1;
1338 return MEMTX_OK;
1339 case A_GERROR_IRQ_CFG2:
1340 *data = s->gerror_irq_cfg2;
1341 return MEMTX_OK;
1342 case A_STRTAB_BASE:
1343 *data = extract64(s->strtab_base, 0, 32);
1344 return MEMTX_OK;
1345 case A_STRTAB_BASE + 4:
1346 *data = extract64(s->strtab_base, 32, 32);
1347 return MEMTX_OK;
1348 case A_STRTAB_BASE_CFG:
1349 *data = s->strtab_base_cfg;
1350 return MEMTX_OK;
1351 case A_CMDQ_BASE:
1352 *data = extract64(s->cmdq.base, 0, 32);
1353 return MEMTX_OK;
1354 case A_CMDQ_BASE + 4:
1355 *data = extract64(s->cmdq.base, 32, 32);
1356 return MEMTX_OK;
1357 case A_CMDQ_PROD:
1358 *data = s->cmdq.prod;
1359 return MEMTX_OK;
1360 case A_CMDQ_CONS:
1361 *data = s->cmdq.cons;
1362 return MEMTX_OK;
1363 case A_EVENTQ_BASE:
1364 *data = extract64(s->eventq.base, 0, 32);
1365 return MEMTX_OK;
1366 case A_EVENTQ_BASE + 4:
1367 *data = extract64(s->eventq.base, 32, 32);
1368 return MEMTX_OK;
1369 case A_EVENTQ_PROD:
1370 *data = s->eventq.prod;
1371 return MEMTX_OK;
1372 case A_EVENTQ_CONS:
1373 *data = s->eventq.cons;
1374 return MEMTX_OK;
1375 default:
1376 *data = 0;
1377 qemu_log_mask(LOG_UNIMP,
1378 "%s unhandled 32-bit access at 0x%"PRIx64" (RAZ)\n",
1379 __func__, offset);
1380 return MEMTX_OK;
1381 }
1382}
1383
1384static MemTxResult smmu_read_mmio(void *opaque, hwaddr offset, uint64_t *data,
1385 unsigned size, MemTxAttrs attrs)
1386{
1387 SMMUState *sys = opaque;
1388 SMMUv3State *s = ARM_SMMUV3(sys);
1389 MemTxResult r;
1390
1391
1392 offset &= ~0x10000;
1393
1394 switch (size) {
1395 case 8:
1396 r = smmu_readll(s, offset, data, attrs);
1397 break;
1398 case 4:
1399 r = smmu_readl(s, offset, data, attrs);
1400 break;
1401 default:
1402 r = MEMTX_ERROR;
1403 break;
1404 }
1405
1406 trace_smmuv3_read_mmio(offset, *data, size, r);
1407 return r;
1408}
1409
1410static const MemoryRegionOps smmu_mem_ops = {
1411 .read_with_attrs = smmu_read_mmio,
1412 .write_with_attrs = smmu_write_mmio,
1413 .endianness = DEVICE_LITTLE_ENDIAN,
1414 .valid = {
1415 .min_access_size = 4,
1416 .max_access_size = 8,
1417 },
1418 .impl = {
1419 .min_access_size = 4,
1420 .max_access_size = 8,
1421 },
1422};
1423
1424static void smmu_init_irq(SMMUv3State *s, SysBusDevice *dev)
1425{
1426 int i;
1427
1428 for (i = 0; i < ARRAY_SIZE(s->irq); i++) {
1429 sysbus_init_irq(dev, &s->irq[i]);
1430 }
1431}
1432
1433static void smmu_reset(DeviceState *dev)
1434{
1435 SMMUv3State *s = ARM_SMMUV3(dev);
1436 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1437
1438 c->parent_reset(dev);
1439
1440 smmuv3_init_regs(s);
1441}
1442
1443static void smmu_realize(DeviceState *d, Error **errp)
1444{
1445 SMMUState *sys = ARM_SMMU(d);
1446 SMMUv3State *s = ARM_SMMUV3(sys);
1447 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1448 SysBusDevice *dev = SYS_BUS_DEVICE(d);
1449 Error *local_err = NULL;
1450
1451 c->parent_realize(d, &local_err);
1452 if (local_err) {
1453 error_propagate(errp, local_err);
1454 return;
1455 }
1456
1457 qemu_mutex_init(&s->mutex);
1458
1459 memory_region_init_io(&sys->iomem, OBJECT(s),
1460 &smmu_mem_ops, sys, TYPE_ARM_SMMUV3, 0x20000);
1461
1462 sys->mrtypename = TYPE_SMMUV3_IOMMU_MEMORY_REGION;
1463
1464 sysbus_init_mmio(dev, &sys->iomem);
1465
1466 smmu_init_irq(s, dev);
1467}
1468
1469static const VMStateDescription vmstate_smmuv3_queue = {
1470 .name = "smmuv3_queue",
1471 .version_id = 1,
1472 .minimum_version_id = 1,
1473 .fields = (VMStateField[]) {
1474 VMSTATE_UINT64(base, SMMUQueue),
1475 VMSTATE_UINT32(prod, SMMUQueue),
1476 VMSTATE_UINT32(cons, SMMUQueue),
1477 VMSTATE_UINT8(log2size, SMMUQueue),
1478 VMSTATE_END_OF_LIST(),
1479 },
1480};
1481
1482static const VMStateDescription vmstate_smmuv3 = {
1483 .name = "smmuv3",
1484 .version_id = 1,
1485 .minimum_version_id = 1,
1486 .priority = MIG_PRI_IOMMU,
1487 .fields = (VMStateField[]) {
1488 VMSTATE_UINT32(features, SMMUv3State),
1489 VMSTATE_UINT8(sid_size, SMMUv3State),
1490 VMSTATE_UINT8(sid_split, SMMUv3State),
1491
1492 VMSTATE_UINT32_ARRAY(cr, SMMUv3State, 3),
1493 VMSTATE_UINT32(cr0ack, SMMUv3State),
1494 VMSTATE_UINT32(statusr, SMMUv3State),
1495 VMSTATE_UINT32(irq_ctrl, SMMUv3State),
1496 VMSTATE_UINT32(gerror, SMMUv3State),
1497 VMSTATE_UINT32(gerrorn, SMMUv3State),
1498 VMSTATE_UINT64(gerror_irq_cfg0, SMMUv3State),
1499 VMSTATE_UINT32(gerror_irq_cfg1, SMMUv3State),
1500 VMSTATE_UINT32(gerror_irq_cfg2, SMMUv3State),
1501 VMSTATE_UINT64(strtab_base, SMMUv3State),
1502 VMSTATE_UINT32(strtab_base_cfg, SMMUv3State),
1503 VMSTATE_UINT64(eventq_irq_cfg0, SMMUv3State),
1504 VMSTATE_UINT32(eventq_irq_cfg1, SMMUv3State),
1505 VMSTATE_UINT32(eventq_irq_cfg2, SMMUv3State),
1506
1507 VMSTATE_STRUCT(cmdq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1508 VMSTATE_STRUCT(eventq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1509
1510 VMSTATE_END_OF_LIST(),
1511 },
1512};
1513
1514static void smmuv3_instance_init(Object *obj)
1515{
1516
1517}
1518
1519static void smmuv3_class_init(ObjectClass *klass, void *data)
1520{
1521 DeviceClass *dc = DEVICE_CLASS(klass);
1522 SMMUv3Class *c = ARM_SMMUV3_CLASS(klass);
1523
1524 dc->vmsd = &vmstate_smmuv3;
1525 device_class_set_parent_reset(dc, smmu_reset, &c->parent_reset);
1526 c->parent_realize = dc->realize;
1527 dc->realize = smmu_realize;
1528}
1529
1530static int smmuv3_notify_flag_changed(IOMMUMemoryRegion *iommu,
1531 IOMMUNotifierFlag old,
1532 IOMMUNotifierFlag new,
1533 Error **errp)
1534{
1535 SMMUDevice *sdev = container_of(iommu, SMMUDevice, iommu);
1536 SMMUv3State *s3 = sdev->smmu;
1537 SMMUState *s = &(s3->smmu_state);
1538
1539 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
1540 error_setg(errp, "SMMUv3 does not support dev-iotlb yet");
1541 return -EINVAL;
1542 }
1543
1544 if (new & IOMMU_NOTIFIER_MAP) {
1545 error_setg(errp,
1546 "device %02x.%02x.%x requires iommu MAP notifier which is "
1547 "not currently supported", pci_bus_num(sdev->bus),
1548 PCI_SLOT(sdev->devfn), PCI_FUNC(sdev->devfn));
1549 return -EINVAL;
1550 }
1551
1552 if (old == IOMMU_NOTIFIER_NONE) {
1553 trace_smmuv3_notify_flag_add(iommu->parent_obj.name);
1554 QLIST_INSERT_HEAD(&s->devices_with_notifiers, sdev, next);
1555 } else if (new == IOMMU_NOTIFIER_NONE) {
1556 trace_smmuv3_notify_flag_del(iommu->parent_obj.name);
1557 QLIST_REMOVE(sdev, next);
1558 }
1559 return 0;
1560}
1561
1562static void smmuv3_iommu_memory_region_class_init(ObjectClass *klass,
1563 void *data)
1564{
1565 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1566
1567 imrc->translate = smmuv3_translate;
1568 imrc->notify_flag_changed = smmuv3_notify_flag_changed;
1569}
1570
1571static const TypeInfo smmuv3_type_info = {
1572 .name = TYPE_ARM_SMMUV3,
1573 .parent = TYPE_ARM_SMMU,
1574 .instance_size = sizeof(SMMUv3State),
1575 .instance_init = smmuv3_instance_init,
1576 .class_size = sizeof(SMMUv3Class),
1577 .class_init = smmuv3_class_init,
1578};
1579
1580static const TypeInfo smmuv3_iommu_memory_region_info = {
1581 .parent = TYPE_IOMMU_MEMORY_REGION,
1582 .name = TYPE_SMMUV3_IOMMU_MEMORY_REGION,
1583 .class_init = smmuv3_iommu_memory_region_class_init,
1584};
1585
1586static void smmuv3_register_types(void)
1587{
1588 type_register(&smmuv3_type_info);
1589 type_register(&smmuv3_iommu_memory_region_info);
1590}
1591
1592type_init(smmuv3_register_types)
1593
1594
