1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include "qemu/osdep.h"
20#include "qemu/bitops.h"
21#include "hw/irq.h"
22#include "hw/sysbus.h"
23#include "migration/vmstate.h"
24#include "hw/qdev-core.h"
25#include "hw/pci/pci.h"
26#include "cpu.h"
27#include "trace.h"
28#include "qemu/log.h"
29#include "qemu/error-report.h"
30#include "qapi/error.h"
31
32#include "hw/arm/smmuv3.h"
33#include "smmuv3-internal.h"
34#include "smmu-internal.h"
35
36
37
38
39
40
41
42
43static void smmuv3_trigger_irq(SMMUv3State *s, SMMUIrq irq,
44 uint32_t gerror_mask)
45{
46
47 bool pulse = false;
48
49 switch (irq) {
50 case SMMU_IRQ_EVTQ:
51 pulse = smmuv3_eventq_irq_enabled(s);
52 break;
53 case SMMU_IRQ_PRIQ:
54 qemu_log_mask(LOG_UNIMP, "PRI not yet supported\n");
55 break;
56 case SMMU_IRQ_CMD_SYNC:
57 pulse = true;
58 break;
59 case SMMU_IRQ_GERROR:
60 {
61 uint32_t pending = s->gerror ^ s->gerrorn;
62 uint32_t new_gerrors = ~pending & gerror_mask;
63
64 if (!new_gerrors) {
65
66 return;
67 }
68 s->gerror ^= new_gerrors;
69 trace_smmuv3_write_gerror(new_gerrors, s->gerror);
70
71 pulse = smmuv3_gerror_irq_enabled(s);
72 break;
73 }
74 }
75 if (pulse) {
76 trace_smmuv3_trigger_irq(irq);
77 qemu_irq_pulse(s->irq[irq]);
78 }
79}
80
81static void smmuv3_write_gerrorn(SMMUv3State *s, uint32_t new_gerrorn)
82{
83 uint32_t pending = s->gerror ^ s->gerrorn;
84 uint32_t toggled = s->gerrorn ^ new_gerrorn;
85
86 if (toggled & ~pending) {
87 qemu_log_mask(LOG_GUEST_ERROR,
88 "guest toggles non pending errors = 0x%x\n",
89 toggled & ~pending);
90 }
91
92
93
94
95
96 s->gerrorn = new_gerrorn;
97
98 trace_smmuv3_write_gerrorn(toggled & pending, s->gerrorn);
99}
100
101static inline MemTxResult queue_read(SMMUQueue *q, Cmd *cmd)
102{
103 dma_addr_t addr = Q_CONS_ENTRY(q);
104 MemTxResult ret;
105 int i;
106
107 ret = dma_memory_read(&address_space_memory, addr, cmd, sizeof(Cmd),
108 MEMTXATTRS_UNSPECIFIED);
109 if (ret != MEMTX_OK) {
110 return ret;
111 }
112 for (i = 0; i < ARRAY_SIZE(cmd->word); i++) {
113 le32_to_cpus(&cmd->word[i]);
114 }
115 return ret;
116}
117
118static MemTxResult queue_write(SMMUQueue *q, Evt *evt_in)
119{
120 dma_addr_t addr = Q_PROD_ENTRY(q);
121 MemTxResult ret;
122 Evt evt = *evt_in;
123 int i;
124
125 for (i = 0; i < ARRAY_SIZE(evt.word); i++) {
126 cpu_to_le32s(&evt.word[i]);
127 }
128 ret = dma_memory_write(&address_space_memory, addr, &evt, sizeof(Evt),
129 MEMTXATTRS_UNSPECIFIED);
130 if (ret != MEMTX_OK) {
131 return ret;
132 }
133
134 queue_prod_incr(q);
135 return MEMTX_OK;
136}
137
138static MemTxResult smmuv3_write_eventq(SMMUv3State *s, Evt *evt)
139{
140 SMMUQueue *q = &s->eventq;
141 MemTxResult r;
142
143 if (!smmuv3_eventq_enabled(s)) {
144 return MEMTX_ERROR;
145 }
146
147 if (smmuv3_q_full(q)) {
148 return MEMTX_ERROR;
149 }
150
151 r = queue_write(q, evt);
152 if (r != MEMTX_OK) {
153 return r;
154 }
155
156 if (!smmuv3_q_empty(q)) {
157 smmuv3_trigger_irq(s, SMMU_IRQ_EVTQ, 0);
158 }
159 return MEMTX_OK;
160}
161
162void smmuv3_record_event(SMMUv3State *s, SMMUEventInfo *info)
163{
164 Evt evt = {};
165 MemTxResult r;
166
167 if (!smmuv3_eventq_enabled(s)) {
168 return;
169 }
170
171 EVT_SET_TYPE(&evt, info->type);
172 EVT_SET_SID(&evt, info->sid);
173
174 switch (info->type) {
175 case SMMU_EVT_NONE:
176 return;
177 case SMMU_EVT_F_UUT:
178 EVT_SET_SSID(&evt, info->u.f_uut.ssid);
179 EVT_SET_SSV(&evt, info->u.f_uut.ssv);
180 EVT_SET_ADDR(&evt, info->u.f_uut.addr);
181 EVT_SET_RNW(&evt, info->u.f_uut.rnw);
182 EVT_SET_PNU(&evt, info->u.f_uut.pnu);
183 EVT_SET_IND(&evt, info->u.f_uut.ind);
184 break;
185 case SMMU_EVT_C_BAD_STREAMID:
186 EVT_SET_SSID(&evt, info->u.c_bad_streamid.ssid);
187 EVT_SET_SSV(&evt, info->u.c_bad_streamid.ssv);
188 break;
189 case SMMU_EVT_F_STE_FETCH:
190 EVT_SET_SSID(&evt, info->u.f_ste_fetch.ssid);
191 EVT_SET_SSV(&evt, info->u.f_ste_fetch.ssv);
192 EVT_SET_ADDR2(&evt, info->u.f_ste_fetch.addr);
193 break;
194 case SMMU_EVT_C_BAD_STE:
195 EVT_SET_SSID(&evt, info->u.c_bad_ste.ssid);
196 EVT_SET_SSV(&evt, info->u.c_bad_ste.ssv);
197 break;
198 case SMMU_EVT_F_STREAM_DISABLED:
199 break;
200 case SMMU_EVT_F_TRANS_FORBIDDEN:
201 EVT_SET_ADDR(&evt, info->u.f_transl_forbidden.addr);
202 EVT_SET_RNW(&evt, info->u.f_transl_forbidden.rnw);
203 break;
204 case SMMU_EVT_C_BAD_SUBSTREAMID:
205 EVT_SET_SSID(&evt, info->u.c_bad_substream.ssid);
206 break;
207 case SMMU_EVT_F_CD_FETCH:
208 EVT_SET_SSID(&evt, info->u.f_cd_fetch.ssid);
209 EVT_SET_SSV(&evt, info->u.f_cd_fetch.ssv);
210 EVT_SET_ADDR(&evt, info->u.f_cd_fetch.addr);
211 break;
212 case SMMU_EVT_C_BAD_CD:
213 EVT_SET_SSID(&evt, info->u.c_bad_cd.ssid);
214 EVT_SET_SSV(&evt, info->u.c_bad_cd.ssv);
215 break;
216 case SMMU_EVT_F_WALK_EABT:
217 case SMMU_EVT_F_TRANSLATION:
218 case SMMU_EVT_F_ADDR_SIZE:
219 case SMMU_EVT_F_ACCESS:
220 case SMMU_EVT_F_PERMISSION:
221 EVT_SET_STALL(&evt, info->u.f_walk_eabt.stall);
222 EVT_SET_STAG(&evt, info->u.f_walk_eabt.stag);
223 EVT_SET_SSID(&evt, info->u.f_walk_eabt.ssid);
224 EVT_SET_SSV(&evt, info->u.f_walk_eabt.ssv);
225 EVT_SET_S2(&evt, info->u.f_walk_eabt.s2);
226 EVT_SET_ADDR(&evt, info->u.f_walk_eabt.addr);
227 EVT_SET_RNW(&evt, info->u.f_walk_eabt.rnw);
228 EVT_SET_PNU(&evt, info->u.f_walk_eabt.pnu);
229 EVT_SET_IND(&evt, info->u.f_walk_eabt.ind);
230 EVT_SET_CLASS(&evt, info->u.f_walk_eabt.class);
231 EVT_SET_ADDR2(&evt, info->u.f_walk_eabt.addr2);
232 break;
233 case SMMU_EVT_F_CFG_CONFLICT:
234 EVT_SET_SSID(&evt, info->u.f_cfg_conflict.ssid);
235 EVT_SET_SSV(&evt, info->u.f_cfg_conflict.ssv);
236 break;
237
238 case SMMU_EVT_F_BAD_ATS_TREQ:
239 case SMMU_EVT_F_TLB_CONFLICT:
240 case SMMU_EVT_E_PAGE_REQ:
241 default:
242 g_assert_not_reached();
243 }
244
245 trace_smmuv3_record_event(smmu_event_string(info->type), info->sid);
246 r = smmuv3_write_eventq(s, &evt);
247 if (r != MEMTX_OK) {
248 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_EVENTQ_ABT_ERR_MASK);
249 }
250 info->recorded = true;
251}
252
253static void smmuv3_init_regs(SMMUv3State *s)
254{
255
256
257
258
259 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1);
260 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTF, 2);
261 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, COHACC, 1);
262 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ASID16, 1);
263 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTENDIAN, 2);
264 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STALL_MODEL, 1);
265
266 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TERM_MODEL, 1);
267
268 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STLEVEL, 1);
269
270 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SIDSIZE, SMMU_IDR1_SIDSIZE);
271 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, EVENTQS, SMMU_EVENTQS);
272 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, CMDQS, SMMU_CMDQS);
273
274 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1);
275 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1);
276 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, BBML, 2);
277
278
279 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1);
280 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN16K, 1);
281 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1);
282 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS);
283
284 s->cmdq.base = deposit64(s->cmdq.base, 0, 5, SMMU_CMDQS);
285 s->cmdq.prod = 0;
286 s->cmdq.cons = 0;
287 s->cmdq.entry_size = sizeof(struct Cmd);
288 s->eventq.base = deposit64(s->eventq.base, 0, 5, SMMU_EVENTQS);
289 s->eventq.prod = 0;
290 s->eventq.cons = 0;
291 s->eventq.entry_size = sizeof(struct Evt);
292
293 s->features = 0;
294 s->sid_split = 0;
295 s->aidr = 0x1;
296 s->cr[0] = 0;
297 s->cr0ack = 0;
298 s->irq_ctrl = 0;
299 s->gerror = 0;
300 s->gerrorn = 0;
301 s->statusr = 0;
302}
303
304static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
305 SMMUEventInfo *event)
306{
307 int ret, i;
308
309 trace_smmuv3_get_ste(addr);
310
311 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
312 MEMTXATTRS_UNSPECIFIED);
313 if (ret != MEMTX_OK) {
314 qemu_log_mask(LOG_GUEST_ERROR,
315 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
316 event->type = SMMU_EVT_F_STE_FETCH;
317 event->u.f_ste_fetch.addr = addr;
318 return -EINVAL;
319 }
320 for (i = 0; i < ARRAY_SIZE(buf->word); i++) {
321 le32_to_cpus(&buf->word[i]);
322 }
323 return 0;
324
325}
326
327
328static int smmu_get_cd(SMMUv3State *s, STE *ste, uint32_t ssid,
329 CD *buf, SMMUEventInfo *event)
330{
331 dma_addr_t addr = STE_CTXPTR(ste);
332 int ret, i;
333
334 trace_smmuv3_get_cd(addr);
335
336 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
337 MEMTXATTRS_UNSPECIFIED);
338 if (ret != MEMTX_OK) {
339 qemu_log_mask(LOG_GUEST_ERROR,
340 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
341 event->type = SMMU_EVT_F_CD_FETCH;
342 event->u.f_ste_fetch.addr = addr;
343 return -EINVAL;
344 }
345 for (i = 0; i < ARRAY_SIZE(buf->word); i++) {
346 le32_to_cpus(&buf->word[i]);
347 }
348 return 0;
349}
350
351
352static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg,
353 STE *ste, SMMUEventInfo *event)
354{
355 uint32_t config;
356
357 if (!STE_VALID(ste)) {
358 if (!event->inval_ste_allowed) {
359 qemu_log_mask(LOG_GUEST_ERROR, "invalid STE\n");
360 }
361 goto bad_ste;
362 }
363
364 config = STE_CONFIG(ste);
365
366 if (STE_CFG_ABORT(config)) {
367 cfg->aborted = true;
368 return 0;
369 }
370
371 if (STE_CFG_BYPASS(config)) {
372 cfg->bypassed = true;
373 return 0;
374 }
375
376 if (STE_CFG_S2_ENABLED(config)) {
377 qemu_log_mask(LOG_UNIMP, "SMMUv3 does not support stage 2 yet\n");
378 goto bad_ste;
379 }
380
381 if (STE_S1CDMAX(ste) != 0) {
382 qemu_log_mask(LOG_UNIMP,
383 "SMMUv3 does not support multiple context descriptors yet\n");
384 goto bad_ste;
385 }
386
387 if (STE_S1STALLD(ste)) {
388 qemu_log_mask(LOG_UNIMP,
389 "SMMUv3 S1 stalling fault model not allowed yet\n");
390 goto bad_ste;
391 }
392 return 0;
393
394bad_ste:
395 event->type = SMMU_EVT_C_BAD_STE;
396 return -EINVAL;
397}
398
399
400
401
402
403
404
405
406
407
408
409
410
411static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste,
412 SMMUEventInfo *event)
413{
414 dma_addr_t addr, strtab_base;
415 uint32_t log2size;
416 int strtab_size_shift;
417 int ret;
418
419 trace_smmuv3_find_ste(sid, s->features, s->sid_split);
420 log2size = FIELD_EX32(s->strtab_base_cfg, STRTAB_BASE_CFG, LOG2SIZE);
421
422
423
424 if (sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE))) {
425 event->type = SMMU_EVT_C_BAD_STREAMID;
426 return -EINVAL;
427 }
428 if (s->features & SMMU_FEATURE_2LVL_STE) {
429 int l1_ste_offset, l2_ste_offset, max_l2_ste, span, i;
430 dma_addr_t l1ptr, l2ptr;
431 STEDesc l1std;
432
433
434
435
436
437 strtab_size_shift = MAX(5, (int)log2size - s->sid_split - 1 + 3);
438 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
439 ~MAKE_64BIT_MASK(0, strtab_size_shift);
440 l1_ste_offset = sid >> s->sid_split;
441 l2_ste_offset = sid & ((1 << s->sid_split) - 1);
442 l1ptr = (dma_addr_t)(strtab_base + l1_ste_offset * sizeof(l1std));
443
444 ret = dma_memory_read(&address_space_memory, l1ptr, &l1std,
445 sizeof(l1std), MEMTXATTRS_UNSPECIFIED);
446 if (ret != MEMTX_OK) {
447 qemu_log_mask(LOG_GUEST_ERROR,
448 "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr);
449 event->type = SMMU_EVT_F_STE_FETCH;
450 event->u.f_ste_fetch.addr = l1ptr;
451 return -EINVAL;
452 }
453 for (i = 0; i < ARRAY_SIZE(l1std.word); i++) {
454 le32_to_cpus(&l1std.word[i]);
455 }
456
457 span = L1STD_SPAN(&l1std);
458
459 if (!span) {
460
461 if (!event->inval_ste_allowed) {
462 qemu_log_mask(LOG_GUEST_ERROR,
463 "invalid sid=%d (L1STD span=0)\n", sid);
464 }
465 event->type = SMMU_EVT_C_BAD_STREAMID;
466 return -EINVAL;
467 }
468 max_l2_ste = (1 << span) - 1;
469 l2ptr = l1std_l2ptr(&l1std);
470 trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset,
471 l2ptr, l2_ste_offset, max_l2_ste);
472 if (l2_ste_offset > max_l2_ste) {
473 qemu_log_mask(LOG_GUEST_ERROR,
474 "l2_ste_offset=%d > max_l2_ste=%d\n",
475 l2_ste_offset, max_l2_ste);
476 event->type = SMMU_EVT_C_BAD_STE;
477 return -EINVAL;
478 }
479 addr = l2ptr + l2_ste_offset * sizeof(*ste);
480 } else {
481 strtab_size_shift = log2size + 5;
482 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
483 ~MAKE_64BIT_MASK(0, strtab_size_shift);
484 addr = strtab_base + sid * sizeof(*ste);
485 }
486
487 if (smmu_get_ste(s, addr, ste, event)) {
488 return -EINVAL;
489 }
490
491 return 0;
492}
493
494static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event)
495{
496 int ret = -EINVAL;
497 int i;
498
499 if (!CD_VALID(cd) || !CD_AARCH64(cd)) {
500 goto bad_cd;
501 }
502 if (!CD_A(cd)) {
503 goto bad_cd;
504 }
505 if (CD_S(cd)) {
506 goto bad_cd;
507 }
508 if (CD_HA(cd) || CD_HD(cd)) {
509 goto bad_cd;
510 }
511
512
513 cfg->aa64 = true;
514 cfg->stage = 1;
515
516 cfg->oas = oas2bits(CD_IPS(cd));
517 cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas);
518 cfg->tbi = CD_TBI(cd);
519 cfg->asid = CD_ASID(cd);
520
521 trace_smmuv3_decode_cd(cfg->oas);
522
523
524 for (i = 0; i <= 1; i++) {
525 int tg, tsz;
526 SMMUTransTableInfo *tt = &cfg->tt[i];
527
528 cfg->tt[i].disabled = CD_EPD(cd, i);
529 if (cfg->tt[i].disabled) {
530 continue;
531 }
532
533 tsz = CD_TSZ(cd, i);
534 if (tsz < 16 || tsz > 39) {
535 goto bad_cd;
536 }
537
538 tg = CD_TG(cd, i);
539 tt->granule_sz = tg2granule(tg, i);
540 if ((tt->granule_sz != 12 && tt->granule_sz != 14 &&
541 tt->granule_sz != 16) || CD_ENDI(cd)) {
542 goto bad_cd;
543 }
544
545 tt->tsz = tsz;
546 tt->ttb = CD_TTB(cd, i);
547 if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) {
548 goto bad_cd;
549 }
550 tt->had = CD_HAD(cd, i);
551 trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had);
552 }
553
554 cfg->record_faults = CD_R(cd);
555
556 return 0;
557
558bad_cd:
559 event->type = SMMU_EVT_C_BAD_CD;
560 return ret;
561}
562
563
564
565
566
567
568
569
570
571
572
573
574static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg,
575 SMMUEventInfo *event)
576{
577 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
578 uint32_t sid = smmu_get_sid(sdev);
579 SMMUv3State *s = sdev->smmu;
580 int ret;
581 STE ste;
582 CD cd;
583
584 ret = smmu_find_ste(s, sid, &ste, event);
585 if (ret) {
586 return ret;
587 }
588
589 ret = decode_ste(s, cfg, &ste, event);
590 if (ret) {
591 return ret;
592 }
593
594 if (cfg->aborted || cfg->bypassed) {
595 return 0;
596 }
597
598 ret = smmu_get_cd(s, &ste, 0 , &cd, event);
599 if (ret) {
600 return ret;
601 }
602
603 return decode_cd(cfg, &cd, event);
604}
605
606
607
608
609
610
611
612
613
614
615
616
617
618static SMMUTransCfg *smmuv3_get_config(SMMUDevice *sdev, SMMUEventInfo *event)
619{
620 SMMUv3State *s = sdev->smmu;
621 SMMUState *bc = &s->smmu_state;
622 SMMUTransCfg *cfg;
623
624 cfg = g_hash_table_lookup(bc->configs, sdev);
625 if (cfg) {
626 sdev->cfg_cache_hits++;
627 trace_smmuv3_config_cache_hit(smmu_get_sid(sdev),
628 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
629 100 * sdev->cfg_cache_hits /
630 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
631 } else {
632 sdev->cfg_cache_misses++;
633 trace_smmuv3_config_cache_miss(smmu_get_sid(sdev),
634 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
635 100 * sdev->cfg_cache_hits /
636 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
637 cfg = g_new0(SMMUTransCfg, 1);
638
639 if (!smmuv3_decode_config(&sdev->iommu, cfg, event)) {
640 g_hash_table_insert(bc->configs, sdev, cfg);
641 } else {
642 g_free(cfg);
643 cfg = NULL;
644 }
645 }
646 return cfg;
647}
648
649static void smmuv3_flush_config(SMMUDevice *sdev)
650{
651 SMMUv3State *s = sdev->smmu;
652 SMMUState *bc = &s->smmu_state;
653
654 trace_smmuv3_config_cache_inv(smmu_get_sid(sdev));
655 g_hash_table_remove(bc->configs, sdev);
656}
657
658static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
659 IOMMUAccessFlags flag, int iommu_idx)
660{
661 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
662 SMMUv3State *s = sdev->smmu;
663 uint32_t sid = smmu_get_sid(sdev);
664 SMMUEventInfo event = {.type = SMMU_EVT_NONE,
665 .sid = sid,
666 .inval_ste_allowed = false};
667 SMMUPTWEventInfo ptw_info = {};
668 SMMUTranslationStatus status;
669 SMMUState *bs = ARM_SMMU(s);
670 uint64_t page_mask, aligned_addr;
671 SMMUTLBEntry *cached_entry = NULL;
672 SMMUTransTableInfo *tt;
673 SMMUTransCfg *cfg = NULL;
674 IOMMUTLBEntry entry = {
675 .target_as = &address_space_memory,
676 .iova = addr,
677 .translated_addr = addr,
678 .addr_mask = ~(hwaddr)0,
679 .perm = IOMMU_NONE,
680 };
681
682 qemu_mutex_lock(&s->mutex);
683
684 if (!smmu_enabled(s)) {
685 status = SMMU_TRANS_DISABLE;
686 goto epilogue;
687 }
688
689 cfg = smmuv3_get_config(sdev, &event);
690 if (!cfg) {
691 status = SMMU_TRANS_ERROR;
692 goto epilogue;
693 }
694
695 if (cfg->aborted) {
696 status = SMMU_TRANS_ABORT;
697 goto epilogue;
698 }
699
700 if (cfg->bypassed) {
701 status = SMMU_TRANS_BYPASS;
702 goto epilogue;
703 }
704
705 tt = select_tt(cfg, addr);
706 if (!tt) {
707 if (cfg->record_faults) {
708 event.type = SMMU_EVT_F_TRANSLATION;
709 event.u.f_translation.addr = addr;
710 event.u.f_translation.rnw = flag & 0x1;
711 }
712 status = SMMU_TRANS_ERROR;
713 goto epilogue;
714 }
715
716 page_mask = (1ULL << (tt->granule_sz)) - 1;
717 aligned_addr = addr & ~page_mask;
718
719 cached_entry = smmu_iotlb_lookup(bs, cfg, tt, aligned_addr);
720 if (cached_entry) {
721 if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) {
722 status = SMMU_TRANS_ERROR;
723 if (cfg->record_faults) {
724 event.type = SMMU_EVT_F_PERMISSION;
725 event.u.f_permission.addr = addr;
726 event.u.f_permission.rnw = flag & 0x1;
727 }
728 } else {
729 status = SMMU_TRANS_SUCCESS;
730 }
731 goto epilogue;
732 }
733
734 cached_entry = g_new0(SMMUTLBEntry, 1);
735
736 if (smmu_ptw(cfg, aligned_addr, flag, cached_entry, &ptw_info)) {
737 g_free(cached_entry);
738 switch (ptw_info.type) {
739 case SMMU_PTW_ERR_WALK_EABT:
740 event.type = SMMU_EVT_F_WALK_EABT;
741 event.u.f_walk_eabt.addr = addr;
742 event.u.f_walk_eabt.rnw = flag & 0x1;
743 event.u.f_walk_eabt.class = 0x1;
744 event.u.f_walk_eabt.addr2 = ptw_info.addr;
745 break;
746 case SMMU_PTW_ERR_TRANSLATION:
747 if (cfg->record_faults) {
748 event.type = SMMU_EVT_F_TRANSLATION;
749 event.u.f_translation.addr = addr;
750 event.u.f_translation.rnw = flag & 0x1;
751 }
752 break;
753 case SMMU_PTW_ERR_ADDR_SIZE:
754 if (cfg->record_faults) {
755 event.type = SMMU_EVT_F_ADDR_SIZE;
756 event.u.f_addr_size.addr = addr;
757 event.u.f_addr_size.rnw = flag & 0x1;
758 }
759 break;
760 case SMMU_PTW_ERR_ACCESS:
761 if (cfg->record_faults) {
762 event.type = SMMU_EVT_F_ACCESS;
763 event.u.f_access.addr = addr;
764 event.u.f_access.rnw = flag & 0x1;
765 }
766 break;
767 case SMMU_PTW_ERR_PERMISSION:
768 if (cfg->record_faults) {
769 event.type = SMMU_EVT_F_PERMISSION;
770 event.u.f_permission.addr = addr;
771 event.u.f_permission.rnw = flag & 0x1;
772 }
773 break;
774 default:
775 g_assert_not_reached();
776 }
777 status = SMMU_TRANS_ERROR;
778 } else {
779 smmu_iotlb_insert(bs, cfg, cached_entry);
780 status = SMMU_TRANS_SUCCESS;
781 }
782
783epilogue:
784 qemu_mutex_unlock(&s->mutex);
785 switch (status) {
786 case SMMU_TRANS_SUCCESS:
787 entry.perm = cached_entry->entry.perm;
788 entry.translated_addr = cached_entry->entry.translated_addr +
789 (addr & cached_entry->entry.addr_mask);
790 entry.addr_mask = cached_entry->entry.addr_mask;
791 trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
792 entry.translated_addr, entry.perm);
793 break;
794 case SMMU_TRANS_DISABLE:
795 entry.perm = flag;
796 entry.addr_mask = ~TARGET_PAGE_MASK;
797 trace_smmuv3_translate_disable(mr->parent_obj.name, sid, addr,
798 entry.perm);
799 break;
800 case SMMU_TRANS_BYPASS:
801 entry.perm = flag;
802 entry.addr_mask = ~TARGET_PAGE_MASK;
803 trace_smmuv3_translate_bypass(mr->parent_obj.name, sid, addr,
804 entry.perm);
805 break;
806 case SMMU_TRANS_ABORT:
807
808 trace_smmuv3_translate_abort(mr->parent_obj.name, sid, addr,
809 entry.perm);
810 break;
811 case SMMU_TRANS_ERROR:
812 qemu_log_mask(LOG_GUEST_ERROR,
813 "%s translation failed for iova=0x%"PRIx64" (%s)\n",
814 mr->parent_obj.name, addr, smmu_event_string(event.type));
815 smmuv3_record_event(s, &event);
816 break;
817 }
818
819 return entry;
820}
821
822
823
824
825
826
827
828
829
830
831
832
833static void smmuv3_notify_iova(IOMMUMemoryRegion *mr,
834 IOMMUNotifier *n,
835 int asid, dma_addr_t iova,
836 uint8_t tg, uint64_t num_pages)
837{
838 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
839 IOMMUTLBEvent event;
840 uint8_t granule;
841
842 if (!tg) {
843 SMMUEventInfo event = {.inval_ste_allowed = true};
844 SMMUTransCfg *cfg = smmuv3_get_config(sdev, &event);
845 SMMUTransTableInfo *tt;
846
847 if (!cfg) {
848 return;
849 }
850
851 if (asid >= 0 && cfg->asid != asid) {
852 return;
853 }
854
855 tt = select_tt(cfg, iova);
856 if (!tt) {
857 return;
858 }
859 granule = tt->granule_sz;
860 } else {
861 granule = tg * 2 + 10;
862 }
863
864 event.type = IOMMU_NOTIFIER_UNMAP;
865 event.entry.target_as = &address_space_memory;
866 event.entry.iova = iova;
867 event.entry.addr_mask = num_pages * (1 << granule) - 1;
868 event.entry.perm = IOMMU_NONE;
869
870 memory_region_notify_iommu_one(n, &event);
871}
872
873
874static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, dma_addr_t iova,
875 uint8_t tg, uint64_t num_pages)
876{
877 SMMUDevice *sdev;
878
879 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
880 IOMMUMemoryRegion *mr = &sdev->iommu;
881 IOMMUNotifier *n;
882
883 trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, iova,
884 tg, num_pages);
885
886 IOMMU_NOTIFIER_FOREACH(n, mr) {
887 smmuv3_notify_iova(mr, n, asid, iova, tg, num_pages);
888 }
889 }
890}
891
892static void smmuv3_s1_range_inval(SMMUState *s, Cmd *cmd)
893{
894 dma_addr_t end, addr = CMD_ADDR(cmd);
895 uint8_t type = CMD_TYPE(cmd);
896 uint16_t vmid = CMD_VMID(cmd);
897 uint8_t scale = CMD_SCALE(cmd);
898 uint8_t num = CMD_NUM(cmd);
899 uint8_t ttl = CMD_TTL(cmd);
900 bool leaf = CMD_LEAF(cmd);
901 uint8_t tg = CMD_TG(cmd);
902 uint64_t num_pages;
903 uint8_t granule;
904 int asid = -1;
905
906 if (type == SMMU_CMD_TLBI_NH_VA) {
907 asid = CMD_ASID(cmd);
908 }
909
910 if (!tg) {
911 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, 1, ttl, leaf);
912 smmuv3_inv_notifiers_iova(s, asid, addr, tg, 1);
913 smmu_iotlb_inv_iova(s, asid, addr, tg, 1, ttl);
914 return;
915 }
916
917
918
919 num_pages = (num + 1) * BIT_ULL(scale);
920 granule = tg * 2 + 10;
921
922
923 end = addr + (num_pages << granule) - 1;
924
925 while (addr != end + 1) {
926 uint64_t mask = dma_aligned_pow2_mask(addr, end, 64);
927
928 num_pages = (mask + 1) >> granule;
929 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf);
930 smmuv3_inv_notifiers_iova(s, asid, addr, tg, num_pages);
931 smmu_iotlb_inv_iova(s, asid, addr, tg, num_pages, ttl);
932 addr += mask + 1;
933 }
934}
935
936static gboolean
937smmuv3_invalidate_ste(gpointer key, gpointer value, gpointer user_data)
938{
939 SMMUDevice *sdev = (SMMUDevice *)key;
940 uint32_t sid = smmu_get_sid(sdev);
941 SMMUSIDRange *sid_range = (SMMUSIDRange *)user_data;
942
943 if (sid < sid_range->start || sid > sid_range->end) {
944 return false;
945 }
946 trace_smmuv3_config_cache_inv(sid);
947 return true;
948}
949
950static int smmuv3_cmdq_consume(SMMUv3State *s)
951{
952 SMMUState *bs = ARM_SMMU(s);
953 SMMUCmdError cmd_error = SMMU_CERROR_NONE;
954 SMMUQueue *q = &s->cmdq;
955 SMMUCommandType type = 0;
956
957 if (!smmuv3_cmdq_enabled(s)) {
958 return 0;
959 }
960
961
962
963
964
965
966
967 while (!smmuv3_q_empty(q)) {
968 uint32_t pending = s->gerror ^ s->gerrorn;
969 Cmd cmd;
970
971 trace_smmuv3_cmdq_consume(Q_PROD(q), Q_CONS(q),
972 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
973
974 if (FIELD_EX32(pending, GERROR, CMDQ_ERR)) {
975 break;
976 }
977
978 if (queue_read(q, &cmd) != MEMTX_OK) {
979 cmd_error = SMMU_CERROR_ABT;
980 break;
981 }
982
983 type = CMD_TYPE(&cmd);
984
985 trace_smmuv3_cmdq_opcode(smmu_cmd_string(type));
986
987 qemu_mutex_lock(&s->mutex);
988 switch (type) {
989 case SMMU_CMD_SYNC:
990 if (CMD_SYNC_CS(&cmd) & CMD_SYNC_SIG_IRQ) {
991 smmuv3_trigger_irq(s, SMMU_IRQ_CMD_SYNC, 0);
992 }
993 break;
994 case SMMU_CMD_PREFETCH_CONFIG:
995 case SMMU_CMD_PREFETCH_ADDR:
996 break;
997 case SMMU_CMD_CFGI_STE:
998 {
999 uint32_t sid = CMD_SID(&cmd);
1000 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
1001 SMMUDevice *sdev;
1002
1003 if (CMD_SSEC(&cmd)) {
1004 cmd_error = SMMU_CERROR_ILL;
1005 break;
1006 }
1007
1008 if (!mr) {
1009 break;
1010 }
1011
1012 trace_smmuv3_cmdq_cfgi_ste(sid);
1013 sdev = container_of(mr, SMMUDevice, iommu);
1014 smmuv3_flush_config(sdev);
1015
1016 break;
1017 }
1018 case SMMU_CMD_CFGI_STE_RANGE:
1019 {
1020 uint32_t sid = CMD_SID(&cmd), mask;
1021 uint8_t range = CMD_STE_RANGE(&cmd);
1022 SMMUSIDRange sid_range;
1023
1024 if (CMD_SSEC(&cmd)) {
1025 cmd_error = SMMU_CERROR_ILL;
1026 break;
1027 }
1028
1029 mask = (1ULL << (range + 1)) - 1;
1030 sid_range.start = sid & ~mask;
1031 sid_range.end = sid_range.start + mask;
1032
1033 trace_smmuv3_cmdq_cfgi_ste_range(sid_range.start, sid_range.end);
1034 g_hash_table_foreach_remove(bs->configs, smmuv3_invalidate_ste,
1035 &sid_range);
1036 break;
1037 }
1038 case SMMU_CMD_CFGI_CD:
1039 case SMMU_CMD_CFGI_CD_ALL:
1040 {
1041 uint32_t sid = CMD_SID(&cmd);
1042 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
1043 SMMUDevice *sdev;
1044
1045 if (CMD_SSEC(&cmd)) {
1046 cmd_error = SMMU_CERROR_ILL;
1047 break;
1048 }
1049
1050 if (!mr) {
1051 break;
1052 }
1053
1054 trace_smmuv3_cmdq_cfgi_cd(sid);
1055 sdev = container_of(mr, SMMUDevice, iommu);
1056 smmuv3_flush_config(sdev);
1057 break;
1058 }
1059 case SMMU_CMD_TLBI_NH_ASID:
1060 {
1061 uint16_t asid = CMD_ASID(&cmd);
1062
1063 trace_smmuv3_cmdq_tlbi_nh_asid(asid);
1064 smmu_inv_notifiers_all(&s->smmu_state);
1065 smmu_iotlb_inv_asid(bs, asid);
1066 break;
1067 }
1068 case SMMU_CMD_TLBI_NH_ALL:
1069 case SMMU_CMD_TLBI_NSNH_ALL:
1070 trace_smmuv3_cmdq_tlbi_nh();
1071 smmu_inv_notifiers_all(&s->smmu_state);
1072 smmu_iotlb_inv_all(bs);
1073 break;
1074 case SMMU_CMD_TLBI_NH_VAA:
1075 case SMMU_CMD_TLBI_NH_VA:
1076 smmuv3_s1_range_inval(bs, &cmd);
1077 break;
1078 case SMMU_CMD_TLBI_EL3_ALL:
1079 case SMMU_CMD_TLBI_EL3_VA:
1080 case SMMU_CMD_TLBI_EL2_ALL:
1081 case SMMU_CMD_TLBI_EL2_ASID:
1082 case SMMU_CMD_TLBI_EL2_VA:
1083 case SMMU_CMD_TLBI_EL2_VAA:
1084 case SMMU_CMD_TLBI_S12_VMALL:
1085 case SMMU_CMD_TLBI_S2_IPA:
1086 case SMMU_CMD_ATC_INV:
1087 case SMMU_CMD_PRI_RESP:
1088 case SMMU_CMD_RESUME:
1089 case SMMU_CMD_STALL_TERM:
1090 trace_smmuv3_unhandled_cmd(type);
1091 break;
1092 default:
1093 cmd_error = SMMU_CERROR_ILL;
1094 qemu_log_mask(LOG_GUEST_ERROR,
1095 "Illegal command type: %d\n", CMD_TYPE(&cmd));
1096 break;
1097 }
1098 qemu_mutex_unlock(&s->mutex);
1099 if (cmd_error) {
1100 break;
1101 }
1102
1103
1104
1105
1106
1107 queue_cons_incr(q);
1108 }
1109
1110 if (cmd_error) {
1111 trace_smmuv3_cmdq_consume_error(smmu_cmd_string(type), cmd_error);
1112 smmu_write_cmdq_err(s, cmd_error);
1113 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_CMDQ_ERR_MASK);
1114 }
1115
1116 trace_smmuv3_cmdq_consume_out(Q_PROD(q), Q_CONS(q),
1117 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1118
1119 return 0;
1120}
1121
1122static MemTxResult smmu_writell(SMMUv3State *s, hwaddr offset,
1123 uint64_t data, MemTxAttrs attrs)
1124{
1125 switch (offset) {
1126 case A_GERROR_IRQ_CFG0:
1127 s->gerror_irq_cfg0 = data;
1128 return MEMTX_OK;
1129 case A_STRTAB_BASE:
1130 s->strtab_base = data;
1131 return MEMTX_OK;
1132 case A_CMDQ_BASE:
1133 s->cmdq.base = data;
1134 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1135 if (s->cmdq.log2size > SMMU_CMDQS) {
1136 s->cmdq.log2size = SMMU_CMDQS;
1137 }
1138 return MEMTX_OK;
1139 case A_EVENTQ_BASE:
1140 s->eventq.base = data;
1141 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1142 if (s->eventq.log2size > SMMU_EVENTQS) {
1143 s->eventq.log2size = SMMU_EVENTQS;
1144 }
1145 return MEMTX_OK;
1146 case A_EVENTQ_IRQ_CFG0:
1147 s->eventq_irq_cfg0 = data;
1148 return MEMTX_OK;
1149 default:
1150 qemu_log_mask(LOG_UNIMP,
1151 "%s Unexpected 64-bit access to 0x%"PRIx64" (WI)\n",
1152 __func__, offset);
1153 return MEMTX_OK;
1154 }
1155}
1156
1157static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
1158 uint64_t data, MemTxAttrs attrs)
1159{
1160 switch (offset) {
1161 case A_CR0:
1162 s->cr[0] = data;
1163 s->cr0ack = data & ~SMMU_CR0_RESERVED;
1164
1165 smmuv3_cmdq_consume(s);
1166 return MEMTX_OK;
1167 case A_CR1:
1168 s->cr[1] = data;
1169 return MEMTX_OK;
1170 case A_CR2:
1171 s->cr[2] = data;
1172 return MEMTX_OK;
1173 case A_IRQ_CTRL:
1174 s->irq_ctrl = data;
1175 return MEMTX_OK;
1176 case A_GERRORN:
1177 smmuv3_write_gerrorn(s, data);
1178
1179
1180
1181
1182 smmuv3_cmdq_consume(s);
1183 return MEMTX_OK;
1184 case A_GERROR_IRQ_CFG0:
1185 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 0, 32, data);
1186 return MEMTX_OK;
1187 case A_GERROR_IRQ_CFG0 + 4:
1188 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 32, 32, data);
1189 return MEMTX_OK;
1190 case A_GERROR_IRQ_CFG1:
1191 s->gerror_irq_cfg1 = data;
1192 return MEMTX_OK;
1193 case A_GERROR_IRQ_CFG2:
1194 s->gerror_irq_cfg2 = data;
1195 return MEMTX_OK;
1196 case A_STRTAB_BASE:
1197 s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
1198 return MEMTX_OK;
1199 case A_STRTAB_BASE + 4:
1200 s->strtab_base = deposit64(s->strtab_base, 32, 32, data);
1201 return MEMTX_OK;
1202 case A_STRTAB_BASE_CFG:
1203 s->strtab_base_cfg = data;
1204 if (FIELD_EX32(data, STRTAB_BASE_CFG, FMT) == 1) {
1205 s->sid_split = FIELD_EX32(data, STRTAB_BASE_CFG, SPLIT);
1206 s->features |= SMMU_FEATURE_2LVL_STE;
1207 }
1208 return MEMTX_OK;
1209 case A_CMDQ_BASE:
1210 s->cmdq.base = deposit64(s->cmdq.base, 0, 32, data);
1211 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1212 if (s->cmdq.log2size > SMMU_CMDQS) {
1213 s->cmdq.log2size = SMMU_CMDQS;
1214 }
1215 return MEMTX_OK;
1216 case A_CMDQ_BASE + 4:
1217 s->cmdq.base = deposit64(s->cmdq.base, 32, 32, data);
1218 return MEMTX_OK;
1219 case A_CMDQ_PROD:
1220 s->cmdq.prod = data;
1221 smmuv3_cmdq_consume(s);
1222 return MEMTX_OK;
1223 case A_CMDQ_CONS:
1224 s->cmdq.cons = data;
1225 return MEMTX_OK;
1226 case A_EVENTQ_BASE:
1227 s->eventq.base = deposit64(s->eventq.base, 0, 32, data);
1228 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1229 if (s->eventq.log2size > SMMU_EVENTQS) {
1230 s->eventq.log2size = SMMU_EVENTQS;
1231 }
1232 return MEMTX_OK;
1233 case A_EVENTQ_BASE + 4:
1234 s->eventq.base = deposit64(s->eventq.base, 32, 32, data);
1235 return MEMTX_OK;
1236 case A_EVENTQ_PROD:
1237 s->eventq.prod = data;
1238 return MEMTX_OK;
1239 case A_EVENTQ_CONS:
1240 s->eventq.cons = data;
1241 return MEMTX_OK;
1242 case A_EVENTQ_IRQ_CFG0:
1243 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 0, 32, data);
1244 return MEMTX_OK;
1245 case A_EVENTQ_IRQ_CFG0 + 4:
1246 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 32, 32, data);
1247 return MEMTX_OK;
1248 case A_EVENTQ_IRQ_CFG1:
1249 s->eventq_irq_cfg1 = data;
1250 return MEMTX_OK;
1251 case A_EVENTQ_IRQ_CFG2:
1252 s->eventq_irq_cfg2 = data;
1253 return MEMTX_OK;
1254 default:
1255 qemu_log_mask(LOG_UNIMP,
1256 "%s Unexpected 32-bit access to 0x%"PRIx64" (WI)\n",
1257 __func__, offset);
1258 return MEMTX_OK;
1259 }
1260}
1261
1262static MemTxResult smmu_write_mmio(void *opaque, hwaddr offset, uint64_t data,
1263 unsigned size, MemTxAttrs attrs)
1264{
1265 SMMUState *sys = opaque;
1266 SMMUv3State *s = ARM_SMMUV3(sys);
1267 MemTxResult r;
1268
1269
1270 offset &= ~0x10000;
1271
1272 switch (size) {
1273 case 8:
1274 r = smmu_writell(s, offset, data, attrs);
1275 break;
1276 case 4:
1277 r = smmu_writel(s, offset, data, attrs);
1278 break;
1279 default:
1280 r = MEMTX_ERROR;
1281 break;
1282 }
1283
1284 trace_smmuv3_write_mmio(offset, data, size, r);
1285 return r;
1286}
1287
1288static MemTxResult smmu_readll(SMMUv3State *s, hwaddr offset,
1289 uint64_t *data, MemTxAttrs attrs)
1290{
1291 switch (offset) {
1292 case A_GERROR_IRQ_CFG0:
1293 *data = s->gerror_irq_cfg0;
1294 return MEMTX_OK;
1295 case A_STRTAB_BASE:
1296 *data = s->strtab_base;
1297 return MEMTX_OK;
1298 case A_CMDQ_BASE:
1299 *data = s->cmdq.base;
1300 return MEMTX_OK;
1301 case A_EVENTQ_BASE:
1302 *data = s->eventq.base;
1303 return MEMTX_OK;
1304 default:
1305 *data = 0;
1306 qemu_log_mask(LOG_UNIMP,
1307 "%s Unexpected 64-bit access to 0x%"PRIx64" (RAZ)\n",
1308 __func__, offset);
1309 return MEMTX_OK;
1310 }
1311}
1312
1313static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
1314 uint64_t *data, MemTxAttrs attrs)
1315{
1316 switch (offset) {
1317 case A_IDREGS ... A_IDREGS + 0x2f:
1318 *data = smmuv3_idreg(offset - A_IDREGS);
1319 return MEMTX_OK;
1320 case A_IDR0 ... A_IDR5:
1321 *data = s->idr[(offset - A_IDR0) / 4];
1322 return MEMTX_OK;
1323 case A_IIDR:
1324 *data = s->iidr;
1325 return MEMTX_OK;
1326 case A_AIDR:
1327 *data = s->aidr;
1328 return MEMTX_OK;
1329 case A_CR0:
1330 *data = s->cr[0];
1331 return MEMTX_OK;
1332 case A_CR0ACK:
1333 *data = s->cr0ack;
1334 return MEMTX_OK;
1335 case A_CR1:
1336 *data = s->cr[1];
1337 return MEMTX_OK;
1338 case A_CR2:
1339 *data = s->cr[2];
1340 return MEMTX_OK;
1341 case A_STATUSR:
1342 *data = s->statusr;
1343 return MEMTX_OK;
1344 case A_IRQ_CTRL:
1345 case A_IRQ_CTRL_ACK:
1346 *data = s->irq_ctrl;
1347 return MEMTX_OK;
1348 case A_GERROR:
1349 *data = s->gerror;
1350 return MEMTX_OK;
1351 case A_GERRORN:
1352 *data = s->gerrorn;
1353 return MEMTX_OK;
1354 case A_GERROR_IRQ_CFG0:
1355 *data = extract64(s->gerror_irq_cfg0, 0, 32);
1356 return MEMTX_OK;
1357 case A_GERROR_IRQ_CFG0 + 4:
1358 *data = extract64(s->gerror_irq_cfg0, 32, 32);
1359 return MEMTX_OK;
1360 case A_GERROR_IRQ_CFG1:
1361 *data = s->gerror_irq_cfg1;
1362 return MEMTX_OK;
1363 case A_GERROR_IRQ_CFG2:
1364 *data = s->gerror_irq_cfg2;
1365 return MEMTX_OK;
1366 case A_STRTAB_BASE:
1367 *data = extract64(s->strtab_base, 0, 32);
1368 return MEMTX_OK;
1369 case A_STRTAB_BASE + 4:
1370 *data = extract64(s->strtab_base, 32, 32);
1371 return MEMTX_OK;
1372 case A_STRTAB_BASE_CFG:
1373 *data = s->strtab_base_cfg;
1374 return MEMTX_OK;
1375 case A_CMDQ_BASE:
1376 *data = extract64(s->cmdq.base, 0, 32);
1377 return MEMTX_OK;
1378 case A_CMDQ_BASE + 4:
1379 *data = extract64(s->cmdq.base, 32, 32);
1380 return MEMTX_OK;
1381 case A_CMDQ_PROD:
1382 *data = s->cmdq.prod;
1383 return MEMTX_OK;
1384 case A_CMDQ_CONS:
1385 *data = s->cmdq.cons;
1386 return MEMTX_OK;
1387 case A_EVENTQ_BASE:
1388 *data = extract64(s->eventq.base, 0, 32);
1389 return MEMTX_OK;
1390 case A_EVENTQ_BASE + 4:
1391 *data = extract64(s->eventq.base, 32, 32);
1392 return MEMTX_OK;
1393 case A_EVENTQ_PROD:
1394 *data = s->eventq.prod;
1395 return MEMTX_OK;
1396 case A_EVENTQ_CONS:
1397 *data = s->eventq.cons;
1398 return MEMTX_OK;
1399 default:
1400 *data = 0;
1401 qemu_log_mask(LOG_UNIMP,
1402 "%s unhandled 32-bit access at 0x%"PRIx64" (RAZ)\n",
1403 __func__, offset);
1404 return MEMTX_OK;
1405 }
1406}
1407
1408static MemTxResult smmu_read_mmio(void *opaque, hwaddr offset, uint64_t *data,
1409 unsigned size, MemTxAttrs attrs)
1410{
1411 SMMUState *sys = opaque;
1412 SMMUv3State *s = ARM_SMMUV3(sys);
1413 MemTxResult r;
1414
1415
1416 offset &= ~0x10000;
1417
1418 switch (size) {
1419 case 8:
1420 r = smmu_readll(s, offset, data, attrs);
1421 break;
1422 case 4:
1423 r = smmu_readl(s, offset, data, attrs);
1424 break;
1425 default:
1426 r = MEMTX_ERROR;
1427 break;
1428 }
1429
1430 trace_smmuv3_read_mmio(offset, *data, size, r);
1431 return r;
1432}
1433
1434static const MemoryRegionOps smmu_mem_ops = {
1435 .read_with_attrs = smmu_read_mmio,
1436 .write_with_attrs = smmu_write_mmio,
1437 .endianness = DEVICE_LITTLE_ENDIAN,
1438 .valid = {
1439 .min_access_size = 4,
1440 .max_access_size = 8,
1441 },
1442 .impl = {
1443 .min_access_size = 4,
1444 .max_access_size = 8,
1445 },
1446};
1447
1448static void smmu_init_irq(SMMUv3State *s, SysBusDevice *dev)
1449{
1450 int i;
1451
1452 for (i = 0; i < ARRAY_SIZE(s->irq); i++) {
1453 sysbus_init_irq(dev, &s->irq[i]);
1454 }
1455}
1456
1457static void smmu_reset(DeviceState *dev)
1458{
1459 SMMUv3State *s = ARM_SMMUV3(dev);
1460 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1461
1462 c->parent_reset(dev);
1463
1464 smmuv3_init_regs(s);
1465}
1466
1467static void smmu_realize(DeviceState *d, Error **errp)
1468{
1469 SMMUState *sys = ARM_SMMU(d);
1470 SMMUv3State *s = ARM_SMMUV3(sys);
1471 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1472 SysBusDevice *dev = SYS_BUS_DEVICE(d);
1473 Error *local_err = NULL;
1474
1475 c->parent_realize(d, &local_err);
1476 if (local_err) {
1477 error_propagate(errp, local_err);
1478 return;
1479 }
1480
1481 qemu_mutex_init(&s->mutex);
1482
1483 memory_region_init_io(&sys->iomem, OBJECT(s),
1484 &smmu_mem_ops, sys, TYPE_ARM_SMMUV3, 0x20000);
1485
1486 sys->mrtypename = TYPE_SMMUV3_IOMMU_MEMORY_REGION;
1487
1488 sysbus_init_mmio(dev, &sys->iomem);
1489
1490 smmu_init_irq(s, dev);
1491}
1492
1493static const VMStateDescription vmstate_smmuv3_queue = {
1494 .name = "smmuv3_queue",
1495 .version_id = 1,
1496 .minimum_version_id = 1,
1497 .fields = (VMStateField[]) {
1498 VMSTATE_UINT64(base, SMMUQueue),
1499 VMSTATE_UINT32(prod, SMMUQueue),
1500 VMSTATE_UINT32(cons, SMMUQueue),
1501 VMSTATE_UINT8(log2size, SMMUQueue),
1502 VMSTATE_END_OF_LIST(),
1503 },
1504};
1505
1506static const VMStateDescription vmstate_smmuv3 = {
1507 .name = "smmuv3",
1508 .version_id = 1,
1509 .minimum_version_id = 1,
1510 .priority = MIG_PRI_IOMMU,
1511 .fields = (VMStateField[]) {
1512 VMSTATE_UINT32(features, SMMUv3State),
1513 VMSTATE_UINT8(sid_size, SMMUv3State),
1514 VMSTATE_UINT8(sid_split, SMMUv3State),
1515
1516 VMSTATE_UINT32_ARRAY(cr, SMMUv3State, 3),
1517 VMSTATE_UINT32(cr0ack, SMMUv3State),
1518 VMSTATE_UINT32(statusr, SMMUv3State),
1519 VMSTATE_UINT32(irq_ctrl, SMMUv3State),
1520 VMSTATE_UINT32(gerror, SMMUv3State),
1521 VMSTATE_UINT32(gerrorn, SMMUv3State),
1522 VMSTATE_UINT64(gerror_irq_cfg0, SMMUv3State),
1523 VMSTATE_UINT32(gerror_irq_cfg1, SMMUv3State),
1524 VMSTATE_UINT32(gerror_irq_cfg2, SMMUv3State),
1525 VMSTATE_UINT64(strtab_base, SMMUv3State),
1526 VMSTATE_UINT32(strtab_base_cfg, SMMUv3State),
1527 VMSTATE_UINT64(eventq_irq_cfg0, SMMUv3State),
1528 VMSTATE_UINT32(eventq_irq_cfg1, SMMUv3State),
1529 VMSTATE_UINT32(eventq_irq_cfg2, SMMUv3State),
1530
1531 VMSTATE_STRUCT(cmdq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1532 VMSTATE_STRUCT(eventq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1533
1534 VMSTATE_END_OF_LIST(),
1535 },
1536};
1537
1538static void smmuv3_instance_init(Object *obj)
1539{
1540
1541}
1542
1543static void smmuv3_class_init(ObjectClass *klass, void *data)
1544{
1545 DeviceClass *dc = DEVICE_CLASS(klass);
1546 SMMUv3Class *c = ARM_SMMUV3_CLASS(klass);
1547
1548 dc->vmsd = &vmstate_smmuv3;
1549 device_class_set_parent_reset(dc, smmu_reset, &c->parent_reset);
1550 c->parent_realize = dc->realize;
1551 dc->realize = smmu_realize;
1552}
1553
1554static int smmuv3_notify_flag_changed(IOMMUMemoryRegion *iommu,
1555 IOMMUNotifierFlag old,
1556 IOMMUNotifierFlag new,
1557 Error **errp)
1558{
1559 SMMUDevice *sdev = container_of(iommu, SMMUDevice, iommu);
1560 SMMUv3State *s3 = sdev->smmu;
1561 SMMUState *s = &(s3->smmu_state);
1562
1563 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
1564 error_setg(errp, "SMMUv3 does not support dev-iotlb yet");
1565 return -EINVAL;
1566 }
1567
1568 if (new & IOMMU_NOTIFIER_MAP) {
1569 error_setg(errp,
1570 "device %02x.%02x.%x requires iommu MAP notifier which is "
1571 "not currently supported", pci_bus_num(sdev->bus),
1572 PCI_SLOT(sdev->devfn), PCI_FUNC(sdev->devfn));
1573 return -EINVAL;
1574 }
1575
1576 if (old == IOMMU_NOTIFIER_NONE) {
1577 trace_smmuv3_notify_flag_add(iommu->parent_obj.name);
1578 QLIST_INSERT_HEAD(&s->devices_with_notifiers, sdev, next);
1579 } else if (new == IOMMU_NOTIFIER_NONE) {
1580 trace_smmuv3_notify_flag_del(iommu->parent_obj.name);
1581 QLIST_REMOVE(sdev, next);
1582 }
1583 return 0;
1584}
1585
1586static void smmuv3_iommu_memory_region_class_init(ObjectClass *klass,
1587 void *data)
1588{
1589 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1590
1591 imrc->translate = smmuv3_translate;
1592 imrc->notify_flag_changed = smmuv3_notify_flag_changed;
1593}
1594
1595static const TypeInfo smmuv3_type_info = {
1596 .name = TYPE_ARM_SMMUV3,
1597 .parent = TYPE_ARM_SMMU,
1598 .instance_size = sizeof(SMMUv3State),
1599 .instance_init = smmuv3_instance_init,
1600 .class_size = sizeof(SMMUv3Class),
1601 .class_init = smmuv3_class_init,
1602};
1603
1604static const TypeInfo smmuv3_iommu_memory_region_info = {
1605 .parent = TYPE_IOMMU_MEMORY_REGION,
1606 .name = TYPE_SMMUV3_IOMMU_MEMORY_REGION,
1607 .class_init = smmuv3_iommu_memory_region_class_init,
1608};
1609
1610static void smmuv3_register_types(void)
1611{
1612 type_register(&smmuv3_type_info);
1613 type_register(&smmuv3_iommu_memory_region_info);
1614}
1615
1616type_init(smmuv3_register_types)
1617
1618
