1
2
3
4
5
6
7
8#ifndef _ARM_SMMU_V3_H
9#define _ARM_SMMU_V3_H
10
11#include <linux/bitfield.h>
12#include <linux/iommu.h>
13#include <linux/kernel.h>
14#include <linux/mmzone.h>
15#include <linux/sizes.h>
16
17
18#define ARM_SMMU_IDR0 0x0
19#define IDR0_ST_LVL GENMASK(28, 27)
20#define IDR0_ST_LVL_2LVL 1
21#define IDR0_STALL_MODEL GENMASK(25, 24)
22#define IDR0_STALL_MODEL_STALL 0
23#define IDR0_STALL_MODEL_FORCE 2
24#define IDR0_TTENDIAN GENMASK(22, 21)
25#define IDR0_TTENDIAN_MIXED 0
26#define IDR0_TTENDIAN_LE 2
27#define IDR0_TTENDIAN_BE 3
28#define IDR0_CD2L (1 << 19)
29#define IDR0_VMID16 (1 << 18)
30#define IDR0_PRI (1 << 16)
31#define IDR0_SEV (1 << 14)
32#define IDR0_MSI (1 << 13)
33#define IDR0_ASID16 (1 << 12)
34#define IDR0_ATS (1 << 10)
35#define IDR0_HYP (1 << 9)
36#define IDR0_COHACC (1 << 4)
37#define IDR0_TTF GENMASK(3, 2)
38#define IDR0_TTF_AARCH64 2
39#define IDR0_TTF_AARCH32_64 3
40#define IDR0_S1P (1 << 1)
41#define IDR0_S2P (1 << 0)
42
43#define ARM_SMMU_IDR1 0x4
44#define IDR1_TABLES_PRESET (1 << 30)
45#define IDR1_QUEUES_PRESET (1 << 29)
46#define IDR1_REL (1 << 28)
47#define IDR1_CMDQS GENMASK(25, 21)
48#define IDR1_EVTQS GENMASK(20, 16)
49#define IDR1_PRIQS GENMASK(15, 11)
50#define IDR1_SSIDSIZE GENMASK(10, 6)
51#define IDR1_SIDSIZE GENMASK(5, 0)
52
53#define ARM_SMMU_IDR3 0xc
54#define IDR3_RIL (1 << 10)
55
56#define ARM_SMMU_IDR5 0x14
57#define IDR5_STALL_MAX GENMASK(31, 16)
58#define IDR5_GRAN64K (1 << 6)
59#define IDR5_GRAN16K (1 << 5)
60#define IDR5_GRAN4K (1 << 4)
61#define IDR5_OAS GENMASK(2, 0)
62#define IDR5_OAS_32_BIT 0
63#define IDR5_OAS_36_BIT 1
64#define IDR5_OAS_40_BIT 2
65#define IDR5_OAS_42_BIT 3
66#define IDR5_OAS_44_BIT 4
67#define IDR5_OAS_48_BIT 5
68#define IDR5_OAS_52_BIT 6
69#define IDR5_VAX GENMASK(11, 10)
70#define IDR5_VAX_52_BIT 1
71
72#define ARM_SMMU_CR0 0x20
73#define CR0_ATSCHK (1 << 4)
74#define CR0_CMDQEN (1 << 3)
75#define CR0_EVTQEN (1 << 2)
76#define CR0_PRIQEN (1 << 1)
77#define CR0_SMMUEN (1 << 0)
78
79#define ARM_SMMU_CR0ACK 0x24
80
81#define ARM_SMMU_CR1 0x28
82#define CR1_TABLE_SH GENMASK(11, 10)
83#define CR1_TABLE_OC GENMASK(9, 8)
84#define CR1_TABLE_IC GENMASK(7, 6)
85#define CR1_QUEUE_SH GENMASK(5, 4)
86#define CR1_QUEUE_OC GENMASK(3, 2)
87#define CR1_QUEUE_IC GENMASK(1, 0)
88
89#define CR1_CACHE_NC 0
90#define CR1_CACHE_WB 1
91#define CR1_CACHE_WT 2
92
93#define ARM_SMMU_CR2 0x2c
94#define CR2_PTM (1 << 2)
95#define CR2_RECINVSID (1 << 1)
96#define CR2_E2H (1 << 0)
97
98#define ARM_SMMU_GBPA 0x44
99#define GBPA_UPDATE (1 << 31)
100#define GBPA_ABORT (1 << 20)
101
102#define ARM_SMMU_IRQ_CTRL 0x50
103#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
104#define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
105#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
106
107#define ARM_SMMU_IRQ_CTRLACK 0x54
108
109#define ARM_SMMU_GERROR 0x60
110#define GERROR_SFM_ERR (1 << 8)
111#define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
112#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
113#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
114#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
115#define GERROR_PRIQ_ABT_ERR (1 << 3)
116#define GERROR_EVTQ_ABT_ERR (1 << 2)
117#define GERROR_CMDQ_ERR (1 << 0)
118#define GERROR_ERR_MASK 0x1fd
119
120#define ARM_SMMU_GERRORN 0x64
121
122#define ARM_SMMU_GERROR_IRQ_CFG0 0x68
123#define ARM_SMMU_GERROR_IRQ_CFG1 0x70
124#define ARM_SMMU_GERROR_IRQ_CFG2 0x74
125
126#define ARM_SMMU_STRTAB_BASE 0x80
127#define STRTAB_BASE_RA (1UL << 62)
128#define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
129
130#define ARM_SMMU_STRTAB_BASE_CFG 0x88
131#define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
132#define STRTAB_BASE_CFG_FMT_LINEAR 0
133#define STRTAB_BASE_CFG_FMT_2LVL 1
134#define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
135#define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
136
137#define ARM_SMMU_CMDQ_BASE 0x90
138#define ARM_SMMU_CMDQ_PROD 0x98
139#define ARM_SMMU_CMDQ_CONS 0x9c
140
141#define ARM_SMMU_EVTQ_BASE 0xa0
142#define ARM_SMMU_EVTQ_PROD 0xa8
143#define ARM_SMMU_EVTQ_CONS 0xac
144#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
145#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
146#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
147
148#define ARM_SMMU_PRIQ_BASE 0xc0
149#define ARM_SMMU_PRIQ_PROD 0xc8
150#define ARM_SMMU_PRIQ_CONS 0xcc
151#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
152#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
153#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
154
155#define ARM_SMMU_REG_SZ 0xe00
156
157
158#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
159#define MSI_CFG2_SH GENMASK(5, 4)
160#define MSI_CFG2_MEMATTR GENMASK(3, 0)
161
162
163#define ARM_SMMU_SH_NSH 0
164#define ARM_SMMU_SH_OSH 2
165#define ARM_SMMU_SH_ISH 3
166#define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
167#define ARM_SMMU_MEMATTR_OIWB 0xf
168
169#define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
170#define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
171#define Q_OVERFLOW_FLAG (1U << 31)
172#define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
173#define Q_ENT(q, p) ((q)->base + \
174 Q_IDX(&((q)->llq), p) * \
175 (q)->ent_dwords)
176
177#define Q_BASE_RWA (1UL << 62)
178#define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
179#define Q_BASE_LOG2SIZE GENMASK(4, 0)
180
181
182#ifdef CONFIG_CMA_ALIGNMENT
183#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
184#else
185#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
186#endif
187
188
189
190
191
192
193
194
195#define STRTAB_L1_SZ_SHIFT 20
196#define STRTAB_SPLIT 8
197
198#define STRTAB_L1_DESC_DWORDS 1
199#define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
200#define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
201
202#define STRTAB_STE_DWORDS 8
203#define STRTAB_STE_0_V (1UL << 0)
204#define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
205#define STRTAB_STE_0_CFG_ABORT 0
206#define STRTAB_STE_0_CFG_BYPASS 4
207#define STRTAB_STE_0_CFG_S1_TRANS 5
208#define STRTAB_STE_0_CFG_S2_TRANS 6
209
210#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
211#define STRTAB_STE_0_S1FMT_LINEAR 0
212#define STRTAB_STE_0_S1FMT_64K_L2 2
213#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
214#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
215
216#define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
217#define STRTAB_STE_1_S1DSS_TERMINATE 0x0
218#define STRTAB_STE_1_S1DSS_BYPASS 0x1
219#define STRTAB_STE_1_S1DSS_SSID0 0x2
220
221#define STRTAB_STE_1_S1C_CACHE_NC 0UL
222#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
223#define STRTAB_STE_1_S1C_CACHE_WT 2UL
224#define STRTAB_STE_1_S1C_CACHE_WB 3UL
225#define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
226#define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
227#define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
228
229#define STRTAB_STE_1_S1STALLD (1UL << 27)
230
231#define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
232#define STRTAB_STE_1_EATS_ABT 0UL
233#define STRTAB_STE_1_EATS_TRANS 1UL
234#define STRTAB_STE_1_EATS_S1CHK 2UL
235
236#define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
237#define STRTAB_STE_1_STRW_NSEL1 0UL
238#define STRTAB_STE_1_STRW_EL2 2UL
239
240#define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
241#define STRTAB_STE_1_SHCFG_INCOMING 1UL
242
243#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
244#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
245#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
246#define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
247#define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
248#define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
249#define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
250#define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
251#define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
252#define STRTAB_STE_2_S2AA64 (1UL << 51)
253#define STRTAB_STE_2_S2ENDI (1UL << 52)
254#define STRTAB_STE_2_S2PTW (1UL << 54)
255#define STRTAB_STE_2_S2R (1UL << 58)
256
257#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
258
259
260
261
262
263
264
265
266#define CTXDESC_SPLIT 10
267#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
268
269#define CTXDESC_L1_DESC_DWORDS 1
270#define CTXDESC_L1_DESC_V (1UL << 0)
271#define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
272
273#define CTXDESC_CD_DWORDS 8
274#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
275#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
276#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
277#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
278#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
279#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
280#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
281
282#define CTXDESC_CD_0_ENDI (1UL << 15)
283#define CTXDESC_CD_0_V (1UL << 31)
284
285#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
286#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
287
288#define CTXDESC_CD_0_AA64 (1UL << 41)
289#define CTXDESC_CD_0_S (1UL << 44)
290#define CTXDESC_CD_0_R (1UL << 45)
291#define CTXDESC_CD_0_A (1UL << 46)
292#define CTXDESC_CD_0_ASET (1UL << 47)
293#define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
294
295#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
296
297
298
299
300
301#define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
302
303
304#define CMDQ_ENT_SZ_SHIFT 4
305#define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
306#define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
307
308#define CMDQ_CONS_ERR GENMASK(30, 24)
309#define CMDQ_ERR_CERROR_NONE_IDX 0
310#define CMDQ_ERR_CERROR_ILL_IDX 1
311#define CMDQ_ERR_CERROR_ABT_IDX 2
312#define CMDQ_ERR_CERROR_ATC_INV_IDX 3
313
314#define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
315
316
317
318
319
320
321#define CMDQ_BATCH_ENTRIES BITS_PER_LONG
322
323#define CMDQ_0_OP GENMASK_ULL(7, 0)
324#define CMDQ_0_SSV (1UL << 11)
325
326#define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
327#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
328#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
329
330#define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
331#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
332#define CMDQ_CFGI_1_LEAF (1UL << 0)
333#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
334
335#define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
336#define CMDQ_TLBI_RANGE_NUM_MAX 31
337#define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
338#define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
339#define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
340#define CMDQ_TLBI_1_LEAF (1UL << 0)
341#define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8)
342#define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10)
343#define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
344#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
345
346#define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
347#define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
348#define CMDQ_ATC_0_GLOBAL (1UL << 9)
349#define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
350#define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
351
352#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
353#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
354#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
355#define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
356
357#define CMDQ_RESUME_0_RESP_TERM 0UL
358#define CMDQ_RESUME_0_RESP_RETRY 1UL
359#define CMDQ_RESUME_0_RESP_ABORT 2UL
360#define CMDQ_RESUME_0_RESP GENMASK_ULL(13, 12)
361#define CMDQ_RESUME_0_SID GENMASK_ULL(63, 32)
362#define CMDQ_RESUME_1_STAG GENMASK_ULL(15, 0)
363
364#define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
365#define CMDQ_SYNC_0_CS_NONE 0
366#define CMDQ_SYNC_0_CS_IRQ 1
367#define CMDQ_SYNC_0_CS_SEV 2
368#define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
369#define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
370#define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
371#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
372
373
374#define EVTQ_ENT_SZ_SHIFT 5
375#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
376#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
377
378#define EVTQ_0_ID GENMASK_ULL(7, 0)
379
380#define EVT_ID_TRANSLATION_FAULT 0x10
381#define EVT_ID_ADDR_SIZE_FAULT 0x11
382#define EVT_ID_ACCESS_FAULT 0x12
383#define EVT_ID_PERMISSION_FAULT 0x13
384
385#define EVTQ_0_SSV (1UL << 11)
386#define EVTQ_0_SSID GENMASK_ULL(31, 12)
387#define EVTQ_0_SID GENMASK_ULL(63, 32)
388#define EVTQ_1_STAG GENMASK_ULL(15, 0)
389#define EVTQ_1_STALL (1UL << 31)
390#define EVTQ_1_PnU (1UL << 33)
391#define EVTQ_1_InD (1UL << 34)
392#define EVTQ_1_RnW (1UL << 35)
393#define EVTQ_1_S2 (1UL << 39)
394#define EVTQ_1_CLASS GENMASK_ULL(41, 40)
395#define EVTQ_1_TT_READ (1UL << 44)
396#define EVTQ_2_ADDR GENMASK_ULL(63, 0)
397#define EVTQ_3_IPA GENMASK_ULL(51, 12)
398
399
400#define PRIQ_ENT_SZ_SHIFT 4
401#define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
402#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
403
404#define PRIQ_0_SID GENMASK_ULL(31, 0)
405#define PRIQ_0_SSID GENMASK_ULL(51, 32)
406#define PRIQ_0_PERM_PRIV (1UL << 58)
407#define PRIQ_0_PERM_EXEC (1UL << 59)
408#define PRIQ_0_PERM_READ (1UL << 60)
409#define PRIQ_0_PERM_WRITE (1UL << 61)
410#define PRIQ_0_PRG_LAST (1UL << 62)
411#define PRIQ_0_SSID_V (1UL << 63)
412
413#define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
414#define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
415
416
417#define ARM_SMMU_POLL_TIMEOUT_US 1000000
418#define ARM_SMMU_POLL_SPIN_COUNT 10
419
420#define MSI_IOVA_BASE 0x8000000
421#define MSI_IOVA_LENGTH 0x100000
422
423enum pri_resp {
424 PRI_RESP_DENY = 0,
425 PRI_RESP_FAIL = 1,
426 PRI_RESP_SUCC = 2,
427};
428
429struct arm_smmu_cmdq_ent {
430
431 u8 opcode;
432 bool substream_valid;
433
434
435 union {
436 #define CMDQ_OP_PREFETCH_CFG 0x1
437 struct {
438 u32 sid;
439 } prefetch;
440
441 #define CMDQ_OP_CFGI_STE 0x3
442 #define CMDQ_OP_CFGI_ALL 0x4
443 #define CMDQ_OP_CFGI_CD 0x5
444 #define CMDQ_OP_CFGI_CD_ALL 0x6
445 struct {
446 u32 sid;
447 u32 ssid;
448 union {
449 bool leaf;
450 u8 span;
451 };
452 } cfgi;
453
454 #define CMDQ_OP_TLBI_NH_ASID 0x11
455 #define CMDQ_OP_TLBI_NH_VA 0x12
456 #define CMDQ_OP_TLBI_EL2_ALL 0x20
457 #define CMDQ_OP_TLBI_EL2_ASID 0x21
458 #define CMDQ_OP_TLBI_EL2_VA 0x22
459 #define CMDQ_OP_TLBI_S12_VMALL 0x28
460 #define CMDQ_OP_TLBI_S2_IPA 0x2a
461 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
462 struct {
463 u8 num;
464 u8 scale;
465 u16 asid;
466 u16 vmid;
467 bool leaf;
468 u8 ttl;
469 u8 tg;
470 u64 addr;
471 } tlbi;
472
473 #define CMDQ_OP_ATC_INV 0x40
474 #define ATC_INV_SIZE_ALL 52
475 struct {
476 u32 sid;
477 u32 ssid;
478 u64 addr;
479 u8 size;
480 bool global;
481 } atc;
482
483 #define CMDQ_OP_PRI_RESP 0x41
484 struct {
485 u32 sid;
486 u32 ssid;
487 u16 grpid;
488 enum pri_resp resp;
489 } pri;
490
491 #define CMDQ_OP_RESUME 0x44
492 struct {
493 u32 sid;
494 u16 stag;
495 u8 resp;
496 } resume;
497
498 #define CMDQ_OP_CMD_SYNC 0x46
499 struct {
500 u64 msiaddr;
501 } sync;
502 };
503};
504
505struct arm_smmu_ll_queue {
506 union {
507 u64 val;
508 struct {
509 u32 prod;
510 u32 cons;
511 };
512 struct {
513 atomic_t prod;
514 atomic_t cons;
515 } atomic;
516 u8 __pad[SMP_CACHE_BYTES];
517 } ____cacheline_aligned_in_smp;
518 u32 max_n_shift;
519};
520
521struct arm_smmu_queue {
522 struct arm_smmu_ll_queue llq;
523 int irq;
524
525 __le64 *base;
526 dma_addr_t base_dma;
527 u64 q_base;
528
529 size_t ent_dwords;
530
531 u32 __iomem *prod_reg;
532 u32 __iomem *cons_reg;
533};
534
535struct arm_smmu_queue_poll {
536 ktime_t timeout;
537 unsigned int delay;
538 unsigned int spin_cnt;
539 bool wfe;
540};
541
542struct arm_smmu_cmdq {
543 struct arm_smmu_queue q;
544 atomic_long_t *valid_map;
545 atomic_t owner_prod;
546 atomic_t lock;
547};
548
549struct arm_smmu_cmdq_batch {
550 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
551 int num;
552};
553
554struct arm_smmu_evtq {
555 struct arm_smmu_queue q;
556 struct iopf_queue *iopf;
557 u32 max_stalls;
558};
559
560struct arm_smmu_priq {
561 struct arm_smmu_queue q;
562};
563
564
565struct arm_smmu_strtab_l1_desc {
566 u8 span;
567
568 __le64 *l2ptr;
569 dma_addr_t l2ptr_dma;
570};
571
572struct arm_smmu_ctx_desc {
573 u16 asid;
574 u64 ttbr;
575 u64 tcr;
576 u64 mair;
577
578 refcount_t refs;
579 struct mm_struct *mm;
580};
581
582struct arm_smmu_l1_ctx_desc {
583 __le64 *l2ptr;
584 dma_addr_t l2ptr_dma;
585};
586
587struct arm_smmu_ctx_desc_cfg {
588 __le64 *cdtab;
589 dma_addr_t cdtab_dma;
590 struct arm_smmu_l1_ctx_desc *l1_desc;
591 unsigned int num_l1_ents;
592};
593
594struct arm_smmu_s1_cfg {
595 struct arm_smmu_ctx_desc_cfg cdcfg;
596 struct arm_smmu_ctx_desc cd;
597 u8 s1fmt;
598 u8 s1cdmax;
599};
600
601struct arm_smmu_s2_cfg {
602 u16 vmid;
603 u64 vttbr;
604 u64 vtcr;
605};
606
607struct arm_smmu_strtab_cfg {
608 __le64 *strtab;
609 dma_addr_t strtab_dma;
610 struct arm_smmu_strtab_l1_desc *l1_desc;
611 unsigned int num_l1_ents;
612
613 u64 strtab_base;
614 u32 strtab_base_cfg;
615};
616
617
618struct arm_smmu_device {
619 struct device *dev;
620 void __iomem *base;
621 void __iomem *page1;
622
623#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
624#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
625#define ARM_SMMU_FEAT_TT_LE (1 << 2)
626#define ARM_SMMU_FEAT_TT_BE (1 << 3)
627#define ARM_SMMU_FEAT_PRI (1 << 4)
628#define ARM_SMMU_FEAT_ATS (1 << 5)
629#define ARM_SMMU_FEAT_SEV (1 << 6)
630#define ARM_SMMU_FEAT_MSI (1 << 7)
631#define ARM_SMMU_FEAT_COHERENCY (1 << 8)
632#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
633#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
634#define ARM_SMMU_FEAT_STALLS (1 << 11)
635#define ARM_SMMU_FEAT_HYP (1 << 12)
636#define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
637#define ARM_SMMU_FEAT_VAX (1 << 14)
638#define ARM_SMMU_FEAT_RANGE_INV (1 << 15)
639#define ARM_SMMU_FEAT_BTM (1 << 16)
640#define ARM_SMMU_FEAT_SVA (1 << 17)
641#define ARM_SMMU_FEAT_E2H (1 << 18)
642 u32 features;
643
644#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
645#define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
646#define ARM_SMMU_OPT_MSIPOLL (1 << 2)
647 u32 options;
648
649 struct arm_smmu_cmdq cmdq;
650 struct arm_smmu_evtq evtq;
651 struct arm_smmu_priq priq;
652
653 int gerr_irq;
654 int combined_irq;
655
656 unsigned long ias;
657 unsigned long oas;
658 unsigned long pgsize_bitmap;
659
660#define ARM_SMMU_MAX_ASIDS (1 << 16)
661 unsigned int asid_bits;
662
663#define ARM_SMMU_MAX_VMIDS (1 << 16)
664 unsigned int vmid_bits;
665 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
666
667 unsigned int ssid_bits;
668 unsigned int sid_bits;
669
670 struct arm_smmu_strtab_cfg strtab_cfg;
671
672
673 struct iommu_device iommu;
674
675 struct rb_root streams;
676 struct mutex streams_mutex;
677};
678
679struct arm_smmu_stream {
680 u32 id;
681 struct arm_smmu_master *master;
682 struct rb_node node;
683};
684
685
686struct arm_smmu_master {
687 struct arm_smmu_device *smmu;
688 struct device *dev;
689 struct arm_smmu_domain *domain;
690 struct list_head domain_head;
691 struct arm_smmu_stream *streams;
692 unsigned int num_streams;
693 bool ats_enabled;
694 bool stall_enabled;
695 bool sva_enabled;
696 bool iopf_enabled;
697 struct list_head bonds;
698 unsigned int ssid_bits;
699};
700
701
702enum arm_smmu_domain_stage {
703 ARM_SMMU_DOMAIN_S1 = 0,
704 ARM_SMMU_DOMAIN_S2,
705 ARM_SMMU_DOMAIN_NESTED,
706 ARM_SMMU_DOMAIN_BYPASS,
707};
708
709struct arm_smmu_domain {
710 struct arm_smmu_device *smmu;
711 struct mutex init_mutex;
712
713 struct io_pgtable_ops *pgtbl_ops;
714 bool stall_enabled;
715 atomic_t nr_ats_masters;
716
717 enum arm_smmu_domain_stage stage;
718 union {
719 struct arm_smmu_s1_cfg s1_cfg;
720 struct arm_smmu_s2_cfg s2_cfg;
721 };
722
723 struct iommu_domain domain;
724
725 struct list_head devices;
726 spinlock_t devices_lock;
727
728 struct list_head mmu_notifiers;
729};
730
731static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
732{
733 return container_of(dom, struct arm_smmu_domain, domain);
734}
735
736extern struct xarray arm_smmu_asid_xa;
737extern struct mutex arm_smmu_asid_lock;
738extern struct arm_smmu_ctx_desc quiet_cd;
739
740int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
741 struct arm_smmu_ctx_desc *cd);
742void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
743void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
744 size_t granule, bool leaf,
745 struct arm_smmu_domain *smmu_domain);
746bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
747int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
748 unsigned long iova, size_t size);
749
750#ifdef CONFIG_ARM_SMMU_V3_SVA
751bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
752bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
753bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
754int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
755int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
756bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
757struct iommu_sva *arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm,
758 void *drvdata);
759void arm_smmu_sva_unbind(struct iommu_sva *handle);
760u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle);
761void arm_smmu_sva_notifier_synchronize(void);
762#else
763static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
764{
765 return false;
766}
767
768static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
769{
770 return false;
771}
772
773static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
774{
775 return false;
776}
777
778static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
779{
780 return -ENODEV;
781}
782
783static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
784{
785 return -ENODEV;
786}
787
788static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
789{
790 return false;
791}
792
793static inline struct iommu_sva *
794arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
795{
796 return ERR_PTR(-ENODEV);
797}
798
799static inline void arm_smmu_sva_unbind(struct iommu_sva *handle) {}
800
801static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
802{
803 return IOMMU_PASID_INVALID;
804}
805
806static inline void arm_smmu_sva_notifier_synchronize(void) {}
807#endif
808#endif
809