1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/acpi.h>
24#include <linux/acpi_iort.h>
25#include <linux/delay.h>
26#include <linux/dma-iommu.h>
27#include <linux/err.h>
28#include <linux/interrupt.h>
29#include <linux/iommu.h>
30#include <linux/iopoll.h>
31#include <linux/module.h>
32#include <linux/msi.h>
33#include <linux/of.h>
34#include <linux/of_address.h>
35#include <linux/of_iommu.h>
36#include <linux/of_platform.h>
37#include <linux/pci.h>
38#include <linux/platform_device.h>
39
40#include <linux/amba/bus.h>
41
42#include "io-pgtable.h"
43
44
45#define ARM_SMMU_IDR0 0x0
46#define IDR0_ST_LVL_SHIFT 27
47#define IDR0_ST_LVL_MASK 0x3
48#define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
49#define IDR0_STALL_MODEL_SHIFT 24
50#define IDR0_STALL_MODEL_MASK 0x3
51#define IDR0_STALL_MODEL_STALL (0 << IDR0_STALL_MODEL_SHIFT)
52#define IDR0_STALL_MODEL_FORCE (2 << IDR0_STALL_MODEL_SHIFT)
53#define IDR0_TTENDIAN_SHIFT 21
54#define IDR0_TTENDIAN_MASK 0x3
55#define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
56#define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT)
57#define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT)
58#define IDR0_CD2L (1 << 19)
59#define IDR0_VMID16 (1 << 18)
60#define IDR0_PRI (1 << 16)
61#define IDR0_SEV (1 << 14)
62#define IDR0_MSI (1 << 13)
63#define IDR0_ASID16 (1 << 12)
64#define IDR0_ATS (1 << 10)
65#define IDR0_HYP (1 << 9)
66#define IDR0_COHACC (1 << 4)
67#define IDR0_TTF_SHIFT 2
68#define IDR0_TTF_MASK 0x3
69#define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT)
70#define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT)
71#define IDR0_S1P (1 << 1)
72#define IDR0_S2P (1 << 0)
73
74#define ARM_SMMU_IDR1 0x4
75#define IDR1_TABLES_PRESET (1 << 30)
76#define IDR1_QUEUES_PRESET (1 << 29)
77#define IDR1_REL (1 << 28)
78#define IDR1_CMDQ_SHIFT 21
79#define IDR1_CMDQ_MASK 0x1f
80#define IDR1_EVTQ_SHIFT 16
81#define IDR1_EVTQ_MASK 0x1f
82#define IDR1_PRIQ_SHIFT 11
83#define IDR1_PRIQ_MASK 0x1f
84#define IDR1_SSID_SHIFT 6
85#define IDR1_SSID_MASK 0x1f
86#define IDR1_SID_SHIFT 0
87#define IDR1_SID_MASK 0x3f
88
89#define ARM_SMMU_IDR5 0x14
90#define IDR5_STALL_MAX_SHIFT 16
91#define IDR5_STALL_MAX_MASK 0xffff
92#define IDR5_GRAN64K (1 << 6)
93#define IDR5_GRAN16K (1 << 5)
94#define IDR5_GRAN4K (1 << 4)
95#define IDR5_OAS_SHIFT 0
96#define IDR5_OAS_MASK 0x7
97#define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT)
98#define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT)
99#define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT)
100#define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT)
101#define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT)
102#define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT)
103
104#define ARM_SMMU_CR0 0x20
105#define CR0_CMDQEN (1 << 3)
106#define CR0_EVTQEN (1 << 2)
107#define CR0_PRIQEN (1 << 1)
108#define CR0_SMMUEN (1 << 0)
109
110#define ARM_SMMU_CR0ACK 0x24
111
112#define ARM_SMMU_CR1 0x28
113#define CR1_SH_NSH 0
114#define CR1_SH_OSH 2
115#define CR1_SH_ISH 3
116#define CR1_CACHE_NC 0
117#define CR1_CACHE_WB 1
118#define CR1_CACHE_WT 2
119#define CR1_TABLE_SH_SHIFT 10
120#define CR1_TABLE_OC_SHIFT 8
121#define CR1_TABLE_IC_SHIFT 6
122#define CR1_QUEUE_SH_SHIFT 4
123#define CR1_QUEUE_OC_SHIFT 2
124#define CR1_QUEUE_IC_SHIFT 0
125
126#define ARM_SMMU_CR2 0x2c
127#define CR2_PTM (1 << 2)
128#define CR2_RECINVSID (1 << 1)
129#define CR2_E2H (1 << 0)
130
131#define ARM_SMMU_GBPA 0x44
132#define GBPA_ABORT (1 << 20)
133#define GBPA_UPDATE (1 << 31)
134
135#define ARM_SMMU_IRQ_CTRL 0x50
136#define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
137#define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
138#define IRQ_CTRL_GERROR_IRQEN (1 << 0)
139
140#define ARM_SMMU_IRQ_CTRLACK 0x54
141
142#define ARM_SMMU_GERROR 0x60
143#define GERROR_SFM_ERR (1 << 8)
144#define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
145#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
146#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
147#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
148#define GERROR_PRIQ_ABT_ERR (1 << 3)
149#define GERROR_EVTQ_ABT_ERR (1 << 2)
150#define GERROR_CMDQ_ERR (1 << 0)
151#define GERROR_ERR_MASK 0xfd
152
153#define ARM_SMMU_GERRORN 0x64
154
155#define ARM_SMMU_GERROR_IRQ_CFG0 0x68
156#define ARM_SMMU_GERROR_IRQ_CFG1 0x70
157#define ARM_SMMU_GERROR_IRQ_CFG2 0x74
158
159#define ARM_SMMU_STRTAB_BASE 0x80
160#define STRTAB_BASE_RA (1UL << 62)
161#define STRTAB_BASE_ADDR_SHIFT 6
162#define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL
163
164#define ARM_SMMU_STRTAB_BASE_CFG 0x88
165#define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0
166#define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f
167#define STRTAB_BASE_CFG_SPLIT_SHIFT 6
168#define STRTAB_BASE_CFG_SPLIT_MASK 0x1f
169#define STRTAB_BASE_CFG_FMT_SHIFT 16
170#define STRTAB_BASE_CFG_FMT_MASK 0x3
171#define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT)
172#define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT)
173
174#define ARM_SMMU_CMDQ_BASE 0x90
175#define ARM_SMMU_CMDQ_PROD 0x98
176#define ARM_SMMU_CMDQ_CONS 0x9c
177
178#define ARM_SMMU_EVTQ_BASE 0xa0
179#define ARM_SMMU_EVTQ_PROD 0x100a8
180#define ARM_SMMU_EVTQ_CONS 0x100ac
181#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
182#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
183#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
184
185#define ARM_SMMU_PRIQ_BASE 0xc0
186#define ARM_SMMU_PRIQ_PROD 0x100c8
187#define ARM_SMMU_PRIQ_CONS 0x100cc
188#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
189#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
190#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
191
192
193#define MSI_CFG0_ADDR_SHIFT 2
194#define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
195#define MSI_CFG2_SH_SHIFT 4
196#define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT)
197#define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT)
198#define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT)
199#define MSI_CFG2_MEMATTR_SHIFT 0
200#define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT)
201
202#define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
203#define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
204#define Q_OVERFLOW_FLAG (1 << 31)
205#define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
206#define Q_ENT(q, p) ((q)->base + \
207 Q_IDX(q, p) * (q)->ent_dwords)
208
209#define Q_BASE_RWA (1UL << 62)
210#define Q_BASE_ADDR_SHIFT 5
211#define Q_BASE_ADDR_MASK 0xfffffffffffUL
212#define Q_BASE_LOG2SIZE_SHIFT 0
213#define Q_BASE_LOG2SIZE_MASK 0x1fUL
214
215
216
217
218
219
220
221
222#define STRTAB_L1_SZ_SHIFT 20
223#define STRTAB_SPLIT 8
224
225#define STRTAB_L1_DESC_DWORDS 1
226#define STRTAB_L1_DESC_SPAN_SHIFT 0
227#define STRTAB_L1_DESC_SPAN_MASK 0x1fUL
228#define STRTAB_L1_DESC_L2PTR_SHIFT 6
229#define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL
230
231#define STRTAB_STE_DWORDS 8
232#define STRTAB_STE_0_V (1UL << 0)
233#define STRTAB_STE_0_CFG_SHIFT 1
234#define STRTAB_STE_0_CFG_MASK 0x7UL
235#define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT)
236#define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT)
237#define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT)
238#define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT)
239
240#define STRTAB_STE_0_S1FMT_SHIFT 4
241#define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT)
242#define STRTAB_STE_0_S1CTXPTR_SHIFT 6
243#define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL
244#define STRTAB_STE_0_S1CDMAX_SHIFT 59
245#define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL
246
247#define STRTAB_STE_1_S1C_CACHE_NC 0UL
248#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
249#define STRTAB_STE_1_S1C_CACHE_WT 2UL
250#define STRTAB_STE_1_S1C_CACHE_WB 3UL
251#define STRTAB_STE_1_S1C_SH_NSH 0UL
252#define STRTAB_STE_1_S1C_SH_OSH 2UL
253#define STRTAB_STE_1_S1C_SH_ISH 3UL
254#define STRTAB_STE_1_S1CIR_SHIFT 2
255#define STRTAB_STE_1_S1COR_SHIFT 4
256#define STRTAB_STE_1_S1CSH_SHIFT 6
257
258#define STRTAB_STE_1_S1STALLD (1UL << 27)
259
260#define STRTAB_STE_1_EATS_ABT 0UL
261#define STRTAB_STE_1_EATS_TRANS 1UL
262#define STRTAB_STE_1_EATS_S1CHK 2UL
263#define STRTAB_STE_1_EATS_SHIFT 28
264
265#define STRTAB_STE_1_STRW_NSEL1 0UL
266#define STRTAB_STE_1_STRW_EL2 2UL
267#define STRTAB_STE_1_STRW_SHIFT 30
268
269#define STRTAB_STE_1_SHCFG_INCOMING 1UL
270#define STRTAB_STE_1_SHCFG_SHIFT 44
271
272#define STRTAB_STE_2_S2VMID_SHIFT 0
273#define STRTAB_STE_2_S2VMID_MASK 0xffffUL
274#define STRTAB_STE_2_VTCR_SHIFT 32
275#define STRTAB_STE_2_VTCR_MASK 0x7ffffUL
276#define STRTAB_STE_2_S2AA64 (1UL << 51)
277#define STRTAB_STE_2_S2ENDI (1UL << 52)
278#define STRTAB_STE_2_S2PTW (1UL << 54)
279#define STRTAB_STE_2_S2R (1UL << 58)
280
281#define STRTAB_STE_3_S2TTB_SHIFT 4
282#define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL
283
284
285#define CTXDESC_CD_DWORDS 8
286#define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0
287#define ARM64_TCR_T0SZ_SHIFT 0
288#define ARM64_TCR_T0SZ_MASK 0x1fUL
289#define CTXDESC_CD_0_TCR_TG0_SHIFT 6
290#define ARM64_TCR_TG0_SHIFT 14
291#define ARM64_TCR_TG0_MASK 0x3UL
292#define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8
293#define ARM64_TCR_IRGN0_SHIFT 8
294#define ARM64_TCR_IRGN0_MASK 0x3UL
295#define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10
296#define ARM64_TCR_ORGN0_SHIFT 10
297#define ARM64_TCR_ORGN0_MASK 0x3UL
298#define CTXDESC_CD_0_TCR_SH0_SHIFT 12
299#define ARM64_TCR_SH0_SHIFT 12
300#define ARM64_TCR_SH0_MASK 0x3UL
301#define CTXDESC_CD_0_TCR_EPD0_SHIFT 14
302#define ARM64_TCR_EPD0_SHIFT 7
303#define ARM64_TCR_EPD0_MASK 0x1UL
304#define CTXDESC_CD_0_TCR_EPD1_SHIFT 30
305#define ARM64_TCR_EPD1_SHIFT 23
306#define ARM64_TCR_EPD1_MASK 0x1UL
307
308#define CTXDESC_CD_0_ENDI (1UL << 15)
309#define CTXDESC_CD_0_V (1UL << 31)
310
311#define CTXDESC_CD_0_TCR_IPS_SHIFT 32
312#define ARM64_TCR_IPS_SHIFT 32
313#define ARM64_TCR_IPS_MASK 0x7UL
314#define CTXDESC_CD_0_TCR_TBI0_SHIFT 38
315#define ARM64_TCR_TBI0_SHIFT 37
316#define ARM64_TCR_TBI0_MASK 0x1UL
317
318#define CTXDESC_CD_0_AA64 (1UL << 41)
319#define CTXDESC_CD_0_R (1UL << 45)
320#define CTXDESC_CD_0_A (1UL << 46)
321#define CTXDESC_CD_0_ASET_SHIFT 47
322#define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT)
323#define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT)
324#define CTXDESC_CD_0_ASID_SHIFT 48
325#define CTXDESC_CD_0_ASID_MASK 0xffffUL
326
327#define CTXDESC_CD_1_TTB0_SHIFT 4
328#define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL
329
330#define CTXDESC_CD_3_MAIR_SHIFT 0
331
332
333#define ARM_SMMU_TCR2CD(tcr, fld) \
334 (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \
335 << CTXDESC_CD_0_TCR_##fld##_SHIFT)
336
337
338#define CMDQ_ENT_DWORDS 2
339#define CMDQ_MAX_SZ_SHIFT 8
340
341#define CMDQ_ERR_SHIFT 24
342#define CMDQ_ERR_MASK 0x7f
343#define CMDQ_ERR_CERROR_NONE_IDX 0
344#define CMDQ_ERR_CERROR_ILL_IDX 1
345#define CMDQ_ERR_CERROR_ABT_IDX 2
346
347#define CMDQ_0_OP_SHIFT 0
348#define CMDQ_0_OP_MASK 0xffUL
349#define CMDQ_0_SSV (1UL << 11)
350
351#define CMDQ_PREFETCH_0_SID_SHIFT 32
352#define CMDQ_PREFETCH_1_SIZE_SHIFT 0
353#define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL
354
355#define CMDQ_CFGI_0_SID_SHIFT 32
356#define CMDQ_CFGI_0_SID_MASK 0xffffffffUL
357#define CMDQ_CFGI_1_LEAF (1UL << 0)
358#define CMDQ_CFGI_1_RANGE_SHIFT 0
359#define CMDQ_CFGI_1_RANGE_MASK 0x1fUL
360
361#define CMDQ_TLBI_0_VMID_SHIFT 32
362#define CMDQ_TLBI_0_ASID_SHIFT 48
363#define CMDQ_TLBI_1_LEAF (1UL << 0)
364#define CMDQ_TLBI_1_VA_MASK ~0xfffUL
365#define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL
366
367#define CMDQ_PRI_0_SSID_SHIFT 12
368#define CMDQ_PRI_0_SSID_MASK 0xfffffUL
369#define CMDQ_PRI_0_SID_SHIFT 32
370#define CMDQ_PRI_0_SID_MASK 0xffffffffUL
371#define CMDQ_PRI_1_GRPID_SHIFT 0
372#define CMDQ_PRI_1_GRPID_MASK 0x1ffUL
373#define CMDQ_PRI_1_RESP_SHIFT 12
374#define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT)
375#define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
376#define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
377
378#define CMDQ_SYNC_0_CS_SHIFT 12
379#define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
380#define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT)
381
382
383#define EVTQ_ENT_DWORDS 4
384#define EVTQ_MAX_SZ_SHIFT 7
385
386#define EVTQ_0_ID_SHIFT 0
387#define EVTQ_0_ID_MASK 0xffUL
388
389
390#define PRIQ_ENT_DWORDS 2
391#define PRIQ_MAX_SZ_SHIFT 8
392
393#define PRIQ_0_SID_SHIFT 0
394#define PRIQ_0_SID_MASK 0xffffffffUL
395#define PRIQ_0_SSID_SHIFT 32
396#define PRIQ_0_SSID_MASK 0xfffffUL
397#define PRIQ_0_PERM_PRIV (1UL << 58)
398#define PRIQ_0_PERM_EXEC (1UL << 59)
399#define PRIQ_0_PERM_READ (1UL << 60)
400#define PRIQ_0_PERM_WRITE (1UL << 61)
401#define PRIQ_0_PRG_LAST (1UL << 62)
402#define PRIQ_0_SSID_V (1UL << 63)
403
404#define PRIQ_1_PRG_IDX_SHIFT 0
405#define PRIQ_1_PRG_IDX_MASK 0x1ffUL
406#define PRIQ_1_ADDR_SHIFT 12
407#define PRIQ_1_ADDR_MASK 0xfffffffffffffUL
408
409
410#define ARM_SMMU_POLL_TIMEOUT_US 100
411
412#define MSI_IOVA_BASE 0x8000000
413#define MSI_IOVA_LENGTH 0x100000
414
415static bool disable_bypass;
416module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
417MODULE_PARM_DESC(disable_bypass,
418 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
419
420enum pri_resp {
421 PRI_RESP_DENY,
422 PRI_RESP_FAIL,
423 PRI_RESP_SUCC,
424};
425
426enum arm_smmu_msi_index {
427 EVTQ_MSI_INDEX,
428 GERROR_MSI_INDEX,
429 PRIQ_MSI_INDEX,
430 ARM_SMMU_MAX_MSIS,
431};
432
433static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
434 [EVTQ_MSI_INDEX] = {
435 ARM_SMMU_EVTQ_IRQ_CFG0,
436 ARM_SMMU_EVTQ_IRQ_CFG1,
437 ARM_SMMU_EVTQ_IRQ_CFG2,
438 },
439 [GERROR_MSI_INDEX] = {
440 ARM_SMMU_GERROR_IRQ_CFG0,
441 ARM_SMMU_GERROR_IRQ_CFG1,
442 ARM_SMMU_GERROR_IRQ_CFG2,
443 },
444 [PRIQ_MSI_INDEX] = {
445 ARM_SMMU_PRIQ_IRQ_CFG0,
446 ARM_SMMU_PRIQ_IRQ_CFG1,
447 ARM_SMMU_PRIQ_IRQ_CFG2,
448 },
449};
450
451struct arm_smmu_cmdq_ent {
452
453 u8 opcode;
454 bool substream_valid;
455
456
457 union {
458 #define CMDQ_OP_PREFETCH_CFG 0x1
459 struct {
460 u32 sid;
461 u8 size;
462 u64 addr;
463 } prefetch;
464
465 #define CMDQ_OP_CFGI_STE 0x3
466 #define CMDQ_OP_CFGI_ALL 0x4
467 struct {
468 u32 sid;
469 union {
470 bool leaf;
471 u8 span;
472 };
473 } cfgi;
474
475 #define CMDQ_OP_TLBI_NH_ASID 0x11
476 #define CMDQ_OP_TLBI_NH_VA 0x12
477 #define CMDQ_OP_TLBI_EL2_ALL 0x20
478 #define CMDQ_OP_TLBI_S12_VMALL 0x28
479 #define CMDQ_OP_TLBI_S2_IPA 0x2a
480 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
481 struct {
482 u16 asid;
483 u16 vmid;
484 bool leaf;
485 u64 addr;
486 } tlbi;
487
488 #define CMDQ_OP_PRI_RESP 0x41
489 struct {
490 u32 sid;
491 u32 ssid;
492 u16 grpid;
493 enum pri_resp resp;
494 } pri;
495
496 #define CMDQ_OP_CMD_SYNC 0x46
497 };
498};
499
500struct arm_smmu_queue {
501 int irq;
502
503 __le64 *base;
504 dma_addr_t base_dma;
505 u64 q_base;
506
507 size_t ent_dwords;
508 u32 max_n_shift;
509 u32 prod;
510 u32 cons;
511
512 u32 __iomem *prod_reg;
513 u32 __iomem *cons_reg;
514};
515
516struct arm_smmu_cmdq {
517 struct arm_smmu_queue q;
518 spinlock_t lock;
519};
520
521struct arm_smmu_evtq {
522 struct arm_smmu_queue q;
523 u32 max_stalls;
524};
525
526struct arm_smmu_priq {
527 struct arm_smmu_queue q;
528};
529
530
531struct arm_smmu_strtab_l1_desc {
532 u8 span;
533
534 __le64 *l2ptr;
535 dma_addr_t l2ptr_dma;
536};
537
538struct arm_smmu_s1_cfg {
539 __le64 *cdptr;
540 dma_addr_t cdptr_dma;
541
542 struct arm_smmu_ctx_desc {
543 u16 asid;
544 u64 ttbr;
545 u64 tcr;
546 u64 mair;
547 } cd;
548};
549
550struct arm_smmu_s2_cfg {
551 u16 vmid;
552 u64 vttbr;
553 u64 vtcr;
554};
555
556struct arm_smmu_strtab_ent {
557
558
559
560
561
562
563
564 bool assigned;
565 struct arm_smmu_s1_cfg *s1_cfg;
566 struct arm_smmu_s2_cfg *s2_cfg;
567};
568
569struct arm_smmu_strtab_cfg {
570 __le64 *strtab;
571 dma_addr_t strtab_dma;
572 struct arm_smmu_strtab_l1_desc *l1_desc;
573 unsigned int num_l1_ents;
574
575 u64 strtab_base;
576 u32 strtab_base_cfg;
577};
578
579
580struct arm_smmu_device {
581 struct device *dev;
582 void __iomem *base;
583
584#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
585#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
586#define ARM_SMMU_FEAT_TT_LE (1 << 2)
587#define ARM_SMMU_FEAT_TT_BE (1 << 3)
588#define ARM_SMMU_FEAT_PRI (1 << 4)
589#define ARM_SMMU_FEAT_ATS (1 << 5)
590#define ARM_SMMU_FEAT_SEV (1 << 6)
591#define ARM_SMMU_FEAT_MSI (1 << 7)
592#define ARM_SMMU_FEAT_COHERENCY (1 << 8)
593#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
594#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
595#define ARM_SMMU_FEAT_STALLS (1 << 11)
596#define ARM_SMMU_FEAT_HYP (1 << 12)
597 u32 features;
598
599#define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
600 u32 options;
601
602 struct arm_smmu_cmdq cmdq;
603 struct arm_smmu_evtq evtq;
604 struct arm_smmu_priq priq;
605
606 int gerr_irq;
607
608 unsigned long ias;
609 unsigned long oas;
610 unsigned long pgsize_bitmap;
611
612#define ARM_SMMU_MAX_ASIDS (1 << 16)
613 unsigned int asid_bits;
614 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
615
616#define ARM_SMMU_MAX_VMIDS (1 << 16)
617 unsigned int vmid_bits;
618 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
619
620 unsigned int ssid_bits;
621 unsigned int sid_bits;
622
623 struct arm_smmu_strtab_cfg strtab_cfg;
624
625
626 struct iommu_device iommu;
627};
628
629
630struct arm_smmu_master_data {
631 struct arm_smmu_device *smmu;
632 struct arm_smmu_strtab_ent ste;
633};
634
635
636enum arm_smmu_domain_stage {
637 ARM_SMMU_DOMAIN_S1 = 0,
638 ARM_SMMU_DOMAIN_S2,
639 ARM_SMMU_DOMAIN_NESTED,
640 ARM_SMMU_DOMAIN_BYPASS,
641};
642
643struct arm_smmu_domain {
644 struct arm_smmu_device *smmu;
645 struct mutex init_mutex;
646
647 struct io_pgtable_ops *pgtbl_ops;
648 spinlock_t pgtbl_lock;
649
650 enum arm_smmu_domain_stage stage;
651 union {
652 struct arm_smmu_s1_cfg s1_cfg;
653 struct arm_smmu_s2_cfg s2_cfg;
654 };
655
656 struct iommu_domain domain;
657};
658
659struct arm_smmu_option_prop {
660 u32 opt;
661 const char *prop;
662};
663
664static struct arm_smmu_option_prop arm_smmu_options[] = {
665 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
666 { 0, NULL},
667};
668
669static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
670{
671 return container_of(dom, struct arm_smmu_domain, domain);
672}
673
674static void parse_driver_options(struct arm_smmu_device *smmu)
675{
676 int i = 0;
677
678 do {
679 if (of_property_read_bool(smmu->dev->of_node,
680 arm_smmu_options[i].prop)) {
681 smmu->options |= arm_smmu_options[i].opt;
682 dev_notice(smmu->dev, "option %s\n",
683 arm_smmu_options[i].prop);
684 }
685 } while (arm_smmu_options[++i].opt);
686}
687
688
689static bool queue_full(struct arm_smmu_queue *q)
690{
691 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
692 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
693}
694
695static bool queue_empty(struct arm_smmu_queue *q)
696{
697 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
698 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
699}
700
701static void queue_sync_cons(struct arm_smmu_queue *q)
702{
703 q->cons = readl_relaxed(q->cons_reg);
704}
705
706static void queue_inc_cons(struct arm_smmu_queue *q)
707{
708 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
709
710 q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
711 writel(q->cons, q->cons_reg);
712}
713
714static int queue_sync_prod(struct arm_smmu_queue *q)
715{
716 int ret = 0;
717 u32 prod = readl_relaxed(q->prod_reg);
718
719 if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
720 ret = -EOVERFLOW;
721
722 q->prod = prod;
723 return ret;
724}
725
726static void queue_inc_prod(struct arm_smmu_queue *q)
727{
728 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
729
730 q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
731 writel(q->prod, q->prod_reg);
732}
733
734
735
736
737
738static int queue_poll_cons(struct arm_smmu_queue *q, bool drain, bool wfe)
739{
740 ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
741
742 while (queue_sync_cons(q), (drain ? !queue_empty(q) : queue_full(q))) {
743 if (ktime_compare(ktime_get(), timeout) > 0)
744 return -ETIMEDOUT;
745
746 if (wfe) {
747 wfe();
748 } else {
749 cpu_relax();
750 udelay(1);
751 }
752 }
753
754 return 0;
755}
756
757static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
758{
759 int i;
760
761 for (i = 0; i < n_dwords; ++i)
762 *dst++ = cpu_to_le64(*src++);
763}
764
765static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
766{
767 if (queue_full(q))
768 return -ENOSPC;
769
770 queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
771 queue_inc_prod(q);
772 return 0;
773}
774
775static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
776{
777 int i;
778
779 for (i = 0; i < n_dwords; ++i)
780 *dst++ = le64_to_cpu(*src++);
781}
782
783static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
784{
785 if (queue_empty(q))
786 return -EAGAIN;
787
788 queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
789 queue_inc_cons(q);
790 return 0;
791}
792
793
794static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
795{
796 memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
797 cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;
798
799 switch (ent->opcode) {
800 case CMDQ_OP_TLBI_EL2_ALL:
801 case CMDQ_OP_TLBI_NSNH_ALL:
802 break;
803 case CMDQ_OP_PREFETCH_CFG:
804 cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;
805 cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;
806 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
807 break;
808 case CMDQ_OP_CFGI_STE:
809 cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;
810 cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;
811 break;
812 case CMDQ_OP_CFGI_ALL:
813
814 cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT;
815 break;
816 case CMDQ_OP_TLBI_NH_VA:
817 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
818 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
819 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
820 break;
821 case CMDQ_OP_TLBI_S2_IPA:
822 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
823 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
824 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
825 break;
826 case CMDQ_OP_TLBI_NH_ASID:
827 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
828
829 case CMDQ_OP_TLBI_S12_VMALL:
830 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
831 break;
832 case CMDQ_OP_PRI_RESP:
833 cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;
834 cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;
835 cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;
836 cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;
837 switch (ent->pri.resp) {
838 case PRI_RESP_DENY:
839 cmd[1] |= CMDQ_PRI_1_RESP_DENY;
840 break;
841 case PRI_RESP_FAIL:
842 cmd[1] |= CMDQ_PRI_1_RESP_FAIL;
843 break;
844 case PRI_RESP_SUCC:
845 cmd[1] |= CMDQ_PRI_1_RESP_SUCC;
846 break;
847 default:
848 return -EINVAL;
849 }
850 break;
851 case CMDQ_OP_CMD_SYNC:
852 cmd[0] |= CMDQ_SYNC_0_CS_SEV;
853 break;
854 default:
855 return -ENOENT;
856 }
857
858 return 0;
859}
860
861static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
862{
863 static const char *cerror_str[] = {
864 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
865 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
866 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
867 };
868
869 int i;
870 u64 cmd[CMDQ_ENT_DWORDS];
871 struct arm_smmu_queue *q = &smmu->cmdq.q;
872 u32 cons = readl_relaxed(q->cons_reg);
873 u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK;
874 struct arm_smmu_cmdq_ent cmd_sync = {
875 .opcode = CMDQ_OP_CMD_SYNC,
876 };
877
878 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
879 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
880
881 switch (idx) {
882 case CMDQ_ERR_CERROR_ABT_IDX:
883 dev_err(smmu->dev, "retrying command fetch\n");
884 case CMDQ_ERR_CERROR_NONE_IDX:
885 return;
886 case CMDQ_ERR_CERROR_ILL_IDX:
887
888 default:
889 break;
890 }
891
892
893
894
895
896 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
897 dev_err(smmu->dev, "skipping command in error state:\n");
898 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
899 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
900
901
902 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
903 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
904 return;
905 }
906
907 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
908}
909
910static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
911 struct arm_smmu_cmdq_ent *ent)
912{
913 u64 cmd[CMDQ_ENT_DWORDS];
914 unsigned long flags;
915 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
916 struct arm_smmu_queue *q = &smmu->cmdq.q;
917
918 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
919 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
920 ent->opcode);
921 return;
922 }
923
924 spin_lock_irqsave(&smmu->cmdq.lock, flags);
925 while (queue_insert_raw(q, cmd) == -ENOSPC) {
926 if (queue_poll_cons(q, false, wfe))
927 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
928 }
929
930 if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, true, wfe))
931 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
932 spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
933}
934
935
936static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
937{
938 u64 val = 0;
939
940
941 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
942 val |= ARM_SMMU_TCR2CD(tcr, TG0);
943 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
944 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
945 val |= ARM_SMMU_TCR2CD(tcr, SH0);
946 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
947 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
948 val |= ARM_SMMU_TCR2CD(tcr, IPS);
949 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
950
951 return val;
952}
953
954static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
955 struct arm_smmu_s1_cfg *cfg)
956{
957 u64 val;
958
959
960
961
962
963 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
964#ifdef __BIG_ENDIAN
965 CTXDESC_CD_0_ENDI |
966#endif
967 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE |
968 CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT |
969 CTXDESC_CD_0_V;
970 cfg->cdptr[0] = cpu_to_le64(val);
971
972 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT;
973 cfg->cdptr[1] = cpu_to_le64(val);
974
975 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT);
976}
977
978
979static void
980arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
981{
982 u64 val = 0;
983
984 val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK)
985 << STRTAB_L1_DESC_SPAN_SHIFT;
986 val |= desc->l2ptr_dma &
987 STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT;
988
989 *dst = cpu_to_le64(val);
990}
991
992static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
993{
994 struct arm_smmu_cmdq_ent cmd = {
995 .opcode = CMDQ_OP_CFGI_STE,
996 .cfgi = {
997 .sid = sid,
998 .leaf = true,
999 },
1000 };
1001
1002 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1003 cmd.opcode = CMDQ_OP_CMD_SYNC;
1004 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1005}
1006
1007static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
1008 __le64 *dst, struct arm_smmu_strtab_ent *ste)
1009{
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026 u64 val = le64_to_cpu(dst[0]);
1027 bool ste_live = false;
1028 struct arm_smmu_cmdq_ent prefetch_cmd = {
1029 .opcode = CMDQ_OP_PREFETCH_CFG,
1030 .prefetch = {
1031 .sid = sid,
1032 },
1033 };
1034
1035 if (val & STRTAB_STE_0_V) {
1036 u64 cfg;
1037
1038 cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT;
1039 switch (cfg) {
1040 case STRTAB_STE_0_CFG_BYPASS:
1041 break;
1042 case STRTAB_STE_0_CFG_S1_TRANS:
1043 case STRTAB_STE_0_CFG_S2_TRANS:
1044 ste_live = true;
1045 break;
1046 case STRTAB_STE_0_CFG_ABORT:
1047 if (disable_bypass)
1048 break;
1049 default:
1050 BUG();
1051 }
1052 }
1053
1054
1055 val = STRTAB_STE_0_V;
1056
1057
1058 if (!ste->assigned || !(ste->s1_cfg || ste->s2_cfg)) {
1059 if (!ste->assigned && disable_bypass)
1060 val |= STRTAB_STE_0_CFG_ABORT;
1061 else
1062 val |= STRTAB_STE_0_CFG_BYPASS;
1063
1064 dst[0] = cpu_to_le64(val);
1065 dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
1066 << STRTAB_STE_1_SHCFG_SHIFT);
1067 dst[2] = 0;
1068 if (ste_live)
1069 arm_smmu_sync_ste_for_sid(smmu, sid);
1070 return;
1071 }
1072
1073 if (ste->s1_cfg) {
1074 BUG_ON(ste_live);
1075 dst[1] = cpu_to_le64(
1076 STRTAB_STE_1_S1C_CACHE_WBRA
1077 << STRTAB_STE_1_S1CIR_SHIFT |
1078 STRTAB_STE_1_S1C_CACHE_WBRA
1079 << STRTAB_STE_1_S1COR_SHIFT |
1080 STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
1081#ifdef CONFIG_PCI_ATS
1082 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
1083#endif
1084 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
1085
1086 if (smmu->features & ARM_SMMU_FEAT_STALLS)
1087 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1088
1089 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
1090 << STRTAB_STE_0_S1CTXPTR_SHIFT) |
1091 STRTAB_STE_0_CFG_S1_TRANS;
1092 }
1093
1094 if (ste->s2_cfg) {
1095 BUG_ON(ste_live);
1096 dst[2] = cpu_to_le64(
1097 ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT |
1098 (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK)
1099 << STRTAB_STE_2_VTCR_SHIFT |
1100#ifdef __BIG_ENDIAN
1101 STRTAB_STE_2_S2ENDI |
1102#endif
1103 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1104 STRTAB_STE_2_S2R);
1105
1106 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr &
1107 STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT);
1108
1109 val |= STRTAB_STE_0_CFG_S2_TRANS;
1110 }
1111
1112 arm_smmu_sync_ste_for_sid(smmu, sid);
1113 dst[0] = cpu_to_le64(val);
1114 arm_smmu_sync_ste_for_sid(smmu, sid);
1115
1116
1117 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1118 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1119}
1120
1121static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1122{
1123 unsigned int i;
1124 struct arm_smmu_strtab_ent ste = { .assigned = false };
1125
1126 for (i = 0; i < nent; ++i) {
1127 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
1128 strtab += STRTAB_STE_DWORDS;
1129 }
1130}
1131
1132static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1133{
1134 size_t size;
1135 void *strtab;
1136 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1137 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1138
1139 if (desc->l2ptr)
1140 return 0;
1141
1142 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1143 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1144
1145 desc->span = STRTAB_SPLIT + 1;
1146 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1147 GFP_KERNEL | __GFP_ZERO);
1148 if (!desc->l2ptr) {
1149 dev_err(smmu->dev,
1150 "failed to allocate l2 stream table for SID %u\n",
1151 sid);
1152 return -ENOMEM;
1153 }
1154
1155 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1156 arm_smmu_write_strtab_l1_desc(strtab, desc);
1157 return 0;
1158}
1159
1160
1161static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1162{
1163 int i;
1164 struct arm_smmu_device *smmu = dev;
1165 struct arm_smmu_queue *q = &smmu->evtq.q;
1166 u64 evt[EVTQ_ENT_DWORDS];
1167
1168 do {
1169 while (!queue_remove_raw(q, evt)) {
1170 u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
1171
1172 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1173 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1174 dev_info(smmu->dev, "\t0x%016llx\n",
1175 (unsigned long long)evt[i]);
1176
1177 }
1178
1179
1180
1181
1182
1183 if (queue_sync_prod(q) == -EOVERFLOW)
1184 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1185 } while (!queue_empty(q));
1186
1187
1188 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1189 return IRQ_HANDLED;
1190}
1191
1192static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1193{
1194 u32 sid, ssid;
1195 u16 grpid;
1196 bool ssv, last;
1197
1198 sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
1199 ssv = evt[0] & PRIQ_0_SSID_V;
1200 ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
1201 last = evt[0] & PRIQ_0_PRG_LAST;
1202 grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
1203
1204 dev_info(smmu->dev, "unexpected PRI request received:\n");
1205 dev_info(smmu->dev,
1206 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1207 sid, ssid, grpid, last ? "L" : "",
1208 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1209 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1210 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1211 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1212 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
1213
1214 if (last) {
1215 struct arm_smmu_cmdq_ent cmd = {
1216 .opcode = CMDQ_OP_PRI_RESP,
1217 .substream_valid = ssv,
1218 .pri = {
1219 .sid = sid,
1220 .ssid = ssid,
1221 .grpid = grpid,
1222 .resp = PRI_RESP_DENY,
1223 },
1224 };
1225
1226 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1227 }
1228}
1229
1230static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1231{
1232 struct arm_smmu_device *smmu = dev;
1233 struct arm_smmu_queue *q = &smmu->priq.q;
1234 u64 evt[PRIQ_ENT_DWORDS];
1235
1236 do {
1237 while (!queue_remove_raw(q, evt))
1238 arm_smmu_handle_ppr(smmu, evt);
1239
1240 if (queue_sync_prod(q) == -EOVERFLOW)
1241 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1242 } while (!queue_empty(q));
1243
1244
1245 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1246 return IRQ_HANDLED;
1247}
1248
1249static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev)
1250{
1251
1252 return IRQ_HANDLED;
1253}
1254
1255static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1256
1257static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1258{
1259 u32 gerror, gerrorn, active;
1260 struct arm_smmu_device *smmu = dev;
1261
1262 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1263 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1264
1265 active = gerror ^ gerrorn;
1266 if (!(active & GERROR_ERR_MASK))
1267 return IRQ_NONE;
1268
1269 dev_warn(smmu->dev,
1270 "unexpected global error reported (0x%08x), this could be serious\n",
1271 active);
1272
1273 if (active & GERROR_SFM_ERR) {
1274 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1275 arm_smmu_device_disable(smmu);
1276 }
1277
1278 if (active & GERROR_MSI_GERROR_ABT_ERR)
1279 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1280
1281 if (active & GERROR_MSI_PRIQ_ABT_ERR)
1282 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1283
1284 if (active & GERROR_MSI_EVTQ_ABT_ERR)
1285 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1286
1287 if (active & GERROR_MSI_CMDQ_ABT_ERR) {
1288 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1289 arm_smmu_cmdq_sync_handler(irq, smmu->dev);
1290 }
1291
1292 if (active & GERROR_PRIQ_ABT_ERR)
1293 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1294
1295 if (active & GERROR_EVTQ_ABT_ERR)
1296 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1297
1298 if (active & GERROR_CMDQ_ERR)
1299 arm_smmu_cmdq_skip_err(smmu);
1300
1301 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1302 return IRQ_HANDLED;
1303}
1304
1305
1306static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1307{
1308 struct arm_smmu_cmdq_ent cmd;
1309
1310 cmd.opcode = CMDQ_OP_CMD_SYNC;
1311 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1312}
1313
1314static void arm_smmu_tlb_sync(void *cookie)
1315{
1316 struct arm_smmu_domain *smmu_domain = cookie;
1317 __arm_smmu_tlb_sync(smmu_domain->smmu);
1318}
1319
1320static void arm_smmu_tlb_inv_context(void *cookie)
1321{
1322 struct arm_smmu_domain *smmu_domain = cookie;
1323 struct arm_smmu_device *smmu = smmu_domain->smmu;
1324 struct arm_smmu_cmdq_ent cmd;
1325
1326 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1327 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1328 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1329 cmd.tlbi.vmid = 0;
1330 } else {
1331 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1332 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1333 }
1334
1335 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1336 __arm_smmu_tlb_sync(smmu);
1337}
1338
1339static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1340 size_t granule, bool leaf, void *cookie)
1341{
1342 struct arm_smmu_domain *smmu_domain = cookie;
1343 struct arm_smmu_device *smmu = smmu_domain->smmu;
1344 struct arm_smmu_cmdq_ent cmd = {
1345 .tlbi = {
1346 .leaf = leaf,
1347 .addr = iova,
1348 },
1349 };
1350
1351 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1352 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
1353 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1354 } else {
1355 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1356 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1357 }
1358
1359 do {
1360 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1361 cmd.tlbi.addr += granule;
1362 } while (size -= granule);
1363}
1364
1365static const struct iommu_gather_ops arm_smmu_gather_ops = {
1366 .tlb_flush_all = arm_smmu_tlb_inv_context,
1367 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1368 .tlb_sync = arm_smmu_tlb_sync,
1369};
1370
1371
1372static bool arm_smmu_capable(enum iommu_cap cap)
1373{
1374 switch (cap) {
1375 case IOMMU_CAP_CACHE_COHERENCY:
1376 return true;
1377 case IOMMU_CAP_NOEXEC:
1378 return true;
1379 default:
1380 return false;
1381 }
1382}
1383
1384static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1385{
1386 struct arm_smmu_domain *smmu_domain;
1387
1388 if (type != IOMMU_DOMAIN_UNMANAGED &&
1389 type != IOMMU_DOMAIN_DMA &&
1390 type != IOMMU_DOMAIN_IDENTITY)
1391 return NULL;
1392
1393
1394
1395
1396
1397
1398 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1399 if (!smmu_domain)
1400 return NULL;
1401
1402 if (type == IOMMU_DOMAIN_DMA &&
1403 iommu_get_dma_cookie(&smmu_domain->domain)) {
1404 kfree(smmu_domain);
1405 return NULL;
1406 }
1407
1408 mutex_init(&smmu_domain->init_mutex);
1409 spin_lock_init(&smmu_domain->pgtbl_lock);
1410 return &smmu_domain->domain;
1411}
1412
1413static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1414{
1415 int idx, size = 1 << span;
1416
1417 do {
1418 idx = find_first_zero_bit(map, size);
1419 if (idx == size)
1420 return -ENOSPC;
1421 } while (test_and_set_bit(idx, map));
1422
1423 return idx;
1424}
1425
1426static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1427{
1428 clear_bit(idx, map);
1429}
1430
1431static void arm_smmu_domain_free(struct iommu_domain *domain)
1432{
1433 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1434 struct arm_smmu_device *smmu = smmu_domain->smmu;
1435
1436 iommu_put_dma_cookie(domain);
1437 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1438
1439
1440 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1441 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1442
1443 if (cfg->cdptr) {
1444 dmam_free_coherent(smmu_domain->smmu->dev,
1445 CTXDESC_CD_DWORDS << 3,
1446 cfg->cdptr,
1447 cfg->cdptr_dma);
1448
1449 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1450 }
1451 } else {
1452 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1453 if (cfg->vmid)
1454 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1455 }
1456
1457 kfree(smmu_domain);
1458}
1459
1460static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1461 struct io_pgtable_cfg *pgtbl_cfg)
1462{
1463 int ret;
1464 int asid;
1465 struct arm_smmu_device *smmu = smmu_domain->smmu;
1466 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1467
1468 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1469 if (asid < 0)
1470 return asid;
1471
1472 cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1473 &cfg->cdptr_dma,
1474 GFP_KERNEL | __GFP_ZERO);
1475 if (!cfg->cdptr) {
1476 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1477 ret = -ENOMEM;
1478 goto out_free_asid;
1479 }
1480
1481 cfg->cd.asid = (u16)asid;
1482 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1483 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1484 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1485 return 0;
1486
1487out_free_asid:
1488 arm_smmu_bitmap_free(smmu->asid_map, asid);
1489 return ret;
1490}
1491
1492static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1493 struct io_pgtable_cfg *pgtbl_cfg)
1494{
1495 int vmid;
1496 struct arm_smmu_device *smmu = smmu_domain->smmu;
1497 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1498
1499 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1500 if (vmid < 0)
1501 return vmid;
1502
1503 cfg->vmid = (u16)vmid;
1504 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1505 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1506 return 0;
1507}
1508
1509static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1510{
1511 int ret;
1512 unsigned long ias, oas;
1513 enum io_pgtable_fmt fmt;
1514 struct io_pgtable_cfg pgtbl_cfg;
1515 struct io_pgtable_ops *pgtbl_ops;
1516 int (*finalise_stage_fn)(struct arm_smmu_domain *,
1517 struct io_pgtable_cfg *);
1518 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1519 struct arm_smmu_device *smmu = smmu_domain->smmu;
1520
1521 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
1522 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
1523 return 0;
1524 }
1525
1526
1527 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1528 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1529 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1530 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1531
1532 switch (smmu_domain->stage) {
1533 case ARM_SMMU_DOMAIN_S1:
1534 ias = VA_BITS;
1535 oas = smmu->ias;
1536 fmt = ARM_64_LPAE_S1;
1537 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1538 break;
1539 case ARM_SMMU_DOMAIN_NESTED:
1540 case ARM_SMMU_DOMAIN_S2:
1541 ias = smmu->ias;
1542 oas = smmu->oas;
1543 fmt = ARM_64_LPAE_S2;
1544 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1545 break;
1546 default:
1547 return -EINVAL;
1548 }
1549
1550 pgtbl_cfg = (struct io_pgtable_cfg) {
1551 .pgsize_bitmap = smmu->pgsize_bitmap,
1552 .ias = ias,
1553 .oas = oas,
1554 .tlb = &arm_smmu_gather_ops,
1555 .iommu_dev = smmu->dev,
1556 };
1557
1558 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1559 if (!pgtbl_ops)
1560 return -ENOMEM;
1561
1562 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1563 domain->geometry.aperture_end = (1UL << ias) - 1;
1564 domain->geometry.force_aperture = true;
1565 smmu_domain->pgtbl_ops = pgtbl_ops;
1566
1567 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1568 if (ret < 0)
1569 free_io_pgtable_ops(pgtbl_ops);
1570
1571 return ret;
1572}
1573
1574static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1575{
1576 __le64 *step;
1577 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1578
1579 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1580 struct arm_smmu_strtab_l1_desc *l1_desc;
1581 int idx;
1582
1583
1584 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1585 l1_desc = &cfg->l1_desc[idx];
1586 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1587 step = &l1_desc->l2ptr[idx];
1588 } else {
1589
1590 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1591 }
1592
1593 return step;
1594}
1595
1596static void arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec)
1597{
1598 int i;
1599 struct arm_smmu_master_data *master = fwspec->iommu_priv;
1600 struct arm_smmu_device *smmu = master->smmu;
1601
1602 for (i = 0; i < fwspec->num_ids; ++i) {
1603 u32 sid = fwspec->ids[i];
1604 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1605
1606 arm_smmu_write_strtab_ent(smmu, sid, step, &master->ste);
1607 }
1608}
1609
1610static void arm_smmu_detach_dev(struct device *dev)
1611{
1612 struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv;
1613
1614 master->ste.assigned = false;
1615 arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
1616}
1617
1618static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1619{
1620 int ret = 0;
1621 struct arm_smmu_device *smmu;
1622 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1623 struct arm_smmu_master_data *master;
1624 struct arm_smmu_strtab_ent *ste;
1625
1626 if (!dev->iommu_fwspec)
1627 return -ENOENT;
1628
1629 master = dev->iommu_fwspec->iommu_priv;
1630 smmu = master->smmu;
1631 ste = &master->ste;
1632
1633
1634 if (ste->assigned)
1635 arm_smmu_detach_dev(dev);
1636
1637 mutex_lock(&smmu_domain->init_mutex);
1638
1639 if (!smmu_domain->smmu) {
1640 smmu_domain->smmu = smmu;
1641 ret = arm_smmu_domain_finalise(domain);
1642 if (ret) {
1643 smmu_domain->smmu = NULL;
1644 goto out_unlock;
1645 }
1646 } else if (smmu_domain->smmu != smmu) {
1647 dev_err(dev,
1648 "cannot attach to SMMU %s (upstream of %s)\n",
1649 dev_name(smmu_domain->smmu->dev),
1650 dev_name(smmu->dev));
1651 ret = -ENXIO;
1652 goto out_unlock;
1653 }
1654
1655 ste->assigned = true;
1656
1657 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) {
1658 ste->s1_cfg = NULL;
1659 ste->s2_cfg = NULL;
1660 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1661 ste->s1_cfg = &smmu_domain->s1_cfg;
1662 ste->s2_cfg = NULL;
1663 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1664 } else {
1665 ste->s1_cfg = NULL;
1666 ste->s2_cfg = &smmu_domain->s2_cfg;
1667 }
1668
1669 arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
1670out_unlock:
1671 mutex_unlock(&smmu_domain->init_mutex);
1672 return ret;
1673}
1674
1675static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1676 phys_addr_t paddr, size_t size, int prot)
1677{
1678 int ret;
1679 unsigned long flags;
1680 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1681 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1682
1683 if (!ops)
1684 return -ENODEV;
1685
1686 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1687 ret = ops->map(ops, iova, paddr, size, prot);
1688 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1689 return ret;
1690}
1691
1692static size_t
1693arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1694{
1695 size_t ret;
1696 unsigned long flags;
1697 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1698 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1699
1700 if (!ops)
1701 return 0;
1702
1703 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1704 ret = ops->unmap(ops, iova, size);
1705 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1706 return ret;
1707}
1708
1709static phys_addr_t
1710arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1711{
1712 phys_addr_t ret;
1713 unsigned long flags;
1714 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1715 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1716
1717 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1718 return iova;
1719
1720 if (!ops)
1721 return 0;
1722
1723 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1724 ret = ops->iova_to_phys(ops, iova);
1725 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1726
1727 return ret;
1728}
1729
1730static struct platform_driver arm_smmu_driver;
1731
1732static int arm_smmu_match_node(struct device *dev, void *data)
1733{
1734 return dev->fwnode == data;
1735}
1736
1737static
1738struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1739{
1740 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1741 fwnode, arm_smmu_match_node);
1742 put_device(dev);
1743 return dev ? dev_get_drvdata(dev) : NULL;
1744}
1745
1746static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
1747{
1748 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
1749
1750 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
1751 limit *= 1UL << STRTAB_SPLIT;
1752
1753 return sid < limit;
1754}
1755
1756static struct iommu_ops arm_smmu_ops;
1757
1758static int arm_smmu_add_device(struct device *dev)
1759{
1760 int i, ret;
1761 struct arm_smmu_device *smmu;
1762 struct arm_smmu_master_data *master;
1763 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1764 struct iommu_group *group;
1765
1766 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1767 return -ENODEV;
1768
1769
1770
1771
1772
1773 if (WARN_ON_ONCE(fwspec->iommu_priv)) {
1774 master = fwspec->iommu_priv;
1775 smmu = master->smmu;
1776 } else {
1777 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1778 if (!smmu)
1779 return -ENODEV;
1780 master = kzalloc(sizeof(*master), GFP_KERNEL);
1781 if (!master)
1782 return -ENOMEM;
1783
1784 master->smmu = smmu;
1785 fwspec->iommu_priv = master;
1786 }
1787
1788
1789 for (i = 0; i < fwspec->num_ids; i++) {
1790 u32 sid = fwspec->ids[i];
1791
1792 if (!arm_smmu_sid_in_range(smmu, sid))
1793 return -ERANGE;
1794
1795
1796 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1797 ret = arm_smmu_init_l2_strtab(smmu, sid);
1798 if (ret)
1799 return ret;
1800 }
1801 }
1802
1803 group = iommu_group_get_for_dev(dev);
1804 if (!IS_ERR(group)) {
1805 iommu_group_put(group);
1806 iommu_device_link(&smmu->iommu, dev);
1807 }
1808
1809 return PTR_ERR_OR_ZERO(group);
1810}
1811
1812static void arm_smmu_remove_device(struct device *dev)
1813{
1814 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1815 struct arm_smmu_master_data *master;
1816 struct arm_smmu_device *smmu;
1817
1818 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1819 return;
1820
1821 master = fwspec->iommu_priv;
1822 smmu = master->smmu;
1823 if (master && master->ste.assigned)
1824 arm_smmu_detach_dev(dev);
1825 iommu_group_remove_device(dev);
1826 iommu_device_unlink(&smmu->iommu, dev);
1827 kfree(master);
1828 iommu_fwspec_free(dev);
1829}
1830
1831static struct iommu_group *arm_smmu_device_group(struct device *dev)
1832{
1833 struct iommu_group *group;
1834
1835
1836
1837
1838
1839
1840 if (dev_is_pci(dev))
1841 group = pci_device_group(dev);
1842 else
1843 group = generic_device_group(dev);
1844
1845 return group;
1846}
1847
1848static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1849 enum iommu_attr attr, void *data)
1850{
1851 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1852
1853 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1854 return -EINVAL;
1855
1856 switch (attr) {
1857 case DOMAIN_ATTR_NESTING:
1858 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1859 return 0;
1860 default:
1861 return -ENODEV;
1862 }
1863}
1864
1865static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1866 enum iommu_attr attr, void *data)
1867{
1868 int ret = 0;
1869 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1870
1871 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1872 return -EINVAL;
1873
1874 mutex_lock(&smmu_domain->init_mutex);
1875
1876 switch (attr) {
1877 case DOMAIN_ATTR_NESTING:
1878 if (smmu_domain->smmu) {
1879 ret = -EPERM;
1880 goto out_unlock;
1881 }
1882
1883 if (*(int *)data)
1884 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1885 else
1886 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1887
1888 break;
1889 default:
1890 ret = -ENODEV;
1891 }
1892
1893out_unlock:
1894 mutex_unlock(&smmu_domain->init_mutex);
1895 return ret;
1896}
1897
1898static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1899{
1900 return iommu_fwspec_add_ids(dev, args->args, 1);
1901}
1902
1903static void arm_smmu_get_resv_regions(struct device *dev,
1904 struct list_head *head)
1905{
1906 struct iommu_resv_region *region;
1907 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1908
1909 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1910 prot, IOMMU_RESV_SW_MSI);
1911 if (!region)
1912 return;
1913
1914 list_add_tail(®ion->list, head);
1915
1916 iommu_dma_get_resv_regions(dev, head);
1917}
1918
1919static void arm_smmu_put_resv_regions(struct device *dev,
1920 struct list_head *head)
1921{
1922 struct iommu_resv_region *entry, *next;
1923
1924 list_for_each_entry_safe(entry, next, head, list)
1925 kfree(entry);
1926}
1927
1928static struct iommu_ops arm_smmu_ops = {
1929 .capable = arm_smmu_capable,
1930 .domain_alloc = arm_smmu_domain_alloc,
1931 .domain_free = arm_smmu_domain_free,
1932 .attach_dev = arm_smmu_attach_dev,
1933 .map = arm_smmu_map,
1934 .unmap = arm_smmu_unmap,
1935 .map_sg = default_iommu_map_sg,
1936 .iova_to_phys = arm_smmu_iova_to_phys,
1937 .add_device = arm_smmu_add_device,
1938 .remove_device = arm_smmu_remove_device,
1939 .device_group = arm_smmu_device_group,
1940 .domain_get_attr = arm_smmu_domain_get_attr,
1941 .domain_set_attr = arm_smmu_domain_set_attr,
1942 .of_xlate = arm_smmu_of_xlate,
1943 .get_resv_regions = arm_smmu_get_resv_regions,
1944 .put_resv_regions = arm_smmu_put_resv_regions,
1945 .pgsize_bitmap = -1UL,
1946};
1947
1948
1949static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
1950 struct arm_smmu_queue *q,
1951 unsigned long prod_off,
1952 unsigned long cons_off,
1953 size_t dwords)
1954{
1955 size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
1956
1957 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
1958 if (!q->base) {
1959 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
1960 qsz);
1961 return -ENOMEM;
1962 }
1963
1964 q->prod_reg = smmu->base + prod_off;
1965 q->cons_reg = smmu->base + cons_off;
1966 q->ent_dwords = dwords;
1967
1968 q->q_base = Q_BASE_RWA;
1969 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;
1970 q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)
1971 << Q_BASE_LOG2SIZE_SHIFT;
1972
1973 q->prod = q->cons = 0;
1974 return 0;
1975}
1976
1977static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
1978{
1979 int ret;
1980
1981
1982 spin_lock_init(&smmu->cmdq.lock);
1983 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
1984 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
1985 if (ret)
1986 return ret;
1987
1988
1989 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
1990 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
1991 if (ret)
1992 return ret;
1993
1994
1995 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
1996 return 0;
1997
1998 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
1999 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
2000}
2001
2002static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2003{
2004 unsigned int i;
2005 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2006 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2007 void *strtab = smmu->strtab_cfg.strtab;
2008
2009 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2010 if (!cfg->l1_desc) {
2011 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2012 return -ENOMEM;
2013 }
2014
2015 for (i = 0; i < cfg->num_l1_ents; ++i) {
2016 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2017 strtab += STRTAB_L1_DESC_DWORDS << 3;
2018 }
2019
2020 return 0;
2021}
2022
2023static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2024{
2025 void *strtab;
2026 u64 reg;
2027 u32 size, l1size;
2028 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2029
2030
2031 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2032 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2033 cfg->num_l1_ents = 1 << size;
2034
2035 size += STRTAB_SPLIT;
2036 if (size < smmu->sid_bits)
2037 dev_warn(smmu->dev,
2038 "2-level strtab only covers %u/%u bits of SID\n",
2039 size, smmu->sid_bits);
2040
2041 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2042 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2043 GFP_KERNEL | __GFP_ZERO);
2044 if (!strtab) {
2045 dev_err(smmu->dev,
2046 "failed to allocate l1 stream table (%u bytes)\n",
2047 size);
2048 return -ENOMEM;
2049 }
2050 cfg->strtab = strtab;
2051
2052
2053 reg = STRTAB_BASE_CFG_FMT_2LVL;
2054 reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2055 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2056 reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK)
2057 << STRTAB_BASE_CFG_SPLIT_SHIFT;
2058 cfg->strtab_base_cfg = reg;
2059
2060 return arm_smmu_init_l1_strtab(smmu);
2061}
2062
2063static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2064{
2065 void *strtab;
2066 u64 reg;
2067 u32 size;
2068 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2069
2070 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2071 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2072 GFP_KERNEL | __GFP_ZERO);
2073 if (!strtab) {
2074 dev_err(smmu->dev,
2075 "failed to allocate linear stream table (%u bytes)\n",
2076 size);
2077 return -ENOMEM;
2078 }
2079 cfg->strtab = strtab;
2080 cfg->num_l1_ents = 1 << smmu->sid_bits;
2081
2082
2083 reg = STRTAB_BASE_CFG_FMT_LINEAR;
2084 reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2085 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2086 cfg->strtab_base_cfg = reg;
2087
2088 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2089 return 0;
2090}
2091
2092static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2093{
2094 u64 reg;
2095 int ret;
2096
2097 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2098 ret = arm_smmu_init_strtab_2lvl(smmu);
2099 else
2100 ret = arm_smmu_init_strtab_linear(smmu);
2101
2102 if (ret)
2103 return ret;
2104
2105
2106 reg = smmu->strtab_cfg.strtab_dma &
2107 STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT;
2108 reg |= STRTAB_BASE_RA;
2109 smmu->strtab_cfg.strtab_base = reg;
2110
2111
2112 set_bit(0, smmu->vmid_map);
2113 return 0;
2114}
2115
2116static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2117{
2118 int ret;
2119
2120 ret = arm_smmu_init_queues(smmu);
2121 if (ret)
2122 return ret;
2123
2124 return arm_smmu_init_strtab(smmu);
2125}
2126
2127static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2128 unsigned int reg_off, unsigned int ack_off)
2129{
2130 u32 reg;
2131
2132 writel_relaxed(val, smmu->base + reg_off);
2133 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2134 1, ARM_SMMU_POLL_TIMEOUT_US);
2135}
2136
2137
2138static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2139{
2140 int ret;
2141 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2142
2143 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2144 1, ARM_SMMU_POLL_TIMEOUT_US);
2145 if (ret)
2146 return ret;
2147
2148 reg &= ~clr;
2149 reg |= set;
2150 writel_relaxed(reg | GBPA_UPDATE, gbpa);
2151 return readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2152 1, ARM_SMMU_POLL_TIMEOUT_US);
2153}
2154
2155static void arm_smmu_free_msis(void *data)
2156{
2157 struct device *dev = data;
2158 platform_msi_domain_free_irqs(dev);
2159}
2160
2161static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2162{
2163 phys_addr_t doorbell;
2164 struct device *dev = msi_desc_to_dev(desc);
2165 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2166 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2167
2168 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2169 doorbell &= MSI_CFG0_ADDR_MASK << MSI_CFG0_ADDR_SHIFT;
2170
2171 writeq_relaxed(doorbell, smmu->base + cfg[0]);
2172 writel_relaxed(msg->data, smmu->base + cfg[1]);
2173 writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2174}
2175
2176static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2177{
2178 struct msi_desc *desc;
2179 int ret, nvec = ARM_SMMU_MAX_MSIS;
2180 struct device *dev = smmu->dev;
2181
2182
2183 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2184 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2185
2186 if (smmu->features & ARM_SMMU_FEAT_PRI)
2187 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2188 else
2189 nvec--;
2190
2191 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2192 return;
2193
2194
2195 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2196 if (ret) {
2197 dev_warn(dev, "failed to allocate MSIs\n");
2198 return;
2199 }
2200
2201 for_each_msi_entry(desc, dev) {
2202 switch (desc->platform.msi_index) {
2203 case EVTQ_MSI_INDEX:
2204 smmu->evtq.q.irq = desc->irq;
2205 break;
2206 case GERROR_MSI_INDEX:
2207 smmu->gerr_irq = desc->irq;
2208 break;
2209 case PRIQ_MSI_INDEX:
2210 smmu->priq.q.irq = desc->irq;
2211 break;
2212 default:
2213 continue;
2214 }
2215 }
2216
2217
2218 devm_add_action(dev, arm_smmu_free_msis, dev);
2219}
2220
2221static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2222{
2223 int ret, irq;
2224 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2225
2226
2227 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2228 ARM_SMMU_IRQ_CTRLACK);
2229 if (ret) {
2230 dev_err(smmu->dev, "failed to disable irqs\n");
2231 return ret;
2232 }
2233
2234 arm_smmu_setup_msis(smmu);
2235
2236
2237 irq = smmu->evtq.q.irq;
2238 if (irq) {
2239 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2240 arm_smmu_evtq_thread,
2241 IRQF_ONESHOT,
2242 "arm-smmu-v3-evtq", smmu);
2243 if (ret < 0)
2244 dev_warn(smmu->dev, "failed to enable evtq irq\n");
2245 }
2246
2247 irq = smmu->cmdq.q.irq;
2248 if (irq) {
2249 ret = devm_request_irq(smmu->dev, irq,
2250 arm_smmu_cmdq_sync_handler, 0,
2251 "arm-smmu-v3-cmdq-sync", smmu);
2252 if (ret < 0)
2253 dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n");
2254 }
2255
2256 irq = smmu->gerr_irq;
2257 if (irq) {
2258 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2259 0, "arm-smmu-v3-gerror", smmu);
2260 if (ret < 0)
2261 dev_warn(smmu->dev, "failed to enable gerror irq\n");
2262 }
2263
2264 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2265 irq = smmu->priq.q.irq;
2266 if (irq) {
2267 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2268 arm_smmu_priq_thread,
2269 IRQF_ONESHOT,
2270 "arm-smmu-v3-priq",
2271 smmu);
2272 if (ret < 0)
2273 dev_warn(smmu->dev,
2274 "failed to enable priq irq\n");
2275 else
2276 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2277 }
2278 }
2279
2280
2281 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2282 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2283 if (ret)
2284 dev_warn(smmu->dev, "failed to enable irqs\n");
2285
2286 return 0;
2287}
2288
2289static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2290{
2291 int ret;
2292
2293 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2294 if (ret)
2295 dev_err(smmu->dev, "failed to clear cr0\n");
2296
2297 return ret;
2298}
2299
2300static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2301{
2302 int ret;
2303 u32 reg, enables;
2304 struct arm_smmu_cmdq_ent cmd;
2305
2306
2307 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2308 if (reg & CR0_SMMUEN)
2309 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2310
2311 ret = arm_smmu_device_disable(smmu);
2312 if (ret)
2313 return ret;
2314
2315
2316 reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) |
2317 (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) |
2318 (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) |
2319 (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) |
2320 (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) |
2321 (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT);
2322 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2323
2324
2325 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2326 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2327
2328
2329 writeq_relaxed(smmu->strtab_cfg.strtab_base,
2330 smmu->base + ARM_SMMU_STRTAB_BASE);
2331 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2332 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2333
2334
2335 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2336 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2337 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2338
2339 enables = CR0_CMDQEN;
2340 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2341 ARM_SMMU_CR0ACK);
2342 if (ret) {
2343 dev_err(smmu->dev, "failed to enable command queue\n");
2344 return ret;
2345 }
2346
2347
2348 cmd.opcode = CMDQ_OP_CFGI_ALL;
2349 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2350 cmd.opcode = CMDQ_OP_CMD_SYNC;
2351 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2352
2353
2354 if (smmu->features & ARM_SMMU_FEAT_HYP) {
2355 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2356 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2357 }
2358
2359 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2360 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2361 cmd.opcode = CMDQ_OP_CMD_SYNC;
2362 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2363
2364
2365 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2366 writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD);
2367 writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS);
2368
2369 enables |= CR0_EVTQEN;
2370 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2371 ARM_SMMU_CR0ACK);
2372 if (ret) {
2373 dev_err(smmu->dev, "failed to enable event queue\n");
2374 return ret;
2375 }
2376
2377
2378 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2379 writeq_relaxed(smmu->priq.q.q_base,
2380 smmu->base + ARM_SMMU_PRIQ_BASE);
2381 writel_relaxed(smmu->priq.q.prod,
2382 smmu->base + ARM_SMMU_PRIQ_PROD);
2383 writel_relaxed(smmu->priq.q.cons,
2384 smmu->base + ARM_SMMU_PRIQ_CONS);
2385
2386 enables |= CR0_PRIQEN;
2387 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2388 ARM_SMMU_CR0ACK);
2389 if (ret) {
2390 dev_err(smmu->dev, "failed to enable PRI queue\n");
2391 return ret;
2392 }
2393 }
2394
2395 ret = arm_smmu_setup_irqs(smmu);
2396 if (ret) {
2397 dev_err(smmu->dev, "failed to setup irqs\n");
2398 return ret;
2399 }
2400
2401
2402
2403 if (!bypass || disable_bypass) {
2404 enables |= CR0_SMMUEN;
2405 } else {
2406 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
2407 if (ret) {
2408 dev_err(smmu->dev, "GBPA not responding to update\n");
2409 return ret;
2410 }
2411 }
2412 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2413 ARM_SMMU_CR0ACK);
2414 if (ret) {
2415 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2416 return ret;
2417 }
2418
2419 return 0;
2420}
2421
2422static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
2423{
2424 u32 reg;
2425 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
2426
2427
2428 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2429
2430
2431 if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL)
2432 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2433
2434 if (reg & IDR0_CD2L)
2435 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2436
2437
2438
2439
2440
2441
2442 switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) {
2443 case IDR0_TTENDIAN_MIXED:
2444 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2445 break;
2446#ifdef __BIG_ENDIAN
2447 case IDR0_TTENDIAN_BE:
2448 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2449 break;
2450#else
2451 case IDR0_TTENDIAN_LE:
2452 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2453 break;
2454#endif
2455 default:
2456 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2457 return -ENXIO;
2458 }
2459
2460
2461 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2462 smmu->features |= ARM_SMMU_FEAT_PRI;
2463
2464 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2465 smmu->features |= ARM_SMMU_FEAT_ATS;
2466
2467 if (reg & IDR0_SEV)
2468 smmu->features |= ARM_SMMU_FEAT_SEV;
2469
2470 if (reg & IDR0_MSI)
2471 smmu->features |= ARM_SMMU_FEAT_MSI;
2472
2473 if (reg & IDR0_HYP)
2474 smmu->features |= ARM_SMMU_FEAT_HYP;
2475
2476
2477
2478
2479
2480 if (!!(reg & IDR0_COHACC) != coherent)
2481 dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
2482 coherent ? "true" : "false");
2483
2484 switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) {
2485 case IDR0_STALL_MODEL_STALL:
2486
2487 case IDR0_STALL_MODEL_FORCE:
2488 smmu->features |= ARM_SMMU_FEAT_STALLS;
2489 }
2490
2491 if (reg & IDR0_S1P)
2492 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2493
2494 if (reg & IDR0_S2P)
2495 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2496
2497 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2498 dev_err(smmu->dev, "no translation support!\n");
2499 return -ENXIO;
2500 }
2501
2502
2503 switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) {
2504 case IDR0_TTF_AARCH32_64:
2505 smmu->ias = 40;
2506
2507 case IDR0_TTF_AARCH64:
2508 break;
2509 default:
2510 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2511 return -ENXIO;
2512 }
2513
2514
2515 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2516 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2517
2518
2519 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2520 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2521 dev_err(smmu->dev, "embedded implementation not supported\n");
2522 return -ENXIO;
2523 }
2524
2525
2526 smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,
2527 reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK);
2528 if (!smmu->cmdq.q.max_n_shift) {
2529
2530 dev_err(smmu->dev, "unit-length command queue not supported\n");
2531 return -ENXIO;
2532 }
2533
2534 smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,
2535 reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK);
2536 smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,
2537 reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK);
2538
2539
2540 smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK;
2541 smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK;
2542
2543
2544
2545
2546
2547 if (smmu->sid_bits <= STRTAB_SPLIT)
2548 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
2549
2550
2551 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2552
2553
2554 smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT
2555 & IDR5_STALL_MAX_MASK;
2556
2557
2558 if (reg & IDR5_GRAN64K)
2559 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2560 if (reg & IDR5_GRAN16K)
2561 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2562 if (reg & IDR5_GRAN4K)
2563 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2564
2565 if (arm_smmu_ops.pgsize_bitmap == -1UL)
2566 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2567 else
2568 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2569
2570
2571 switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) {
2572 case IDR5_OAS_32_BIT:
2573 smmu->oas = 32;
2574 break;
2575 case IDR5_OAS_36_BIT:
2576 smmu->oas = 36;
2577 break;
2578 case IDR5_OAS_40_BIT:
2579 smmu->oas = 40;
2580 break;
2581 case IDR5_OAS_42_BIT:
2582 smmu->oas = 42;
2583 break;
2584 case IDR5_OAS_44_BIT:
2585 smmu->oas = 44;
2586 break;
2587 default:
2588 dev_info(smmu->dev,
2589 "unknown output address size. Truncating to 48-bit\n");
2590
2591 case IDR5_OAS_48_BIT:
2592 smmu->oas = 48;
2593 }
2594
2595
2596 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2597 dev_warn(smmu->dev,
2598 "failed to set DMA mask for table walker\n");
2599
2600 smmu->ias = max(smmu->ias, smmu->oas);
2601
2602 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2603 smmu->ias, smmu->oas, smmu->features);
2604 return 0;
2605}
2606
2607#ifdef CONFIG_ACPI
2608static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2609 struct arm_smmu_device *smmu)
2610{
2611 struct acpi_iort_smmu_v3 *iort_smmu;
2612 struct device *dev = smmu->dev;
2613 struct acpi_iort_node *node;
2614
2615 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
2616
2617
2618 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
2619
2620 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
2621 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2622
2623 return 0;
2624}
2625#else
2626static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2627 struct arm_smmu_device *smmu)
2628{
2629 return -ENODEV;
2630}
2631#endif
2632
2633static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2634 struct arm_smmu_device *smmu)
2635{
2636 struct device *dev = &pdev->dev;
2637 u32 cells;
2638 int ret = -EINVAL;
2639
2640 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
2641 dev_err(dev, "missing #iommu-cells property\n");
2642 else if (cells != 1)
2643 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
2644 else
2645 ret = 0;
2646
2647 parse_driver_options(smmu);
2648
2649 if (of_dma_is_coherent(dev->of_node))
2650 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2651
2652 return ret;
2653}
2654
2655static int arm_smmu_device_probe(struct platform_device *pdev)
2656{
2657 int irq, ret;
2658 struct resource *res;
2659 resource_size_t ioaddr;
2660 struct arm_smmu_device *smmu;
2661 struct device *dev = &pdev->dev;
2662 bool bypass;
2663
2664 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2665 if (!smmu) {
2666 dev_err(dev, "failed to allocate arm_smmu_device\n");
2667 return -ENOMEM;
2668 }
2669 smmu->dev = dev;
2670
2671
2672 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2673 if (resource_size(res) + 1 < SZ_128K) {
2674 dev_err(dev, "MMIO region too small (%pr)\n", res);
2675 return -EINVAL;
2676 }
2677 ioaddr = res->start;
2678
2679 smmu->base = devm_ioremap_resource(dev, res);
2680 if (IS_ERR(smmu->base))
2681 return PTR_ERR(smmu->base);
2682
2683
2684 irq = platform_get_irq_byname(pdev, "eventq");
2685 if (irq > 0)
2686 smmu->evtq.q.irq = irq;
2687
2688 irq = platform_get_irq_byname(pdev, "priq");
2689 if (irq > 0)
2690 smmu->priq.q.irq = irq;
2691
2692 irq = platform_get_irq_byname(pdev, "cmdq-sync");
2693 if (irq > 0)
2694 smmu->cmdq.q.irq = irq;
2695
2696 irq = platform_get_irq_byname(pdev, "gerror");
2697 if (irq > 0)
2698 smmu->gerr_irq = irq;
2699
2700 if (dev->of_node) {
2701 ret = arm_smmu_device_dt_probe(pdev, smmu);
2702 } else {
2703 ret = arm_smmu_device_acpi_probe(pdev, smmu);
2704 if (ret == -ENODEV)
2705 return ret;
2706 }
2707
2708
2709 bypass = !!ret;
2710
2711
2712 ret = arm_smmu_device_hw_probe(smmu);
2713 if (ret)
2714 return ret;
2715
2716
2717 ret = arm_smmu_init_structures(smmu);
2718 if (ret)
2719 return ret;
2720
2721
2722 platform_set_drvdata(pdev, smmu);
2723
2724
2725 ret = arm_smmu_device_reset(smmu, bypass);
2726 if (ret)
2727 return ret;
2728
2729
2730 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
2731 "smmu3.%pa", &ioaddr);
2732 if (ret)
2733 return ret;
2734
2735 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2736 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2737
2738 ret = iommu_device_register(&smmu->iommu);
2739
2740#ifdef CONFIG_PCI
2741 if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
2742 pci_request_acs();
2743 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2744 if (ret)
2745 return ret;
2746 }
2747#endif
2748#ifdef CONFIG_ARM_AMBA
2749 if (amba_bustype.iommu_ops != &arm_smmu_ops) {
2750 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2751 if (ret)
2752 return ret;
2753 }
2754#endif
2755 if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
2756 ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2757 if (ret)
2758 return ret;
2759 }
2760 return 0;
2761}
2762
2763static int arm_smmu_device_remove(struct platform_device *pdev)
2764{
2765 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2766
2767 arm_smmu_device_disable(smmu);
2768 return 0;
2769}
2770
2771static struct of_device_id arm_smmu_of_match[] = {
2772 { .compatible = "arm,smmu-v3", },
2773 { },
2774};
2775MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2776
2777static struct platform_driver arm_smmu_driver = {
2778 .driver = {
2779 .name = "arm-smmu-v3",
2780 .of_match_table = of_match_ptr(arm_smmu_of_match),
2781 },
2782 .probe = arm_smmu_device_probe,
2783 .remove = arm_smmu_device_remove,
2784};
2785module_platform_driver(arm_smmu_driver);
2786
2787IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3", NULL);
2788
2789MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2790MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2791MODULE_LICENSE("GPL v2");
2792