1 // SPDX-License-Identifier: GPL-2.0
3 * IOMMU API for ARM architected SMMUv3 implementations.
5 * Copyright (C) 2015 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver is powered by bad coffee and bombay mix.
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/bitops.h>
16 #include <linux/crash_dump.h>
17 #include <linux/delay.h>
18 #include <linux/dma-iommu.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/init.h>
25 #include <linux/moduleparam.h>
26 #include <linux/msi.h>
28 #include <linux/of_address.h>
29 #include <linux/of_iommu.h>
30 #include <linux/of_platform.h>
31 #include <linux/pci.h>
32 #include <linux/pci-ats.h>
33 #include <linux/platform_device.h>
35 #include <linux/amba/bus.h>
38 #define ARM_SMMU_IDR0 0x0
39 #define IDR0_ST_LVL GENMASK(28, 27)
40 #define IDR0_ST_LVL_2LVL 1
41 #define IDR0_STALL_MODEL GENMASK(25, 24)
42 #define IDR0_STALL_MODEL_STALL 0
43 #define IDR0_STALL_MODEL_FORCE 2
44 #define IDR0_TTENDIAN GENMASK(22, 21)
45 #define IDR0_TTENDIAN_MIXED 0
46 #define IDR0_TTENDIAN_LE 2
47 #define IDR0_TTENDIAN_BE 3
48 #define IDR0_CD2L (1 << 19)
49 #define IDR0_VMID16 (1 << 18)
50 #define IDR0_PRI (1 << 16)
51 #define IDR0_SEV (1 << 14)
52 #define IDR0_MSI (1 << 13)
53 #define IDR0_ASID16 (1 << 12)
54 #define IDR0_ATS (1 << 10)
55 #define IDR0_HYP (1 << 9)
56 #define IDR0_COHACC (1 << 4)
57 #define IDR0_TTF GENMASK(3, 2)
58 #define IDR0_TTF_AARCH64 2
59 #define IDR0_TTF_AARCH32_64 3
60 #define IDR0_S1P (1 << 1)
61 #define IDR0_S2P (1 << 0)
63 #define ARM_SMMU_IDR1 0x4
64 #define IDR1_TABLES_PRESET (1 << 30)
65 #define IDR1_QUEUES_PRESET (1 << 29)
66 #define IDR1_REL (1 << 28)
67 #define IDR1_CMDQS GENMASK(25, 21)
68 #define IDR1_EVTQS GENMASK(20, 16)
69 #define IDR1_PRIQS GENMASK(15, 11)
70 #define IDR1_SSIDSIZE GENMASK(10, 6)
71 #define IDR1_SIDSIZE GENMASK(5, 0)
73 #define ARM_SMMU_IDR5 0x14
74 #define IDR5_STALL_MAX GENMASK(31, 16)
75 #define IDR5_GRAN64K (1 << 6)
76 #define IDR5_GRAN16K (1 << 5)
77 #define IDR5_GRAN4K (1 << 4)
78 #define IDR5_OAS GENMASK(2, 0)
79 #define IDR5_OAS_32_BIT 0
80 #define IDR5_OAS_36_BIT 1
81 #define IDR5_OAS_40_BIT 2
82 #define IDR5_OAS_42_BIT 3
83 #define IDR5_OAS_44_BIT 4
84 #define IDR5_OAS_48_BIT 5
85 #define IDR5_OAS_52_BIT 6
86 #define IDR5_VAX GENMASK(11, 10)
87 #define IDR5_VAX_52_BIT 1
89 #define ARM_SMMU_CR0 0x20
90 #define CR0_ATSCHK (1 << 4)
91 #define CR0_CMDQEN (1 << 3)
92 #define CR0_EVTQEN (1 << 2)
93 #define CR0_PRIQEN (1 << 1)
94 #define CR0_SMMUEN (1 << 0)
96 #define ARM_SMMU_CR0ACK 0x24
98 #define ARM_SMMU_CR1 0x28
99 #define CR1_TABLE_SH GENMASK(11, 10)
100 #define CR1_TABLE_OC GENMASK(9, 8)
101 #define CR1_TABLE_IC GENMASK(7, 6)
102 #define CR1_QUEUE_SH GENMASK(5, 4)
103 #define CR1_QUEUE_OC GENMASK(3, 2)
104 #define CR1_QUEUE_IC GENMASK(1, 0)
105 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
106 #define CR1_CACHE_NC 0
107 #define CR1_CACHE_WB 1
108 #define CR1_CACHE_WT 2
110 #define ARM_SMMU_CR2 0x2c
111 #define CR2_PTM (1 << 2)
112 #define CR2_RECINVSID (1 << 1)
113 #define CR2_E2H (1 << 0)
115 #define ARM_SMMU_GBPA 0x44
116 #define GBPA_UPDATE (1 << 31)
117 #define GBPA_ABORT (1 << 20)
119 #define ARM_SMMU_IRQ_CTRL 0x50
120 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
121 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
122 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
124 #define ARM_SMMU_IRQ_CTRLACK 0x54
126 #define ARM_SMMU_GERROR 0x60
127 #define GERROR_SFM_ERR (1 << 8)
128 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
129 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
130 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
131 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
132 #define GERROR_PRIQ_ABT_ERR (1 << 3)
133 #define GERROR_EVTQ_ABT_ERR (1 << 2)
134 #define GERROR_CMDQ_ERR (1 << 0)
135 #define GERROR_ERR_MASK 0xfd
137 #define ARM_SMMU_GERRORN 0x64
139 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
140 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
141 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
143 #define ARM_SMMU_STRTAB_BASE 0x80
144 #define STRTAB_BASE_RA (1UL << 62)
145 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
147 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
148 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
149 #define STRTAB_BASE_CFG_FMT_LINEAR 0
150 #define STRTAB_BASE_CFG_FMT_2LVL 1
151 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
152 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
154 #define ARM_SMMU_CMDQ_BASE 0x90
155 #define ARM_SMMU_CMDQ_PROD 0x98
156 #define ARM_SMMU_CMDQ_CONS 0x9c
158 #define ARM_SMMU_EVTQ_BASE 0xa0
159 #define ARM_SMMU_EVTQ_PROD 0x100a8
160 #define ARM_SMMU_EVTQ_CONS 0x100ac
161 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
162 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
163 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
165 #define ARM_SMMU_PRIQ_BASE 0xc0
166 #define ARM_SMMU_PRIQ_PROD 0x100c8
167 #define ARM_SMMU_PRIQ_CONS 0x100cc
168 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
169 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
170 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
172 /* Common MSI config fields */
173 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
174 #define MSI_CFG2_SH GENMASK(5, 4)
175 #define MSI_CFG2_MEMATTR GENMASK(3, 0)
177 /* Common memory attribute values */
178 #define ARM_SMMU_SH_NSH 0
179 #define ARM_SMMU_SH_OSH 2
180 #define ARM_SMMU_SH_ISH 3
181 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
182 #define ARM_SMMU_MEMATTR_OIWB 0xf
184 #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
185 #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
186 #define Q_OVERFLOW_FLAG (1 << 31)
187 #define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
188 #define Q_ENT(q, p) ((q)->base + \
189 Q_IDX(q, p) * (q)->ent_dwords)
191 #define Q_BASE_RWA (1UL << 62)
192 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
193 #define Q_BASE_LOG2SIZE GENMASK(4, 0)
198 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
199 * 2lvl: 128k L1 entries,
200 * 256 lazy entries per table (each table covers a PCI bus)
202 #define STRTAB_L1_SZ_SHIFT 20
203 #define STRTAB_SPLIT 8
205 #define STRTAB_L1_DESC_DWORDS 1
206 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
207 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
209 #define STRTAB_STE_DWORDS 8
210 #define STRTAB_STE_0_V (1UL << 0)
211 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
212 #define STRTAB_STE_0_CFG_ABORT 0
213 #define STRTAB_STE_0_CFG_BYPASS 4
214 #define STRTAB_STE_0_CFG_S1_TRANS 5
215 #define STRTAB_STE_0_CFG_S2_TRANS 6
217 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
218 #define STRTAB_STE_0_S1FMT_LINEAR 0
219 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
220 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
222 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
223 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
224 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
225 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
226 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
227 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
228 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
230 #define STRTAB_STE_1_S1STALLD (1UL << 27)
232 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
233 #define STRTAB_STE_1_EATS_ABT 0UL
234 #define STRTAB_STE_1_EATS_TRANS 1UL
235 #define STRTAB_STE_1_EATS_S1CHK 2UL
237 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
238 #define STRTAB_STE_1_STRW_NSEL1 0UL
239 #define STRTAB_STE_1_STRW_EL2 2UL
241 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
242 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
244 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
245 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
246 #define STRTAB_STE_2_S2AA64 (1UL << 51)
247 #define STRTAB_STE_2_S2ENDI (1UL << 52)
248 #define STRTAB_STE_2_S2PTW (1UL << 54)
249 #define STRTAB_STE_2_S2R (1UL << 58)
251 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
253 /* Context descriptor (stage-1 only) */
254 #define CTXDESC_CD_DWORDS 8
255 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
256 #define ARM64_TCR_T0SZ GENMASK_ULL(5, 0)
257 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
258 #define ARM64_TCR_TG0 GENMASK_ULL(15, 14)
259 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
260 #define ARM64_TCR_IRGN0 GENMASK_ULL(9, 8)
261 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
262 #define ARM64_TCR_ORGN0 GENMASK_ULL(11, 10)
263 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
264 #define ARM64_TCR_SH0 GENMASK_ULL(13, 12)
265 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
266 #define ARM64_TCR_EPD0 (1ULL << 7)
267 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
268 #define ARM64_TCR_EPD1 (1ULL << 23)
270 #define CTXDESC_CD_0_ENDI (1UL << 15)
271 #define CTXDESC_CD_0_V (1UL << 31)
273 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
274 #define ARM64_TCR_IPS GENMASK_ULL(34, 32)
275 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
276 #define ARM64_TCR_TBI0 (1ULL << 37)
278 #define CTXDESC_CD_0_AA64 (1UL << 41)
279 #define CTXDESC_CD_0_S (1UL << 44)
280 #define CTXDESC_CD_0_R (1UL << 45)
281 #define CTXDESC_CD_0_A (1UL << 46)
282 #define CTXDESC_CD_0_ASET (1UL << 47)
283 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
285 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
287 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
288 #define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \
289 FIELD_GET(ARM64_TCR_##fld, tcr))
292 #define CMDQ_ENT_DWORDS 2
293 #define CMDQ_MAX_SZ_SHIFT 8
295 #define CMDQ_CONS_ERR GENMASK(30, 24)
296 #define CMDQ_ERR_CERROR_NONE_IDX 0
297 #define CMDQ_ERR_CERROR_ILL_IDX 1
298 #define CMDQ_ERR_CERROR_ABT_IDX 2
299 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
301 #define CMDQ_0_OP GENMASK_ULL(7, 0)
302 #define CMDQ_0_SSV (1UL << 11)
304 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
305 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
306 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
308 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
309 #define CMDQ_CFGI_1_LEAF (1UL << 0)
310 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
312 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
313 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
314 #define CMDQ_TLBI_1_LEAF (1UL << 0)
315 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
316 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
318 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
319 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
320 #define CMDQ_ATC_0_GLOBAL (1UL << 9)
321 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
322 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
324 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
325 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
326 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
327 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
329 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
330 #define CMDQ_SYNC_0_CS_NONE 0
331 #define CMDQ_SYNC_0_CS_IRQ 1
332 #define CMDQ_SYNC_0_CS_SEV 2
333 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
334 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
335 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
336 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
339 #define EVTQ_ENT_DWORDS 4
340 #define EVTQ_MAX_SZ_SHIFT 7
342 #define EVTQ_0_ID GENMASK_ULL(7, 0)
345 #define PRIQ_ENT_DWORDS 2
346 #define PRIQ_MAX_SZ_SHIFT 8
348 #define PRIQ_0_SID GENMASK_ULL(31, 0)
349 #define PRIQ_0_SSID GENMASK_ULL(51, 32)
350 #define PRIQ_0_PERM_PRIV (1UL << 58)
351 #define PRIQ_0_PERM_EXEC (1UL << 59)
352 #define PRIQ_0_PERM_READ (1UL << 60)
353 #define PRIQ_0_PERM_WRITE (1UL << 61)
354 #define PRIQ_0_PRG_LAST (1UL << 62)
355 #define PRIQ_0_SSID_V (1UL << 63)
357 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
358 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
360 /* High-level queue structures */
361 #define ARM_SMMU_POLL_TIMEOUT_US 100
362 #define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US 1000000 /* 1s! */
363 #define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT 10
365 #define MSI_IOVA_BASE 0x8000000
366 #define MSI_IOVA_LENGTH 0x100000
369 * not really modular, but the easiest way to keep compat with existing
370 * bootargs behaviour is to continue using module_param_named here.
372 static bool disable_bypass = 1;
373 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
374 MODULE_PARM_DESC(disable_bypass,
375 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
383 enum arm_smmu_msi_index {
390 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
392 ARM_SMMU_EVTQ_IRQ_CFG0,
393 ARM_SMMU_EVTQ_IRQ_CFG1,
394 ARM_SMMU_EVTQ_IRQ_CFG2,
396 [GERROR_MSI_INDEX] = {
397 ARM_SMMU_GERROR_IRQ_CFG0,
398 ARM_SMMU_GERROR_IRQ_CFG1,
399 ARM_SMMU_GERROR_IRQ_CFG2,
402 ARM_SMMU_PRIQ_IRQ_CFG0,
403 ARM_SMMU_PRIQ_IRQ_CFG1,
404 ARM_SMMU_PRIQ_IRQ_CFG2,
408 struct arm_smmu_cmdq_ent {
411 bool substream_valid;
413 /* Command-specific fields */
415 #define CMDQ_OP_PREFETCH_CFG 0x1
422 #define CMDQ_OP_CFGI_STE 0x3
423 #define CMDQ_OP_CFGI_ALL 0x4
432 #define CMDQ_OP_TLBI_NH_ASID 0x11
433 #define CMDQ_OP_TLBI_NH_VA 0x12
434 #define CMDQ_OP_TLBI_EL2_ALL 0x20
435 #define CMDQ_OP_TLBI_S12_VMALL 0x28
436 #define CMDQ_OP_TLBI_S2_IPA 0x2a
437 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
445 #define CMDQ_OP_ATC_INV 0x40
446 #define ATC_INV_SIZE_ALL 52
455 #define CMDQ_OP_PRI_RESP 0x41
463 #define CMDQ_OP_CMD_SYNC 0x46
471 struct arm_smmu_queue {
472 int irq; /* Wired interrupt */
483 u32 __iomem *prod_reg;
484 u32 __iomem *cons_reg;
487 struct arm_smmu_cmdq {
488 struct arm_smmu_queue q;
492 struct arm_smmu_evtq {
493 struct arm_smmu_queue q;
497 struct arm_smmu_priq {
498 struct arm_smmu_queue q;
501 /* High-level stream table and context descriptor structures */
502 struct arm_smmu_strtab_l1_desc {
506 dma_addr_t l2ptr_dma;
509 struct arm_smmu_s1_cfg {
511 dma_addr_t cdptr_dma;
513 struct arm_smmu_ctx_desc {
521 struct arm_smmu_s2_cfg {
527 struct arm_smmu_strtab_cfg {
529 dma_addr_t strtab_dma;
530 struct arm_smmu_strtab_l1_desc *l1_desc;
531 unsigned int num_l1_ents;
537 /* An SMMUv3 instance */
538 struct arm_smmu_device {
542 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
543 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
544 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
545 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
546 #define ARM_SMMU_FEAT_PRI (1 << 4)
547 #define ARM_SMMU_FEAT_ATS (1 << 5)
548 #define ARM_SMMU_FEAT_SEV (1 << 6)
549 #define ARM_SMMU_FEAT_MSI (1 << 7)
550 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
551 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
552 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
553 #define ARM_SMMU_FEAT_STALLS (1 << 11)
554 #define ARM_SMMU_FEAT_HYP (1 << 12)
555 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
556 #define ARM_SMMU_FEAT_VAX (1 << 14)
559 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
560 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
563 struct arm_smmu_cmdq cmdq;
564 struct arm_smmu_evtq evtq;
565 struct arm_smmu_priq priq;
572 unsigned long ias; /* IPA */
573 unsigned long oas; /* PA */
574 unsigned long pgsize_bitmap;
576 #define ARM_SMMU_MAX_ASIDS (1 << 16)
577 unsigned int asid_bits;
578 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
580 #define ARM_SMMU_MAX_VMIDS (1 << 16)
581 unsigned int vmid_bits;
582 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
584 unsigned int ssid_bits;
585 unsigned int sid_bits;
587 struct arm_smmu_strtab_cfg strtab_cfg;
589 /* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
595 /* IOMMU core code handle */
596 struct iommu_device iommu;
599 /* SMMU private data for each master */
600 struct arm_smmu_master {
601 struct arm_smmu_device *smmu;
603 struct arm_smmu_domain *domain;
604 struct list_head domain_head;
606 unsigned int num_sids;
610 /* SMMU private data for an IOMMU domain */
611 enum arm_smmu_domain_stage {
612 ARM_SMMU_DOMAIN_S1 = 0,
614 ARM_SMMU_DOMAIN_NESTED,
615 ARM_SMMU_DOMAIN_BYPASS,
618 struct arm_smmu_domain {
619 struct arm_smmu_device *smmu;
620 struct mutex init_mutex; /* Protects smmu pointer */
622 struct io_pgtable_ops *pgtbl_ops;
625 enum arm_smmu_domain_stage stage;
627 struct arm_smmu_s1_cfg s1_cfg;
628 struct arm_smmu_s2_cfg s2_cfg;
631 struct iommu_domain domain;
633 struct list_head devices;
634 spinlock_t devices_lock;
637 struct arm_smmu_option_prop {
642 static struct arm_smmu_option_prop arm_smmu_options[] = {
643 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
644 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
648 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
649 struct arm_smmu_device *smmu)
651 if ((offset > SZ_64K) &&
652 (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
655 return smmu->base + offset;
658 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
660 return container_of(dom, struct arm_smmu_domain, domain);
663 static void parse_driver_options(struct arm_smmu_device *smmu)
668 if (of_property_read_bool(smmu->dev->of_node,
669 arm_smmu_options[i].prop)) {
670 smmu->options |= arm_smmu_options[i].opt;
671 dev_notice(smmu->dev, "option %s\n",
672 arm_smmu_options[i].prop);
674 } while (arm_smmu_options[++i].opt);
677 /* Low-level queue manipulation functions */
678 static bool queue_full(struct arm_smmu_queue *q)
680 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
681 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
684 static bool queue_empty(struct arm_smmu_queue *q)
686 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
687 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
690 static void queue_sync_cons(struct arm_smmu_queue *q)
692 q->cons = readl_relaxed(q->cons_reg);
695 static void queue_inc_cons(struct arm_smmu_queue *q)
697 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
699 q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
702 * Ensure that all CPU accesses (reads and writes) to the queue
703 * are complete before we update the cons pointer.
706 writel_relaxed(q->cons, q->cons_reg);
709 static int queue_sync_prod(struct arm_smmu_queue *q)
712 u32 prod = readl_relaxed(q->prod_reg);
714 if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
721 static void queue_inc_prod(struct arm_smmu_queue *q)
723 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
725 q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
726 writel(q->prod, q->prod_reg);
730 * Wait for the SMMU to consume items. If sync is true, wait until the queue
731 * is empty. Otherwise, wait until there is at least one free slot.
733 static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
736 unsigned int delay = 1, spin_cnt = 0;
738 /* Wait longer if it's a CMD_SYNC */
739 timeout = ktime_add_us(ktime_get(), sync ?
740 ARM_SMMU_CMDQ_SYNC_TIMEOUT_US :
741 ARM_SMMU_POLL_TIMEOUT_US);
743 while (queue_sync_cons(q), (sync ? !queue_empty(q) : queue_full(q))) {
744 if (ktime_compare(ktime_get(), timeout) > 0)
749 } else if (++spin_cnt < ARM_SMMU_CMDQ_SYNC_SPIN_COUNT) {
762 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
766 for (i = 0; i < n_dwords; ++i)
767 *dst++ = cpu_to_le64(*src++);
770 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
775 queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
780 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
784 for (i = 0; i < n_dwords; ++i)
785 *dst++ = le64_to_cpu(*src++);
788 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
793 queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
798 /* High-level queue accessors */
799 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
801 memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
802 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
804 switch (ent->opcode) {
805 case CMDQ_OP_TLBI_EL2_ALL:
806 case CMDQ_OP_TLBI_NSNH_ALL:
808 case CMDQ_OP_PREFETCH_CFG:
809 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
810 cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
811 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
813 case CMDQ_OP_CFGI_STE:
814 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
815 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
817 case CMDQ_OP_CFGI_ALL:
818 /* Cover the entire SID range */
819 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
821 case CMDQ_OP_TLBI_NH_VA:
822 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
823 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
824 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
826 case CMDQ_OP_TLBI_S2_IPA:
827 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
828 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
829 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
831 case CMDQ_OP_TLBI_NH_ASID:
832 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
834 case CMDQ_OP_TLBI_S12_VMALL:
835 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
837 case CMDQ_OP_ATC_INV:
838 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
839 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
840 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
841 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
842 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
843 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
845 case CMDQ_OP_PRI_RESP:
846 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
847 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
848 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
849 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
850 switch (ent->pri.resp) {
858 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
860 case CMDQ_OP_CMD_SYNC:
861 if (ent->sync.msiaddr)
862 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
864 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
865 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
866 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
868 * Commands are written little-endian, but we want the SMMU to
869 * receive MSIData, and thus write it back to memory, in CPU
870 * byte order, so big-endian needs an extra byteswap here.
872 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIDATA,
873 cpu_to_le32(ent->sync.msidata));
874 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
883 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
885 static const char *cerror_str[] = {
886 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
887 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
888 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
889 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
893 u64 cmd[CMDQ_ENT_DWORDS];
894 struct arm_smmu_queue *q = &smmu->cmdq.q;
895 u32 cons = readl_relaxed(q->cons_reg);
896 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
897 struct arm_smmu_cmdq_ent cmd_sync = {
898 .opcode = CMDQ_OP_CMD_SYNC,
901 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
902 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
905 case CMDQ_ERR_CERROR_ABT_IDX:
906 dev_err(smmu->dev, "retrying command fetch\n");
907 case CMDQ_ERR_CERROR_NONE_IDX:
909 case CMDQ_ERR_CERROR_ATC_INV_IDX:
911 * ATC Invalidation Completion timeout. CONS is still pointing
912 * at the CMD_SYNC. Attempt to complete other pending commands
913 * by repeating the CMD_SYNC, though we might well end up back
914 * here since the ATC invalidation may still be pending.
917 case CMDQ_ERR_CERROR_ILL_IDX:
924 * We may have concurrent producers, so we need to be careful
925 * not to touch any of the shadow cmdq state.
927 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
928 dev_err(smmu->dev, "skipping command in error state:\n");
929 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
930 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
932 /* Convert the erroneous command into a CMD_SYNC */
933 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
934 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
938 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
941 static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device *smmu, u64 *cmd)
943 struct arm_smmu_queue *q = &smmu->cmdq.q;
944 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
946 smmu->prev_cmd_opcode = FIELD_GET(CMDQ_0_OP, cmd[0]);
948 while (queue_insert_raw(q, cmd) == -ENOSPC) {
949 if (queue_poll_cons(q, false, wfe))
950 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
954 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
955 struct arm_smmu_cmdq_ent *ent)
957 u64 cmd[CMDQ_ENT_DWORDS];
960 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
961 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
966 spin_lock_irqsave(&smmu->cmdq.lock, flags);
967 arm_smmu_cmdq_insert_cmd(smmu, cmd);
968 spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
972 * The difference between val and sync_idx is bounded by the maximum size of
973 * a queue at 2^20 entries, so 32 bits is plenty for wrap-safe arithmetic.
975 static int __arm_smmu_sync_poll_msi(struct arm_smmu_device *smmu, u32 sync_idx)
980 timeout = ktime_add_us(ktime_get(), ARM_SMMU_CMDQ_SYNC_TIMEOUT_US);
981 val = smp_cond_load_acquire(&smmu->sync_count,
982 (int)(VAL - sync_idx) >= 0 ||
983 !ktime_before(ktime_get(), timeout));
985 return (int)(val - sync_idx) < 0 ? -ETIMEDOUT : 0;
988 static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device *smmu)
990 u64 cmd[CMDQ_ENT_DWORDS];
992 struct arm_smmu_cmdq_ent ent = {
993 .opcode = CMDQ_OP_CMD_SYNC,
995 .msiaddr = virt_to_phys(&smmu->sync_count),
999 spin_lock_irqsave(&smmu->cmdq.lock, flags);
1001 /* Piggy-back on the previous command if it's a SYNC */
1002 if (smmu->prev_cmd_opcode == CMDQ_OP_CMD_SYNC) {
1003 ent.sync.msidata = smmu->sync_nr;
1005 ent.sync.msidata = ++smmu->sync_nr;
1006 arm_smmu_cmdq_build_cmd(cmd, &ent);
1007 arm_smmu_cmdq_insert_cmd(smmu, cmd);
1010 spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1012 return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata);
1015 static int __arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1017 u64 cmd[CMDQ_ENT_DWORDS];
1018 unsigned long flags;
1019 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
1020 struct arm_smmu_cmdq_ent ent = { .opcode = CMDQ_OP_CMD_SYNC };
1023 arm_smmu_cmdq_build_cmd(cmd, &ent);
1025 spin_lock_irqsave(&smmu->cmdq.lock, flags);
1026 arm_smmu_cmdq_insert_cmd(smmu, cmd);
1027 ret = queue_poll_cons(&smmu->cmdq.q, true, wfe);
1028 spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1033 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1036 bool msi = (smmu->features & ARM_SMMU_FEAT_MSI) &&
1037 (smmu->features & ARM_SMMU_FEAT_COHERENCY);
1039 ret = msi ? __arm_smmu_cmdq_issue_sync_msi(smmu)
1040 : __arm_smmu_cmdq_issue_sync(smmu);
1042 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
1046 /* Context descriptor manipulation functions */
1047 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
1051 /* Repack the TCR. Just care about TTBR0 for now */
1052 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
1053 val |= ARM_SMMU_TCR2CD(tcr, TG0);
1054 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
1055 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
1056 val |= ARM_SMMU_TCR2CD(tcr, SH0);
1057 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
1058 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
1059 val |= ARM_SMMU_TCR2CD(tcr, IPS);
1060 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
1065 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
1066 struct arm_smmu_s1_cfg *cfg)
1071 * We don't need to issue any invalidation here, as we'll invalidate
1072 * the STE when installing the new entry anyway.
1074 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
1078 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
1079 CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) |
1082 /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1083 if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1084 val |= CTXDESC_CD_0_S;
1086 cfg->cdptr[0] = cpu_to_le64(val);
1088 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
1089 cfg->cdptr[1] = cpu_to_le64(val);
1091 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
1094 /* Stream table manipulation functions */
1096 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1100 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1101 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1103 *dst = cpu_to_le64(val);
1106 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1108 struct arm_smmu_cmdq_ent cmd = {
1109 .opcode = CMDQ_OP_CFGI_STE,
1116 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1117 arm_smmu_cmdq_issue_sync(smmu);
1120 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1124 * This is hideously complicated, but we only really care about
1125 * three cases at the moment:
1127 * 1. Invalid (all zero) -> bypass/fault (init)
1128 * 2. Bypass/fault -> translation/bypass (attach)
1129 * 3. Translation/bypass -> bypass/fault (detach)
1131 * Given that we can't update the STE atomically and the SMMU
1132 * doesn't read the thing in a defined order, that leaves us
1133 * with the following maintenance requirements:
1135 * 1. Update Config, return (init time STEs aren't live)
1136 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1137 * 3. Update Config, sync
1139 u64 val = le64_to_cpu(dst[0]);
1140 bool ste_live = false;
1141 struct arm_smmu_device *smmu = NULL;
1142 struct arm_smmu_s1_cfg *s1_cfg = NULL;
1143 struct arm_smmu_s2_cfg *s2_cfg = NULL;
1144 struct arm_smmu_domain *smmu_domain = NULL;
1145 struct arm_smmu_cmdq_ent prefetch_cmd = {
1146 .opcode = CMDQ_OP_PREFETCH_CFG,
1153 smmu_domain = master->domain;
1154 smmu = master->smmu;
1158 switch (smmu_domain->stage) {
1159 case ARM_SMMU_DOMAIN_S1:
1160 s1_cfg = &smmu_domain->s1_cfg;
1162 case ARM_SMMU_DOMAIN_S2:
1163 case ARM_SMMU_DOMAIN_NESTED:
1164 s2_cfg = &smmu_domain->s2_cfg;
1171 if (val & STRTAB_STE_0_V) {
1172 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1173 case STRTAB_STE_0_CFG_BYPASS:
1175 case STRTAB_STE_0_CFG_S1_TRANS:
1176 case STRTAB_STE_0_CFG_S2_TRANS:
1179 case STRTAB_STE_0_CFG_ABORT:
1183 BUG(); /* STE corruption */
1187 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1188 val = STRTAB_STE_0_V;
1191 if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1192 if (!smmu_domain && disable_bypass)
1193 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1195 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1197 dst[0] = cpu_to_le64(val);
1198 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1199 STRTAB_STE_1_SHCFG_INCOMING));
1200 dst[2] = 0; /* Nuke the VMID */
1202 * The SMMU can perform negative caching, so we must sync
1203 * the STE regardless of whether the old value was live.
1206 arm_smmu_sync_ste_for_sid(smmu, sid);
1212 dst[1] = cpu_to_le64(
1213 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1214 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1215 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1216 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1218 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1219 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1220 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1222 val |= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1223 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);
1228 dst[2] = cpu_to_le64(
1229 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1230 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1232 STRTAB_STE_2_S2ENDI |
1234 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1237 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1239 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1242 if (master->ats_enabled)
1243 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1244 STRTAB_STE_1_EATS_TRANS));
1246 arm_smmu_sync_ste_for_sid(smmu, sid);
1247 dst[0] = cpu_to_le64(val);
1248 arm_smmu_sync_ste_for_sid(smmu, sid);
1250 /* It's likely that we'll want to use the new STE soon */
1251 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1252 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1255 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1259 for (i = 0; i < nent; ++i) {
1260 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1261 strtab += STRTAB_STE_DWORDS;
1265 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1269 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1270 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1275 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1276 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1278 desc->span = STRTAB_SPLIT + 1;
1279 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1280 GFP_KERNEL | __GFP_ZERO);
1283 "failed to allocate l2 stream table for SID %u\n",
1288 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1289 arm_smmu_write_strtab_l1_desc(strtab, desc);
1293 /* IRQ and event handlers */
1294 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1297 struct arm_smmu_device *smmu = dev;
1298 struct arm_smmu_queue *q = &smmu->evtq.q;
1299 u64 evt[EVTQ_ENT_DWORDS];
1302 while (!queue_remove_raw(q, evt)) {
1303 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1305 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1306 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1307 dev_info(smmu->dev, "\t0x%016llx\n",
1308 (unsigned long long)evt[i]);
1313 * Not much we can do on overflow, so scream and pretend we're
1316 if (queue_sync_prod(q) == -EOVERFLOW)
1317 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1318 } while (!queue_empty(q));
1320 /* Sync our overflow flag, as we believe we're up to speed */
1321 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1325 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1331 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1332 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1333 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
1334 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1335 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1337 dev_info(smmu->dev, "unexpected PRI request received:\n");
1339 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1340 sid, ssid, grpid, last ? "L" : "",
1341 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1342 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1343 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1344 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1345 evt[1] & PRIQ_1_ADDR_MASK);
1348 struct arm_smmu_cmdq_ent cmd = {
1349 .opcode = CMDQ_OP_PRI_RESP,
1350 .substream_valid = ssv,
1355 .resp = PRI_RESP_DENY,
1359 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1363 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1365 struct arm_smmu_device *smmu = dev;
1366 struct arm_smmu_queue *q = &smmu->priq.q;
1367 u64 evt[PRIQ_ENT_DWORDS];
1370 while (!queue_remove_raw(q, evt))
1371 arm_smmu_handle_ppr(smmu, evt);
1373 if (queue_sync_prod(q) == -EOVERFLOW)
1374 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1375 } while (!queue_empty(q));
1377 /* Sync our overflow flag, as we believe we're up to speed */
1378 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1379 writel(q->cons, q->cons_reg);
1383 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1385 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1387 u32 gerror, gerrorn, active;
1388 struct arm_smmu_device *smmu = dev;
1390 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1391 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1393 active = gerror ^ gerrorn;
1394 if (!(active & GERROR_ERR_MASK))
1395 return IRQ_NONE; /* No errors pending */
1398 "unexpected global error reported (0x%08x), this could be serious\n",
1401 if (active & GERROR_SFM_ERR) {
1402 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1403 arm_smmu_device_disable(smmu);
1406 if (active & GERROR_MSI_GERROR_ABT_ERR)
1407 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1409 if (active & GERROR_MSI_PRIQ_ABT_ERR)
1410 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1412 if (active & GERROR_MSI_EVTQ_ABT_ERR)
1413 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1415 if (active & GERROR_MSI_CMDQ_ABT_ERR)
1416 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1418 if (active & GERROR_PRIQ_ABT_ERR)
1419 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1421 if (active & GERROR_EVTQ_ABT_ERR)
1422 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1424 if (active & GERROR_CMDQ_ERR)
1425 arm_smmu_cmdq_skip_err(smmu);
1427 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1431 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1433 struct arm_smmu_device *smmu = dev;
1435 arm_smmu_evtq_thread(irq, dev);
1436 if (smmu->features & ARM_SMMU_FEAT_PRI)
1437 arm_smmu_priq_thread(irq, dev);
1442 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1444 arm_smmu_gerror_handler(irq, dev);
1445 return IRQ_WAKE_THREAD;
1449 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1450 struct arm_smmu_cmdq_ent *cmd)
1454 /* ATC invalidates are always on 4096-bytes pages */
1455 size_t inval_grain_shift = 12;
1456 unsigned long page_start, page_end;
1458 *cmd = (struct arm_smmu_cmdq_ent) {
1459 .opcode = CMDQ_OP_ATC_INV,
1460 .substream_valid = !!ssid,
1465 cmd->atc.size = ATC_INV_SIZE_ALL;
1469 page_start = iova >> inval_grain_shift;
1470 page_end = (iova + size - 1) >> inval_grain_shift;
1473 * In an ATS Invalidate Request, the address must be aligned on the
1474 * range size, which must be a power of two number of page sizes. We
1475 * thus have to choose between grossly over-invalidating the region, or
1476 * splitting the invalidation into multiple commands. For simplicity
1477 * we'll go with the first solution, but should refine it in the future
1478 * if multiple commands are shown to be more efficient.
1480 * Find the smallest power of two that covers the range. The most
1481 * significant differing bit between the start and end addresses,
1482 * fls(start ^ end), indicates the required span. For example:
1484 * We want to invalidate pages [8; 11]. This is already the ideal range:
1485 * x = 0b1000 ^ 0b1011 = 0b11
1486 * span = 1 << fls(x) = 4
1488 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1489 * x = 0b0111 ^ 0b1010 = 0b1101
1490 * span = 1 << fls(x) = 16
1492 log2_span = fls_long(page_start ^ page_end);
1493 span_mask = (1ULL << log2_span) - 1;
1495 page_start &= ~span_mask;
1497 cmd->atc.addr = page_start << inval_grain_shift;
1498 cmd->atc.size = log2_span;
1501 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
1502 struct arm_smmu_cmdq_ent *cmd)
1506 if (!master->ats_enabled)
1509 for (i = 0; i < master->num_sids; i++) {
1510 cmd->atc.sid = master->sids[i];
1511 arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
1514 return arm_smmu_cmdq_issue_sync(master->smmu);
1517 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
1518 int ssid, unsigned long iova, size_t size)
1521 unsigned long flags;
1522 struct arm_smmu_cmdq_ent cmd;
1523 struct arm_smmu_master *master;
1525 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1528 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1530 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1531 list_for_each_entry(master, &smmu_domain->devices, domain_head)
1532 ret |= arm_smmu_atc_inv_master(master, &cmd);
1533 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1535 return ret ? -ETIMEDOUT : 0;
1538 /* IO_PGTABLE API */
1539 static void arm_smmu_tlb_sync(void *cookie)
1541 struct arm_smmu_domain *smmu_domain = cookie;
1543 arm_smmu_cmdq_issue_sync(smmu_domain->smmu);
1546 static void arm_smmu_tlb_inv_context(void *cookie)
1548 struct arm_smmu_domain *smmu_domain = cookie;
1549 struct arm_smmu_device *smmu = smmu_domain->smmu;
1550 struct arm_smmu_cmdq_ent cmd;
1552 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1553 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1554 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1557 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1558 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1562 * NOTE: when io-pgtable is in non-strict mode, we may get here with
1563 * PTEs previously cleared by unmaps on the current CPU not yet visible
1564 * to the SMMU. We are relying on the DSB implicit in queue_inc_prod()
1565 * to guarantee those are observed before the TLBI. Do be careful, 007.
1567 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1568 arm_smmu_cmdq_issue_sync(smmu);
1571 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1572 size_t granule, bool leaf, void *cookie)
1574 struct arm_smmu_domain *smmu_domain = cookie;
1575 struct arm_smmu_device *smmu = smmu_domain->smmu;
1576 struct arm_smmu_cmdq_ent cmd = {
1583 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1584 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
1585 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1587 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1588 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1592 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1593 cmd.tlbi.addr += granule;
1594 } while (size -= granule);
1597 static const struct iommu_gather_ops arm_smmu_gather_ops = {
1598 .tlb_flush_all = arm_smmu_tlb_inv_context,
1599 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1600 .tlb_sync = arm_smmu_tlb_sync,
1604 static bool arm_smmu_capable(enum iommu_cap cap)
1607 case IOMMU_CAP_CACHE_COHERENCY:
1609 case IOMMU_CAP_NOEXEC:
1616 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1618 struct arm_smmu_domain *smmu_domain;
1620 if (type != IOMMU_DOMAIN_UNMANAGED &&
1621 type != IOMMU_DOMAIN_DMA &&
1622 type != IOMMU_DOMAIN_IDENTITY)
1626 * Allocate the domain and initialise some of its data structures.
1627 * We can't really do anything meaningful until we've added a
1630 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1634 if (type == IOMMU_DOMAIN_DMA &&
1635 iommu_get_dma_cookie(&smmu_domain->domain)) {
1640 mutex_init(&smmu_domain->init_mutex);
1641 INIT_LIST_HEAD(&smmu_domain->devices);
1642 spin_lock_init(&smmu_domain->devices_lock);
1644 return &smmu_domain->domain;
1647 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1649 int idx, size = 1 << span;
1652 idx = find_first_zero_bit(map, size);
1655 } while (test_and_set_bit(idx, map));
1660 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1662 clear_bit(idx, map);
1665 static void arm_smmu_domain_free(struct iommu_domain *domain)
1667 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1668 struct arm_smmu_device *smmu = smmu_domain->smmu;
1670 iommu_put_dma_cookie(domain);
1671 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1673 /* Free the CD and ASID, if we allocated them */
1674 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1675 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1678 dmam_free_coherent(smmu_domain->smmu->dev,
1679 CTXDESC_CD_DWORDS << 3,
1683 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1686 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1688 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1694 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1695 struct io_pgtable_cfg *pgtbl_cfg)
1699 struct arm_smmu_device *smmu = smmu_domain->smmu;
1700 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1702 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1706 cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1708 GFP_KERNEL | __GFP_ZERO);
1710 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1715 cfg->cd.asid = (u16)asid;
1716 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1717 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1718 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1722 arm_smmu_bitmap_free(smmu->asid_map, asid);
1726 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1727 struct io_pgtable_cfg *pgtbl_cfg)
1730 struct arm_smmu_device *smmu = smmu_domain->smmu;
1731 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1733 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1737 cfg->vmid = (u16)vmid;
1738 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1739 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1743 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1746 unsigned long ias, oas;
1747 enum io_pgtable_fmt fmt;
1748 struct io_pgtable_cfg pgtbl_cfg;
1749 struct io_pgtable_ops *pgtbl_ops;
1750 int (*finalise_stage_fn)(struct arm_smmu_domain *,
1751 struct io_pgtable_cfg *);
1752 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1753 struct arm_smmu_device *smmu = smmu_domain->smmu;
1755 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
1756 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
1760 /* Restrict the stage to what we can actually support */
1761 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1762 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1763 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1764 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1766 switch (smmu_domain->stage) {
1767 case ARM_SMMU_DOMAIN_S1:
1768 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
1769 ias = min_t(unsigned long, ias, VA_BITS);
1771 fmt = ARM_64_LPAE_S1;
1772 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1774 case ARM_SMMU_DOMAIN_NESTED:
1775 case ARM_SMMU_DOMAIN_S2:
1778 fmt = ARM_64_LPAE_S2;
1779 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1785 pgtbl_cfg = (struct io_pgtable_cfg) {
1786 .pgsize_bitmap = smmu->pgsize_bitmap,
1789 .tlb = &arm_smmu_gather_ops,
1790 .iommu_dev = smmu->dev,
1793 if (smmu->features & ARM_SMMU_FEAT_COHERENCY)
1794 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
1796 if (smmu_domain->non_strict)
1797 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
1799 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1803 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1804 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
1805 domain->geometry.force_aperture = true;
1807 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1809 free_io_pgtable_ops(pgtbl_ops);
1813 smmu_domain->pgtbl_ops = pgtbl_ops;
1817 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1820 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1822 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1823 struct arm_smmu_strtab_l1_desc *l1_desc;
1826 /* Two-level walk */
1827 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1828 l1_desc = &cfg->l1_desc[idx];
1829 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1830 step = &l1_desc->l2ptr[idx];
1832 /* Simple linear lookup */
1833 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1839 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
1842 struct arm_smmu_device *smmu = master->smmu;
1844 for (i = 0; i < master->num_sids; ++i) {
1845 u32 sid = master->sids[i];
1846 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1848 /* Bridged PCI devices may end up with duplicated IDs */
1849 for (j = 0; j < i; j++)
1850 if (master->sids[j] == sid)
1855 arm_smmu_write_strtab_ent(master, sid, step);
1859 static int arm_smmu_enable_ats(struct arm_smmu_master *master)
1863 struct pci_dev *pdev;
1864 struct arm_smmu_device *smmu = master->smmu;
1865 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
1867 if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
1868 !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
1871 pdev = to_pci_dev(master->dev);
1872 if (pdev->untrusted)
1875 /* Smallest Translation Unit: log2 of the smallest supported granule */
1876 stu = __ffs(smmu->pgsize_bitmap);
1878 ret = pci_enable_ats(pdev, stu);
1882 master->ats_enabled = true;
1886 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
1888 if (!master->ats_enabled || !dev_is_pci(master->dev))
1891 pci_disable_ats(to_pci_dev(master->dev));
1892 master->ats_enabled = false;
1895 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
1897 unsigned long flags;
1898 struct arm_smmu_domain *smmu_domain = master->domain;
1903 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1904 list_del(&master->domain_head);
1905 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1907 master->domain = NULL;
1908 arm_smmu_install_ste_for_dev(master);
1910 /* Disabling ATS invalidates all ATC entries */
1911 arm_smmu_disable_ats(master);
1914 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1917 unsigned long flags;
1918 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1919 struct arm_smmu_device *smmu;
1920 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1921 struct arm_smmu_master *master;
1926 master = fwspec->iommu_priv;
1927 smmu = master->smmu;
1929 arm_smmu_detach_dev(master);
1931 mutex_lock(&smmu_domain->init_mutex);
1933 if (!smmu_domain->smmu) {
1934 smmu_domain->smmu = smmu;
1935 ret = arm_smmu_domain_finalise(domain);
1937 smmu_domain->smmu = NULL;
1940 } else if (smmu_domain->smmu != smmu) {
1942 "cannot attach to SMMU %s (upstream of %s)\n",
1943 dev_name(smmu_domain->smmu->dev),
1944 dev_name(smmu->dev));
1949 master->domain = smmu_domain;
1951 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1952 list_add(&master->domain_head, &smmu_domain->devices);
1953 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1955 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
1956 arm_smmu_enable_ats(master);
1958 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1959 arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
1961 arm_smmu_install_ste_for_dev(master);
1963 mutex_unlock(&smmu_domain->init_mutex);
1967 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1968 phys_addr_t paddr, size_t size, int prot)
1970 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1975 return ops->map(ops, iova, paddr, size, prot);
1979 arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1982 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1983 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1988 ret = ops->unmap(ops, iova, size);
1989 if (ret && arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size))
1995 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1997 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1999 if (smmu_domain->smmu)
2000 arm_smmu_tlb_inv_context(smmu_domain);
2003 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
2005 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
2008 arm_smmu_cmdq_issue_sync(smmu);
2012 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2014 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2016 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2022 return ops->iova_to_phys(ops, iova);
2025 static struct platform_driver arm_smmu_driver;
2027 static int arm_smmu_match_node(struct device *dev, void *data)
2029 return dev->fwnode == data;
2033 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2035 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
2036 fwnode, arm_smmu_match_node);
2038 return dev ? dev_get_drvdata(dev) : NULL;
2041 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2043 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2045 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2046 limit *= 1UL << STRTAB_SPLIT;
2051 static struct iommu_ops arm_smmu_ops;
2053 static int arm_smmu_add_device(struct device *dev)
2056 struct arm_smmu_device *smmu;
2057 struct arm_smmu_master *master;
2058 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2059 struct iommu_group *group;
2061 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2064 * We _can_ actually withstand dodgy bus code re-calling add_device()
2065 * without an intervening remove_device()/of_xlate() sequence, but
2066 * we're not going to do so quietly...
2068 if (WARN_ON_ONCE(fwspec->iommu_priv)) {
2069 master = fwspec->iommu_priv;
2070 smmu = master->smmu;
2072 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2075 master = kzalloc(sizeof(*master), GFP_KERNEL);
2080 master->smmu = smmu;
2081 master->sids = fwspec->ids;
2082 master->num_sids = fwspec->num_ids;
2083 fwspec->iommu_priv = master;
2086 /* Check the SIDs are in range of the SMMU and our stream table */
2087 for (i = 0; i < master->num_sids; i++) {
2088 u32 sid = master->sids[i];
2090 if (!arm_smmu_sid_in_range(smmu, sid))
2093 /* Ensure l2 strtab is initialised */
2094 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2095 ret = arm_smmu_init_l2_strtab(smmu, sid);
2101 group = iommu_group_get_for_dev(dev);
2102 if (!IS_ERR(group)) {
2103 iommu_group_put(group);
2104 iommu_device_link(&smmu->iommu, dev);
2107 return PTR_ERR_OR_ZERO(group);
2110 static void arm_smmu_remove_device(struct device *dev)
2112 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2113 struct arm_smmu_master *master;
2114 struct arm_smmu_device *smmu;
2116 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2119 master = fwspec->iommu_priv;
2120 smmu = master->smmu;
2121 arm_smmu_detach_dev(master);
2122 iommu_group_remove_device(dev);
2123 iommu_device_unlink(&smmu->iommu, dev);
2125 iommu_fwspec_free(dev);
2128 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2130 struct iommu_group *group;
2133 * We don't support devices sharing stream IDs other than PCI RID
2134 * aliases, since the necessary ID-to-device lookup becomes rather
2135 * impractical given a potential sparse 32-bit stream ID space.
2137 if (dev_is_pci(dev))
2138 group = pci_device_group(dev);
2140 group = generic_device_group(dev);
2145 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
2146 enum iommu_attr attr, void *data)
2148 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2150 switch (domain->type) {
2151 case IOMMU_DOMAIN_UNMANAGED:
2153 case DOMAIN_ATTR_NESTING:
2154 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
2160 case IOMMU_DOMAIN_DMA:
2162 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2163 *(int *)data = smmu_domain->non_strict;
2174 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
2175 enum iommu_attr attr, void *data)
2178 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2180 mutex_lock(&smmu_domain->init_mutex);
2182 switch (domain->type) {
2183 case IOMMU_DOMAIN_UNMANAGED:
2185 case DOMAIN_ATTR_NESTING:
2186 if (smmu_domain->smmu) {
2192 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2194 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2200 case IOMMU_DOMAIN_DMA:
2202 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2203 smmu_domain->non_strict = *(int *)data;
2214 mutex_unlock(&smmu_domain->init_mutex);
2218 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2220 return iommu_fwspec_add_ids(dev, args->args, 1);
2223 static void arm_smmu_get_resv_regions(struct device *dev,
2224 struct list_head *head)
2226 struct iommu_resv_region *region;
2227 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2229 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2230 prot, IOMMU_RESV_SW_MSI);
2234 list_add_tail(®ion->list, head);
2236 iommu_dma_get_resv_regions(dev, head);
2239 static void arm_smmu_put_resv_regions(struct device *dev,
2240 struct list_head *head)
2242 struct iommu_resv_region *entry, *next;
2244 list_for_each_entry_safe(entry, next, head, list)
2248 static struct iommu_ops arm_smmu_ops = {
2249 .capable = arm_smmu_capable,
2250 .domain_alloc = arm_smmu_domain_alloc,
2251 .domain_free = arm_smmu_domain_free,
2252 .attach_dev = arm_smmu_attach_dev,
2253 .map = arm_smmu_map,
2254 .unmap = arm_smmu_unmap,
2255 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
2256 .iotlb_sync = arm_smmu_iotlb_sync,
2257 .iova_to_phys = arm_smmu_iova_to_phys,
2258 .add_device = arm_smmu_add_device,
2259 .remove_device = arm_smmu_remove_device,
2260 .device_group = arm_smmu_device_group,
2261 .domain_get_attr = arm_smmu_domain_get_attr,
2262 .domain_set_attr = arm_smmu_domain_set_attr,
2263 .of_xlate = arm_smmu_of_xlate,
2264 .get_resv_regions = arm_smmu_get_resv_regions,
2265 .put_resv_regions = arm_smmu_put_resv_regions,
2266 .pgsize_bitmap = -1UL, /* Restricted during device attach */
2269 /* Probing and initialisation functions */
2270 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2271 struct arm_smmu_queue *q,
2272 unsigned long prod_off,
2273 unsigned long cons_off,
2276 size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
2278 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
2280 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
2285 q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu);
2286 q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu);
2287 q->ent_dwords = dwords;
2289 q->q_base = Q_BASE_RWA;
2290 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2291 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->max_n_shift);
2293 q->prod = q->cons = 0;
2297 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2302 spin_lock_init(&smmu->cmdq.lock);
2303 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
2304 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
2309 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
2310 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
2315 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2318 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2319 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
2322 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2325 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2326 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2327 void *strtab = smmu->strtab_cfg.strtab;
2329 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2330 if (!cfg->l1_desc) {
2331 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2335 for (i = 0; i < cfg->num_l1_ents; ++i) {
2336 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2337 strtab += STRTAB_L1_DESC_DWORDS << 3;
2343 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2348 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2350 /* Calculate the L1 size, capped to the SIDSIZE. */
2351 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2352 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2353 cfg->num_l1_ents = 1 << size;
2355 size += STRTAB_SPLIT;
2356 if (size < smmu->sid_bits)
2358 "2-level strtab only covers %u/%u bits of SID\n",
2359 size, smmu->sid_bits);
2361 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2362 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2363 GFP_KERNEL | __GFP_ZERO);
2366 "failed to allocate l1 stream table (%u bytes)\n",
2370 cfg->strtab = strtab;
2372 /* Configure strtab_base_cfg for 2 levels */
2373 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
2374 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
2375 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
2376 cfg->strtab_base_cfg = reg;
2378 return arm_smmu_init_l1_strtab(smmu);
2381 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2386 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2388 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2389 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2390 GFP_KERNEL | __GFP_ZERO);
2393 "failed to allocate linear stream table (%u bytes)\n",
2397 cfg->strtab = strtab;
2398 cfg->num_l1_ents = 1 << smmu->sid_bits;
2400 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2401 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
2402 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
2403 cfg->strtab_base_cfg = reg;
2405 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2409 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2414 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2415 ret = arm_smmu_init_strtab_2lvl(smmu);
2417 ret = arm_smmu_init_strtab_linear(smmu);
2422 /* Set the strtab base address */
2423 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
2424 reg |= STRTAB_BASE_RA;
2425 smmu->strtab_cfg.strtab_base = reg;
2427 /* Allocate the first VMID for stage-2 bypass STEs */
2428 set_bit(0, smmu->vmid_map);
2432 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2436 ret = arm_smmu_init_queues(smmu);
2440 return arm_smmu_init_strtab(smmu);
2443 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2444 unsigned int reg_off, unsigned int ack_off)
2448 writel_relaxed(val, smmu->base + reg_off);
2449 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2450 1, ARM_SMMU_POLL_TIMEOUT_US);
2453 /* GBPA is "special" */
2454 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2457 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2459 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2460 1, ARM_SMMU_POLL_TIMEOUT_US);
2466 writel_relaxed(reg | GBPA_UPDATE, gbpa);
2467 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2468 1, ARM_SMMU_POLL_TIMEOUT_US);
2471 dev_err(smmu->dev, "GBPA not responding to update\n");
2475 static void arm_smmu_free_msis(void *data)
2477 struct device *dev = data;
2478 platform_msi_domain_free_irqs(dev);
2481 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2483 phys_addr_t doorbell;
2484 struct device *dev = msi_desc_to_dev(desc);
2485 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2486 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2488 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2489 doorbell &= MSI_CFG0_ADDR_MASK;
2491 writeq_relaxed(doorbell, smmu->base + cfg[0]);
2492 writel_relaxed(msg->data, smmu->base + cfg[1]);
2493 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2496 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2498 struct msi_desc *desc;
2499 int ret, nvec = ARM_SMMU_MAX_MSIS;
2500 struct device *dev = smmu->dev;
2502 /* Clear the MSI address regs */
2503 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2504 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2506 if (smmu->features & ARM_SMMU_FEAT_PRI)
2507 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2511 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2514 if (!dev->msi_domain) {
2515 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
2519 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2520 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2522 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
2526 for_each_msi_entry(desc, dev) {
2527 switch (desc->platform.msi_index) {
2528 case EVTQ_MSI_INDEX:
2529 smmu->evtq.q.irq = desc->irq;
2531 case GERROR_MSI_INDEX:
2532 smmu->gerr_irq = desc->irq;
2534 case PRIQ_MSI_INDEX:
2535 smmu->priq.q.irq = desc->irq;
2537 default: /* Unknown */
2542 /* Add callback to free MSIs on teardown */
2543 devm_add_action(dev, arm_smmu_free_msis, dev);
2546 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
2550 arm_smmu_setup_msis(smmu);
2552 /* Request interrupt lines */
2553 irq = smmu->evtq.q.irq;
2555 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2556 arm_smmu_evtq_thread,
2558 "arm-smmu-v3-evtq", smmu);
2560 dev_warn(smmu->dev, "failed to enable evtq irq\n");
2562 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
2565 irq = smmu->gerr_irq;
2567 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2568 0, "arm-smmu-v3-gerror", smmu);
2570 dev_warn(smmu->dev, "failed to enable gerror irq\n");
2572 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
2575 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2576 irq = smmu->priq.q.irq;
2578 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2579 arm_smmu_priq_thread,
2585 "failed to enable priq irq\n");
2587 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
2592 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2595 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2597 /* Disable IRQs first */
2598 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2599 ARM_SMMU_IRQ_CTRLACK);
2601 dev_err(smmu->dev, "failed to disable irqs\n");
2605 irq = smmu->combined_irq;
2608 * Cavium ThunderX2 implementation doesn't support unique irq
2609 * lines. Use a single irq line for all the SMMUv3 interrupts.
2611 ret = devm_request_threaded_irq(smmu->dev, irq,
2612 arm_smmu_combined_irq_handler,
2613 arm_smmu_combined_irq_thread,
2615 "arm-smmu-v3-combined-irq", smmu);
2617 dev_warn(smmu->dev, "failed to enable combined irq\n");
2619 arm_smmu_setup_unique_irqs(smmu);
2621 if (smmu->features & ARM_SMMU_FEAT_PRI)
2622 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2624 /* Enable interrupt generation on the SMMU */
2625 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2626 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2628 dev_warn(smmu->dev, "failed to enable irqs\n");
2633 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2637 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2639 dev_err(smmu->dev, "failed to clear cr0\n");
2644 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2648 struct arm_smmu_cmdq_ent cmd;
2650 /* Clear CR0 and sync (disables SMMU and queue processing) */
2651 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2652 if (reg & CR0_SMMUEN) {
2653 if (is_kdump_kernel()) {
2654 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
2655 arm_smmu_device_disable(smmu);
2659 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2662 ret = arm_smmu_device_disable(smmu);
2666 /* CR1 (table and queue memory attributes) */
2667 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
2668 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
2669 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
2670 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
2671 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
2672 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
2673 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2675 /* CR2 (random crap) */
2676 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2677 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2680 writeq_relaxed(smmu->strtab_cfg.strtab_base,
2681 smmu->base + ARM_SMMU_STRTAB_BASE);
2682 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2683 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2686 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2687 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2688 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2690 enables = CR0_CMDQEN;
2691 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2694 dev_err(smmu->dev, "failed to enable command queue\n");
2698 /* Invalidate any cached configuration */
2699 cmd.opcode = CMDQ_OP_CFGI_ALL;
2700 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2701 arm_smmu_cmdq_issue_sync(smmu);
2703 /* Invalidate any stale TLB entries */
2704 if (smmu->features & ARM_SMMU_FEAT_HYP) {
2705 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2706 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2709 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2710 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2711 arm_smmu_cmdq_issue_sync(smmu);
2714 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2715 writel_relaxed(smmu->evtq.q.prod,
2716 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
2717 writel_relaxed(smmu->evtq.q.cons,
2718 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
2720 enables |= CR0_EVTQEN;
2721 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2724 dev_err(smmu->dev, "failed to enable event queue\n");
2729 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2730 writeq_relaxed(smmu->priq.q.q_base,
2731 smmu->base + ARM_SMMU_PRIQ_BASE);
2732 writel_relaxed(smmu->priq.q.prod,
2733 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
2734 writel_relaxed(smmu->priq.q.cons,
2735 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
2737 enables |= CR0_PRIQEN;
2738 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2741 dev_err(smmu->dev, "failed to enable PRI queue\n");
2746 if (smmu->features & ARM_SMMU_FEAT_ATS) {
2747 enables |= CR0_ATSCHK;
2748 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2751 dev_err(smmu->dev, "failed to enable ATS check\n");
2756 ret = arm_smmu_setup_irqs(smmu);
2758 dev_err(smmu->dev, "failed to setup irqs\n");
2763 /* Enable the SMMU interface, or ensure bypass */
2764 if (!bypass || disable_bypass) {
2765 enables |= CR0_SMMUEN;
2767 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
2771 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2774 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2781 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
2784 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
2787 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2789 /* 2-level structures */
2790 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
2791 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2793 if (reg & IDR0_CD2L)
2794 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2797 * Translation table endianness.
2798 * We currently require the same endianness as the CPU, but this
2799 * could be changed later by adding a new IO_PGTABLE_QUIRK.
2801 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
2802 case IDR0_TTENDIAN_MIXED:
2803 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2806 case IDR0_TTENDIAN_BE:
2807 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2810 case IDR0_TTENDIAN_LE:
2811 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2815 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2819 /* Boolean feature flags */
2820 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2821 smmu->features |= ARM_SMMU_FEAT_PRI;
2823 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2824 smmu->features |= ARM_SMMU_FEAT_ATS;
2827 smmu->features |= ARM_SMMU_FEAT_SEV;
2830 smmu->features |= ARM_SMMU_FEAT_MSI;
2833 smmu->features |= ARM_SMMU_FEAT_HYP;
2836 * The coherency feature as set by FW is used in preference to the ID
2837 * register, but warn on mismatch.
2839 if (!!(reg & IDR0_COHACC) != coherent)
2840 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
2841 coherent ? "true" : "false");
2843 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
2844 case IDR0_STALL_MODEL_FORCE:
2845 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
2847 case IDR0_STALL_MODEL_STALL:
2848 smmu->features |= ARM_SMMU_FEAT_STALLS;
2852 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2855 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2857 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2858 dev_err(smmu->dev, "no translation support!\n");
2862 /* We only support the AArch64 table format at present */
2863 switch (FIELD_GET(IDR0_TTF, reg)) {
2864 case IDR0_TTF_AARCH32_64:
2867 case IDR0_TTF_AARCH64:
2870 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2874 /* ASID/VMID sizes */
2875 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2876 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2879 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2880 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2881 dev_err(smmu->dev, "embedded implementation not supported\n");
2885 /* Queue sizes, capped at 4k */
2886 smmu->cmdq.q.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
2887 FIELD_GET(IDR1_CMDQS, reg));
2888 if (!smmu->cmdq.q.max_n_shift) {
2889 /* Odd alignment restrictions on the base, so ignore for now */
2890 dev_err(smmu->dev, "unit-length command queue not supported\n");
2894 smmu->evtq.q.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
2895 FIELD_GET(IDR1_EVTQS, reg));
2896 smmu->priq.q.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
2897 FIELD_GET(IDR1_PRIQS, reg));
2899 /* SID/SSID sizes */
2900 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
2901 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
2904 * If the SMMU supports fewer bits than would fill a single L2 stream
2905 * table, use a linear table instead.
2907 if (smmu->sid_bits <= STRTAB_SPLIT)
2908 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
2911 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2913 /* Maximum number of outstanding stalls */
2914 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
2917 if (reg & IDR5_GRAN64K)
2918 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2919 if (reg & IDR5_GRAN16K)
2920 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2921 if (reg & IDR5_GRAN4K)
2922 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2924 /* Input address size */
2925 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
2926 smmu->features |= ARM_SMMU_FEAT_VAX;
2928 /* Output address size */
2929 switch (FIELD_GET(IDR5_OAS, reg)) {
2930 case IDR5_OAS_32_BIT:
2933 case IDR5_OAS_36_BIT:
2936 case IDR5_OAS_40_BIT:
2939 case IDR5_OAS_42_BIT:
2942 case IDR5_OAS_44_BIT:
2945 case IDR5_OAS_52_BIT:
2947 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
2951 "unknown output address size. Truncating to 48-bit\n");
2953 case IDR5_OAS_48_BIT:
2957 if (arm_smmu_ops.pgsize_bitmap == -1UL)
2958 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2960 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2962 /* Set the DMA mask for our table walker */
2963 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2965 "failed to set DMA mask for table walker\n");
2967 smmu->ias = max(smmu->ias, smmu->oas);
2969 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2970 smmu->ias, smmu->oas, smmu->features);
2975 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
2978 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
2979 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
2981 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
2982 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
2986 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
2989 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2990 struct arm_smmu_device *smmu)
2992 struct acpi_iort_smmu_v3 *iort_smmu;
2993 struct device *dev = smmu->dev;
2994 struct acpi_iort_node *node;
2996 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
2998 /* Retrieve SMMUv3 specific data */
2999 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3001 acpi_smmu_get_options(iort_smmu->model, smmu);
3003 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3004 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3009 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3010 struct arm_smmu_device *smmu)
3016 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3017 struct arm_smmu_device *smmu)
3019 struct device *dev = &pdev->dev;
3023 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3024 dev_err(dev, "missing #iommu-cells property\n");
3025 else if (cells != 1)
3026 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3030 parse_driver_options(smmu);
3032 if (of_dma_is_coherent(dev->of_node))
3033 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3038 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3040 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3046 static int arm_smmu_device_probe(struct platform_device *pdev)
3049 struct resource *res;
3050 resource_size_t ioaddr;
3051 struct arm_smmu_device *smmu;
3052 struct device *dev = &pdev->dev;
3055 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3057 dev_err(dev, "failed to allocate arm_smmu_device\n");
3063 ret = arm_smmu_device_dt_probe(pdev, smmu);
3065 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3070 /* Set bypass mode according to firmware probing result */
3074 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3075 if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {
3076 dev_err(dev, "MMIO region too small (%pr)\n", res);
3079 ioaddr = res->start;
3081 smmu->base = devm_ioremap_resource(dev, res);
3082 if (IS_ERR(smmu->base))
3083 return PTR_ERR(smmu->base);
3085 /* Interrupt lines */
3087 irq = platform_get_irq_byname(pdev, "combined");
3089 smmu->combined_irq = irq;
3091 irq = platform_get_irq_byname(pdev, "eventq");
3093 smmu->evtq.q.irq = irq;
3095 irq = platform_get_irq_byname(pdev, "priq");
3097 smmu->priq.q.irq = irq;
3099 irq = platform_get_irq_byname(pdev, "gerror");
3101 smmu->gerr_irq = irq;
3104 ret = arm_smmu_device_hw_probe(smmu);
3108 /* Initialise in-memory data structures */
3109 ret = arm_smmu_init_structures(smmu);
3113 /* Record our private device structure */
3114 platform_set_drvdata(pdev, smmu);
3116 /* Reset the device */
3117 ret = arm_smmu_device_reset(smmu, bypass);
3121 /* And we're up. Go go go! */
3122 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3123 "smmu3.%pa", &ioaddr);
3127 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
3128 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
3130 ret = iommu_device_register(&smmu->iommu);
3132 dev_err(dev, "Failed to register iommu\n");
3137 if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
3139 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
3144 #ifdef CONFIG_ARM_AMBA
3145 if (amba_bustype.iommu_ops != &arm_smmu_ops) {
3146 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
3151 if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
3152 ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
3159 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3161 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3163 arm_smmu_device_disable(smmu);
3166 static const struct of_device_id arm_smmu_of_match[] = {
3167 { .compatible = "arm,smmu-v3", },
3171 static struct platform_driver arm_smmu_driver = {
3173 .name = "arm-smmu-v3",
3174 .of_match_table = of_match_ptr(arm_smmu_of_match),
3175 .suppress_bind_attrs = true,
3177 .probe = arm_smmu_device_probe,
3178 .shutdown = arm_smmu_device_shutdown,
3180 builtin_platform_driver(arm_smmu_driver);