1 // SPDX-License-Identifier: GPL-2.0
3 * IOMMU API for ARM architected SMMUv3 implementations.
5 * Copyright (C) 2015 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver is powered by bad coffee and bombay mix.
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/bitops.h>
16 #include <linux/crash_dump.h>
17 #include <linux/delay.h>
18 #include <linux/dma-iommu.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/module.h>
25 #include <linux/msi.h>
27 #include <linux/of_address.h>
28 #include <linux/of_iommu.h>
29 #include <linux/of_platform.h>
30 #include <linux/pci.h>
31 #include <linux/pci-ats.h>
32 #include <linux/platform_device.h>
34 #include <linux/amba/bus.h>
37 #define ARM_SMMU_IDR0 0x0
38 #define IDR0_ST_LVL GENMASK(28, 27)
39 #define IDR0_ST_LVL_2LVL 1
40 #define IDR0_STALL_MODEL GENMASK(25, 24)
41 #define IDR0_STALL_MODEL_STALL 0
42 #define IDR0_STALL_MODEL_FORCE 2
43 #define IDR0_TTENDIAN GENMASK(22, 21)
44 #define IDR0_TTENDIAN_MIXED 0
45 #define IDR0_TTENDIAN_LE 2
46 #define IDR0_TTENDIAN_BE 3
47 #define IDR0_CD2L (1 << 19)
48 #define IDR0_VMID16 (1 << 18)
49 #define IDR0_PRI (1 << 16)
50 #define IDR0_SEV (1 << 14)
51 #define IDR0_MSI (1 << 13)
52 #define IDR0_ASID16 (1 << 12)
53 #define IDR0_ATS (1 << 10)
54 #define IDR0_HYP (1 << 9)
55 #define IDR0_COHACC (1 << 4)
56 #define IDR0_TTF GENMASK(3, 2)
57 #define IDR0_TTF_AARCH64 2
58 #define IDR0_TTF_AARCH32_64 3
59 #define IDR0_S1P (1 << 1)
60 #define IDR0_S2P (1 << 0)
62 #define ARM_SMMU_IDR1 0x4
63 #define IDR1_TABLES_PRESET (1 << 30)
64 #define IDR1_QUEUES_PRESET (1 << 29)
65 #define IDR1_REL (1 << 28)
66 #define IDR1_CMDQS GENMASK(25, 21)
67 #define IDR1_EVTQS GENMASK(20, 16)
68 #define IDR1_PRIQS GENMASK(15, 11)
69 #define IDR1_SSIDSIZE GENMASK(10, 6)
70 #define IDR1_SIDSIZE GENMASK(5, 0)
72 #define ARM_SMMU_IDR5 0x14
73 #define IDR5_STALL_MAX GENMASK(31, 16)
74 #define IDR5_GRAN64K (1 << 6)
75 #define IDR5_GRAN16K (1 << 5)
76 #define IDR5_GRAN4K (1 << 4)
77 #define IDR5_OAS GENMASK(2, 0)
78 #define IDR5_OAS_32_BIT 0
79 #define IDR5_OAS_36_BIT 1
80 #define IDR5_OAS_40_BIT 2
81 #define IDR5_OAS_42_BIT 3
82 #define IDR5_OAS_44_BIT 4
83 #define IDR5_OAS_48_BIT 5
84 #define IDR5_OAS_52_BIT 6
85 #define IDR5_VAX GENMASK(11, 10)
86 #define IDR5_VAX_52_BIT 1
88 #define ARM_SMMU_CR0 0x20
89 #define CR0_ATSCHK (1 << 4)
90 #define CR0_CMDQEN (1 << 3)
91 #define CR0_EVTQEN (1 << 2)
92 #define CR0_PRIQEN (1 << 1)
93 #define CR0_SMMUEN (1 << 0)
95 #define ARM_SMMU_CR0ACK 0x24
97 #define ARM_SMMU_CR1 0x28
98 #define CR1_TABLE_SH GENMASK(11, 10)
99 #define CR1_TABLE_OC GENMASK(9, 8)
100 #define CR1_TABLE_IC GENMASK(7, 6)
101 #define CR1_QUEUE_SH GENMASK(5, 4)
102 #define CR1_QUEUE_OC GENMASK(3, 2)
103 #define CR1_QUEUE_IC GENMASK(1, 0)
104 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
105 #define CR1_CACHE_NC 0
106 #define CR1_CACHE_WB 1
107 #define CR1_CACHE_WT 2
109 #define ARM_SMMU_CR2 0x2c
110 #define CR2_PTM (1 << 2)
111 #define CR2_RECINVSID (1 << 1)
112 #define CR2_E2H (1 << 0)
114 #define ARM_SMMU_GBPA 0x44
115 #define GBPA_UPDATE (1 << 31)
116 #define GBPA_ABORT (1 << 20)
118 #define ARM_SMMU_IRQ_CTRL 0x50
119 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
120 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
121 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
123 #define ARM_SMMU_IRQ_CTRLACK 0x54
125 #define ARM_SMMU_GERROR 0x60
126 #define GERROR_SFM_ERR (1 << 8)
127 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
128 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
129 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
130 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
131 #define GERROR_PRIQ_ABT_ERR (1 << 3)
132 #define GERROR_EVTQ_ABT_ERR (1 << 2)
133 #define GERROR_CMDQ_ERR (1 << 0)
134 #define GERROR_ERR_MASK 0xfd
136 #define ARM_SMMU_GERRORN 0x64
138 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
139 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
140 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
142 #define ARM_SMMU_STRTAB_BASE 0x80
143 #define STRTAB_BASE_RA (1UL << 62)
144 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
146 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
147 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
148 #define STRTAB_BASE_CFG_FMT_LINEAR 0
149 #define STRTAB_BASE_CFG_FMT_2LVL 1
150 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
151 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
153 #define ARM_SMMU_CMDQ_BASE 0x90
154 #define ARM_SMMU_CMDQ_PROD 0x98
155 #define ARM_SMMU_CMDQ_CONS 0x9c
157 #define ARM_SMMU_EVTQ_BASE 0xa0
158 #define ARM_SMMU_EVTQ_PROD 0x100a8
159 #define ARM_SMMU_EVTQ_CONS 0x100ac
160 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
161 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
162 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
164 #define ARM_SMMU_PRIQ_BASE 0xc0
165 #define ARM_SMMU_PRIQ_PROD 0x100c8
166 #define ARM_SMMU_PRIQ_CONS 0x100cc
167 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
168 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
169 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
171 /* Common MSI config fields */
172 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
173 #define MSI_CFG2_SH GENMASK(5, 4)
174 #define MSI_CFG2_MEMATTR GENMASK(3, 0)
176 /* Common memory attribute values */
177 #define ARM_SMMU_SH_NSH 0
178 #define ARM_SMMU_SH_OSH 2
179 #define ARM_SMMU_SH_ISH 3
180 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
181 #define ARM_SMMU_MEMATTR_OIWB 0xf
183 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
184 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
185 #define Q_OVERFLOW_FLAG (1U << 31)
186 #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
187 #define Q_ENT(q, p) ((q)->base + \
188 Q_IDX(&((q)->llq), p) * \
191 #define Q_BASE_RWA (1UL << 62)
192 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
193 #define Q_BASE_LOG2SIZE GENMASK(4, 0)
195 /* Ensure DMA allocations are naturally aligned */
196 #ifdef CONFIG_CMA_ALIGNMENT
197 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
199 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
205 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
206 * 2lvl: 128k L1 entries,
207 * 256 lazy entries per table (each table covers a PCI bus)
209 #define STRTAB_L1_SZ_SHIFT 20
210 #define STRTAB_SPLIT 8
212 #define STRTAB_L1_DESC_DWORDS 1
213 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
214 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
216 #define STRTAB_STE_DWORDS 8
217 #define STRTAB_STE_0_V (1UL << 0)
218 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
219 #define STRTAB_STE_0_CFG_ABORT 0
220 #define STRTAB_STE_0_CFG_BYPASS 4
221 #define STRTAB_STE_0_CFG_S1_TRANS 5
222 #define STRTAB_STE_0_CFG_S2_TRANS 6
224 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
225 #define STRTAB_STE_0_S1FMT_LINEAR 0
226 #define STRTAB_STE_0_S1FMT_64K_L2 2
227 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
228 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
230 #define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
231 #define STRTAB_STE_1_S1DSS_TERMINATE 0x0
232 #define STRTAB_STE_1_S1DSS_BYPASS 0x1
233 #define STRTAB_STE_1_S1DSS_SSID0 0x2
235 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
236 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
237 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
238 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
239 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
240 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
241 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
243 #define STRTAB_STE_1_S1STALLD (1UL << 27)
245 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
246 #define STRTAB_STE_1_EATS_ABT 0UL
247 #define STRTAB_STE_1_EATS_TRANS 1UL
248 #define STRTAB_STE_1_EATS_S1CHK 2UL
250 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
251 #define STRTAB_STE_1_STRW_NSEL1 0UL
252 #define STRTAB_STE_1_STRW_EL2 2UL
254 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
255 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
257 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
258 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
259 #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
260 #define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
261 #define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
262 #define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
263 #define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
264 #define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
265 #define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
266 #define STRTAB_STE_2_S2AA64 (1UL << 51)
267 #define STRTAB_STE_2_S2ENDI (1UL << 52)
268 #define STRTAB_STE_2_S2PTW (1UL << 54)
269 #define STRTAB_STE_2_S2R (1UL << 58)
271 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
274 * Context descriptors.
276 * Linear: when less than 1024 SSIDs are supported
277 * 2lvl: at most 1024 L1 entries,
278 * 1024 lazy entries per table.
280 #define CTXDESC_SPLIT 10
281 #define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
283 #define CTXDESC_L1_DESC_DWORDS 1
284 #define CTXDESC_L1_DESC_V (1UL << 0)
285 #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
287 #define CTXDESC_CD_DWORDS 8
288 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
289 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
290 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
291 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
292 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
293 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
294 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
296 #define CTXDESC_CD_0_ENDI (1UL << 15)
297 #define CTXDESC_CD_0_V (1UL << 31)
299 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
300 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
302 #define CTXDESC_CD_0_AA64 (1UL << 41)
303 #define CTXDESC_CD_0_S (1UL << 44)
304 #define CTXDESC_CD_0_R (1UL << 45)
305 #define CTXDESC_CD_0_A (1UL << 46)
306 #define CTXDESC_CD_0_ASET (1UL << 47)
307 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
309 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
312 * When the SMMU only supports linear context descriptor tables, pick a
313 * reasonable size limit (64kB).
315 #define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
318 #define CMDQ_ENT_SZ_SHIFT 4
319 #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
320 #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
322 #define CMDQ_CONS_ERR GENMASK(30, 24)
323 #define CMDQ_ERR_CERROR_NONE_IDX 0
324 #define CMDQ_ERR_CERROR_ILL_IDX 1
325 #define CMDQ_ERR_CERROR_ABT_IDX 2
326 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
328 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
331 * This is used to size the command queue and therefore must be at least
332 * BITS_PER_LONG so that the valid_map works correctly (it relies on the
333 * total number of queue entries being a multiple of BITS_PER_LONG).
335 #define CMDQ_BATCH_ENTRIES BITS_PER_LONG
337 #define CMDQ_0_OP GENMASK_ULL(7, 0)
338 #define CMDQ_0_SSV (1UL << 11)
340 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
341 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
342 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
344 #define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
345 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
346 #define CMDQ_CFGI_1_LEAF (1UL << 0)
347 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
349 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
350 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
351 #define CMDQ_TLBI_1_LEAF (1UL << 0)
352 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
353 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
355 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
356 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
357 #define CMDQ_ATC_0_GLOBAL (1UL << 9)
358 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
359 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
361 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
362 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
363 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
364 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
366 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
367 #define CMDQ_SYNC_0_CS_NONE 0
368 #define CMDQ_SYNC_0_CS_IRQ 1
369 #define CMDQ_SYNC_0_CS_SEV 2
370 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
371 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
372 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
373 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
376 #define EVTQ_ENT_SZ_SHIFT 5
377 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
378 #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
380 #define EVTQ_0_ID GENMASK_ULL(7, 0)
383 #define PRIQ_ENT_SZ_SHIFT 4
384 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
385 #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
387 #define PRIQ_0_SID GENMASK_ULL(31, 0)
388 #define PRIQ_0_SSID GENMASK_ULL(51, 32)
389 #define PRIQ_0_PERM_PRIV (1UL << 58)
390 #define PRIQ_0_PERM_EXEC (1UL << 59)
391 #define PRIQ_0_PERM_READ (1UL << 60)
392 #define PRIQ_0_PERM_WRITE (1UL << 61)
393 #define PRIQ_0_PRG_LAST (1UL << 62)
394 #define PRIQ_0_SSID_V (1UL << 63)
396 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
397 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
399 /* High-level queue structures */
400 #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
401 #define ARM_SMMU_POLL_SPIN_COUNT 10
403 #define MSI_IOVA_BASE 0x8000000
404 #define MSI_IOVA_LENGTH 0x100000
406 static bool disable_bypass = 1;
407 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
408 MODULE_PARM_DESC(disable_bypass,
409 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
417 enum arm_smmu_msi_index {
424 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
426 ARM_SMMU_EVTQ_IRQ_CFG0,
427 ARM_SMMU_EVTQ_IRQ_CFG1,
428 ARM_SMMU_EVTQ_IRQ_CFG2,
430 [GERROR_MSI_INDEX] = {
431 ARM_SMMU_GERROR_IRQ_CFG0,
432 ARM_SMMU_GERROR_IRQ_CFG1,
433 ARM_SMMU_GERROR_IRQ_CFG2,
436 ARM_SMMU_PRIQ_IRQ_CFG0,
437 ARM_SMMU_PRIQ_IRQ_CFG1,
438 ARM_SMMU_PRIQ_IRQ_CFG2,
442 struct arm_smmu_cmdq_ent {
445 bool substream_valid;
447 /* Command-specific fields */
449 #define CMDQ_OP_PREFETCH_CFG 0x1
456 #define CMDQ_OP_CFGI_STE 0x3
457 #define CMDQ_OP_CFGI_ALL 0x4
458 #define CMDQ_OP_CFGI_CD 0x5
459 #define CMDQ_OP_CFGI_CD_ALL 0x6
469 #define CMDQ_OP_TLBI_NH_ASID 0x11
470 #define CMDQ_OP_TLBI_NH_VA 0x12
471 #define CMDQ_OP_TLBI_EL2_ALL 0x20
472 #define CMDQ_OP_TLBI_S12_VMALL 0x28
473 #define CMDQ_OP_TLBI_S2_IPA 0x2a
474 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
482 #define CMDQ_OP_ATC_INV 0x40
483 #define ATC_INV_SIZE_ALL 52
492 #define CMDQ_OP_PRI_RESP 0x41
500 #define CMDQ_OP_CMD_SYNC 0x46
507 struct arm_smmu_ll_queue {
518 u8 __pad[SMP_CACHE_BYTES];
519 } ____cacheline_aligned_in_smp;
523 struct arm_smmu_queue {
524 struct arm_smmu_ll_queue llq;
525 int irq; /* Wired interrupt */
533 u32 __iomem *prod_reg;
534 u32 __iomem *cons_reg;
537 struct arm_smmu_queue_poll {
540 unsigned int spin_cnt;
544 struct arm_smmu_cmdq {
545 struct arm_smmu_queue q;
546 atomic_long_t *valid_map;
551 struct arm_smmu_evtq {
552 struct arm_smmu_queue q;
556 struct arm_smmu_priq {
557 struct arm_smmu_queue q;
560 /* High-level stream table and context descriptor structures */
561 struct arm_smmu_strtab_l1_desc {
565 dma_addr_t l2ptr_dma;
568 struct arm_smmu_ctx_desc {
575 struct arm_smmu_l1_ctx_desc {
577 dma_addr_t l2ptr_dma;
580 struct arm_smmu_ctx_desc_cfg {
582 dma_addr_t cdtab_dma;
583 struct arm_smmu_l1_ctx_desc *l1_desc;
584 unsigned int num_l1_ents;
587 struct arm_smmu_s1_cfg {
588 struct arm_smmu_ctx_desc_cfg cdcfg;
589 struct arm_smmu_ctx_desc cd;
594 struct arm_smmu_s2_cfg {
600 struct arm_smmu_strtab_cfg {
602 dma_addr_t strtab_dma;
603 struct arm_smmu_strtab_l1_desc *l1_desc;
604 unsigned int num_l1_ents;
610 /* An SMMUv3 instance */
611 struct arm_smmu_device {
615 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
616 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
617 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
618 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
619 #define ARM_SMMU_FEAT_PRI (1 << 4)
620 #define ARM_SMMU_FEAT_ATS (1 << 5)
621 #define ARM_SMMU_FEAT_SEV (1 << 6)
622 #define ARM_SMMU_FEAT_MSI (1 << 7)
623 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
624 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
625 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
626 #define ARM_SMMU_FEAT_STALLS (1 << 11)
627 #define ARM_SMMU_FEAT_HYP (1 << 12)
628 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
629 #define ARM_SMMU_FEAT_VAX (1 << 14)
632 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
633 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
636 struct arm_smmu_cmdq cmdq;
637 struct arm_smmu_evtq evtq;
638 struct arm_smmu_priq priq;
643 unsigned long ias; /* IPA */
644 unsigned long oas; /* PA */
645 unsigned long pgsize_bitmap;
647 #define ARM_SMMU_MAX_ASIDS (1 << 16)
648 unsigned int asid_bits;
649 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
651 #define ARM_SMMU_MAX_VMIDS (1 << 16)
652 unsigned int vmid_bits;
653 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
655 unsigned int ssid_bits;
656 unsigned int sid_bits;
658 struct arm_smmu_strtab_cfg strtab_cfg;
660 /* IOMMU core code handle */
661 struct iommu_device iommu;
664 /* SMMU private data for each master */
665 struct arm_smmu_master {
666 struct arm_smmu_device *smmu;
668 struct arm_smmu_domain *domain;
669 struct list_head domain_head;
671 unsigned int num_sids;
673 unsigned int ssid_bits;
676 /* SMMU private data for an IOMMU domain */
677 enum arm_smmu_domain_stage {
678 ARM_SMMU_DOMAIN_S1 = 0,
680 ARM_SMMU_DOMAIN_NESTED,
681 ARM_SMMU_DOMAIN_BYPASS,
684 struct arm_smmu_domain {
685 struct arm_smmu_device *smmu;
686 struct mutex init_mutex; /* Protects smmu pointer */
688 struct io_pgtable_ops *pgtbl_ops;
690 atomic_t nr_ats_masters;
692 enum arm_smmu_domain_stage stage;
694 struct arm_smmu_s1_cfg s1_cfg;
695 struct arm_smmu_s2_cfg s2_cfg;
698 struct iommu_domain domain;
700 struct list_head devices;
701 spinlock_t devices_lock;
704 struct arm_smmu_option_prop {
709 static struct arm_smmu_option_prop arm_smmu_options[] = {
710 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
711 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
715 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
716 struct arm_smmu_device *smmu)
718 if ((offset > SZ_64K) &&
719 (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
722 return smmu->base + offset;
725 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
727 return container_of(dom, struct arm_smmu_domain, domain);
730 static void parse_driver_options(struct arm_smmu_device *smmu)
735 if (of_property_read_bool(smmu->dev->of_node,
736 arm_smmu_options[i].prop)) {
737 smmu->options |= arm_smmu_options[i].opt;
738 dev_notice(smmu->dev, "option %s\n",
739 arm_smmu_options[i].prop);
741 } while (arm_smmu_options[++i].opt);
744 /* Low-level queue manipulation functions */
745 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
747 u32 space, prod, cons;
749 prod = Q_IDX(q, q->prod);
750 cons = Q_IDX(q, q->cons);
752 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
753 space = (1 << q->max_n_shift) - (prod - cons);
760 static bool queue_full(struct arm_smmu_ll_queue *q)
762 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
763 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
766 static bool queue_empty(struct arm_smmu_ll_queue *q)
768 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
769 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
772 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
774 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
775 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
776 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
777 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
780 static void queue_sync_cons_out(struct arm_smmu_queue *q)
783 * Ensure that all CPU accesses (reads and writes) to the queue
784 * are complete before we update the cons pointer.
787 writel_relaxed(q->llq.cons, q->cons_reg);
790 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
792 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
793 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
796 static int queue_sync_prod_in(struct arm_smmu_queue *q)
799 u32 prod = readl_relaxed(q->prod_reg);
801 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
808 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
810 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
811 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
814 static void queue_poll_init(struct arm_smmu_device *smmu,
815 struct arm_smmu_queue_poll *qp)
819 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
820 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
823 static int queue_poll(struct arm_smmu_queue_poll *qp)
825 if (ktime_compare(ktime_get(), qp->timeout) > 0)
830 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
841 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
845 for (i = 0; i < n_dwords; ++i)
846 *dst++ = cpu_to_le64(*src++);
849 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
853 for (i = 0; i < n_dwords; ++i)
854 *dst++ = le64_to_cpu(*src++);
857 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
859 if (queue_empty(&q->llq))
862 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
863 queue_inc_cons(&q->llq);
864 queue_sync_cons_out(q);
868 /* High-level queue accessors */
869 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
871 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
872 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
874 switch (ent->opcode) {
875 case CMDQ_OP_TLBI_EL2_ALL:
876 case CMDQ_OP_TLBI_NSNH_ALL:
878 case CMDQ_OP_PREFETCH_CFG:
879 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
880 cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
881 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
883 case CMDQ_OP_CFGI_CD:
884 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
886 case CMDQ_OP_CFGI_STE:
887 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
888 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
890 case CMDQ_OP_CFGI_CD_ALL:
891 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
893 case CMDQ_OP_CFGI_ALL:
894 /* Cover the entire SID range */
895 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
897 case CMDQ_OP_TLBI_NH_VA:
898 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
899 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
900 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
901 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
903 case CMDQ_OP_TLBI_S2_IPA:
904 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
905 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
906 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
908 case CMDQ_OP_TLBI_NH_ASID:
909 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
911 case CMDQ_OP_TLBI_S12_VMALL:
912 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
914 case CMDQ_OP_ATC_INV:
915 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
916 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
917 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
918 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
919 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
920 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
922 case CMDQ_OP_PRI_RESP:
923 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
924 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
925 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
926 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
927 switch (ent->pri.resp) {
935 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
937 case CMDQ_OP_CMD_SYNC:
938 if (ent->sync.msiaddr) {
939 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
940 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
942 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
944 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
945 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
954 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
957 struct arm_smmu_queue *q = &smmu->cmdq.q;
958 struct arm_smmu_cmdq_ent ent = {
959 .opcode = CMDQ_OP_CMD_SYNC,
963 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
964 * payload, so the write will zero the entire command on that platform.
966 if (smmu->features & ARM_SMMU_FEAT_MSI &&
967 smmu->features & ARM_SMMU_FEAT_COHERENCY) {
968 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
972 arm_smmu_cmdq_build_cmd(cmd, &ent);
975 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
977 static const char *cerror_str[] = {
978 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
979 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
980 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
981 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
985 u64 cmd[CMDQ_ENT_DWORDS];
986 struct arm_smmu_queue *q = &smmu->cmdq.q;
987 u32 cons = readl_relaxed(q->cons_reg);
988 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
989 struct arm_smmu_cmdq_ent cmd_sync = {
990 .opcode = CMDQ_OP_CMD_SYNC,
993 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
994 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
997 case CMDQ_ERR_CERROR_ABT_IDX:
998 dev_err(smmu->dev, "retrying command fetch\n");
999 case CMDQ_ERR_CERROR_NONE_IDX:
1001 case CMDQ_ERR_CERROR_ATC_INV_IDX:
1003 * ATC Invalidation Completion timeout. CONS is still pointing
1004 * at the CMD_SYNC. Attempt to complete other pending commands
1005 * by repeating the CMD_SYNC, though we might well end up back
1006 * here since the ATC invalidation may still be pending.
1009 case CMDQ_ERR_CERROR_ILL_IDX:
1016 * We may have concurrent producers, so we need to be careful
1017 * not to touch any of the shadow cmdq state.
1019 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
1020 dev_err(smmu->dev, "skipping command in error state:\n");
1021 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
1022 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
1024 /* Convert the erroneous command into a CMD_SYNC */
1025 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
1026 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
1030 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
1034 * Command queue locking.
1035 * This is a form of bastardised rwlock with the following major changes:
1037 * - The only LOCK routines are exclusive_trylock() and shared_lock().
1038 * Neither have barrier semantics, and instead provide only a control
1041 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
1042 * fails if the caller appears to be the last lock holder (yes, this is
1043 * racy). All successful UNLOCK routines have RELEASE semantics.
1045 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
1050 * We can try to avoid the cmpxchg() loop by simply incrementing the
1051 * lock counter. When held in exclusive state, the lock counter is set
1052 * to INT_MIN so these increments won't hurt as the value will remain
1055 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
1059 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
1060 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
1063 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
1065 (void)atomic_dec_return_release(&cmdq->lock);
1068 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
1070 if (atomic_read(&cmdq->lock) == 1)
1073 arm_smmu_cmdq_shared_unlock(cmdq);
1077 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
1080 local_irq_save(flags); \
1081 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
1083 local_irq_restore(flags); \
1087 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
1089 atomic_set_release(&cmdq->lock, 0); \
1090 local_irq_restore(flags); \
1095 * Command queue insertion.
1096 * This is made fiddly by our attempts to achieve some sort of scalability
1097 * since there is one queue shared amongst all of the CPUs in the system. If
1098 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
1099 * then you'll *love* this monstrosity.
1101 * The basic idea is to split the queue up into ranges of commands that are
1102 * owned by a given CPU; the owner may not have written all of the commands
1103 * itself, but is responsible for advancing the hardware prod pointer when
1104 * the time comes. The algorithm is roughly:
1106 * 1. Allocate some space in the queue. At this point we also discover
1107 * whether the head of the queue is currently owned by another CPU,
1108 * or whether we are the owner.
1110 * 2. Write our commands into our allocated slots in the queue.
1112 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
1114 * 4. If we are an owner:
1115 * a. Wait for the previous owner to finish.
1116 * b. Mark the queue head as unowned, which tells us the range
1117 * that we are responsible for publishing.
1118 * c. Wait for all commands in our owned range to become valid.
1119 * d. Advance the hardware prod pointer.
1120 * e. Tell the next owner we've finished.
1122 * 5. If we are inserting a CMD_SYNC (we may or may not have been an
1123 * owner), then we need to stick around until it has completed:
1124 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
1125 * to clear the first 4 bytes.
1126 * b. Otherwise, we spin waiting for the hardware cons pointer to
1127 * advance past our command.
1129 * The devil is in the details, particularly the use of locking for handling
1130 * SYNC completion and freeing up space in the queue before we think that it is
1133 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
1134 u32 sprod, u32 eprod, bool set)
1136 u32 swidx, sbidx, ewidx, ebidx;
1137 struct arm_smmu_ll_queue llq = {
1138 .max_n_shift = cmdq->q.llq.max_n_shift,
1142 ewidx = BIT_WORD(Q_IDX(&llq, eprod));
1143 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
1145 while (llq.prod != eprod) {
1148 u32 limit = BITS_PER_LONG;
1150 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
1151 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
1153 ptr = &cmdq->valid_map[swidx];
1155 if ((swidx == ewidx) && (sbidx < ebidx))
1158 mask = GENMASK(limit - 1, sbidx);
1161 * The valid bit is the inverse of the wrap bit. This means
1162 * that a zero-initialised queue is invalid and, after marking
1163 * all entries as valid, they become invalid again when we
1167 atomic_long_xor(mask, ptr);
1169 unsigned long valid;
1171 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
1172 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
1175 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
1179 /* Mark all entries in the range [sprod, eprod) as valid */
1180 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
1181 u32 sprod, u32 eprod)
1183 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
1186 /* Wait for all entries in the range [sprod, eprod) to become valid */
1187 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
1188 u32 sprod, u32 eprod)
1190 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
1193 /* Wait for the command queue to become non-full */
1194 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
1195 struct arm_smmu_ll_queue *llq)
1197 unsigned long flags;
1198 struct arm_smmu_queue_poll qp;
1199 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1203 * Try to update our copy of cons by grabbing exclusive cmdq access. If
1204 * that fails, spin until somebody else updates it for us.
1206 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
1207 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
1208 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
1209 llq->val = READ_ONCE(cmdq->q.llq.val);
1213 queue_poll_init(smmu, &qp);
1215 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1216 if (!queue_full(llq))
1219 ret = queue_poll(&qp);
1226 * Wait until the SMMU signals a CMD_SYNC completion MSI.
1227 * Must be called with the cmdq lock held in some capacity.
1229 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
1230 struct arm_smmu_ll_queue *llq)
1233 struct arm_smmu_queue_poll qp;
1234 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1235 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
1237 queue_poll_init(smmu, &qp);
1240 * The MSI won't generate an event, since it's being written back
1241 * into the command queue.
1244 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
1245 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
1250 * Wait until the SMMU cons index passes llq->prod.
1251 * Must be called with the cmdq lock held in some capacity.
1253 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
1254 struct arm_smmu_ll_queue *llq)
1256 struct arm_smmu_queue_poll qp;
1257 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1258 u32 prod = llq->prod;
1261 queue_poll_init(smmu, &qp);
1262 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1264 if (queue_consumed(llq, prod))
1267 ret = queue_poll(&qp);
1270 * This needs to be a readl() so that our subsequent call
1271 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
1273 * Specifically, we need to ensure that we observe all
1274 * shared_lock()s by other CMD_SYNCs that share our owner,
1275 * so that a failing call to tryunlock() means that we're
1276 * the last one out and therefore we can safely advance
1277 * cmdq->q.llq.cons. Roughly speaking:
1279 * CPU 0 CPU1 CPU2 (us)
1289 * <control dependency>
1295 * Requires us to see CPU 0's shared_lock() acquisition.
1297 llq->cons = readl(cmdq->q.cons_reg);
1303 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
1304 struct arm_smmu_ll_queue *llq)
1306 if (smmu->features & ARM_SMMU_FEAT_MSI &&
1307 smmu->features & ARM_SMMU_FEAT_COHERENCY)
1308 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
1310 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
1313 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
1317 struct arm_smmu_ll_queue llq = {
1318 .max_n_shift = cmdq->q.llq.max_n_shift,
1322 for (i = 0; i < n; ++i) {
1323 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
1325 prod = queue_inc_prod_n(&llq, i);
1326 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
1331 * This is the actual insertion function, and provides the following
1332 * ordering guarantees to callers:
1334 * - There is a dma_wmb() before publishing any commands to the queue.
1335 * This can be relied upon to order prior writes to data structures
1336 * in memory (such as a CD or an STE) before the command.
1338 * - On completion of a CMD_SYNC, there is a control dependency.
1339 * This can be relied upon to order subsequent writes to memory (e.g.
1340 * freeing an IOVA) after completion of the CMD_SYNC.
1342 * - Command insertion is totally ordered, so if two CPUs each race to
1343 * insert their own list of commands then all of the commands from one
1344 * CPU will appear before any of the commands from the other CPU.
1346 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
1347 u64 *cmds, int n, bool sync)
1349 u64 cmd_sync[CMDQ_ENT_DWORDS];
1351 unsigned long flags;
1353 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1354 struct arm_smmu_ll_queue llq = {
1355 .max_n_shift = cmdq->q.llq.max_n_shift,
1359 /* 1. Allocate some space in the queue */
1360 local_irq_save(flags);
1361 llq.val = READ_ONCE(cmdq->q.llq.val);
1365 while (!queue_has_space(&llq, n + sync)) {
1366 local_irq_restore(flags);
1367 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
1368 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
1369 local_irq_save(flags);
1372 head.cons = llq.cons;
1373 head.prod = queue_inc_prod_n(&llq, n + sync) |
1374 CMDQ_PROD_OWNED_FLAG;
1376 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
1382 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
1383 head.prod &= ~CMDQ_PROD_OWNED_FLAG;
1384 llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
1387 * 2. Write our commands into the queue
1388 * Dependency ordering from the cmpxchg() loop above.
1390 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
1392 prod = queue_inc_prod_n(&llq, n);
1393 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
1394 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
1397 * In order to determine completion of our CMD_SYNC, we must
1398 * ensure that the queue can't wrap twice without us noticing.
1399 * We achieve that by taking the cmdq lock as shared before
1400 * marking our slot as valid.
1402 arm_smmu_cmdq_shared_lock(cmdq);
1405 /* 3. Mark our slots as valid, ensuring commands are visible first */
1407 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
1409 /* 4. If we are the owner, take control of the SMMU hardware */
1411 /* a. Wait for previous owner to finish */
1412 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
1414 /* b. Stop gathering work by clearing the owned flag */
1415 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
1416 &cmdq->q.llq.atomic.prod);
1417 prod &= ~CMDQ_PROD_OWNED_FLAG;
1420 * c. Wait for any gathered work to be written to the queue.
1421 * Note that we read our own entries so that we have the control
1422 * dependency required by (d).
1424 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
1427 * d. Advance the hardware prod pointer
1428 * Control dependency ordering from the entries becoming valid.
1430 writel_relaxed(prod, cmdq->q.prod_reg);
1433 * e. Tell the next owner we're done
1434 * Make sure we've updated the hardware first, so that we don't
1435 * race to update prod and potentially move it backwards.
1437 atomic_set_release(&cmdq->owner_prod, prod);
1440 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
1442 llq.prod = queue_inc_prod_n(&llq, n);
1443 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
1445 dev_err_ratelimited(smmu->dev,
1446 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
1448 readl_relaxed(cmdq->q.prod_reg),
1449 readl_relaxed(cmdq->q.cons_reg));
1453 * Try to unlock the cmq lock. This will fail if we're the last
1454 * reader, in which case we can safely update cmdq->q.llq.cons
1456 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
1457 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
1458 arm_smmu_cmdq_shared_unlock(cmdq);
1462 local_irq_restore(flags);
1466 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
1467 struct arm_smmu_cmdq_ent *ent)
1469 u64 cmd[CMDQ_ENT_DWORDS];
1471 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
1472 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
1477 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
1480 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1482 return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
1485 /* Context descriptor manipulation functions */
1486 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
1487 int ssid, bool leaf)
1490 unsigned long flags;
1491 struct arm_smmu_master *master;
1492 struct arm_smmu_device *smmu = smmu_domain->smmu;
1493 struct arm_smmu_cmdq_ent cmd = {
1494 .opcode = CMDQ_OP_CFGI_CD,
1501 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1502 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
1503 for (i = 0; i < master->num_sids; i++) {
1504 cmd.cfgi.sid = master->sids[i];
1505 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1508 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1510 arm_smmu_cmdq_issue_sync(smmu);
1513 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
1514 struct arm_smmu_l1_ctx_desc *l1_desc)
1516 size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1518 l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
1519 &l1_desc->l2ptr_dma, GFP_KERNEL);
1520 if (!l1_desc->l2ptr) {
1522 "failed to allocate context descriptor table\n");
1528 static void arm_smmu_write_cd_l1_desc(__le64 *dst,
1529 struct arm_smmu_l1_ctx_desc *l1_desc)
1531 u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |
1534 WRITE_ONCE(*dst, cpu_to_le64(val));
1537 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,
1542 struct arm_smmu_l1_ctx_desc *l1_desc;
1543 struct arm_smmu_device *smmu = smmu_domain->smmu;
1544 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1546 if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1547 return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
1549 idx = ssid >> CTXDESC_SPLIT;
1550 l1_desc = &cdcfg->l1_desc[idx];
1551 if (!l1_desc->l2ptr) {
1552 if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
1555 l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
1556 arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
1557 /* An invalid L1CD can be cached */
1558 arm_smmu_sync_cd(smmu_domain, ssid, false);
1560 idx = ssid & (CTXDESC_L2_ENTRIES - 1);
1561 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
1564 static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain,
1565 int ssid, struct arm_smmu_ctx_desc *cd)
1568 * This function handles the following cases:
1570 * (1) Install primary CD, for normal DMA traffic (SSID = 0).
1571 * (2) Install a secondary CD, for SID+SSID traffic.
1572 * (3) Update ASID of a CD. Atomically write the first 64 bits of the
1573 * CD, then invalidate the old entry and mappings.
1574 * (4) Remove a secondary CD.
1579 struct arm_smmu_device *smmu = smmu_domain->smmu;
1581 if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
1584 cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
1588 val = le64_to_cpu(cdptr[0]);
1589 cd_live = !!(val & CTXDESC_CD_0_V);
1591 if (!cd) { /* (4) */
1593 } else if (cd_live) { /* (3) */
1594 val &= ~CTXDESC_CD_0_ASID;
1595 val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
1597 * Until CD+TLB invalidation, both ASIDs may be used for tagging
1598 * this substream's traffic
1600 } else { /* (1) and (2) */
1601 cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
1603 cdptr[3] = cpu_to_le64(cd->mair);
1606 * STE is live, and the SMMU might read dwords of this CD in any
1607 * order. Ensure that it observes valid values before reading
1610 arm_smmu_sync_cd(smmu_domain, ssid, true);
1616 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
1618 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
1621 /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1622 if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1623 val |= CTXDESC_CD_0_S;
1627 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
1628 * "Configuration structures and configuration invalidation completion"
1630 * The size of single-copy atomic reads made by the SMMU is
1631 * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
1632 * field within an aligned 64-bit span of a structure can be altered
1633 * without first making the structure invalid.
1635 WRITE_ONCE(cdptr[0], cpu_to_le64(val));
1636 arm_smmu_sync_cd(smmu_domain, ssid, true);
1640 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
1644 size_t max_contexts;
1645 struct arm_smmu_device *smmu = smmu_domain->smmu;
1646 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1647 struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
1649 max_contexts = 1 << cfg->s1cdmax;
1651 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1652 max_contexts <= CTXDESC_L2_ENTRIES) {
1653 cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1654 cdcfg->num_l1_ents = max_contexts;
1656 l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
1658 cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1659 cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
1660 CTXDESC_L2_ENTRIES);
1662 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
1663 sizeof(*cdcfg->l1_desc),
1665 if (!cdcfg->l1_desc)
1668 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1671 cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
1673 if (!cdcfg->cdtab) {
1674 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1682 if (cdcfg->l1_desc) {
1683 devm_kfree(smmu->dev, cdcfg->l1_desc);
1684 cdcfg->l1_desc = NULL;
1689 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
1692 size_t size, l1size;
1693 struct arm_smmu_device *smmu = smmu_domain->smmu;
1694 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1696 if (cdcfg->l1_desc) {
1697 size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1699 for (i = 0; i < cdcfg->num_l1_ents; i++) {
1700 if (!cdcfg->l1_desc[i].l2ptr)
1703 dmam_free_coherent(smmu->dev, size,
1704 cdcfg->l1_desc[i].l2ptr,
1705 cdcfg->l1_desc[i].l2ptr_dma);
1707 devm_kfree(smmu->dev, cdcfg->l1_desc);
1708 cdcfg->l1_desc = NULL;
1710 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1712 l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
1715 dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
1716 cdcfg->cdtab_dma = 0;
1717 cdcfg->cdtab = NULL;
1720 /* Stream table manipulation functions */
1722 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1726 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1727 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1729 *dst = cpu_to_le64(val);
1732 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1734 struct arm_smmu_cmdq_ent cmd = {
1735 .opcode = CMDQ_OP_CFGI_STE,
1742 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1743 arm_smmu_cmdq_issue_sync(smmu);
1746 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1750 * This is hideously complicated, but we only really care about
1751 * three cases at the moment:
1753 * 1. Invalid (all zero) -> bypass/fault (init)
1754 * 2. Bypass/fault -> translation/bypass (attach)
1755 * 3. Translation/bypass -> bypass/fault (detach)
1757 * Given that we can't update the STE atomically and the SMMU
1758 * doesn't read the thing in a defined order, that leaves us
1759 * with the following maintenance requirements:
1761 * 1. Update Config, return (init time STEs aren't live)
1762 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1763 * 3. Update Config, sync
1765 u64 val = le64_to_cpu(dst[0]);
1766 bool ste_live = false;
1767 struct arm_smmu_device *smmu = NULL;
1768 struct arm_smmu_s1_cfg *s1_cfg = NULL;
1769 struct arm_smmu_s2_cfg *s2_cfg = NULL;
1770 struct arm_smmu_domain *smmu_domain = NULL;
1771 struct arm_smmu_cmdq_ent prefetch_cmd = {
1772 .opcode = CMDQ_OP_PREFETCH_CFG,
1779 smmu_domain = master->domain;
1780 smmu = master->smmu;
1784 switch (smmu_domain->stage) {
1785 case ARM_SMMU_DOMAIN_S1:
1786 s1_cfg = &smmu_domain->s1_cfg;
1788 case ARM_SMMU_DOMAIN_S2:
1789 case ARM_SMMU_DOMAIN_NESTED:
1790 s2_cfg = &smmu_domain->s2_cfg;
1797 if (val & STRTAB_STE_0_V) {
1798 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1799 case STRTAB_STE_0_CFG_BYPASS:
1801 case STRTAB_STE_0_CFG_S1_TRANS:
1802 case STRTAB_STE_0_CFG_S2_TRANS:
1805 case STRTAB_STE_0_CFG_ABORT:
1806 BUG_ON(!disable_bypass);
1809 BUG(); /* STE corruption */
1813 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1814 val = STRTAB_STE_0_V;
1817 if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1818 if (!smmu_domain && disable_bypass)
1819 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1821 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1823 dst[0] = cpu_to_le64(val);
1824 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1825 STRTAB_STE_1_SHCFG_INCOMING));
1826 dst[2] = 0; /* Nuke the VMID */
1828 * The SMMU can perform negative caching, so we must sync
1829 * the STE regardless of whether the old value was live.
1832 arm_smmu_sync_ste_for_sid(smmu, sid);
1838 dst[1] = cpu_to_le64(
1839 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
1840 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1841 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1842 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1843 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1845 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1846 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1847 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1849 val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1850 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1851 FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |
1852 FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1857 dst[2] = cpu_to_le64(
1858 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1859 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1861 STRTAB_STE_2_S2ENDI |
1863 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1866 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1868 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1871 if (master->ats_enabled)
1872 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1873 STRTAB_STE_1_EATS_TRANS));
1875 arm_smmu_sync_ste_for_sid(smmu, sid);
1876 /* See comment in arm_smmu_write_ctx_desc() */
1877 WRITE_ONCE(dst[0], cpu_to_le64(val));
1878 arm_smmu_sync_ste_for_sid(smmu, sid);
1880 /* It's likely that we'll want to use the new STE soon */
1881 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1882 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1885 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1889 for (i = 0; i < nent; ++i) {
1890 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1891 strtab += STRTAB_STE_DWORDS;
1895 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1899 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1900 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1905 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1906 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1908 desc->span = STRTAB_SPLIT + 1;
1909 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1913 "failed to allocate l2 stream table for SID %u\n",
1918 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1919 arm_smmu_write_strtab_l1_desc(strtab, desc);
1923 /* IRQ and event handlers */
1924 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1927 struct arm_smmu_device *smmu = dev;
1928 struct arm_smmu_queue *q = &smmu->evtq.q;
1929 struct arm_smmu_ll_queue *llq = &q->llq;
1930 u64 evt[EVTQ_ENT_DWORDS];
1933 while (!queue_remove_raw(q, evt)) {
1934 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1936 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1937 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1938 dev_info(smmu->dev, "\t0x%016llx\n",
1939 (unsigned long long)evt[i]);
1944 * Not much we can do on overflow, so scream and pretend we're
1947 if (queue_sync_prod_in(q) == -EOVERFLOW)
1948 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1949 } while (!queue_empty(llq));
1951 /* Sync our overflow flag, as we believe we're up to speed */
1952 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
1953 Q_IDX(llq, llq->cons);
1957 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1963 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1964 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1965 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
1966 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1967 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1969 dev_info(smmu->dev, "unexpected PRI request received:\n");
1971 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1972 sid, ssid, grpid, last ? "L" : "",
1973 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1974 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1975 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1976 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1977 evt[1] & PRIQ_1_ADDR_MASK);
1980 struct arm_smmu_cmdq_ent cmd = {
1981 .opcode = CMDQ_OP_PRI_RESP,
1982 .substream_valid = ssv,
1987 .resp = PRI_RESP_DENY,
1991 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1995 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1997 struct arm_smmu_device *smmu = dev;
1998 struct arm_smmu_queue *q = &smmu->priq.q;
1999 struct arm_smmu_ll_queue *llq = &q->llq;
2000 u64 evt[PRIQ_ENT_DWORDS];
2003 while (!queue_remove_raw(q, evt))
2004 arm_smmu_handle_ppr(smmu, evt);
2006 if (queue_sync_prod_in(q) == -EOVERFLOW)
2007 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
2008 } while (!queue_empty(llq));
2010 /* Sync our overflow flag, as we believe we're up to speed */
2011 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
2012 Q_IDX(llq, llq->cons);
2013 queue_sync_cons_out(q);
2017 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
2019 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
2021 u32 gerror, gerrorn, active;
2022 struct arm_smmu_device *smmu = dev;
2024 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
2025 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
2027 active = gerror ^ gerrorn;
2028 if (!(active & GERROR_ERR_MASK))
2029 return IRQ_NONE; /* No errors pending */
2032 "unexpected global error reported (0x%08x), this could be serious\n",
2035 if (active & GERROR_SFM_ERR) {
2036 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
2037 arm_smmu_device_disable(smmu);
2040 if (active & GERROR_MSI_GERROR_ABT_ERR)
2041 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
2043 if (active & GERROR_MSI_PRIQ_ABT_ERR)
2044 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
2046 if (active & GERROR_MSI_EVTQ_ABT_ERR)
2047 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
2049 if (active & GERROR_MSI_CMDQ_ABT_ERR)
2050 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
2052 if (active & GERROR_PRIQ_ABT_ERR)
2053 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
2055 if (active & GERROR_EVTQ_ABT_ERR)
2056 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
2058 if (active & GERROR_CMDQ_ERR)
2059 arm_smmu_cmdq_skip_err(smmu);
2061 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
2065 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
2067 struct arm_smmu_device *smmu = dev;
2069 arm_smmu_evtq_thread(irq, dev);
2070 if (smmu->features & ARM_SMMU_FEAT_PRI)
2071 arm_smmu_priq_thread(irq, dev);
2076 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
2078 arm_smmu_gerror_handler(irq, dev);
2079 return IRQ_WAKE_THREAD;
2083 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
2084 struct arm_smmu_cmdq_ent *cmd)
2088 /* ATC invalidates are always on 4096-bytes pages */
2089 size_t inval_grain_shift = 12;
2090 unsigned long page_start, page_end;
2092 *cmd = (struct arm_smmu_cmdq_ent) {
2093 .opcode = CMDQ_OP_ATC_INV,
2094 .substream_valid = !!ssid,
2099 cmd->atc.size = ATC_INV_SIZE_ALL;
2103 page_start = iova >> inval_grain_shift;
2104 page_end = (iova + size - 1) >> inval_grain_shift;
2107 * In an ATS Invalidate Request, the address must be aligned on the
2108 * range size, which must be a power of two number of page sizes. We
2109 * thus have to choose between grossly over-invalidating the region, or
2110 * splitting the invalidation into multiple commands. For simplicity
2111 * we'll go with the first solution, but should refine it in the future
2112 * if multiple commands are shown to be more efficient.
2114 * Find the smallest power of two that covers the range. The most
2115 * significant differing bit between the start and end addresses,
2116 * fls(start ^ end), indicates the required span. For example:
2118 * We want to invalidate pages [8; 11]. This is already the ideal range:
2119 * x = 0b1000 ^ 0b1011 = 0b11
2120 * span = 1 << fls(x) = 4
2122 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
2123 * x = 0b0111 ^ 0b1010 = 0b1101
2124 * span = 1 << fls(x) = 16
2126 log2_span = fls_long(page_start ^ page_end);
2127 span_mask = (1ULL << log2_span) - 1;
2129 page_start &= ~span_mask;
2131 cmd->atc.addr = page_start << inval_grain_shift;
2132 cmd->atc.size = log2_span;
2135 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
2136 struct arm_smmu_cmdq_ent *cmd)
2140 if (!master->ats_enabled)
2143 for (i = 0; i < master->num_sids; i++) {
2144 cmd->atc.sid = master->sids[i];
2145 arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
2148 return arm_smmu_cmdq_issue_sync(master->smmu);
2151 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
2152 int ssid, unsigned long iova, size_t size)
2155 unsigned long flags;
2156 struct arm_smmu_cmdq_ent cmd;
2157 struct arm_smmu_master *master;
2159 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
2163 * Ensure that we've completed prior invalidation of the main TLBs
2164 * before we read 'nr_ats_masters' in case of a concurrent call to
2165 * arm_smmu_enable_ats():
2167 * // unmap() // arm_smmu_enable_ats()
2168 * TLBI+SYNC atomic_inc(&nr_ats_masters);
2170 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel()
2172 * Ensures that we always see the incremented 'nr_ats_masters' count if
2173 * ATS was enabled at the PCI device before completion of the TLBI.
2176 if (!atomic_read(&smmu_domain->nr_ats_masters))
2179 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
2181 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2182 list_for_each_entry(master, &smmu_domain->devices, domain_head)
2183 ret |= arm_smmu_atc_inv_master(master, &cmd);
2184 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2186 return ret ? -ETIMEDOUT : 0;
2189 /* IO_PGTABLE API */
2190 static void arm_smmu_tlb_inv_context(void *cookie)
2192 struct arm_smmu_domain *smmu_domain = cookie;
2193 struct arm_smmu_device *smmu = smmu_domain->smmu;
2194 struct arm_smmu_cmdq_ent cmd;
2196 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2197 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
2198 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
2201 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
2202 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2206 * NOTE: when io-pgtable is in non-strict mode, we may get here with
2207 * PTEs previously cleared by unmaps on the current CPU not yet visible
2208 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
2209 * insertion to guarantee those are observed before the TLBI. Do be
2212 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2213 arm_smmu_cmdq_issue_sync(smmu);
2214 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2217 static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
2218 size_t granule, bool leaf,
2219 struct arm_smmu_domain *smmu_domain)
2221 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
2222 struct arm_smmu_device *smmu = smmu_domain->smmu;
2223 unsigned long start = iova, end = iova + size;
2225 struct arm_smmu_cmdq_ent cmd = {
2234 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2235 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
2236 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
2238 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
2239 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2242 while (iova < end) {
2243 if (i == CMDQ_BATCH_ENTRIES) {
2244 arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, false);
2248 cmd.tlbi.addr = iova;
2249 arm_smmu_cmdq_build_cmd(&cmds[i * CMDQ_ENT_DWORDS], &cmd);
2254 arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, true);
2257 * Unfortunately, this can't be leaf-only since we may have
2258 * zapped an entire table.
2260 arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);
2263 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2264 unsigned long iova, size_t granule,
2267 struct arm_smmu_domain *smmu_domain = cookie;
2268 struct iommu_domain *domain = &smmu_domain->domain;
2270 iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2273 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2274 size_t granule, void *cookie)
2276 arm_smmu_tlb_inv_range(iova, size, granule, false, cookie);
2279 static void arm_smmu_tlb_inv_leaf(unsigned long iova, size_t size,
2280 size_t granule, void *cookie)
2282 arm_smmu_tlb_inv_range(iova, size, granule, true, cookie);
2285 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2286 .tlb_flush_all = arm_smmu_tlb_inv_context,
2287 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2288 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
2289 .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2293 static bool arm_smmu_capable(enum iommu_cap cap)
2296 case IOMMU_CAP_CACHE_COHERENCY:
2298 case IOMMU_CAP_NOEXEC:
2305 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
2307 struct arm_smmu_domain *smmu_domain;
2309 if (type != IOMMU_DOMAIN_UNMANAGED &&
2310 type != IOMMU_DOMAIN_DMA &&
2311 type != IOMMU_DOMAIN_IDENTITY)
2315 * Allocate the domain and initialise some of its data structures.
2316 * We can't really do anything meaningful until we've added a
2319 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2323 if (type == IOMMU_DOMAIN_DMA &&
2324 iommu_get_dma_cookie(&smmu_domain->domain)) {
2329 mutex_init(&smmu_domain->init_mutex);
2330 INIT_LIST_HEAD(&smmu_domain->devices);
2331 spin_lock_init(&smmu_domain->devices_lock);
2333 return &smmu_domain->domain;
2336 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
2338 int idx, size = 1 << span;
2341 idx = find_first_zero_bit(map, size);
2344 } while (test_and_set_bit(idx, map));
2349 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
2351 clear_bit(idx, map);
2354 static void arm_smmu_domain_free(struct iommu_domain *domain)
2356 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2357 struct arm_smmu_device *smmu = smmu_domain->smmu;
2359 iommu_put_dma_cookie(domain);
2360 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2362 /* Free the CD and ASID, if we allocated them */
2363 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2364 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2366 if (cfg->cdcfg.cdtab) {
2367 arm_smmu_free_cd_tables(smmu_domain);
2368 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
2371 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2373 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
2379 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
2380 struct arm_smmu_master *master,
2381 struct io_pgtable_cfg *pgtbl_cfg)
2385 struct arm_smmu_device *smmu = smmu_domain->smmu;
2386 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2387 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2389 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
2393 cfg->s1cdmax = master->ssid_bits;
2395 ret = arm_smmu_alloc_cd_tables(smmu_domain);
2399 cfg->cd.asid = (u16)asid;
2400 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
2401 cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
2402 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
2403 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
2404 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
2405 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
2406 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
2407 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
2408 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
2411 * Note that this will end up calling arm_smmu_sync_cd() before
2412 * the master has been added to the devices list for this domain.
2413 * This isn't an issue because the STE hasn't been installed yet.
2415 ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd);
2417 goto out_free_cd_tables;
2422 arm_smmu_free_cd_tables(smmu_domain);
2424 arm_smmu_bitmap_free(smmu->asid_map, asid);
2428 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
2429 struct arm_smmu_master *master,
2430 struct io_pgtable_cfg *pgtbl_cfg)
2433 struct arm_smmu_device *smmu = smmu_domain->smmu;
2434 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2435 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2437 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
2441 vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2442 cfg->vmid = (u16)vmid;
2443 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2444 cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
2445 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
2446 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
2447 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
2448 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
2449 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
2450 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2454 static int arm_smmu_domain_finalise(struct iommu_domain *domain,
2455 struct arm_smmu_master *master)
2458 unsigned long ias, oas;
2459 enum io_pgtable_fmt fmt;
2460 struct io_pgtable_cfg pgtbl_cfg;
2461 struct io_pgtable_ops *pgtbl_ops;
2462 int (*finalise_stage_fn)(struct arm_smmu_domain *,
2463 struct arm_smmu_master *,
2464 struct io_pgtable_cfg *);
2465 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2466 struct arm_smmu_device *smmu = smmu_domain->smmu;
2468 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2469 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2473 /* Restrict the stage to what we can actually support */
2474 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2475 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2476 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2477 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2479 switch (smmu_domain->stage) {
2480 case ARM_SMMU_DOMAIN_S1:
2481 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2482 ias = min_t(unsigned long, ias, VA_BITS);
2484 fmt = ARM_64_LPAE_S1;
2485 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2487 case ARM_SMMU_DOMAIN_NESTED:
2488 case ARM_SMMU_DOMAIN_S2:
2491 fmt = ARM_64_LPAE_S2;
2492 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2498 pgtbl_cfg = (struct io_pgtable_cfg) {
2499 .pgsize_bitmap = smmu->pgsize_bitmap,
2502 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2503 .tlb = &arm_smmu_flush_ops,
2504 .iommu_dev = smmu->dev,
2507 if (smmu_domain->non_strict)
2508 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
2510 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2514 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2515 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2516 domain->geometry.force_aperture = true;
2518 ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
2520 free_io_pgtable_ops(pgtbl_ops);
2524 smmu_domain->pgtbl_ops = pgtbl_ops;
2528 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2531 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2533 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2534 struct arm_smmu_strtab_l1_desc *l1_desc;
2537 /* Two-level walk */
2538 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
2539 l1_desc = &cfg->l1_desc[idx];
2540 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2541 step = &l1_desc->l2ptr[idx];
2543 /* Simple linear lookup */
2544 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2550 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
2553 struct arm_smmu_device *smmu = master->smmu;
2555 for (i = 0; i < master->num_sids; ++i) {
2556 u32 sid = master->sids[i];
2557 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
2559 /* Bridged PCI devices may end up with duplicated IDs */
2560 for (j = 0; j < i; j++)
2561 if (master->sids[j] == sid)
2566 arm_smmu_write_strtab_ent(master, sid, step);
2570 #ifdef CONFIG_PCI_ATS
2571 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2573 struct pci_dev *pdev;
2574 struct arm_smmu_device *smmu = master->smmu;
2575 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2577 if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
2578 !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
2581 pdev = to_pci_dev(master->dev);
2582 return !pdev->untrusted && pdev->ats_cap;
2585 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2591 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2594 struct pci_dev *pdev;
2595 struct arm_smmu_device *smmu = master->smmu;
2596 struct arm_smmu_domain *smmu_domain = master->domain;
2598 /* Don't enable ATS at the endpoint if it's not enabled in the STE */
2599 if (!master->ats_enabled)
2602 /* Smallest Translation Unit: log2 of the smallest supported granule */
2603 stu = __ffs(smmu->pgsize_bitmap);
2604 pdev = to_pci_dev(master->dev);
2606 atomic_inc(&smmu_domain->nr_ats_masters);
2607 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2608 if (pci_enable_ats(pdev, stu))
2609 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2612 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
2614 struct arm_smmu_cmdq_ent cmd;
2615 struct arm_smmu_domain *smmu_domain = master->domain;
2617 if (!master->ats_enabled)
2620 pci_disable_ats(to_pci_dev(master->dev));
2622 * Ensure ATS is disabled at the endpoint before we issue the
2623 * ATC invalidation via the SMMU.
2626 arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
2627 arm_smmu_atc_inv_master(master, &cmd);
2628 atomic_dec(&smmu_domain->nr_ats_masters);
2631 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
2633 unsigned long flags;
2634 struct arm_smmu_domain *smmu_domain = master->domain;
2639 arm_smmu_disable_ats(master);
2641 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2642 list_del(&master->domain_head);
2643 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2645 master->domain = NULL;
2646 master->ats_enabled = false;
2647 arm_smmu_install_ste_for_dev(master);
2650 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2653 unsigned long flags;
2654 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2655 struct arm_smmu_device *smmu;
2656 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2657 struct arm_smmu_master *master;
2662 master = fwspec->iommu_priv;
2663 smmu = master->smmu;
2665 arm_smmu_detach_dev(master);
2667 mutex_lock(&smmu_domain->init_mutex);
2669 if (!smmu_domain->smmu) {
2670 smmu_domain->smmu = smmu;
2671 ret = arm_smmu_domain_finalise(domain, master);
2673 smmu_domain->smmu = NULL;
2676 } else if (smmu_domain->smmu != smmu) {
2678 "cannot attach to SMMU %s (upstream of %s)\n",
2679 dev_name(smmu_domain->smmu->dev),
2680 dev_name(smmu->dev));
2683 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2684 master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
2686 "cannot attach to incompatible domain (%u SSID bits != %u)\n",
2687 smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
2692 master->domain = smmu_domain;
2694 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2695 master->ats_enabled = arm_smmu_ats_supported(master);
2697 arm_smmu_install_ste_for_dev(master);
2699 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2700 list_add(&master->domain_head, &smmu_domain->devices);
2701 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2703 arm_smmu_enable_ats(master);
2706 mutex_unlock(&smmu_domain->init_mutex);
2710 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
2711 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2713 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2718 return ops->map(ops, iova, paddr, size, prot);
2721 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
2722 size_t size, struct iommu_iotlb_gather *gather)
2724 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2725 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2730 return ops->unmap(ops, iova, size, gather);
2733 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
2735 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2737 if (smmu_domain->smmu)
2738 arm_smmu_tlb_inv_context(smmu_domain);
2741 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
2742 struct iommu_iotlb_gather *gather)
2744 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2746 arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start,
2747 gather->pgsize, true, smmu_domain);
2751 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2753 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2755 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2761 return ops->iova_to_phys(ops, iova);
2764 static struct platform_driver arm_smmu_driver;
2767 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2769 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
2772 return dev ? dev_get_drvdata(dev) : NULL;
2775 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2777 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2779 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2780 limit *= 1UL << STRTAB_SPLIT;
2785 static struct iommu_ops arm_smmu_ops;
2787 static int arm_smmu_add_device(struct device *dev)
2790 struct arm_smmu_device *smmu;
2791 struct arm_smmu_master *master;
2792 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2793 struct iommu_group *group;
2795 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2798 if (WARN_ON_ONCE(fwspec->iommu_priv))
2801 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2805 master = kzalloc(sizeof(*master), GFP_KERNEL);
2810 master->smmu = smmu;
2811 master->sids = fwspec->ids;
2812 master->num_sids = fwspec->num_ids;
2813 fwspec->iommu_priv = master;
2815 /* Check the SIDs are in range of the SMMU and our stream table */
2816 for (i = 0; i < master->num_sids; i++) {
2817 u32 sid = master->sids[i];
2819 if (!arm_smmu_sid_in_range(smmu, sid)) {
2821 goto err_free_master;
2824 /* Ensure l2 strtab is initialised */
2825 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2826 ret = arm_smmu_init_l2_strtab(smmu, sid);
2828 goto err_free_master;
2832 master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits);
2834 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
2835 master->ssid_bits = min_t(u8, master->ssid_bits,
2836 CTXDESC_LINEAR_CDMAX);
2838 ret = iommu_device_link(&smmu->iommu, dev);
2840 goto err_free_master;
2842 group = iommu_group_get_for_dev(dev);
2843 if (IS_ERR(group)) {
2844 ret = PTR_ERR(group);
2848 iommu_group_put(group);
2852 iommu_device_unlink(&smmu->iommu, dev);
2855 fwspec->iommu_priv = NULL;
2859 static void arm_smmu_remove_device(struct device *dev)
2861 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2862 struct arm_smmu_master *master;
2863 struct arm_smmu_device *smmu;
2865 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2868 master = fwspec->iommu_priv;
2869 smmu = master->smmu;
2870 arm_smmu_detach_dev(master);
2871 iommu_group_remove_device(dev);
2872 iommu_device_unlink(&smmu->iommu, dev);
2874 iommu_fwspec_free(dev);
2877 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2879 struct iommu_group *group;
2882 * We don't support devices sharing stream IDs other than PCI RID
2883 * aliases, since the necessary ID-to-device lookup becomes rather
2884 * impractical given a potential sparse 32-bit stream ID space.
2886 if (dev_is_pci(dev))
2887 group = pci_device_group(dev);
2889 group = generic_device_group(dev);
2894 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
2895 enum iommu_attr attr, void *data)
2897 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2899 switch (domain->type) {
2900 case IOMMU_DOMAIN_UNMANAGED:
2902 case DOMAIN_ATTR_NESTING:
2903 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
2909 case IOMMU_DOMAIN_DMA:
2911 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2912 *(int *)data = smmu_domain->non_strict;
2923 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
2924 enum iommu_attr attr, void *data)
2927 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2929 mutex_lock(&smmu_domain->init_mutex);
2931 switch (domain->type) {
2932 case IOMMU_DOMAIN_UNMANAGED:
2934 case DOMAIN_ATTR_NESTING:
2935 if (smmu_domain->smmu) {
2941 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2943 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2949 case IOMMU_DOMAIN_DMA:
2951 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2952 smmu_domain->non_strict = *(int *)data;
2963 mutex_unlock(&smmu_domain->init_mutex);
2967 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2969 return iommu_fwspec_add_ids(dev, args->args, 1);
2972 static void arm_smmu_get_resv_regions(struct device *dev,
2973 struct list_head *head)
2975 struct iommu_resv_region *region;
2976 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2978 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2979 prot, IOMMU_RESV_SW_MSI);
2983 list_add_tail(®ion->list, head);
2985 iommu_dma_get_resv_regions(dev, head);
2988 static struct iommu_ops arm_smmu_ops = {
2989 .capable = arm_smmu_capable,
2990 .domain_alloc = arm_smmu_domain_alloc,
2991 .domain_free = arm_smmu_domain_free,
2992 .attach_dev = arm_smmu_attach_dev,
2993 .map = arm_smmu_map,
2994 .unmap = arm_smmu_unmap,
2995 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
2996 .iotlb_sync = arm_smmu_iotlb_sync,
2997 .iova_to_phys = arm_smmu_iova_to_phys,
2998 .add_device = arm_smmu_add_device,
2999 .remove_device = arm_smmu_remove_device,
3000 .device_group = arm_smmu_device_group,
3001 .domain_get_attr = arm_smmu_domain_get_attr,
3002 .domain_set_attr = arm_smmu_domain_set_attr,
3003 .of_xlate = arm_smmu_of_xlate,
3004 .get_resv_regions = arm_smmu_get_resv_regions,
3005 .put_resv_regions = generic_iommu_put_resv_regions,
3006 .pgsize_bitmap = -1UL, /* Restricted during device attach */
3009 /* Probing and initialisation functions */
3010 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
3011 struct arm_smmu_queue *q,
3012 unsigned long prod_off,
3013 unsigned long cons_off,
3014 size_t dwords, const char *name)
3019 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
3020 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
3022 if (q->base || qsz < PAGE_SIZE)
3025 q->llq.max_n_shift--;
3030 "failed to allocate queue (0x%zx bytes) for %s\n",
3035 if (!WARN_ON(q->base_dma & (qsz - 1))) {
3036 dev_info(smmu->dev, "allocated %u entries for %s\n",
3037 1 << q->llq.max_n_shift, name);
3040 q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu);
3041 q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu);
3042 q->ent_dwords = dwords;
3044 q->q_base = Q_BASE_RWA;
3045 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
3046 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
3048 q->llq.prod = q->llq.cons = 0;
3052 static void arm_smmu_cmdq_free_bitmap(void *data)
3054 unsigned long *bitmap = data;
3055 bitmap_free(bitmap);
3058 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
3061 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
3062 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
3063 atomic_long_t *bitmap;
3065 atomic_set(&cmdq->owner_prod, 0);
3066 atomic_set(&cmdq->lock, 0);
3068 bitmap = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
3070 dev_err(smmu->dev, "failed to allocate cmdq bitmap\n");
3073 cmdq->valid_map = bitmap;
3074 devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap);
3080 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
3085 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
3086 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS,
3091 ret = arm_smmu_cmdq_init(smmu);
3096 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
3097 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS,
3103 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
3106 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
3107 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS,
3111 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
3114 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3115 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
3116 void *strtab = smmu->strtab_cfg.strtab;
3118 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
3119 if (!cfg->l1_desc) {
3120 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
3124 for (i = 0; i < cfg->num_l1_ents; ++i) {
3125 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
3126 strtab += STRTAB_L1_DESC_DWORDS << 3;
3132 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3137 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3139 /* Calculate the L1 size, capped to the SIDSIZE. */
3140 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
3141 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
3142 cfg->num_l1_ents = 1 << size;
3144 size += STRTAB_SPLIT;
3145 if (size < smmu->sid_bits)
3147 "2-level strtab only covers %u/%u bits of SID\n",
3148 size, smmu->sid_bits);
3150 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
3151 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
3155 "failed to allocate l1 stream table (%u bytes)\n",
3159 cfg->strtab = strtab;
3161 /* Configure strtab_base_cfg for 2 levels */
3162 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
3163 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
3164 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
3165 cfg->strtab_base_cfg = reg;
3167 return arm_smmu_init_l1_strtab(smmu);
3170 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3175 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3177 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
3178 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
3182 "failed to allocate linear stream table (%u bytes)\n",
3186 cfg->strtab = strtab;
3187 cfg->num_l1_ents = 1 << smmu->sid_bits;
3189 /* Configure strtab_base_cfg for a linear table covering all SIDs */
3190 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
3191 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
3192 cfg->strtab_base_cfg = reg;
3194 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
3198 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3203 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3204 ret = arm_smmu_init_strtab_2lvl(smmu);
3206 ret = arm_smmu_init_strtab_linear(smmu);
3211 /* Set the strtab base address */
3212 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
3213 reg |= STRTAB_BASE_RA;
3214 smmu->strtab_cfg.strtab_base = reg;
3216 /* Allocate the first VMID for stage-2 bypass STEs */
3217 set_bit(0, smmu->vmid_map);
3221 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3225 ret = arm_smmu_init_queues(smmu);
3229 return arm_smmu_init_strtab(smmu);
3232 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3233 unsigned int reg_off, unsigned int ack_off)
3237 writel_relaxed(val, smmu->base + reg_off);
3238 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3239 1, ARM_SMMU_POLL_TIMEOUT_US);
3242 /* GBPA is "special" */
3243 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3246 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3248 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3249 1, ARM_SMMU_POLL_TIMEOUT_US);
3255 writel_relaxed(reg | GBPA_UPDATE, gbpa);
3256 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3257 1, ARM_SMMU_POLL_TIMEOUT_US);
3260 dev_err(smmu->dev, "GBPA not responding to update\n");
3264 static void arm_smmu_free_msis(void *data)
3266 struct device *dev = data;
3267 platform_msi_domain_free_irqs(dev);
3270 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3272 phys_addr_t doorbell;
3273 struct device *dev = msi_desc_to_dev(desc);
3274 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3275 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
3277 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3278 doorbell &= MSI_CFG0_ADDR_MASK;
3280 writeq_relaxed(doorbell, smmu->base + cfg[0]);
3281 writel_relaxed(msg->data, smmu->base + cfg[1]);
3282 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3285 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3287 struct msi_desc *desc;
3288 int ret, nvec = ARM_SMMU_MAX_MSIS;
3289 struct device *dev = smmu->dev;
3291 /* Clear the MSI address regs */
3292 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3293 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3295 if (smmu->features & ARM_SMMU_FEAT_PRI)
3296 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3300 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3303 if (!dev->msi_domain) {
3304 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3308 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3309 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3311 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3315 for_each_msi_entry(desc, dev) {
3316 switch (desc->platform.msi_index) {
3317 case EVTQ_MSI_INDEX:
3318 smmu->evtq.q.irq = desc->irq;
3320 case GERROR_MSI_INDEX:
3321 smmu->gerr_irq = desc->irq;
3323 case PRIQ_MSI_INDEX:
3324 smmu->priq.q.irq = desc->irq;
3326 default: /* Unknown */
3331 /* Add callback to free MSIs on teardown */
3332 devm_add_action(dev, arm_smmu_free_msis, dev);
3335 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3339 arm_smmu_setup_msis(smmu);
3341 /* Request interrupt lines */
3342 irq = smmu->evtq.q.irq;
3344 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3345 arm_smmu_evtq_thread,
3347 "arm-smmu-v3-evtq", smmu);
3349 dev_warn(smmu->dev, "failed to enable evtq irq\n");
3351 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3354 irq = smmu->gerr_irq;
3356 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3357 0, "arm-smmu-v3-gerror", smmu);
3359 dev_warn(smmu->dev, "failed to enable gerror irq\n");
3361 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3364 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3365 irq = smmu->priq.q.irq;
3367 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3368 arm_smmu_priq_thread,
3374 "failed to enable priq irq\n");
3376 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3381 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3384 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3386 /* Disable IRQs first */
3387 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3388 ARM_SMMU_IRQ_CTRLACK);
3390 dev_err(smmu->dev, "failed to disable irqs\n");
3394 irq = smmu->combined_irq;
3397 * Cavium ThunderX2 implementation doesn't support unique irq
3398 * lines. Use a single irq line for all the SMMUv3 interrupts.
3400 ret = devm_request_threaded_irq(smmu->dev, irq,
3401 arm_smmu_combined_irq_handler,
3402 arm_smmu_combined_irq_thread,
3404 "arm-smmu-v3-combined-irq", smmu);
3406 dev_warn(smmu->dev, "failed to enable combined irq\n");
3408 arm_smmu_setup_unique_irqs(smmu);
3410 if (smmu->features & ARM_SMMU_FEAT_PRI)
3411 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3413 /* Enable interrupt generation on the SMMU */
3414 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3415 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3417 dev_warn(smmu->dev, "failed to enable irqs\n");
3422 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3426 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3428 dev_err(smmu->dev, "failed to clear cr0\n");
3433 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
3437 struct arm_smmu_cmdq_ent cmd;
3439 /* Clear CR0 and sync (disables SMMU and queue processing) */
3440 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3441 if (reg & CR0_SMMUEN) {
3442 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3443 WARN_ON(is_kdump_kernel() && !disable_bypass);
3444 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3447 ret = arm_smmu_device_disable(smmu);
3451 /* CR1 (table and queue memory attributes) */
3452 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3453 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3454 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3455 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3456 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3457 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3458 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3460 /* CR2 (random crap) */
3461 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
3462 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3465 writeq_relaxed(smmu->strtab_cfg.strtab_base,
3466 smmu->base + ARM_SMMU_STRTAB_BASE);
3467 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3468 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3471 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3472 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3473 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3475 enables = CR0_CMDQEN;
3476 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3479 dev_err(smmu->dev, "failed to enable command queue\n");
3483 /* Invalidate any cached configuration */
3484 cmd.opcode = CMDQ_OP_CFGI_ALL;
3485 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3486 arm_smmu_cmdq_issue_sync(smmu);
3488 /* Invalidate any stale TLB entries */
3489 if (smmu->features & ARM_SMMU_FEAT_HYP) {
3490 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3491 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3494 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
3495 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3496 arm_smmu_cmdq_issue_sync(smmu);
3499 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3500 writel_relaxed(smmu->evtq.q.llq.prod,
3501 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
3502 writel_relaxed(smmu->evtq.q.llq.cons,
3503 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
3505 enables |= CR0_EVTQEN;
3506 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3509 dev_err(smmu->dev, "failed to enable event queue\n");
3514 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3515 writeq_relaxed(smmu->priq.q.q_base,
3516 smmu->base + ARM_SMMU_PRIQ_BASE);
3517 writel_relaxed(smmu->priq.q.llq.prod,
3518 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
3519 writel_relaxed(smmu->priq.q.llq.cons,
3520 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
3522 enables |= CR0_PRIQEN;
3523 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3526 dev_err(smmu->dev, "failed to enable PRI queue\n");
3531 if (smmu->features & ARM_SMMU_FEAT_ATS) {
3532 enables |= CR0_ATSCHK;
3533 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3536 dev_err(smmu->dev, "failed to enable ATS check\n");
3541 ret = arm_smmu_setup_irqs(smmu);
3543 dev_err(smmu->dev, "failed to setup irqs\n");
3547 if (is_kdump_kernel())
3548 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
3550 /* Enable the SMMU interface, or ensure bypass */
3551 if (!bypass || disable_bypass) {
3552 enables |= CR0_SMMUEN;
3554 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
3558 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3561 dev_err(smmu->dev, "failed to enable SMMU interface\n");
3568 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
3571 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3574 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3576 /* 2-level structures */
3577 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
3578 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3580 if (reg & IDR0_CD2L)
3581 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3584 * Translation table endianness.
3585 * We currently require the same endianness as the CPU, but this
3586 * could be changed later by adding a new IO_PGTABLE_QUIRK.
3588 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
3589 case IDR0_TTENDIAN_MIXED:
3590 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3593 case IDR0_TTENDIAN_BE:
3594 smmu->features |= ARM_SMMU_FEAT_TT_BE;
3597 case IDR0_TTENDIAN_LE:
3598 smmu->features |= ARM_SMMU_FEAT_TT_LE;
3602 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3606 /* Boolean feature flags */
3607 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
3608 smmu->features |= ARM_SMMU_FEAT_PRI;
3610 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
3611 smmu->features |= ARM_SMMU_FEAT_ATS;
3614 smmu->features |= ARM_SMMU_FEAT_SEV;
3617 smmu->features |= ARM_SMMU_FEAT_MSI;
3620 smmu->features |= ARM_SMMU_FEAT_HYP;
3623 * The coherency feature as set by FW is used in preference to the ID
3624 * register, but warn on mismatch.
3626 if (!!(reg & IDR0_COHACC) != coherent)
3627 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3628 coherent ? "true" : "false");
3630 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
3631 case IDR0_STALL_MODEL_FORCE:
3632 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3634 case IDR0_STALL_MODEL_STALL:
3635 smmu->features |= ARM_SMMU_FEAT_STALLS;
3639 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3642 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3644 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
3645 dev_err(smmu->dev, "no translation support!\n");
3649 /* We only support the AArch64 table format at present */
3650 switch (FIELD_GET(IDR0_TTF, reg)) {
3651 case IDR0_TTF_AARCH32_64:
3654 case IDR0_TTF_AARCH64:
3657 dev_err(smmu->dev, "AArch64 table format not supported!\n");
3661 /* ASID/VMID sizes */
3662 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3663 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3666 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3667 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
3668 dev_err(smmu->dev, "embedded implementation not supported\n");
3672 /* Queue sizes, capped to ensure natural alignment */
3673 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3674 FIELD_GET(IDR1_CMDQS, reg));
3675 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3677 * We don't support splitting up batches, so one batch of
3678 * commands plus an extra sync needs to fit inside the command
3679 * queue. There's also no way we can handle the weird alignment
3680 * restrictions on the base pointer for a unit-length queue.
3682 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3683 CMDQ_BATCH_ENTRIES);
3687 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3688 FIELD_GET(IDR1_EVTQS, reg));
3689 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3690 FIELD_GET(IDR1_PRIQS, reg));
3692 /* SID/SSID sizes */
3693 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3694 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3697 * If the SMMU supports fewer bits than would fill a single L2 stream
3698 * table, use a linear table instead.
3700 if (smmu->sid_bits <= STRTAB_SPLIT)
3701 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3704 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3706 /* Maximum number of outstanding stalls */
3707 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3710 if (reg & IDR5_GRAN64K)
3711 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3712 if (reg & IDR5_GRAN16K)
3713 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3714 if (reg & IDR5_GRAN4K)
3715 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3717 /* Input address size */
3718 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
3719 smmu->features |= ARM_SMMU_FEAT_VAX;
3721 /* Output address size */
3722 switch (FIELD_GET(IDR5_OAS, reg)) {
3723 case IDR5_OAS_32_BIT:
3726 case IDR5_OAS_36_BIT:
3729 case IDR5_OAS_40_BIT:
3732 case IDR5_OAS_42_BIT:
3735 case IDR5_OAS_44_BIT:
3738 case IDR5_OAS_52_BIT:
3740 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
3744 "unknown output address size. Truncating to 48-bit\n");
3746 case IDR5_OAS_48_BIT:
3750 if (arm_smmu_ops.pgsize_bitmap == -1UL)
3751 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3753 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3755 /* Set the DMA mask for our table walker */
3756 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3758 "failed to set DMA mask for table walker\n");
3760 smmu->ias = max(smmu->ias, smmu->oas);
3762 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3763 smmu->ias, smmu->oas, smmu->features);
3768 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
3771 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
3772 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3774 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3775 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3779 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3782 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3783 struct arm_smmu_device *smmu)
3785 struct acpi_iort_smmu_v3 *iort_smmu;
3786 struct device *dev = smmu->dev;
3787 struct acpi_iort_node *node;
3789 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3791 /* Retrieve SMMUv3 specific data */
3792 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3794 acpi_smmu_get_options(iort_smmu->model, smmu);
3796 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3797 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3802 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3803 struct arm_smmu_device *smmu)
3809 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3810 struct arm_smmu_device *smmu)
3812 struct device *dev = &pdev->dev;
3816 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3817 dev_err(dev, "missing #iommu-cells property\n");
3818 else if (cells != 1)
3819 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3823 parse_driver_options(smmu);
3825 if (of_dma_is_coherent(dev->of_node))
3826 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3831 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3833 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3839 static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
3844 if (pci_bus_type.iommu_ops != ops) {
3845 err = bus_set_iommu(&pci_bus_type, ops);
3850 #ifdef CONFIG_ARM_AMBA
3851 if (amba_bustype.iommu_ops != ops) {
3852 err = bus_set_iommu(&amba_bustype, ops);
3854 goto err_reset_pci_ops;
3857 if (platform_bus_type.iommu_ops != ops) {
3858 err = bus_set_iommu(&platform_bus_type, ops);
3860 goto err_reset_amba_ops;
3866 #ifdef CONFIG_ARM_AMBA
3867 bus_set_iommu(&amba_bustype, NULL);
3869 err_reset_pci_ops: __maybe_unused;
3871 bus_set_iommu(&pci_bus_type, NULL);
3876 static int arm_smmu_device_probe(struct platform_device *pdev)
3879 struct resource *res;
3880 resource_size_t ioaddr;
3881 struct arm_smmu_device *smmu;
3882 struct device *dev = &pdev->dev;
3885 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3887 dev_err(dev, "failed to allocate arm_smmu_device\n");
3893 ret = arm_smmu_device_dt_probe(pdev, smmu);
3895 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3900 /* Set bypass mode according to firmware probing result */
3904 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3905 if (resource_size(res) < arm_smmu_resource_size(smmu)) {
3906 dev_err(dev, "MMIO region too small (%pr)\n", res);
3909 ioaddr = res->start;
3911 smmu->base = devm_ioremap_resource(dev, res);
3912 if (IS_ERR(smmu->base))
3913 return PTR_ERR(smmu->base);
3915 /* Interrupt lines */
3917 irq = platform_get_irq_byname_optional(pdev, "combined");
3919 smmu->combined_irq = irq;
3921 irq = platform_get_irq_byname_optional(pdev, "eventq");
3923 smmu->evtq.q.irq = irq;
3925 irq = platform_get_irq_byname_optional(pdev, "priq");
3927 smmu->priq.q.irq = irq;
3929 irq = platform_get_irq_byname_optional(pdev, "gerror");
3931 smmu->gerr_irq = irq;
3934 ret = arm_smmu_device_hw_probe(smmu);
3938 /* Initialise in-memory data structures */
3939 ret = arm_smmu_init_structures(smmu);
3943 /* Record our private device structure */
3944 platform_set_drvdata(pdev, smmu);
3946 /* Reset the device */
3947 ret = arm_smmu_device_reset(smmu, bypass);
3951 /* And we're up. Go go go! */
3952 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3953 "smmu3.%pa", &ioaddr);
3957 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
3958 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
3960 ret = iommu_device_register(&smmu->iommu);
3962 dev_err(dev, "Failed to register iommu\n");
3966 return arm_smmu_set_bus_ops(&arm_smmu_ops);
3969 static int arm_smmu_device_remove(struct platform_device *pdev)
3971 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3973 arm_smmu_set_bus_ops(NULL);
3974 iommu_device_unregister(&smmu->iommu);
3975 iommu_device_sysfs_remove(&smmu->iommu);
3976 arm_smmu_device_disable(smmu);
3981 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3983 arm_smmu_device_remove(pdev);
3986 static const struct of_device_id arm_smmu_of_match[] = {
3987 { .compatible = "arm,smmu-v3", },
3990 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
3992 static struct platform_driver arm_smmu_driver = {
3994 .name = "arm-smmu-v3",
3995 .of_match_table = arm_smmu_of_match,
3996 .suppress_bind_attrs = true,
3998 .probe = arm_smmu_device_probe,
3999 .remove = arm_smmu_device_remove,
4000 .shutdown = arm_smmu_device_shutdown,
4002 module_platform_driver(arm_smmu_driver);
4004 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
4005 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
4006 MODULE_ALIAS("platform:arm-smmu-v3");
4007 MODULE_LICENSE("GPL v2");