1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * IOMMU API for ARM architected SMMUv3 implementations.
5 * Copyright (C) 2015 ARM Limited
11 #include <linux/bitfield.h>
12 #include <linux/iommu.h>
13 #include <linux/kernel.h>
14 #include <linux/mmzone.h>
15 #include <linux/sizes.h>
18 #define ARM_SMMU_IDR0 0x0
19 #define IDR0_ST_LVL GENMASK(28, 27)
20 #define IDR0_ST_LVL_2LVL 1
21 #define IDR0_STALL_MODEL GENMASK(25, 24)
22 #define IDR0_STALL_MODEL_STALL 0
23 #define IDR0_STALL_MODEL_FORCE 2
24 #define IDR0_TTENDIAN GENMASK(22, 21)
25 #define IDR0_TTENDIAN_MIXED 0
26 #define IDR0_TTENDIAN_LE 2
27 #define IDR0_TTENDIAN_BE 3
28 #define IDR0_CD2L (1 << 19)
29 #define IDR0_VMID16 (1 << 18)
30 #define IDR0_PRI (1 << 16)
31 #define IDR0_SEV (1 << 14)
32 #define IDR0_MSI (1 << 13)
33 #define IDR0_ASID16 (1 << 12)
34 #define IDR0_ATS (1 << 10)
35 #define IDR0_HYP (1 << 9)
36 #define IDR0_COHACC (1 << 4)
37 #define IDR0_TTF GENMASK(3, 2)
38 #define IDR0_TTF_AARCH64 2
39 #define IDR0_TTF_AARCH32_64 3
40 #define IDR0_S1P (1 << 1)
41 #define IDR0_S2P (1 << 0)
43 #define ARM_SMMU_IDR1 0x4
44 #define IDR1_TABLES_PRESET (1 << 30)
45 #define IDR1_QUEUES_PRESET (1 << 29)
46 #define IDR1_REL (1 << 28)
47 #define IDR1_CMDQS GENMASK(25, 21)
48 #define IDR1_EVTQS GENMASK(20, 16)
49 #define IDR1_PRIQS GENMASK(15, 11)
50 #define IDR1_SSIDSIZE GENMASK(10, 6)
51 #define IDR1_SIDSIZE GENMASK(5, 0)
53 #define ARM_SMMU_IDR3 0xc
54 #define IDR3_RIL (1 << 10)
56 #define ARM_SMMU_IDR5 0x14
57 #define IDR5_STALL_MAX GENMASK(31, 16)
58 #define IDR5_GRAN64K (1 << 6)
59 #define IDR5_GRAN16K (1 << 5)
60 #define IDR5_GRAN4K (1 << 4)
61 #define IDR5_OAS GENMASK(2, 0)
62 #define IDR5_OAS_32_BIT 0
63 #define IDR5_OAS_36_BIT 1
64 #define IDR5_OAS_40_BIT 2
65 #define IDR5_OAS_42_BIT 3
66 #define IDR5_OAS_44_BIT 4
67 #define IDR5_OAS_48_BIT 5
68 #define IDR5_OAS_52_BIT 6
69 #define IDR5_VAX GENMASK(11, 10)
70 #define IDR5_VAX_52_BIT 1
72 #define ARM_SMMU_IIDR 0x18
73 #define IIDR_PRODUCTID GENMASK(31, 20)
74 #define IIDR_VARIANT GENMASK(19, 16)
75 #define IIDR_REVISION GENMASK(15, 12)
76 #define IIDR_IMPLEMENTER GENMASK(11, 0)
78 #define ARM_SMMU_CR0 0x20
79 #define CR0_ATSCHK (1 << 4)
80 #define CR0_CMDQEN (1 << 3)
81 #define CR0_EVTQEN (1 << 2)
82 #define CR0_PRIQEN (1 << 1)
83 #define CR0_SMMUEN (1 << 0)
85 #define ARM_SMMU_CR0ACK 0x24
87 #define ARM_SMMU_CR1 0x28
88 #define CR1_TABLE_SH GENMASK(11, 10)
89 #define CR1_TABLE_OC GENMASK(9, 8)
90 #define CR1_TABLE_IC GENMASK(7, 6)
91 #define CR1_QUEUE_SH GENMASK(5, 4)
92 #define CR1_QUEUE_OC GENMASK(3, 2)
93 #define CR1_QUEUE_IC GENMASK(1, 0)
94 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
95 #define CR1_CACHE_NC 0
96 #define CR1_CACHE_WB 1
97 #define CR1_CACHE_WT 2
99 #define ARM_SMMU_CR2 0x2c
100 #define CR2_PTM (1 << 2)
101 #define CR2_RECINVSID (1 << 1)
102 #define CR2_E2H (1 << 0)
104 #define ARM_SMMU_GBPA 0x44
105 #define GBPA_UPDATE (1 << 31)
106 #define GBPA_ABORT (1 << 20)
108 #define ARM_SMMU_IRQ_CTRL 0x50
109 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
110 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
111 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
113 #define ARM_SMMU_IRQ_CTRLACK 0x54
115 #define ARM_SMMU_GERROR 0x60
116 #define GERROR_SFM_ERR (1 << 8)
117 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
118 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
119 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
120 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
121 #define GERROR_PRIQ_ABT_ERR (1 << 3)
122 #define GERROR_EVTQ_ABT_ERR (1 << 2)
123 #define GERROR_CMDQ_ERR (1 << 0)
124 #define GERROR_ERR_MASK 0x1fd
126 #define ARM_SMMU_GERRORN 0x64
128 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
129 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
130 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
132 #define ARM_SMMU_STRTAB_BASE 0x80
133 #define STRTAB_BASE_RA (1UL << 62)
134 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
136 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
137 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
138 #define STRTAB_BASE_CFG_FMT_LINEAR 0
139 #define STRTAB_BASE_CFG_FMT_2LVL 1
140 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
141 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
143 #define ARM_SMMU_CMDQ_BASE 0x90
144 #define ARM_SMMU_CMDQ_PROD 0x98
145 #define ARM_SMMU_CMDQ_CONS 0x9c
147 #define ARM_SMMU_EVTQ_BASE 0xa0
148 #define ARM_SMMU_EVTQ_PROD 0xa8
149 #define ARM_SMMU_EVTQ_CONS 0xac
150 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
151 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
152 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
154 #define ARM_SMMU_PRIQ_BASE 0xc0
155 #define ARM_SMMU_PRIQ_PROD 0xc8
156 #define ARM_SMMU_PRIQ_CONS 0xcc
157 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
158 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
159 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
161 #define ARM_SMMU_REG_SZ 0xe00
163 /* Common MSI config fields */
164 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
165 #define MSI_CFG2_SH GENMASK(5, 4)
166 #define MSI_CFG2_MEMATTR GENMASK(3, 0)
168 /* Common memory attribute values */
169 #define ARM_SMMU_SH_NSH 0
170 #define ARM_SMMU_SH_OSH 2
171 #define ARM_SMMU_SH_ISH 3
172 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
173 #define ARM_SMMU_MEMATTR_OIWB 0xf
175 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
176 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
177 #define Q_OVERFLOW_FLAG (1U << 31)
178 #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
179 #define Q_ENT(q, p) ((q)->base + \
180 Q_IDX(&((q)->llq), p) * \
183 #define Q_BASE_RWA (1UL << 62)
184 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
185 #define Q_BASE_LOG2SIZE GENMASK(4, 0)
187 /* Ensure DMA allocations are naturally aligned */
188 #ifdef CONFIG_CMA_ALIGNMENT
189 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
191 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER)
197 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
198 * 2lvl: 128k L1 entries,
199 * 256 lazy entries per table (each table covers a PCI bus)
201 #define STRTAB_L1_SZ_SHIFT 20
202 #define STRTAB_SPLIT 8
204 #define STRTAB_L1_DESC_DWORDS 1
205 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
206 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
208 #define STRTAB_STE_DWORDS 8
209 #define STRTAB_STE_0_V (1UL << 0)
210 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
211 #define STRTAB_STE_0_CFG_ABORT 0
212 #define STRTAB_STE_0_CFG_BYPASS 4
213 #define STRTAB_STE_0_CFG_S1_TRANS 5
214 #define STRTAB_STE_0_CFG_S2_TRANS 6
216 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
217 #define STRTAB_STE_0_S1FMT_LINEAR 0
218 #define STRTAB_STE_0_S1FMT_64K_L2 2
219 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
220 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
222 #define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
223 #define STRTAB_STE_1_S1DSS_TERMINATE 0x0
224 #define STRTAB_STE_1_S1DSS_BYPASS 0x1
225 #define STRTAB_STE_1_S1DSS_SSID0 0x2
227 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
228 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
229 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
230 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
231 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
232 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
233 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
235 #define STRTAB_STE_1_S1STALLD (1UL << 27)
237 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
238 #define STRTAB_STE_1_EATS_ABT 0UL
239 #define STRTAB_STE_1_EATS_TRANS 1UL
240 #define STRTAB_STE_1_EATS_S1CHK 2UL
242 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
243 #define STRTAB_STE_1_STRW_NSEL1 0UL
244 #define STRTAB_STE_1_STRW_EL2 2UL
246 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
247 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
249 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
250 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
251 #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
252 #define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
253 #define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
254 #define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
255 #define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
256 #define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
257 #define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
258 #define STRTAB_STE_2_S2AA64 (1UL << 51)
259 #define STRTAB_STE_2_S2ENDI (1UL << 52)
260 #define STRTAB_STE_2_S2PTW (1UL << 54)
261 #define STRTAB_STE_2_S2R (1UL << 58)
263 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
266 * Context descriptors.
268 * Linear: when less than 1024 SSIDs are supported
269 * 2lvl: at most 1024 L1 entries,
270 * 1024 lazy entries per table.
272 #define CTXDESC_SPLIT 10
273 #define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
275 #define CTXDESC_L1_DESC_DWORDS 1
276 #define CTXDESC_L1_DESC_V (1UL << 0)
277 #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
279 #define CTXDESC_CD_DWORDS 8
280 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
281 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
282 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
283 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
284 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
285 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
286 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
288 #define CTXDESC_CD_0_ENDI (1UL << 15)
289 #define CTXDESC_CD_0_V (1UL << 31)
291 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
292 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
294 #define CTXDESC_CD_0_AA64 (1UL << 41)
295 #define CTXDESC_CD_0_S (1UL << 44)
296 #define CTXDESC_CD_0_R (1UL << 45)
297 #define CTXDESC_CD_0_A (1UL << 46)
298 #define CTXDESC_CD_0_ASET (1UL << 47)
299 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
301 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
304 * When the SMMU only supports linear context descriptor tables, pick a
305 * reasonable size limit (64kB).
307 #define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
310 #define CMDQ_ENT_SZ_SHIFT 4
311 #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
312 #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
314 #define CMDQ_CONS_ERR GENMASK(30, 24)
315 #define CMDQ_ERR_CERROR_NONE_IDX 0
316 #define CMDQ_ERR_CERROR_ILL_IDX 1
317 #define CMDQ_ERR_CERROR_ABT_IDX 2
318 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
320 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
323 * This is used to size the command queue and therefore must be at least
324 * BITS_PER_LONG so that the valid_map works correctly (it relies on the
325 * total number of queue entries being a multiple of BITS_PER_LONG).
327 #define CMDQ_BATCH_ENTRIES BITS_PER_LONG
329 #define CMDQ_0_OP GENMASK_ULL(7, 0)
330 #define CMDQ_0_SSV (1UL << 11)
332 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
333 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
334 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
336 #define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
337 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
338 #define CMDQ_CFGI_1_LEAF (1UL << 0)
339 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
341 #define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
342 #define CMDQ_TLBI_RANGE_NUM_MAX 31
343 #define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
344 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
345 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
346 #define CMDQ_TLBI_1_LEAF (1UL << 0)
347 #define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8)
348 #define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10)
349 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
350 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
352 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
353 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
354 #define CMDQ_ATC_0_GLOBAL (1UL << 9)
355 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
356 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
358 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
359 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
360 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
361 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
363 #define CMDQ_RESUME_0_RESP_TERM 0UL
364 #define CMDQ_RESUME_0_RESP_RETRY 1UL
365 #define CMDQ_RESUME_0_RESP_ABORT 2UL
366 #define CMDQ_RESUME_0_RESP GENMASK_ULL(13, 12)
367 #define CMDQ_RESUME_0_SID GENMASK_ULL(63, 32)
368 #define CMDQ_RESUME_1_STAG GENMASK_ULL(15, 0)
370 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
371 #define CMDQ_SYNC_0_CS_NONE 0
372 #define CMDQ_SYNC_0_CS_IRQ 1
373 #define CMDQ_SYNC_0_CS_SEV 2
374 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
375 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
376 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
377 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
380 #define EVTQ_ENT_SZ_SHIFT 5
381 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
382 #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
384 #define EVTQ_0_ID GENMASK_ULL(7, 0)
386 #define EVT_ID_TRANSLATION_FAULT 0x10
387 #define EVT_ID_ADDR_SIZE_FAULT 0x11
388 #define EVT_ID_ACCESS_FAULT 0x12
389 #define EVT_ID_PERMISSION_FAULT 0x13
391 #define EVTQ_0_SSV (1UL << 11)
392 #define EVTQ_0_SSID GENMASK_ULL(31, 12)
393 #define EVTQ_0_SID GENMASK_ULL(63, 32)
394 #define EVTQ_1_STAG GENMASK_ULL(15, 0)
395 #define EVTQ_1_STALL (1UL << 31)
396 #define EVTQ_1_PnU (1UL << 33)
397 #define EVTQ_1_InD (1UL << 34)
398 #define EVTQ_1_RnW (1UL << 35)
399 #define EVTQ_1_S2 (1UL << 39)
400 #define EVTQ_1_CLASS GENMASK_ULL(41, 40)
401 #define EVTQ_1_TT_READ (1UL << 44)
402 #define EVTQ_2_ADDR GENMASK_ULL(63, 0)
403 #define EVTQ_3_IPA GENMASK_ULL(51, 12)
406 #define PRIQ_ENT_SZ_SHIFT 4
407 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
408 #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
410 #define PRIQ_0_SID GENMASK_ULL(31, 0)
411 #define PRIQ_0_SSID GENMASK_ULL(51, 32)
412 #define PRIQ_0_PERM_PRIV (1UL << 58)
413 #define PRIQ_0_PERM_EXEC (1UL << 59)
414 #define PRIQ_0_PERM_READ (1UL << 60)
415 #define PRIQ_0_PERM_WRITE (1UL << 61)
416 #define PRIQ_0_PRG_LAST (1UL << 62)
417 #define PRIQ_0_SSID_V (1UL << 63)
419 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
420 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
422 /* High-level queue structures */
423 #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
424 #define ARM_SMMU_POLL_SPIN_COUNT 10
426 #define MSI_IOVA_BASE 0x8000000
427 #define MSI_IOVA_LENGTH 0x100000
435 struct arm_smmu_cmdq_ent {
438 bool substream_valid;
440 /* Command-specific fields */
442 #define CMDQ_OP_PREFETCH_CFG 0x1
447 #define CMDQ_OP_CFGI_STE 0x3
448 #define CMDQ_OP_CFGI_ALL 0x4
449 #define CMDQ_OP_CFGI_CD 0x5
450 #define CMDQ_OP_CFGI_CD_ALL 0x6
460 #define CMDQ_OP_TLBI_NH_ASID 0x11
461 #define CMDQ_OP_TLBI_NH_VA 0x12
462 #define CMDQ_OP_TLBI_EL2_ALL 0x20
463 #define CMDQ_OP_TLBI_EL2_ASID 0x21
464 #define CMDQ_OP_TLBI_EL2_VA 0x22
465 #define CMDQ_OP_TLBI_S12_VMALL 0x28
466 #define CMDQ_OP_TLBI_S2_IPA 0x2a
467 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
479 #define CMDQ_OP_ATC_INV 0x40
480 #define ATC_INV_SIZE_ALL 52
489 #define CMDQ_OP_PRI_RESP 0x41
497 #define CMDQ_OP_RESUME 0x44
504 #define CMDQ_OP_CMD_SYNC 0x46
511 struct arm_smmu_ll_queue {
522 u8 __pad[SMP_CACHE_BYTES];
523 } ____cacheline_aligned_in_smp;
527 struct arm_smmu_queue {
528 struct arm_smmu_ll_queue llq;
529 int irq; /* Wired interrupt */
537 u32 __iomem *prod_reg;
538 u32 __iomem *cons_reg;
541 struct arm_smmu_queue_poll {
544 unsigned int spin_cnt;
548 struct arm_smmu_cmdq {
549 struct arm_smmu_queue q;
550 atomic_long_t *valid_map;
555 struct arm_smmu_cmdq_batch {
556 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
560 struct arm_smmu_evtq {
561 struct arm_smmu_queue q;
562 struct iopf_queue *iopf;
566 struct arm_smmu_priq {
567 struct arm_smmu_queue q;
570 /* High-level stream table and context descriptor structures */
571 struct arm_smmu_strtab_l1_desc {
575 dma_addr_t l2ptr_dma;
578 struct arm_smmu_ctx_desc {
585 struct mm_struct *mm;
588 struct arm_smmu_l1_ctx_desc {
590 dma_addr_t l2ptr_dma;
593 struct arm_smmu_ctx_desc_cfg {
595 dma_addr_t cdtab_dma;
596 struct arm_smmu_l1_ctx_desc *l1_desc;
597 unsigned int num_l1_ents;
600 struct arm_smmu_s1_cfg {
601 struct arm_smmu_ctx_desc_cfg cdcfg;
602 struct arm_smmu_ctx_desc cd;
607 struct arm_smmu_s2_cfg {
613 struct arm_smmu_strtab_cfg {
615 dma_addr_t strtab_dma;
616 struct arm_smmu_strtab_l1_desc *l1_desc;
617 unsigned int num_l1_ents;
623 /* An SMMUv3 instance */
624 struct arm_smmu_device {
629 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
630 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
631 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
632 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
633 #define ARM_SMMU_FEAT_PRI (1 << 4)
634 #define ARM_SMMU_FEAT_ATS (1 << 5)
635 #define ARM_SMMU_FEAT_SEV (1 << 6)
636 #define ARM_SMMU_FEAT_MSI (1 << 7)
637 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
638 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
639 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
640 #define ARM_SMMU_FEAT_STALLS (1 << 11)
641 #define ARM_SMMU_FEAT_HYP (1 << 12)
642 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
643 #define ARM_SMMU_FEAT_VAX (1 << 14)
644 #define ARM_SMMU_FEAT_RANGE_INV (1 << 15)
645 #define ARM_SMMU_FEAT_BTM (1 << 16)
646 #define ARM_SMMU_FEAT_SVA (1 << 17)
647 #define ARM_SMMU_FEAT_E2H (1 << 18)
648 #define ARM_SMMU_FEAT_NESTING (1 << 19)
651 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
652 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
653 #define ARM_SMMU_OPT_MSIPOLL (1 << 2)
654 #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3)
657 struct arm_smmu_cmdq cmdq;
658 struct arm_smmu_evtq evtq;
659 struct arm_smmu_priq priq;
664 unsigned long ias; /* IPA */
665 unsigned long oas; /* PA */
666 unsigned long pgsize_bitmap;
668 #define ARM_SMMU_MAX_ASIDS (1 << 16)
669 unsigned int asid_bits;
671 #define ARM_SMMU_MAX_VMIDS (1 << 16)
672 unsigned int vmid_bits;
673 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
675 unsigned int ssid_bits;
676 unsigned int sid_bits;
678 struct arm_smmu_strtab_cfg strtab_cfg;
680 /* IOMMU core code handle */
681 struct iommu_device iommu;
683 struct rb_root streams;
684 struct mutex streams_mutex;
687 struct arm_smmu_stream {
689 struct arm_smmu_master *master;
693 /* SMMU private data for each master */
694 struct arm_smmu_master {
695 struct arm_smmu_device *smmu;
697 struct arm_smmu_domain *domain;
698 struct list_head domain_head;
699 struct arm_smmu_stream *streams;
700 unsigned int num_streams;
705 struct list_head bonds;
706 unsigned int ssid_bits;
709 /* SMMU private data for an IOMMU domain */
710 enum arm_smmu_domain_stage {
711 ARM_SMMU_DOMAIN_S1 = 0,
713 ARM_SMMU_DOMAIN_NESTED,
714 ARM_SMMU_DOMAIN_BYPASS,
717 struct arm_smmu_domain {
718 struct arm_smmu_device *smmu;
719 struct mutex init_mutex; /* Protects smmu pointer */
721 struct io_pgtable_ops *pgtbl_ops;
723 atomic_t nr_ats_masters;
725 enum arm_smmu_domain_stage stage;
727 struct arm_smmu_s1_cfg s1_cfg;
728 struct arm_smmu_s2_cfg s2_cfg;
731 struct iommu_domain domain;
733 struct list_head devices;
734 spinlock_t devices_lock;
736 struct list_head mmu_notifiers;
739 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
741 return container_of(dom, struct arm_smmu_domain, domain);
744 extern struct xarray arm_smmu_asid_xa;
745 extern struct mutex arm_smmu_asid_lock;
746 extern struct arm_smmu_ctx_desc quiet_cd;
748 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
749 struct arm_smmu_ctx_desc *cd);
750 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
751 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
752 size_t granule, bool leaf,
753 struct arm_smmu_domain *smmu_domain);
754 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
755 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
756 unsigned long iova, size_t size);
758 #ifdef CONFIG_ARM_SMMU_V3_SVA
759 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
760 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
761 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
762 int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
763 int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
764 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
765 void arm_smmu_sva_notifier_synchronize(void);
766 struct iommu_domain *arm_smmu_sva_domain_alloc(void);
767 void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
768 struct device *dev, ioasid_t id);
769 #else /* CONFIG_ARM_SMMU_V3_SVA */
770 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
775 static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
780 static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
785 static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
790 static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
795 static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
800 static inline void arm_smmu_sva_notifier_synchronize(void) {}
802 static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void)
807 static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
812 #endif /* CONFIG_ARM_SMMU_V3_SVA */
813 #endif /* _ARM_SMMU_V3_H */