iommu/io-pgtable: Replace IO_PGTABLE_QUIRK_NO_DMA with specific flag
[sfrench/cifs-2.6.git] / drivers / iommu / arm-smmu-v3.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * IOMMU API for ARM architected SMMUv3 implementations.
4  *
5  * Copyright (C) 2015 ARM Limited
6  *
7  * Author: Will Deacon <will.deacon@arm.com>
8  *
9  * This driver is powered by bad coffee and bombay mix.
10  */
11
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/bitops.h>
16 #include <linux/crash_dump.h>
17 #include <linux/delay.h>
18 #include <linux/dma-iommu.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/init.h>
25 #include <linux/moduleparam.h>
26 #include <linux/msi.h>
27 #include <linux/of.h>
28 #include <linux/of_address.h>
29 #include <linux/of_iommu.h>
30 #include <linux/of_platform.h>
31 #include <linux/pci.h>
32 #include <linux/pci-ats.h>
33 #include <linux/platform_device.h>
34
35 #include <linux/amba/bus.h>
36
37 /* MMIO registers */
38 #define ARM_SMMU_IDR0                   0x0
39 #define IDR0_ST_LVL                     GENMASK(28, 27)
40 #define IDR0_ST_LVL_2LVL                1
41 #define IDR0_STALL_MODEL                GENMASK(25, 24)
42 #define IDR0_STALL_MODEL_STALL          0
43 #define IDR0_STALL_MODEL_FORCE          2
44 #define IDR0_TTENDIAN                   GENMASK(22, 21)
45 #define IDR0_TTENDIAN_MIXED             0
46 #define IDR0_TTENDIAN_LE                2
47 #define IDR0_TTENDIAN_BE                3
48 #define IDR0_CD2L                       (1 << 19)
49 #define IDR0_VMID16                     (1 << 18)
50 #define IDR0_PRI                        (1 << 16)
51 #define IDR0_SEV                        (1 << 14)
52 #define IDR0_MSI                        (1 << 13)
53 #define IDR0_ASID16                     (1 << 12)
54 #define IDR0_ATS                        (1 << 10)
55 #define IDR0_HYP                        (1 << 9)
56 #define IDR0_COHACC                     (1 << 4)
57 #define IDR0_TTF                        GENMASK(3, 2)
58 #define IDR0_TTF_AARCH64                2
59 #define IDR0_TTF_AARCH32_64             3
60 #define IDR0_S1P                        (1 << 1)
61 #define IDR0_S2P                        (1 << 0)
62
63 #define ARM_SMMU_IDR1                   0x4
64 #define IDR1_TABLES_PRESET              (1 << 30)
65 #define IDR1_QUEUES_PRESET              (1 << 29)
66 #define IDR1_REL                        (1 << 28)
67 #define IDR1_CMDQS                      GENMASK(25, 21)
68 #define IDR1_EVTQS                      GENMASK(20, 16)
69 #define IDR1_PRIQS                      GENMASK(15, 11)
70 #define IDR1_SSIDSIZE                   GENMASK(10, 6)
71 #define IDR1_SIDSIZE                    GENMASK(5, 0)
72
73 #define ARM_SMMU_IDR5                   0x14
74 #define IDR5_STALL_MAX                  GENMASK(31, 16)
75 #define IDR5_GRAN64K                    (1 << 6)
76 #define IDR5_GRAN16K                    (1 << 5)
77 #define IDR5_GRAN4K                     (1 << 4)
78 #define IDR5_OAS                        GENMASK(2, 0)
79 #define IDR5_OAS_32_BIT                 0
80 #define IDR5_OAS_36_BIT                 1
81 #define IDR5_OAS_40_BIT                 2
82 #define IDR5_OAS_42_BIT                 3
83 #define IDR5_OAS_44_BIT                 4
84 #define IDR5_OAS_48_BIT                 5
85 #define IDR5_OAS_52_BIT                 6
86 #define IDR5_VAX                        GENMASK(11, 10)
87 #define IDR5_VAX_52_BIT                 1
88
89 #define ARM_SMMU_CR0                    0x20
90 #define CR0_ATSCHK                      (1 << 4)
91 #define CR0_CMDQEN                      (1 << 3)
92 #define CR0_EVTQEN                      (1 << 2)
93 #define CR0_PRIQEN                      (1 << 1)
94 #define CR0_SMMUEN                      (1 << 0)
95
96 #define ARM_SMMU_CR0ACK                 0x24
97
98 #define ARM_SMMU_CR1                    0x28
99 #define CR1_TABLE_SH                    GENMASK(11, 10)
100 #define CR1_TABLE_OC                    GENMASK(9, 8)
101 #define CR1_TABLE_IC                    GENMASK(7, 6)
102 #define CR1_QUEUE_SH                    GENMASK(5, 4)
103 #define CR1_QUEUE_OC                    GENMASK(3, 2)
104 #define CR1_QUEUE_IC                    GENMASK(1, 0)
105 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
106 #define CR1_CACHE_NC                    0
107 #define CR1_CACHE_WB                    1
108 #define CR1_CACHE_WT                    2
109
110 #define ARM_SMMU_CR2                    0x2c
111 #define CR2_PTM                         (1 << 2)
112 #define CR2_RECINVSID                   (1 << 1)
113 #define CR2_E2H                         (1 << 0)
114
115 #define ARM_SMMU_GBPA                   0x44
116 #define GBPA_UPDATE                     (1 << 31)
117 #define GBPA_ABORT                      (1 << 20)
118
119 #define ARM_SMMU_IRQ_CTRL               0x50
120 #define IRQ_CTRL_EVTQ_IRQEN             (1 << 2)
121 #define IRQ_CTRL_PRIQ_IRQEN             (1 << 1)
122 #define IRQ_CTRL_GERROR_IRQEN           (1 << 0)
123
124 #define ARM_SMMU_IRQ_CTRLACK            0x54
125
126 #define ARM_SMMU_GERROR                 0x60
127 #define GERROR_SFM_ERR                  (1 << 8)
128 #define GERROR_MSI_GERROR_ABT_ERR       (1 << 7)
129 #define GERROR_MSI_PRIQ_ABT_ERR         (1 << 6)
130 #define GERROR_MSI_EVTQ_ABT_ERR         (1 << 5)
131 #define GERROR_MSI_CMDQ_ABT_ERR         (1 << 4)
132 #define GERROR_PRIQ_ABT_ERR             (1 << 3)
133 #define GERROR_EVTQ_ABT_ERR             (1 << 2)
134 #define GERROR_CMDQ_ERR                 (1 << 0)
135 #define GERROR_ERR_MASK                 0xfd
136
137 #define ARM_SMMU_GERRORN                0x64
138
139 #define ARM_SMMU_GERROR_IRQ_CFG0        0x68
140 #define ARM_SMMU_GERROR_IRQ_CFG1        0x70
141 #define ARM_SMMU_GERROR_IRQ_CFG2        0x74
142
143 #define ARM_SMMU_STRTAB_BASE            0x80
144 #define STRTAB_BASE_RA                  (1UL << 62)
145 #define STRTAB_BASE_ADDR_MASK           GENMASK_ULL(51, 6)
146
147 #define ARM_SMMU_STRTAB_BASE_CFG        0x88
148 #define STRTAB_BASE_CFG_FMT             GENMASK(17, 16)
149 #define STRTAB_BASE_CFG_FMT_LINEAR      0
150 #define STRTAB_BASE_CFG_FMT_2LVL        1
151 #define STRTAB_BASE_CFG_SPLIT           GENMASK(10, 6)
152 #define STRTAB_BASE_CFG_LOG2SIZE        GENMASK(5, 0)
153
154 #define ARM_SMMU_CMDQ_BASE              0x90
155 #define ARM_SMMU_CMDQ_PROD              0x98
156 #define ARM_SMMU_CMDQ_CONS              0x9c
157
158 #define ARM_SMMU_EVTQ_BASE              0xa0
159 #define ARM_SMMU_EVTQ_PROD              0x100a8
160 #define ARM_SMMU_EVTQ_CONS              0x100ac
161 #define ARM_SMMU_EVTQ_IRQ_CFG0          0xb0
162 #define ARM_SMMU_EVTQ_IRQ_CFG1          0xb8
163 #define ARM_SMMU_EVTQ_IRQ_CFG2          0xbc
164
165 #define ARM_SMMU_PRIQ_BASE              0xc0
166 #define ARM_SMMU_PRIQ_PROD              0x100c8
167 #define ARM_SMMU_PRIQ_CONS              0x100cc
168 #define ARM_SMMU_PRIQ_IRQ_CFG0          0xd0
169 #define ARM_SMMU_PRIQ_IRQ_CFG1          0xd8
170 #define ARM_SMMU_PRIQ_IRQ_CFG2          0xdc
171
172 /* Common MSI config fields */
173 #define MSI_CFG0_ADDR_MASK              GENMASK_ULL(51, 2)
174 #define MSI_CFG2_SH                     GENMASK(5, 4)
175 #define MSI_CFG2_MEMATTR                GENMASK(3, 0)
176
177 /* Common memory attribute values */
178 #define ARM_SMMU_SH_NSH                 0
179 #define ARM_SMMU_SH_OSH                 2
180 #define ARM_SMMU_SH_ISH                 3
181 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE   0x1
182 #define ARM_SMMU_MEMATTR_OIWB           0xf
183
184 #define Q_IDX(q, p)                     ((p) & ((1 << (q)->max_n_shift) - 1))
185 #define Q_WRP(q, p)                     ((p) & (1 << (q)->max_n_shift))
186 #define Q_OVERFLOW_FLAG                 (1 << 31)
187 #define Q_OVF(q, p)                     ((p) & Q_OVERFLOW_FLAG)
188 #define Q_ENT(q, p)                     ((q)->base +                    \
189                                          Q_IDX(q, p) * (q)->ent_dwords)
190
191 #define Q_BASE_RWA                      (1UL << 62)
192 #define Q_BASE_ADDR_MASK                GENMASK_ULL(51, 5)
193 #define Q_BASE_LOG2SIZE                 GENMASK(4, 0)
194 #define Q_MAX_SZ_SHIFT                  (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
195
196 /*
197  * Stream table.
198  *
199  * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
200  * 2lvl: 128k L1 entries,
201  *       256 lazy entries per table (each table covers a PCI bus)
202  */
203 #define STRTAB_L1_SZ_SHIFT              20
204 #define STRTAB_SPLIT                    8
205
206 #define STRTAB_L1_DESC_DWORDS           1
207 #define STRTAB_L1_DESC_SPAN             GENMASK_ULL(4, 0)
208 #define STRTAB_L1_DESC_L2PTR_MASK       GENMASK_ULL(51, 6)
209
210 #define STRTAB_STE_DWORDS               8
211 #define STRTAB_STE_0_V                  (1UL << 0)
212 #define STRTAB_STE_0_CFG                GENMASK_ULL(3, 1)
213 #define STRTAB_STE_0_CFG_ABORT          0
214 #define STRTAB_STE_0_CFG_BYPASS         4
215 #define STRTAB_STE_0_CFG_S1_TRANS       5
216 #define STRTAB_STE_0_CFG_S2_TRANS       6
217
218 #define STRTAB_STE_0_S1FMT              GENMASK_ULL(5, 4)
219 #define STRTAB_STE_0_S1FMT_LINEAR       0
220 #define STRTAB_STE_0_S1CTXPTR_MASK      GENMASK_ULL(51, 6)
221 #define STRTAB_STE_0_S1CDMAX            GENMASK_ULL(63, 59)
222
223 #define STRTAB_STE_1_S1C_CACHE_NC       0UL
224 #define STRTAB_STE_1_S1C_CACHE_WBRA     1UL
225 #define STRTAB_STE_1_S1C_CACHE_WT       2UL
226 #define STRTAB_STE_1_S1C_CACHE_WB       3UL
227 #define STRTAB_STE_1_S1CIR              GENMASK_ULL(3, 2)
228 #define STRTAB_STE_1_S1COR              GENMASK_ULL(5, 4)
229 #define STRTAB_STE_1_S1CSH              GENMASK_ULL(7, 6)
230
231 #define STRTAB_STE_1_S1STALLD           (1UL << 27)
232
233 #define STRTAB_STE_1_EATS               GENMASK_ULL(29, 28)
234 #define STRTAB_STE_1_EATS_ABT           0UL
235 #define STRTAB_STE_1_EATS_TRANS         1UL
236 #define STRTAB_STE_1_EATS_S1CHK         2UL
237
238 #define STRTAB_STE_1_STRW               GENMASK_ULL(31, 30)
239 #define STRTAB_STE_1_STRW_NSEL1         0UL
240 #define STRTAB_STE_1_STRW_EL2           2UL
241
242 #define STRTAB_STE_1_SHCFG              GENMASK_ULL(45, 44)
243 #define STRTAB_STE_1_SHCFG_INCOMING     1UL
244
245 #define STRTAB_STE_2_S2VMID             GENMASK_ULL(15, 0)
246 #define STRTAB_STE_2_VTCR               GENMASK_ULL(50, 32)
247 #define STRTAB_STE_2_S2AA64             (1UL << 51)
248 #define STRTAB_STE_2_S2ENDI             (1UL << 52)
249 #define STRTAB_STE_2_S2PTW              (1UL << 54)
250 #define STRTAB_STE_2_S2R                (1UL << 58)
251
252 #define STRTAB_STE_3_S2TTB_MASK         GENMASK_ULL(51, 4)
253
254 /* Context descriptor (stage-1 only) */
255 #define CTXDESC_CD_DWORDS               8
256 #define CTXDESC_CD_0_TCR_T0SZ           GENMASK_ULL(5, 0)
257 #define ARM64_TCR_T0SZ                  GENMASK_ULL(5, 0)
258 #define CTXDESC_CD_0_TCR_TG0            GENMASK_ULL(7, 6)
259 #define ARM64_TCR_TG0                   GENMASK_ULL(15, 14)
260 #define CTXDESC_CD_0_TCR_IRGN0          GENMASK_ULL(9, 8)
261 #define ARM64_TCR_IRGN0                 GENMASK_ULL(9, 8)
262 #define CTXDESC_CD_0_TCR_ORGN0          GENMASK_ULL(11, 10)
263 #define ARM64_TCR_ORGN0                 GENMASK_ULL(11, 10)
264 #define CTXDESC_CD_0_TCR_SH0            GENMASK_ULL(13, 12)
265 #define ARM64_TCR_SH0                   GENMASK_ULL(13, 12)
266 #define CTXDESC_CD_0_TCR_EPD0           (1ULL << 14)
267 #define ARM64_TCR_EPD0                  (1ULL << 7)
268 #define CTXDESC_CD_0_TCR_EPD1           (1ULL << 30)
269 #define ARM64_TCR_EPD1                  (1ULL << 23)
270
271 #define CTXDESC_CD_0_ENDI               (1UL << 15)
272 #define CTXDESC_CD_0_V                  (1UL << 31)
273
274 #define CTXDESC_CD_0_TCR_IPS            GENMASK_ULL(34, 32)
275 #define ARM64_TCR_IPS                   GENMASK_ULL(34, 32)
276 #define CTXDESC_CD_0_TCR_TBI0           (1ULL << 38)
277 #define ARM64_TCR_TBI0                  (1ULL << 37)
278
279 #define CTXDESC_CD_0_AA64               (1UL << 41)
280 #define CTXDESC_CD_0_S                  (1UL << 44)
281 #define CTXDESC_CD_0_R                  (1UL << 45)
282 #define CTXDESC_CD_0_A                  (1UL << 46)
283 #define CTXDESC_CD_0_ASET               (1UL << 47)
284 #define CTXDESC_CD_0_ASID               GENMASK_ULL(63, 48)
285
286 #define CTXDESC_CD_1_TTB0_MASK          GENMASK_ULL(51, 4)
287
288 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
289 #define ARM_SMMU_TCR2CD(tcr, fld)       FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \
290                                         FIELD_GET(ARM64_TCR_##fld, tcr))
291
292 /* Command queue */
293 #define CMDQ_ENT_SZ_SHIFT               4
294 #define CMDQ_ENT_DWORDS                 ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
295 #define CMDQ_MAX_SZ_SHIFT               (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
296
297 #define CMDQ_CONS_ERR                   GENMASK(30, 24)
298 #define CMDQ_ERR_CERROR_NONE_IDX        0
299 #define CMDQ_ERR_CERROR_ILL_IDX         1
300 #define CMDQ_ERR_CERROR_ABT_IDX         2
301 #define CMDQ_ERR_CERROR_ATC_INV_IDX     3
302
303 #define CMDQ_0_OP                       GENMASK_ULL(7, 0)
304 #define CMDQ_0_SSV                      (1UL << 11)
305
306 #define CMDQ_PREFETCH_0_SID             GENMASK_ULL(63, 32)
307 #define CMDQ_PREFETCH_1_SIZE            GENMASK_ULL(4, 0)
308 #define CMDQ_PREFETCH_1_ADDR_MASK       GENMASK_ULL(63, 12)
309
310 #define CMDQ_CFGI_0_SID                 GENMASK_ULL(63, 32)
311 #define CMDQ_CFGI_1_LEAF                (1UL << 0)
312 #define CMDQ_CFGI_1_RANGE               GENMASK_ULL(4, 0)
313
314 #define CMDQ_TLBI_0_VMID                GENMASK_ULL(47, 32)
315 #define CMDQ_TLBI_0_ASID                GENMASK_ULL(63, 48)
316 #define CMDQ_TLBI_1_LEAF                (1UL << 0)
317 #define CMDQ_TLBI_1_VA_MASK             GENMASK_ULL(63, 12)
318 #define CMDQ_TLBI_1_IPA_MASK            GENMASK_ULL(51, 12)
319
320 #define CMDQ_ATC_0_SSID                 GENMASK_ULL(31, 12)
321 #define CMDQ_ATC_0_SID                  GENMASK_ULL(63, 32)
322 #define CMDQ_ATC_0_GLOBAL               (1UL << 9)
323 #define CMDQ_ATC_1_SIZE                 GENMASK_ULL(5, 0)
324 #define CMDQ_ATC_1_ADDR_MASK            GENMASK_ULL(63, 12)
325
326 #define CMDQ_PRI_0_SSID                 GENMASK_ULL(31, 12)
327 #define CMDQ_PRI_0_SID                  GENMASK_ULL(63, 32)
328 #define CMDQ_PRI_1_GRPID                GENMASK_ULL(8, 0)
329 #define CMDQ_PRI_1_RESP                 GENMASK_ULL(13, 12)
330
331 #define CMDQ_SYNC_0_CS                  GENMASK_ULL(13, 12)
332 #define CMDQ_SYNC_0_CS_NONE             0
333 #define CMDQ_SYNC_0_CS_IRQ              1
334 #define CMDQ_SYNC_0_CS_SEV              2
335 #define CMDQ_SYNC_0_MSH                 GENMASK_ULL(23, 22)
336 #define CMDQ_SYNC_0_MSIATTR             GENMASK_ULL(27, 24)
337 #define CMDQ_SYNC_0_MSIDATA             GENMASK_ULL(63, 32)
338 #define CMDQ_SYNC_1_MSIADDR_MASK        GENMASK_ULL(51, 2)
339
340 /* Event queue */
341 #define EVTQ_ENT_SZ_SHIFT               5
342 #define EVTQ_ENT_DWORDS                 ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
343 #define EVTQ_MAX_SZ_SHIFT               (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
344
345 #define EVTQ_0_ID                       GENMASK_ULL(7, 0)
346
347 /* PRI queue */
348 #define PRIQ_ENT_SZ_SHIFT               4
349 #define PRIQ_ENT_DWORDS                 ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
350 #define PRIQ_MAX_SZ_SHIFT               (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
351
352 #define PRIQ_0_SID                      GENMASK_ULL(31, 0)
353 #define PRIQ_0_SSID                     GENMASK_ULL(51, 32)
354 #define PRIQ_0_PERM_PRIV                (1UL << 58)
355 #define PRIQ_0_PERM_EXEC                (1UL << 59)
356 #define PRIQ_0_PERM_READ                (1UL << 60)
357 #define PRIQ_0_PERM_WRITE               (1UL << 61)
358 #define PRIQ_0_PRG_LAST                 (1UL << 62)
359 #define PRIQ_0_SSID_V                   (1UL << 63)
360
361 #define PRIQ_1_PRG_IDX                  GENMASK_ULL(8, 0)
362 #define PRIQ_1_ADDR_MASK                GENMASK_ULL(63, 12)
363
364 /* High-level queue structures */
365 #define ARM_SMMU_POLL_TIMEOUT_US        100
366 #define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US   1000000 /* 1s! */
367 #define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT   10
368
369 #define MSI_IOVA_BASE                   0x8000000
370 #define MSI_IOVA_LENGTH                 0x100000
371
372 /*
373  * not really modular, but the easiest way to keep compat with existing
374  * bootargs behaviour is to continue using module_param_named here.
375  */
376 static bool disable_bypass = 1;
377 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
378 MODULE_PARM_DESC(disable_bypass,
379         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
380
381 enum pri_resp {
382         PRI_RESP_DENY = 0,
383         PRI_RESP_FAIL = 1,
384         PRI_RESP_SUCC = 2,
385 };
386
387 enum arm_smmu_msi_index {
388         EVTQ_MSI_INDEX,
389         GERROR_MSI_INDEX,
390         PRIQ_MSI_INDEX,
391         ARM_SMMU_MAX_MSIS,
392 };
393
394 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
395         [EVTQ_MSI_INDEX] = {
396                 ARM_SMMU_EVTQ_IRQ_CFG0,
397                 ARM_SMMU_EVTQ_IRQ_CFG1,
398                 ARM_SMMU_EVTQ_IRQ_CFG2,
399         },
400         [GERROR_MSI_INDEX] = {
401                 ARM_SMMU_GERROR_IRQ_CFG0,
402                 ARM_SMMU_GERROR_IRQ_CFG1,
403                 ARM_SMMU_GERROR_IRQ_CFG2,
404         },
405         [PRIQ_MSI_INDEX] = {
406                 ARM_SMMU_PRIQ_IRQ_CFG0,
407                 ARM_SMMU_PRIQ_IRQ_CFG1,
408                 ARM_SMMU_PRIQ_IRQ_CFG2,
409         },
410 };
411
412 struct arm_smmu_cmdq_ent {
413         /* Common fields */
414         u8                              opcode;
415         bool                            substream_valid;
416
417         /* Command-specific fields */
418         union {
419                 #define CMDQ_OP_PREFETCH_CFG    0x1
420                 struct {
421                         u32                     sid;
422                         u8                      size;
423                         u64                     addr;
424                 } prefetch;
425
426                 #define CMDQ_OP_CFGI_STE        0x3
427                 #define CMDQ_OP_CFGI_ALL        0x4
428                 struct {
429                         u32                     sid;
430                         union {
431                                 bool            leaf;
432                                 u8              span;
433                         };
434                 } cfgi;
435
436                 #define CMDQ_OP_TLBI_NH_ASID    0x11
437                 #define CMDQ_OP_TLBI_NH_VA      0x12
438                 #define CMDQ_OP_TLBI_EL2_ALL    0x20
439                 #define CMDQ_OP_TLBI_S12_VMALL  0x28
440                 #define CMDQ_OP_TLBI_S2_IPA     0x2a
441                 #define CMDQ_OP_TLBI_NSNH_ALL   0x30
442                 struct {
443                         u16                     asid;
444                         u16                     vmid;
445                         bool                    leaf;
446                         u64                     addr;
447                 } tlbi;
448
449                 #define CMDQ_OP_ATC_INV         0x40
450                 #define ATC_INV_SIZE_ALL        52
451                 struct {
452                         u32                     sid;
453                         u32                     ssid;
454                         u64                     addr;
455                         u8                      size;
456                         bool                    global;
457                 } atc;
458
459                 #define CMDQ_OP_PRI_RESP        0x41
460                 struct {
461                         u32                     sid;
462                         u32                     ssid;
463                         u16                     grpid;
464                         enum pri_resp           resp;
465                 } pri;
466
467                 #define CMDQ_OP_CMD_SYNC        0x46
468                 struct {
469                         u32                     msidata;
470                         u64                     msiaddr;
471                 } sync;
472         };
473 };
474
475 struct arm_smmu_queue {
476         int                             irq; /* Wired interrupt */
477
478         __le64                          *base;
479         dma_addr_t                      base_dma;
480         u64                             q_base;
481
482         size_t                          ent_dwords;
483         u32                             max_n_shift;
484         u32                             prod;
485         u32                             cons;
486
487         u32 __iomem                     *prod_reg;
488         u32 __iomem                     *cons_reg;
489 };
490
491 struct arm_smmu_cmdq {
492         struct arm_smmu_queue           q;
493         spinlock_t                      lock;
494 };
495
496 struct arm_smmu_evtq {
497         struct arm_smmu_queue           q;
498         u32                             max_stalls;
499 };
500
501 struct arm_smmu_priq {
502         struct arm_smmu_queue           q;
503 };
504
505 /* High-level stream table and context descriptor structures */
506 struct arm_smmu_strtab_l1_desc {
507         u8                              span;
508
509         __le64                          *l2ptr;
510         dma_addr_t                      l2ptr_dma;
511 };
512
513 struct arm_smmu_s1_cfg {
514         __le64                          *cdptr;
515         dma_addr_t                      cdptr_dma;
516
517         struct arm_smmu_ctx_desc {
518                 u16     asid;
519                 u64     ttbr;
520                 u64     tcr;
521                 u64     mair;
522         }                               cd;
523 };
524
525 struct arm_smmu_s2_cfg {
526         u16                             vmid;
527         u64                             vttbr;
528         u64                             vtcr;
529 };
530
531 struct arm_smmu_strtab_cfg {
532         __le64                          *strtab;
533         dma_addr_t                      strtab_dma;
534         struct arm_smmu_strtab_l1_desc  *l1_desc;
535         unsigned int                    num_l1_ents;
536
537         u64                             strtab_base;
538         u32                             strtab_base_cfg;
539 };
540
541 /* An SMMUv3 instance */
542 struct arm_smmu_device {
543         struct device                   *dev;
544         void __iomem                    *base;
545
546 #define ARM_SMMU_FEAT_2_LVL_STRTAB      (1 << 0)
547 #define ARM_SMMU_FEAT_2_LVL_CDTAB       (1 << 1)
548 #define ARM_SMMU_FEAT_TT_LE             (1 << 2)
549 #define ARM_SMMU_FEAT_TT_BE             (1 << 3)
550 #define ARM_SMMU_FEAT_PRI               (1 << 4)
551 #define ARM_SMMU_FEAT_ATS               (1 << 5)
552 #define ARM_SMMU_FEAT_SEV               (1 << 6)
553 #define ARM_SMMU_FEAT_MSI               (1 << 7)
554 #define ARM_SMMU_FEAT_COHERENCY         (1 << 8)
555 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 9)
556 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 10)
557 #define ARM_SMMU_FEAT_STALLS            (1 << 11)
558 #define ARM_SMMU_FEAT_HYP               (1 << 12)
559 #define ARM_SMMU_FEAT_STALL_FORCE       (1 << 13)
560 #define ARM_SMMU_FEAT_VAX               (1 << 14)
561         u32                             features;
562
563 #define ARM_SMMU_OPT_SKIP_PREFETCH      (1 << 0)
564 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY    (1 << 1)
565         u32                             options;
566
567         struct arm_smmu_cmdq            cmdq;
568         struct arm_smmu_evtq            evtq;
569         struct arm_smmu_priq            priq;
570
571         int                             gerr_irq;
572         int                             combined_irq;
573         u32                             sync_nr;
574         u8                              prev_cmd_opcode;
575
576         unsigned long                   ias; /* IPA */
577         unsigned long                   oas; /* PA */
578         unsigned long                   pgsize_bitmap;
579
580 #define ARM_SMMU_MAX_ASIDS              (1 << 16)
581         unsigned int                    asid_bits;
582         DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
583
584 #define ARM_SMMU_MAX_VMIDS              (1 << 16)
585         unsigned int                    vmid_bits;
586         DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
587
588         unsigned int                    ssid_bits;
589         unsigned int                    sid_bits;
590
591         struct arm_smmu_strtab_cfg      strtab_cfg;
592
593         /* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
594         union {
595                 u32                     sync_count;
596                 u64                     padding;
597         };
598
599         /* IOMMU core code handle */
600         struct iommu_device             iommu;
601 };
602
603 /* SMMU private data for each master */
604 struct arm_smmu_master {
605         struct arm_smmu_device          *smmu;
606         struct device                   *dev;
607         struct arm_smmu_domain          *domain;
608         struct list_head                domain_head;
609         u32                             *sids;
610         unsigned int                    num_sids;
611         bool                            ats_enabled             :1;
612 };
613
614 /* SMMU private data for an IOMMU domain */
615 enum arm_smmu_domain_stage {
616         ARM_SMMU_DOMAIN_S1 = 0,
617         ARM_SMMU_DOMAIN_S2,
618         ARM_SMMU_DOMAIN_NESTED,
619         ARM_SMMU_DOMAIN_BYPASS,
620 };
621
622 struct arm_smmu_domain {
623         struct arm_smmu_device          *smmu;
624         struct mutex                    init_mutex; /* Protects smmu pointer */
625
626         struct io_pgtable_ops           *pgtbl_ops;
627         bool                            non_strict;
628
629         enum arm_smmu_domain_stage      stage;
630         union {
631                 struct arm_smmu_s1_cfg  s1_cfg;
632                 struct arm_smmu_s2_cfg  s2_cfg;
633         };
634
635         struct iommu_domain             domain;
636
637         struct list_head                devices;
638         spinlock_t                      devices_lock;
639 };
640
641 struct arm_smmu_option_prop {
642         u32 opt;
643         const char *prop;
644 };
645
646 static struct arm_smmu_option_prop arm_smmu_options[] = {
647         { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
648         { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
649         { 0, NULL},
650 };
651
652 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
653                                                  struct arm_smmu_device *smmu)
654 {
655         if ((offset > SZ_64K) &&
656             (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
657                 offset -= SZ_64K;
658
659         return smmu->base + offset;
660 }
661
662 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
663 {
664         return container_of(dom, struct arm_smmu_domain, domain);
665 }
666
667 static void parse_driver_options(struct arm_smmu_device *smmu)
668 {
669         int i = 0;
670
671         do {
672                 if (of_property_read_bool(smmu->dev->of_node,
673                                                 arm_smmu_options[i].prop)) {
674                         smmu->options |= arm_smmu_options[i].opt;
675                         dev_notice(smmu->dev, "option %s\n",
676                                 arm_smmu_options[i].prop);
677                 }
678         } while (arm_smmu_options[++i].opt);
679 }
680
681 /* Low-level queue manipulation functions */
682 static bool queue_full(struct arm_smmu_queue *q)
683 {
684         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
685                Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
686 }
687
688 static bool queue_empty(struct arm_smmu_queue *q)
689 {
690         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
691                Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
692 }
693
694 static void queue_sync_cons(struct arm_smmu_queue *q)
695 {
696         q->cons = readl_relaxed(q->cons_reg);
697 }
698
699 static void queue_inc_cons(struct arm_smmu_queue *q)
700 {
701         u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
702
703         q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
704
705         /*
706          * Ensure that all CPU accesses (reads and writes) to the queue
707          * are complete before we update the cons pointer.
708          */
709         mb();
710         writel_relaxed(q->cons, q->cons_reg);
711 }
712
713 static int queue_sync_prod(struct arm_smmu_queue *q)
714 {
715         int ret = 0;
716         u32 prod = readl_relaxed(q->prod_reg);
717
718         if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
719                 ret = -EOVERFLOW;
720
721         q->prod = prod;
722         return ret;
723 }
724
725 static void queue_inc_prod(struct arm_smmu_queue *q)
726 {
727         u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
728
729         q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
730         writel(q->prod, q->prod_reg);
731 }
732
733 /*
734  * Wait for the SMMU to consume items. If sync is true, wait until the queue
735  * is empty. Otherwise, wait until there is at least one free slot.
736  */
737 static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
738 {
739         ktime_t timeout;
740         unsigned int delay = 1, spin_cnt = 0;
741
742         /* Wait longer if it's a CMD_SYNC */
743         timeout = ktime_add_us(ktime_get(), sync ?
744                                             ARM_SMMU_CMDQ_SYNC_TIMEOUT_US :
745                                             ARM_SMMU_POLL_TIMEOUT_US);
746
747         while (queue_sync_cons(q), (sync ? !queue_empty(q) : queue_full(q))) {
748                 if (ktime_compare(ktime_get(), timeout) > 0)
749                         return -ETIMEDOUT;
750
751                 if (wfe) {
752                         wfe();
753                 } else if (++spin_cnt < ARM_SMMU_CMDQ_SYNC_SPIN_COUNT) {
754                         cpu_relax();
755                         continue;
756                 } else {
757                         udelay(delay);
758                         delay *= 2;
759                         spin_cnt = 0;
760                 }
761         }
762
763         return 0;
764 }
765
766 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
767 {
768         int i;
769
770         for (i = 0; i < n_dwords; ++i)
771                 *dst++ = cpu_to_le64(*src++);
772 }
773
774 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
775 {
776         if (queue_full(q))
777                 return -ENOSPC;
778
779         queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
780         queue_inc_prod(q);
781         return 0;
782 }
783
784 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
785 {
786         int i;
787
788         for (i = 0; i < n_dwords; ++i)
789                 *dst++ = le64_to_cpu(*src++);
790 }
791
792 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
793 {
794         if (queue_empty(q))
795                 return -EAGAIN;
796
797         queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
798         queue_inc_cons(q);
799         return 0;
800 }
801
802 /* High-level queue accessors */
803 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
804 {
805         memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
806         cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
807
808         switch (ent->opcode) {
809         case CMDQ_OP_TLBI_EL2_ALL:
810         case CMDQ_OP_TLBI_NSNH_ALL:
811                 break;
812         case CMDQ_OP_PREFETCH_CFG:
813                 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
814                 cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
815                 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
816                 break;
817         case CMDQ_OP_CFGI_STE:
818                 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
819                 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
820                 break;
821         case CMDQ_OP_CFGI_ALL:
822                 /* Cover the entire SID range */
823                 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
824                 break;
825         case CMDQ_OP_TLBI_NH_VA:
826                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
827                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
828                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
829                 break;
830         case CMDQ_OP_TLBI_S2_IPA:
831                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
832                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
833                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
834                 break;
835         case CMDQ_OP_TLBI_NH_ASID:
836                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
837                 /* Fallthrough */
838         case CMDQ_OP_TLBI_S12_VMALL:
839                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
840                 break;
841         case CMDQ_OP_ATC_INV:
842                 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
843                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
844                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
845                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
846                 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
847                 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
848                 break;
849         case CMDQ_OP_PRI_RESP:
850                 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
851                 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
852                 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
853                 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
854                 switch (ent->pri.resp) {
855                 case PRI_RESP_DENY:
856                 case PRI_RESP_FAIL:
857                 case PRI_RESP_SUCC:
858                         break;
859                 default:
860                         return -EINVAL;
861                 }
862                 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
863                 break;
864         case CMDQ_OP_CMD_SYNC:
865                 if (ent->sync.msiaddr)
866                         cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
867                 else
868                         cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
869                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
870                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
871                 /*
872                  * Commands are written little-endian, but we want the SMMU to
873                  * receive MSIData, and thus write it back to memory, in CPU
874                  * byte order, so big-endian needs an extra byteswap here.
875                  */
876                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIDATA,
877                                      cpu_to_le32(ent->sync.msidata));
878                 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
879                 break;
880         default:
881                 return -ENOENT;
882         }
883
884         return 0;
885 }
886
887 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
888 {
889         static const char *cerror_str[] = {
890                 [CMDQ_ERR_CERROR_NONE_IDX]      = "No error",
891                 [CMDQ_ERR_CERROR_ILL_IDX]       = "Illegal command",
892                 [CMDQ_ERR_CERROR_ABT_IDX]       = "Abort on command fetch",
893                 [CMDQ_ERR_CERROR_ATC_INV_IDX]   = "ATC invalidate timeout",
894         };
895
896         int i;
897         u64 cmd[CMDQ_ENT_DWORDS];
898         struct arm_smmu_queue *q = &smmu->cmdq.q;
899         u32 cons = readl_relaxed(q->cons_reg);
900         u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
901         struct arm_smmu_cmdq_ent cmd_sync = {
902                 .opcode = CMDQ_OP_CMD_SYNC,
903         };
904
905         dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
906                 idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
907
908         switch (idx) {
909         case CMDQ_ERR_CERROR_ABT_IDX:
910                 dev_err(smmu->dev, "retrying command fetch\n");
911         case CMDQ_ERR_CERROR_NONE_IDX:
912                 return;
913         case CMDQ_ERR_CERROR_ATC_INV_IDX:
914                 /*
915                  * ATC Invalidation Completion timeout. CONS is still pointing
916                  * at the CMD_SYNC. Attempt to complete other pending commands
917                  * by repeating the CMD_SYNC, though we might well end up back
918                  * here since the ATC invalidation may still be pending.
919                  */
920                 return;
921         case CMDQ_ERR_CERROR_ILL_IDX:
922                 /* Fallthrough */
923         default:
924                 break;
925         }
926
927         /*
928          * We may have concurrent producers, so we need to be careful
929          * not to touch any of the shadow cmdq state.
930          */
931         queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
932         dev_err(smmu->dev, "skipping command in error state:\n");
933         for (i = 0; i < ARRAY_SIZE(cmd); ++i)
934                 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
935
936         /* Convert the erroneous command into a CMD_SYNC */
937         if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
938                 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
939                 return;
940         }
941
942         queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
943 }
944
945 static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device *smmu, u64 *cmd)
946 {
947         struct arm_smmu_queue *q = &smmu->cmdq.q;
948         bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
949
950         smmu->prev_cmd_opcode = FIELD_GET(CMDQ_0_OP, cmd[0]);
951
952         while (queue_insert_raw(q, cmd) == -ENOSPC) {
953                 if (queue_poll_cons(q, false, wfe))
954                         dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
955         }
956 }
957
958 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
959                                     struct arm_smmu_cmdq_ent *ent)
960 {
961         u64 cmd[CMDQ_ENT_DWORDS];
962         unsigned long flags;
963
964         if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
965                 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
966                          ent->opcode);
967                 return;
968         }
969
970         spin_lock_irqsave(&smmu->cmdq.lock, flags);
971         arm_smmu_cmdq_insert_cmd(smmu, cmd);
972         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
973 }
974
975 /*
976  * The difference between val and sync_idx is bounded by the maximum size of
977  * a queue at 2^20 entries, so 32 bits is plenty for wrap-safe arithmetic.
978  */
979 static int __arm_smmu_sync_poll_msi(struct arm_smmu_device *smmu, u32 sync_idx)
980 {
981         ktime_t timeout;
982         u32 val;
983
984         timeout = ktime_add_us(ktime_get(), ARM_SMMU_CMDQ_SYNC_TIMEOUT_US);
985         val = smp_cond_load_acquire(&smmu->sync_count,
986                                     (int)(VAL - sync_idx) >= 0 ||
987                                     !ktime_before(ktime_get(), timeout));
988
989         return (int)(val - sync_idx) < 0 ? -ETIMEDOUT : 0;
990 }
991
992 static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device *smmu)
993 {
994         u64 cmd[CMDQ_ENT_DWORDS];
995         unsigned long flags;
996         struct arm_smmu_cmdq_ent ent = {
997                 .opcode = CMDQ_OP_CMD_SYNC,
998                 .sync   = {
999                         .msiaddr = virt_to_phys(&smmu->sync_count),
1000                 },
1001         };
1002
1003         spin_lock_irqsave(&smmu->cmdq.lock, flags);
1004
1005         /* Piggy-back on the previous command if it's a SYNC */
1006         if (smmu->prev_cmd_opcode == CMDQ_OP_CMD_SYNC) {
1007                 ent.sync.msidata = smmu->sync_nr;
1008         } else {
1009                 ent.sync.msidata = ++smmu->sync_nr;
1010                 arm_smmu_cmdq_build_cmd(cmd, &ent);
1011                 arm_smmu_cmdq_insert_cmd(smmu, cmd);
1012         }
1013
1014         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1015
1016         return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata);
1017 }
1018
1019 static int __arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1020 {
1021         u64 cmd[CMDQ_ENT_DWORDS];
1022         unsigned long flags;
1023         bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
1024         struct arm_smmu_cmdq_ent ent = { .opcode = CMDQ_OP_CMD_SYNC };
1025         int ret;
1026
1027         arm_smmu_cmdq_build_cmd(cmd, &ent);
1028
1029         spin_lock_irqsave(&smmu->cmdq.lock, flags);
1030         arm_smmu_cmdq_insert_cmd(smmu, cmd);
1031         ret = queue_poll_cons(&smmu->cmdq.q, true, wfe);
1032         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1033
1034         return ret;
1035 }
1036
1037 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1038 {
1039         int ret;
1040         bool msi = (smmu->features & ARM_SMMU_FEAT_MSI) &&
1041                    (smmu->features & ARM_SMMU_FEAT_COHERENCY);
1042
1043         ret = msi ? __arm_smmu_cmdq_issue_sync_msi(smmu)
1044                   : __arm_smmu_cmdq_issue_sync(smmu);
1045         if (ret)
1046                 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
1047         return ret;
1048 }
1049
1050 /* Context descriptor manipulation functions */
1051 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
1052 {
1053         u64 val = 0;
1054
1055         /* Repack the TCR. Just care about TTBR0 for now */
1056         val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
1057         val |= ARM_SMMU_TCR2CD(tcr, TG0);
1058         val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
1059         val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
1060         val |= ARM_SMMU_TCR2CD(tcr, SH0);
1061         val |= ARM_SMMU_TCR2CD(tcr, EPD0);
1062         val |= ARM_SMMU_TCR2CD(tcr, EPD1);
1063         val |= ARM_SMMU_TCR2CD(tcr, IPS);
1064
1065         return val;
1066 }
1067
1068 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
1069                                     struct arm_smmu_s1_cfg *cfg)
1070 {
1071         u64 val;
1072
1073         /*
1074          * We don't need to issue any invalidation here, as we'll invalidate
1075          * the STE when installing the new entry anyway.
1076          */
1077         val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
1078 #ifdef __BIG_ENDIAN
1079               CTXDESC_CD_0_ENDI |
1080 #endif
1081               CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
1082               CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) |
1083               CTXDESC_CD_0_V;
1084
1085         /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1086         if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1087                 val |= CTXDESC_CD_0_S;
1088
1089         cfg->cdptr[0] = cpu_to_le64(val);
1090
1091         val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
1092         cfg->cdptr[1] = cpu_to_le64(val);
1093
1094         cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
1095 }
1096
1097 /* Stream table manipulation functions */
1098 static void
1099 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1100 {
1101         u64 val = 0;
1102
1103         val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1104         val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1105
1106         *dst = cpu_to_le64(val);
1107 }
1108
1109 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1110 {
1111         struct arm_smmu_cmdq_ent cmd = {
1112                 .opcode = CMDQ_OP_CFGI_STE,
1113                 .cfgi   = {
1114                         .sid    = sid,
1115                         .leaf   = true,
1116                 },
1117         };
1118
1119         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1120         arm_smmu_cmdq_issue_sync(smmu);
1121 }
1122
1123 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1124                                       __le64 *dst)
1125 {
1126         /*
1127          * This is hideously complicated, but we only really care about
1128          * three cases at the moment:
1129          *
1130          * 1. Invalid (all zero) -> bypass/fault (init)
1131          * 2. Bypass/fault -> translation/bypass (attach)
1132          * 3. Translation/bypass -> bypass/fault (detach)
1133          *
1134          * Given that we can't update the STE atomically and the SMMU
1135          * doesn't read the thing in a defined order, that leaves us
1136          * with the following maintenance requirements:
1137          *
1138          * 1. Update Config, return (init time STEs aren't live)
1139          * 2. Write everything apart from dword 0, sync, write dword 0, sync
1140          * 3. Update Config, sync
1141          */
1142         u64 val = le64_to_cpu(dst[0]);
1143         bool ste_live = false;
1144         struct arm_smmu_device *smmu = NULL;
1145         struct arm_smmu_s1_cfg *s1_cfg = NULL;
1146         struct arm_smmu_s2_cfg *s2_cfg = NULL;
1147         struct arm_smmu_domain *smmu_domain = NULL;
1148         struct arm_smmu_cmdq_ent prefetch_cmd = {
1149                 .opcode         = CMDQ_OP_PREFETCH_CFG,
1150                 .prefetch       = {
1151                         .sid    = sid,
1152                 },
1153         };
1154
1155         if (master) {
1156                 smmu_domain = master->domain;
1157                 smmu = master->smmu;
1158         }
1159
1160         if (smmu_domain) {
1161                 switch (smmu_domain->stage) {
1162                 case ARM_SMMU_DOMAIN_S1:
1163                         s1_cfg = &smmu_domain->s1_cfg;
1164                         break;
1165                 case ARM_SMMU_DOMAIN_S2:
1166                 case ARM_SMMU_DOMAIN_NESTED:
1167                         s2_cfg = &smmu_domain->s2_cfg;
1168                         break;
1169                 default:
1170                         break;
1171                 }
1172         }
1173
1174         if (val & STRTAB_STE_0_V) {
1175                 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1176                 case STRTAB_STE_0_CFG_BYPASS:
1177                         break;
1178                 case STRTAB_STE_0_CFG_S1_TRANS:
1179                 case STRTAB_STE_0_CFG_S2_TRANS:
1180                         ste_live = true;
1181                         break;
1182                 case STRTAB_STE_0_CFG_ABORT:
1183                         if (disable_bypass)
1184                                 break;
1185                 default:
1186                         BUG(); /* STE corruption */
1187                 }
1188         }
1189
1190         /* Nuke the existing STE_0 value, as we're going to rewrite it */
1191         val = STRTAB_STE_0_V;
1192
1193         /* Bypass/fault */
1194         if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1195                 if (!smmu_domain && disable_bypass)
1196                         val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1197                 else
1198                         val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1199
1200                 dst[0] = cpu_to_le64(val);
1201                 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1202                                                 STRTAB_STE_1_SHCFG_INCOMING));
1203                 dst[2] = 0; /* Nuke the VMID */
1204                 /*
1205                  * The SMMU can perform negative caching, so we must sync
1206                  * the STE regardless of whether the old value was live.
1207                  */
1208                 if (smmu)
1209                         arm_smmu_sync_ste_for_sid(smmu, sid);
1210                 return;
1211         }
1212
1213         if (s1_cfg) {
1214                 BUG_ON(ste_live);
1215                 dst[1] = cpu_to_le64(
1216                          FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1217                          FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1218                          FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1219                          FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1220
1221                 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1222                    !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1223                         dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1224
1225                 val |= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1226                         FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);
1227         }
1228
1229         if (s2_cfg) {
1230                 BUG_ON(ste_live);
1231                 dst[2] = cpu_to_le64(
1232                          FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1233                          FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1234 #ifdef __BIG_ENDIAN
1235                          STRTAB_STE_2_S2ENDI |
1236 #endif
1237                          STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1238                          STRTAB_STE_2_S2R);
1239
1240                 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1241
1242                 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1243         }
1244
1245         if (master->ats_enabled)
1246                 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1247                                                  STRTAB_STE_1_EATS_TRANS));
1248
1249         arm_smmu_sync_ste_for_sid(smmu, sid);
1250         dst[0] = cpu_to_le64(val);
1251         arm_smmu_sync_ste_for_sid(smmu, sid);
1252
1253         /* It's likely that we'll want to use the new STE soon */
1254         if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1255                 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1256 }
1257
1258 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1259 {
1260         unsigned int i;
1261
1262         for (i = 0; i < nent; ++i) {
1263                 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1264                 strtab += STRTAB_STE_DWORDS;
1265         }
1266 }
1267
1268 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1269 {
1270         size_t size;
1271         void *strtab;
1272         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1273         struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1274
1275         if (desc->l2ptr)
1276                 return 0;
1277
1278         size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1279         strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1280
1281         desc->span = STRTAB_SPLIT + 1;
1282         desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1283                                           GFP_KERNEL | __GFP_ZERO);
1284         if (!desc->l2ptr) {
1285                 dev_err(smmu->dev,
1286                         "failed to allocate l2 stream table for SID %u\n",
1287                         sid);
1288                 return -ENOMEM;
1289         }
1290
1291         arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1292         arm_smmu_write_strtab_l1_desc(strtab, desc);
1293         return 0;
1294 }
1295
1296 /* IRQ and event handlers */
1297 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1298 {
1299         int i;
1300         struct arm_smmu_device *smmu = dev;
1301         struct arm_smmu_queue *q = &smmu->evtq.q;
1302         u64 evt[EVTQ_ENT_DWORDS];
1303
1304         do {
1305                 while (!queue_remove_raw(q, evt)) {
1306                         u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1307
1308                         dev_info(smmu->dev, "event 0x%02x received:\n", id);
1309                         for (i = 0; i < ARRAY_SIZE(evt); ++i)
1310                                 dev_info(smmu->dev, "\t0x%016llx\n",
1311                                          (unsigned long long)evt[i]);
1312
1313                 }
1314
1315                 /*
1316                  * Not much we can do on overflow, so scream and pretend we're
1317                  * trying harder.
1318                  */
1319                 if (queue_sync_prod(q) == -EOVERFLOW)
1320                         dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1321         } while (!queue_empty(q));
1322
1323         /* Sync our overflow flag, as we believe we're up to speed */
1324         q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1325         return IRQ_HANDLED;
1326 }
1327
1328 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1329 {
1330         u32 sid, ssid;
1331         u16 grpid;
1332         bool ssv, last;
1333
1334         sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1335         ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1336         ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
1337         last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1338         grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1339
1340         dev_info(smmu->dev, "unexpected PRI request received:\n");
1341         dev_info(smmu->dev,
1342                  "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1343                  sid, ssid, grpid, last ? "L" : "",
1344                  evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1345                  evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1346                  evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1347                  evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1348                  evt[1] & PRIQ_1_ADDR_MASK);
1349
1350         if (last) {
1351                 struct arm_smmu_cmdq_ent cmd = {
1352                         .opcode                 = CMDQ_OP_PRI_RESP,
1353                         .substream_valid        = ssv,
1354                         .pri                    = {
1355                                 .sid    = sid,
1356                                 .ssid   = ssid,
1357                                 .grpid  = grpid,
1358                                 .resp   = PRI_RESP_DENY,
1359                         },
1360                 };
1361
1362                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1363         }
1364 }
1365
1366 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1367 {
1368         struct arm_smmu_device *smmu = dev;
1369         struct arm_smmu_queue *q = &smmu->priq.q;
1370         u64 evt[PRIQ_ENT_DWORDS];
1371
1372         do {
1373                 while (!queue_remove_raw(q, evt))
1374                         arm_smmu_handle_ppr(smmu, evt);
1375
1376                 if (queue_sync_prod(q) == -EOVERFLOW)
1377                         dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1378         } while (!queue_empty(q));
1379
1380         /* Sync our overflow flag, as we believe we're up to speed */
1381         q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1382         writel(q->cons, q->cons_reg);
1383         return IRQ_HANDLED;
1384 }
1385
1386 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1387
1388 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1389 {
1390         u32 gerror, gerrorn, active;
1391         struct arm_smmu_device *smmu = dev;
1392
1393         gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1394         gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1395
1396         active = gerror ^ gerrorn;
1397         if (!(active & GERROR_ERR_MASK))
1398                 return IRQ_NONE; /* No errors pending */
1399
1400         dev_warn(smmu->dev,
1401                  "unexpected global error reported (0x%08x), this could be serious\n",
1402                  active);
1403
1404         if (active & GERROR_SFM_ERR) {
1405                 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1406                 arm_smmu_device_disable(smmu);
1407         }
1408
1409         if (active & GERROR_MSI_GERROR_ABT_ERR)
1410                 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1411
1412         if (active & GERROR_MSI_PRIQ_ABT_ERR)
1413                 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1414
1415         if (active & GERROR_MSI_EVTQ_ABT_ERR)
1416                 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1417
1418         if (active & GERROR_MSI_CMDQ_ABT_ERR)
1419                 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1420
1421         if (active & GERROR_PRIQ_ABT_ERR)
1422                 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1423
1424         if (active & GERROR_EVTQ_ABT_ERR)
1425                 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1426
1427         if (active & GERROR_CMDQ_ERR)
1428                 arm_smmu_cmdq_skip_err(smmu);
1429
1430         writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1431         return IRQ_HANDLED;
1432 }
1433
1434 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1435 {
1436         struct arm_smmu_device *smmu = dev;
1437
1438         arm_smmu_evtq_thread(irq, dev);
1439         if (smmu->features & ARM_SMMU_FEAT_PRI)
1440                 arm_smmu_priq_thread(irq, dev);
1441
1442         return IRQ_HANDLED;
1443 }
1444
1445 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1446 {
1447         arm_smmu_gerror_handler(irq, dev);
1448         return IRQ_WAKE_THREAD;
1449 }
1450
1451 static void
1452 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1453                         struct arm_smmu_cmdq_ent *cmd)
1454 {
1455         size_t log2_span;
1456         size_t span_mask;
1457         /* ATC invalidates are always on 4096-bytes pages */
1458         size_t inval_grain_shift = 12;
1459         unsigned long page_start, page_end;
1460
1461         *cmd = (struct arm_smmu_cmdq_ent) {
1462                 .opcode                 = CMDQ_OP_ATC_INV,
1463                 .substream_valid        = !!ssid,
1464                 .atc.ssid               = ssid,
1465         };
1466
1467         if (!size) {
1468                 cmd->atc.size = ATC_INV_SIZE_ALL;
1469                 return;
1470         }
1471
1472         page_start      = iova >> inval_grain_shift;
1473         page_end        = (iova + size - 1) >> inval_grain_shift;
1474
1475         /*
1476          * In an ATS Invalidate Request, the address must be aligned on the
1477          * range size, which must be a power of two number of page sizes. We
1478          * thus have to choose between grossly over-invalidating the region, or
1479          * splitting the invalidation into multiple commands. For simplicity
1480          * we'll go with the first solution, but should refine it in the future
1481          * if multiple commands are shown to be more efficient.
1482          *
1483          * Find the smallest power of two that covers the range. The most
1484          * significant differing bit between the start and end addresses,
1485          * fls(start ^ end), indicates the required span. For example:
1486          *
1487          * We want to invalidate pages [8; 11]. This is already the ideal range:
1488          *              x = 0b1000 ^ 0b1011 = 0b11
1489          *              span = 1 << fls(x) = 4
1490          *
1491          * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1492          *              x = 0b0111 ^ 0b1010 = 0b1101
1493          *              span = 1 << fls(x) = 16
1494          */
1495         log2_span       = fls_long(page_start ^ page_end);
1496         span_mask       = (1ULL << log2_span) - 1;
1497
1498         page_start      &= ~span_mask;
1499
1500         cmd->atc.addr   = page_start << inval_grain_shift;
1501         cmd->atc.size   = log2_span;
1502 }
1503
1504 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
1505                                    struct arm_smmu_cmdq_ent *cmd)
1506 {
1507         int i;
1508
1509         if (!master->ats_enabled)
1510                 return 0;
1511
1512         for (i = 0; i < master->num_sids; i++) {
1513                 cmd->atc.sid = master->sids[i];
1514                 arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
1515         }
1516
1517         return arm_smmu_cmdq_issue_sync(master->smmu);
1518 }
1519
1520 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
1521                                    int ssid, unsigned long iova, size_t size)
1522 {
1523         int ret = 0;
1524         unsigned long flags;
1525         struct arm_smmu_cmdq_ent cmd;
1526         struct arm_smmu_master *master;
1527
1528         if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1529                 return 0;
1530
1531         arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1532
1533         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1534         list_for_each_entry(master, &smmu_domain->devices, domain_head)
1535                 ret |= arm_smmu_atc_inv_master(master, &cmd);
1536         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1537
1538         return ret ? -ETIMEDOUT : 0;
1539 }
1540
1541 /* IO_PGTABLE API */
1542 static void arm_smmu_tlb_sync(void *cookie)
1543 {
1544         struct arm_smmu_domain *smmu_domain = cookie;
1545
1546         arm_smmu_cmdq_issue_sync(smmu_domain->smmu);
1547 }
1548
1549 static void arm_smmu_tlb_inv_context(void *cookie)
1550 {
1551         struct arm_smmu_domain *smmu_domain = cookie;
1552         struct arm_smmu_device *smmu = smmu_domain->smmu;
1553         struct arm_smmu_cmdq_ent cmd;
1554
1555         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1556                 cmd.opcode      = CMDQ_OP_TLBI_NH_ASID;
1557                 cmd.tlbi.asid   = smmu_domain->s1_cfg.cd.asid;
1558                 cmd.tlbi.vmid   = 0;
1559         } else {
1560                 cmd.opcode      = CMDQ_OP_TLBI_S12_VMALL;
1561                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1562         }
1563
1564         /*
1565          * NOTE: when io-pgtable is in non-strict mode, we may get here with
1566          * PTEs previously cleared by unmaps on the current CPU not yet visible
1567          * to the SMMU. We are relying on the DSB implicit in queue_inc_prod()
1568          * to guarantee those are observed before the TLBI. Do be careful, 007.
1569          */
1570         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1571         arm_smmu_cmdq_issue_sync(smmu);
1572 }
1573
1574 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1575                                           size_t granule, bool leaf, void *cookie)
1576 {
1577         struct arm_smmu_domain *smmu_domain = cookie;
1578         struct arm_smmu_device *smmu = smmu_domain->smmu;
1579         struct arm_smmu_cmdq_ent cmd = {
1580                 .tlbi = {
1581                         .leaf   = leaf,
1582                         .addr   = iova,
1583                 },
1584         };
1585
1586         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1587                 cmd.opcode      = CMDQ_OP_TLBI_NH_VA;
1588                 cmd.tlbi.asid   = smmu_domain->s1_cfg.cd.asid;
1589         } else {
1590                 cmd.opcode      = CMDQ_OP_TLBI_S2_IPA;
1591                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1592         }
1593
1594         do {
1595                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1596                 cmd.tlbi.addr += granule;
1597         } while (size -= granule);
1598 }
1599
1600 static const struct iommu_gather_ops arm_smmu_gather_ops = {
1601         .tlb_flush_all  = arm_smmu_tlb_inv_context,
1602         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
1603         .tlb_sync       = arm_smmu_tlb_sync,
1604 };
1605
1606 /* IOMMU API */
1607 static bool arm_smmu_capable(enum iommu_cap cap)
1608 {
1609         switch (cap) {
1610         case IOMMU_CAP_CACHE_COHERENCY:
1611                 return true;
1612         case IOMMU_CAP_NOEXEC:
1613                 return true;
1614         default:
1615                 return false;
1616         }
1617 }
1618
1619 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1620 {
1621         struct arm_smmu_domain *smmu_domain;
1622
1623         if (type != IOMMU_DOMAIN_UNMANAGED &&
1624             type != IOMMU_DOMAIN_DMA &&
1625             type != IOMMU_DOMAIN_IDENTITY)
1626                 return NULL;
1627
1628         /*
1629          * Allocate the domain and initialise some of its data structures.
1630          * We can't really do anything meaningful until we've added a
1631          * master.
1632          */
1633         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1634         if (!smmu_domain)
1635                 return NULL;
1636
1637         if (type == IOMMU_DOMAIN_DMA &&
1638             iommu_get_dma_cookie(&smmu_domain->domain)) {
1639                 kfree(smmu_domain);
1640                 return NULL;
1641         }
1642
1643         mutex_init(&smmu_domain->init_mutex);
1644         INIT_LIST_HEAD(&smmu_domain->devices);
1645         spin_lock_init(&smmu_domain->devices_lock);
1646
1647         return &smmu_domain->domain;
1648 }
1649
1650 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1651 {
1652         int idx, size = 1 << span;
1653
1654         do {
1655                 idx = find_first_zero_bit(map, size);
1656                 if (idx == size)
1657                         return -ENOSPC;
1658         } while (test_and_set_bit(idx, map));
1659
1660         return idx;
1661 }
1662
1663 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1664 {
1665         clear_bit(idx, map);
1666 }
1667
1668 static void arm_smmu_domain_free(struct iommu_domain *domain)
1669 {
1670         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1671         struct arm_smmu_device *smmu = smmu_domain->smmu;
1672
1673         iommu_put_dma_cookie(domain);
1674         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1675
1676         /* Free the CD and ASID, if we allocated them */
1677         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1678                 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1679
1680                 if (cfg->cdptr) {
1681                         dmam_free_coherent(smmu_domain->smmu->dev,
1682                                            CTXDESC_CD_DWORDS << 3,
1683                                            cfg->cdptr,
1684                                            cfg->cdptr_dma);
1685
1686                         arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1687                 }
1688         } else {
1689                 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1690                 if (cfg->vmid)
1691                         arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1692         }
1693
1694         kfree(smmu_domain);
1695 }
1696
1697 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1698                                        struct io_pgtable_cfg *pgtbl_cfg)
1699 {
1700         int ret;
1701         int asid;
1702         struct arm_smmu_device *smmu = smmu_domain->smmu;
1703         struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1704
1705         asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1706         if (asid < 0)
1707                 return asid;
1708
1709         cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1710                                          &cfg->cdptr_dma,
1711                                          GFP_KERNEL | __GFP_ZERO);
1712         if (!cfg->cdptr) {
1713                 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1714                 ret = -ENOMEM;
1715                 goto out_free_asid;
1716         }
1717
1718         cfg->cd.asid    = (u16)asid;
1719         cfg->cd.ttbr    = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1720         cfg->cd.tcr     = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1721         cfg->cd.mair    = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1722         return 0;
1723
1724 out_free_asid:
1725         arm_smmu_bitmap_free(smmu->asid_map, asid);
1726         return ret;
1727 }
1728
1729 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1730                                        struct io_pgtable_cfg *pgtbl_cfg)
1731 {
1732         int vmid;
1733         struct arm_smmu_device *smmu = smmu_domain->smmu;
1734         struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1735
1736         vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1737         if (vmid < 0)
1738                 return vmid;
1739
1740         cfg->vmid       = (u16)vmid;
1741         cfg->vttbr      = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1742         cfg->vtcr       = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1743         return 0;
1744 }
1745
1746 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1747 {
1748         int ret;
1749         unsigned long ias, oas;
1750         enum io_pgtable_fmt fmt;
1751         struct io_pgtable_cfg pgtbl_cfg;
1752         struct io_pgtable_ops *pgtbl_ops;
1753         int (*finalise_stage_fn)(struct arm_smmu_domain *,
1754                                  struct io_pgtable_cfg *);
1755         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1756         struct arm_smmu_device *smmu = smmu_domain->smmu;
1757
1758         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
1759                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
1760                 return 0;
1761         }
1762
1763         /* Restrict the stage to what we can actually support */
1764         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1765                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1766         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1767                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1768
1769         switch (smmu_domain->stage) {
1770         case ARM_SMMU_DOMAIN_S1:
1771                 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
1772                 ias = min_t(unsigned long, ias, VA_BITS);
1773                 oas = smmu->ias;
1774                 fmt = ARM_64_LPAE_S1;
1775                 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1776                 break;
1777         case ARM_SMMU_DOMAIN_NESTED:
1778         case ARM_SMMU_DOMAIN_S2:
1779                 ias = smmu->ias;
1780                 oas = smmu->oas;
1781                 fmt = ARM_64_LPAE_S2;
1782                 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1783                 break;
1784         default:
1785                 return -EINVAL;
1786         }
1787
1788         pgtbl_cfg = (struct io_pgtable_cfg) {
1789                 .pgsize_bitmap  = smmu->pgsize_bitmap,
1790                 .ias            = ias,
1791                 .oas            = oas,
1792                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENCY,
1793                 .tlb            = &arm_smmu_gather_ops,
1794                 .iommu_dev      = smmu->dev,
1795         };
1796
1797         if (smmu_domain->non_strict)
1798                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
1799
1800         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1801         if (!pgtbl_ops)
1802                 return -ENOMEM;
1803
1804         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1805         domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
1806         domain->geometry.force_aperture = true;
1807
1808         ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1809         if (ret < 0) {
1810                 free_io_pgtable_ops(pgtbl_ops);
1811                 return ret;
1812         }
1813
1814         smmu_domain->pgtbl_ops = pgtbl_ops;
1815         return 0;
1816 }
1817
1818 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1819 {
1820         __le64 *step;
1821         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1822
1823         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1824                 struct arm_smmu_strtab_l1_desc *l1_desc;
1825                 int idx;
1826
1827                 /* Two-level walk */
1828                 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1829                 l1_desc = &cfg->l1_desc[idx];
1830                 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1831                 step = &l1_desc->l2ptr[idx];
1832         } else {
1833                 /* Simple linear lookup */
1834                 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1835         }
1836
1837         return step;
1838 }
1839
1840 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
1841 {
1842         int i, j;
1843         struct arm_smmu_device *smmu = master->smmu;
1844
1845         for (i = 0; i < master->num_sids; ++i) {
1846                 u32 sid = master->sids[i];
1847                 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1848
1849                 /* Bridged PCI devices may end up with duplicated IDs */
1850                 for (j = 0; j < i; j++)
1851                         if (master->sids[j] == sid)
1852                                 break;
1853                 if (j < i)
1854                         continue;
1855
1856                 arm_smmu_write_strtab_ent(master, sid, step);
1857         }
1858 }
1859
1860 static int arm_smmu_enable_ats(struct arm_smmu_master *master)
1861 {
1862         int ret;
1863         size_t stu;
1864         struct pci_dev *pdev;
1865         struct arm_smmu_device *smmu = master->smmu;
1866         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
1867
1868         if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
1869             !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
1870                 return -ENXIO;
1871
1872         pdev = to_pci_dev(master->dev);
1873         if (pdev->untrusted)
1874                 return -EPERM;
1875
1876         /* Smallest Translation Unit: log2 of the smallest supported granule */
1877         stu = __ffs(smmu->pgsize_bitmap);
1878
1879         ret = pci_enable_ats(pdev, stu);
1880         if (ret)
1881                 return ret;
1882
1883         master->ats_enabled = true;
1884         return 0;
1885 }
1886
1887 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
1888 {
1889         if (!master->ats_enabled || !dev_is_pci(master->dev))
1890                 return;
1891
1892         pci_disable_ats(to_pci_dev(master->dev));
1893         master->ats_enabled = false;
1894 }
1895
1896 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
1897 {
1898         unsigned long flags;
1899         struct arm_smmu_domain *smmu_domain = master->domain;
1900
1901         if (!smmu_domain)
1902                 return;
1903
1904         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1905         list_del(&master->domain_head);
1906         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1907
1908         master->domain = NULL;
1909         arm_smmu_install_ste_for_dev(master);
1910
1911         /* Disabling ATS invalidates all ATC entries */
1912         arm_smmu_disable_ats(master);
1913 }
1914
1915 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1916 {
1917         int ret = 0;
1918         unsigned long flags;
1919         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1920         struct arm_smmu_device *smmu;
1921         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1922         struct arm_smmu_master *master;
1923
1924         if (!fwspec)
1925                 return -ENOENT;
1926
1927         master = fwspec->iommu_priv;
1928         smmu = master->smmu;
1929
1930         arm_smmu_detach_dev(master);
1931
1932         mutex_lock(&smmu_domain->init_mutex);
1933
1934         if (!smmu_domain->smmu) {
1935                 smmu_domain->smmu = smmu;
1936                 ret = arm_smmu_domain_finalise(domain);
1937                 if (ret) {
1938                         smmu_domain->smmu = NULL;
1939                         goto out_unlock;
1940                 }
1941         } else if (smmu_domain->smmu != smmu) {
1942                 dev_err(dev,
1943                         "cannot attach to SMMU %s (upstream of %s)\n",
1944                         dev_name(smmu_domain->smmu->dev),
1945                         dev_name(smmu->dev));
1946                 ret = -ENXIO;
1947                 goto out_unlock;
1948         }
1949
1950         master->domain = smmu_domain;
1951
1952         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1953         list_add(&master->domain_head, &smmu_domain->devices);
1954         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1955
1956         if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
1957                 arm_smmu_enable_ats(master);
1958
1959         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1960                 arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
1961
1962         arm_smmu_install_ste_for_dev(master);
1963 out_unlock:
1964         mutex_unlock(&smmu_domain->init_mutex);
1965         return ret;
1966 }
1967
1968 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1969                         phys_addr_t paddr, size_t size, int prot)
1970 {
1971         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1972
1973         if (!ops)
1974                 return -ENODEV;
1975
1976         return ops->map(ops, iova, paddr, size, prot);
1977 }
1978
1979 static size_t
1980 arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1981 {
1982         int ret;
1983         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1984         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1985
1986         if (!ops)
1987                 return 0;
1988
1989         ret = ops->unmap(ops, iova, size);
1990         if (ret && arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size))
1991                 return 0;
1992
1993         return ret;
1994 }
1995
1996 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1997 {
1998         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1999
2000         if (smmu_domain->smmu)
2001                 arm_smmu_tlb_inv_context(smmu_domain);
2002 }
2003
2004 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
2005 {
2006         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
2007
2008         if (smmu)
2009                 arm_smmu_cmdq_issue_sync(smmu);
2010 }
2011
2012 static phys_addr_t
2013 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2014 {
2015         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2016
2017         if (domain->type == IOMMU_DOMAIN_IDENTITY)
2018                 return iova;
2019
2020         if (!ops)
2021                 return 0;
2022
2023         return ops->iova_to_phys(ops, iova);
2024 }
2025
2026 static struct platform_driver arm_smmu_driver;
2027
2028 static int arm_smmu_match_node(struct device *dev, void *data)
2029 {
2030         return dev->fwnode == data;
2031 }
2032
2033 static
2034 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2035 {
2036         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
2037                                                 fwnode, arm_smmu_match_node);
2038         put_device(dev);
2039         return dev ? dev_get_drvdata(dev) : NULL;
2040 }
2041
2042 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2043 {
2044         unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2045
2046         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2047                 limit *= 1UL << STRTAB_SPLIT;
2048
2049         return sid < limit;
2050 }
2051
2052 static struct iommu_ops arm_smmu_ops;
2053
2054 static int arm_smmu_add_device(struct device *dev)
2055 {
2056         int i, ret;
2057         struct arm_smmu_device *smmu;
2058         struct arm_smmu_master *master;
2059         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2060         struct iommu_group *group;
2061
2062         if (!fwspec || fwspec->ops != &arm_smmu_ops)
2063                 return -ENODEV;
2064         /*
2065          * We _can_ actually withstand dodgy bus code re-calling add_device()
2066          * without an intervening remove_device()/of_xlate() sequence, but
2067          * we're not going to do so quietly...
2068          */
2069         if (WARN_ON_ONCE(fwspec->iommu_priv)) {
2070                 master = fwspec->iommu_priv;
2071                 smmu = master->smmu;
2072         } else {
2073                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2074                 if (!smmu)
2075                         return -ENODEV;
2076                 master = kzalloc(sizeof(*master), GFP_KERNEL);
2077                 if (!master)
2078                         return -ENOMEM;
2079
2080                 master->dev = dev;
2081                 master->smmu = smmu;
2082                 master->sids = fwspec->ids;
2083                 master->num_sids = fwspec->num_ids;
2084                 fwspec->iommu_priv = master;
2085         }
2086
2087         /* Check the SIDs are in range of the SMMU and our stream table */
2088         for (i = 0; i < master->num_sids; i++) {
2089                 u32 sid = master->sids[i];
2090
2091                 if (!arm_smmu_sid_in_range(smmu, sid))
2092                         return -ERANGE;
2093
2094                 /* Ensure l2 strtab is initialised */
2095                 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2096                         ret = arm_smmu_init_l2_strtab(smmu, sid);
2097                         if (ret)
2098                                 return ret;
2099                 }
2100         }
2101
2102         group = iommu_group_get_for_dev(dev);
2103         if (!IS_ERR(group)) {
2104                 iommu_group_put(group);
2105                 iommu_device_link(&smmu->iommu, dev);
2106         }
2107
2108         return PTR_ERR_OR_ZERO(group);
2109 }
2110
2111 static void arm_smmu_remove_device(struct device *dev)
2112 {
2113         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2114         struct arm_smmu_master *master;
2115         struct arm_smmu_device *smmu;
2116
2117         if (!fwspec || fwspec->ops != &arm_smmu_ops)
2118                 return;
2119
2120         master = fwspec->iommu_priv;
2121         smmu = master->smmu;
2122         arm_smmu_detach_dev(master);
2123         iommu_group_remove_device(dev);
2124         iommu_device_unlink(&smmu->iommu, dev);
2125         kfree(master);
2126         iommu_fwspec_free(dev);
2127 }
2128
2129 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2130 {
2131         struct iommu_group *group;
2132
2133         /*
2134          * We don't support devices sharing stream IDs other than PCI RID
2135          * aliases, since the necessary ID-to-device lookup becomes rather
2136          * impractical given a potential sparse 32-bit stream ID space.
2137          */
2138         if (dev_is_pci(dev))
2139                 group = pci_device_group(dev);
2140         else
2141                 group = generic_device_group(dev);
2142
2143         return group;
2144 }
2145
2146 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
2147                                     enum iommu_attr attr, void *data)
2148 {
2149         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2150
2151         switch (domain->type) {
2152         case IOMMU_DOMAIN_UNMANAGED:
2153                 switch (attr) {
2154                 case DOMAIN_ATTR_NESTING:
2155                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
2156                         return 0;
2157                 default:
2158                         return -ENODEV;
2159                 }
2160                 break;
2161         case IOMMU_DOMAIN_DMA:
2162                 switch (attr) {
2163                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2164                         *(int *)data = smmu_domain->non_strict;
2165                         return 0;
2166                 default:
2167                         return -ENODEV;
2168                 }
2169                 break;
2170         default:
2171                 return -EINVAL;
2172         }
2173 }
2174
2175 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
2176                                     enum iommu_attr attr, void *data)
2177 {
2178         int ret = 0;
2179         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2180
2181         mutex_lock(&smmu_domain->init_mutex);
2182
2183         switch (domain->type) {
2184         case IOMMU_DOMAIN_UNMANAGED:
2185                 switch (attr) {
2186                 case DOMAIN_ATTR_NESTING:
2187                         if (smmu_domain->smmu) {
2188                                 ret = -EPERM;
2189                                 goto out_unlock;
2190                         }
2191
2192                         if (*(int *)data)
2193                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2194                         else
2195                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2196                         break;
2197                 default:
2198                         ret = -ENODEV;
2199                 }
2200                 break;
2201         case IOMMU_DOMAIN_DMA:
2202                 switch(attr) {
2203                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2204                         smmu_domain->non_strict = *(int *)data;
2205                         break;
2206                 default:
2207                         ret = -ENODEV;
2208                 }
2209                 break;
2210         default:
2211                 ret = -EINVAL;
2212         }
2213
2214 out_unlock:
2215         mutex_unlock(&smmu_domain->init_mutex);
2216         return ret;
2217 }
2218
2219 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2220 {
2221         return iommu_fwspec_add_ids(dev, args->args, 1);
2222 }
2223
2224 static void arm_smmu_get_resv_regions(struct device *dev,
2225                                       struct list_head *head)
2226 {
2227         struct iommu_resv_region *region;
2228         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2229
2230         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2231                                          prot, IOMMU_RESV_SW_MSI);
2232         if (!region)
2233                 return;
2234
2235         list_add_tail(&region->list, head);
2236
2237         iommu_dma_get_resv_regions(dev, head);
2238 }
2239
2240 static void arm_smmu_put_resv_regions(struct device *dev,
2241                                       struct list_head *head)
2242 {
2243         struct iommu_resv_region *entry, *next;
2244
2245         list_for_each_entry_safe(entry, next, head, list)
2246                 kfree(entry);
2247 }
2248
2249 static struct iommu_ops arm_smmu_ops = {
2250         .capable                = arm_smmu_capable,
2251         .domain_alloc           = arm_smmu_domain_alloc,
2252         .domain_free            = arm_smmu_domain_free,
2253         .attach_dev             = arm_smmu_attach_dev,
2254         .map                    = arm_smmu_map,
2255         .unmap                  = arm_smmu_unmap,
2256         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
2257         .iotlb_sync             = arm_smmu_iotlb_sync,
2258         .iova_to_phys           = arm_smmu_iova_to_phys,
2259         .add_device             = arm_smmu_add_device,
2260         .remove_device          = arm_smmu_remove_device,
2261         .device_group           = arm_smmu_device_group,
2262         .domain_get_attr        = arm_smmu_domain_get_attr,
2263         .domain_set_attr        = arm_smmu_domain_set_attr,
2264         .of_xlate               = arm_smmu_of_xlate,
2265         .get_resv_regions       = arm_smmu_get_resv_regions,
2266         .put_resv_regions       = arm_smmu_put_resv_regions,
2267         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
2268 };
2269
2270 /* Probing and initialisation functions */
2271 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2272                                    struct arm_smmu_queue *q,
2273                                    unsigned long prod_off,
2274                                    unsigned long cons_off,
2275                                    size_t dwords, const char *name)
2276 {
2277         size_t qsz;
2278
2279         do {
2280                 qsz = ((1 << q->max_n_shift) * dwords) << 3;
2281                 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2282                                               GFP_KERNEL);
2283                 if (q->base || qsz < PAGE_SIZE)
2284                         break;
2285
2286                 q->max_n_shift--;
2287         } while (1);
2288
2289         if (!q->base) {
2290                 dev_err(smmu->dev,
2291                         "failed to allocate queue (0x%zx bytes) for %s\n",
2292                         qsz, name);
2293                 return -ENOMEM;
2294         }
2295
2296         if (!WARN_ON(q->base_dma & (qsz - 1))) {
2297                 dev_info(smmu->dev, "allocated %u entries for %s\n",
2298                          1 << q->max_n_shift, name);
2299         }
2300
2301         q->prod_reg     = arm_smmu_page1_fixup(prod_off, smmu);
2302         q->cons_reg     = arm_smmu_page1_fixup(cons_off, smmu);
2303         q->ent_dwords   = dwords;
2304
2305         q->q_base  = Q_BASE_RWA;
2306         q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2307         q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->max_n_shift);
2308
2309         q->prod = q->cons = 0;
2310         return 0;
2311 }
2312
2313 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2314 {
2315         int ret;
2316
2317         /* cmdq */
2318         spin_lock_init(&smmu->cmdq.lock);
2319         ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
2320                                       ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS,
2321                                       "cmdq");
2322         if (ret)
2323                 return ret;
2324
2325         /* evtq */
2326         ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
2327                                       ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS,
2328                                       "evtq");
2329         if (ret)
2330                 return ret;
2331
2332         /* priq */
2333         if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2334                 return 0;
2335
2336         return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2337                                        ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS,
2338                                        "priq");
2339 }
2340
2341 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2342 {
2343         unsigned int i;
2344         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2345         size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2346         void *strtab = smmu->strtab_cfg.strtab;
2347
2348         cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2349         if (!cfg->l1_desc) {
2350                 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2351                 return -ENOMEM;
2352         }
2353
2354         for (i = 0; i < cfg->num_l1_ents; ++i) {
2355                 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2356                 strtab += STRTAB_L1_DESC_DWORDS << 3;
2357         }
2358
2359         return 0;
2360 }
2361
2362 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2363 {
2364         void *strtab;
2365         u64 reg;
2366         u32 size, l1size;
2367         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2368
2369         /* Calculate the L1 size, capped to the SIDSIZE. */
2370         size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2371         size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2372         cfg->num_l1_ents = 1 << size;
2373
2374         size += STRTAB_SPLIT;
2375         if (size < smmu->sid_bits)
2376                 dev_warn(smmu->dev,
2377                          "2-level strtab only covers %u/%u bits of SID\n",
2378                          size, smmu->sid_bits);
2379
2380         l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2381         strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2382                                      GFP_KERNEL | __GFP_ZERO);
2383         if (!strtab) {
2384                 dev_err(smmu->dev,
2385                         "failed to allocate l1 stream table (%u bytes)\n",
2386                         size);
2387                 return -ENOMEM;
2388         }
2389         cfg->strtab = strtab;
2390
2391         /* Configure strtab_base_cfg for 2 levels */
2392         reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
2393         reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
2394         reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
2395         cfg->strtab_base_cfg = reg;
2396
2397         return arm_smmu_init_l1_strtab(smmu);
2398 }
2399
2400 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2401 {
2402         void *strtab;
2403         u64 reg;
2404         u32 size;
2405         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2406
2407         size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2408         strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2409                                      GFP_KERNEL | __GFP_ZERO);
2410         if (!strtab) {
2411                 dev_err(smmu->dev,
2412                         "failed to allocate linear stream table (%u bytes)\n",
2413                         size);
2414                 return -ENOMEM;
2415         }
2416         cfg->strtab = strtab;
2417         cfg->num_l1_ents = 1 << smmu->sid_bits;
2418
2419         /* Configure strtab_base_cfg for a linear table covering all SIDs */
2420         reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
2421         reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
2422         cfg->strtab_base_cfg = reg;
2423
2424         arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2425         return 0;
2426 }
2427
2428 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2429 {
2430         u64 reg;
2431         int ret;
2432
2433         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2434                 ret = arm_smmu_init_strtab_2lvl(smmu);
2435         else
2436                 ret = arm_smmu_init_strtab_linear(smmu);
2437
2438         if (ret)
2439                 return ret;
2440
2441         /* Set the strtab base address */
2442         reg  = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
2443         reg |= STRTAB_BASE_RA;
2444         smmu->strtab_cfg.strtab_base = reg;
2445
2446         /* Allocate the first VMID for stage-2 bypass STEs */
2447         set_bit(0, smmu->vmid_map);
2448         return 0;
2449 }
2450
2451 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2452 {
2453         int ret;
2454
2455         ret = arm_smmu_init_queues(smmu);
2456         if (ret)
2457                 return ret;
2458
2459         return arm_smmu_init_strtab(smmu);
2460 }
2461
2462 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2463                                    unsigned int reg_off, unsigned int ack_off)
2464 {
2465         u32 reg;
2466
2467         writel_relaxed(val, smmu->base + reg_off);
2468         return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2469                                           1, ARM_SMMU_POLL_TIMEOUT_US);
2470 }
2471
2472 /* GBPA is "special" */
2473 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2474 {
2475         int ret;
2476         u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2477
2478         ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2479                                          1, ARM_SMMU_POLL_TIMEOUT_US);
2480         if (ret)
2481                 return ret;
2482
2483         reg &= ~clr;
2484         reg |= set;
2485         writel_relaxed(reg | GBPA_UPDATE, gbpa);
2486         ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2487                                          1, ARM_SMMU_POLL_TIMEOUT_US);
2488
2489         if (ret)
2490                 dev_err(smmu->dev, "GBPA not responding to update\n");
2491         return ret;
2492 }
2493
2494 static void arm_smmu_free_msis(void *data)
2495 {
2496         struct device *dev = data;
2497         platform_msi_domain_free_irqs(dev);
2498 }
2499
2500 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2501 {
2502         phys_addr_t doorbell;
2503         struct device *dev = msi_desc_to_dev(desc);
2504         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2505         phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2506
2507         doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2508         doorbell &= MSI_CFG0_ADDR_MASK;
2509
2510         writeq_relaxed(doorbell, smmu->base + cfg[0]);
2511         writel_relaxed(msg->data, smmu->base + cfg[1]);
2512         writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2513 }
2514
2515 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2516 {
2517         struct msi_desc *desc;
2518         int ret, nvec = ARM_SMMU_MAX_MSIS;
2519         struct device *dev = smmu->dev;
2520
2521         /* Clear the MSI address regs */
2522         writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2523         writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2524
2525         if (smmu->features & ARM_SMMU_FEAT_PRI)
2526                 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2527         else
2528                 nvec--;
2529
2530         if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2531                 return;
2532
2533         if (!dev->msi_domain) {
2534                 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
2535                 return;
2536         }
2537
2538         /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2539         ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2540         if (ret) {
2541                 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
2542                 return;
2543         }
2544
2545         for_each_msi_entry(desc, dev) {
2546                 switch (desc->platform.msi_index) {
2547                 case EVTQ_MSI_INDEX:
2548                         smmu->evtq.q.irq = desc->irq;
2549                         break;
2550                 case GERROR_MSI_INDEX:
2551                         smmu->gerr_irq = desc->irq;
2552                         break;
2553                 case PRIQ_MSI_INDEX:
2554                         smmu->priq.q.irq = desc->irq;
2555                         break;
2556                 default:        /* Unknown */
2557                         continue;
2558                 }
2559         }
2560
2561         /* Add callback to free MSIs on teardown */
2562         devm_add_action(dev, arm_smmu_free_msis, dev);
2563 }
2564
2565 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
2566 {
2567         int irq, ret;
2568
2569         arm_smmu_setup_msis(smmu);
2570
2571         /* Request interrupt lines */
2572         irq = smmu->evtq.q.irq;
2573         if (irq) {
2574                 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2575                                                 arm_smmu_evtq_thread,
2576                                                 IRQF_ONESHOT,
2577                                                 "arm-smmu-v3-evtq", smmu);
2578                 if (ret < 0)
2579                         dev_warn(smmu->dev, "failed to enable evtq irq\n");
2580         } else {
2581                 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
2582         }
2583
2584         irq = smmu->gerr_irq;
2585         if (irq) {
2586                 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2587                                        0, "arm-smmu-v3-gerror", smmu);
2588                 if (ret < 0)
2589                         dev_warn(smmu->dev, "failed to enable gerror irq\n");
2590         } else {
2591                 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
2592         }
2593
2594         if (smmu->features & ARM_SMMU_FEAT_PRI) {
2595                 irq = smmu->priq.q.irq;
2596                 if (irq) {
2597                         ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2598                                                         arm_smmu_priq_thread,
2599                                                         IRQF_ONESHOT,
2600                                                         "arm-smmu-v3-priq",
2601                                                         smmu);
2602                         if (ret < 0)
2603                                 dev_warn(smmu->dev,
2604                                          "failed to enable priq irq\n");
2605                 } else {
2606                         dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
2607                 }
2608         }
2609 }
2610
2611 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2612 {
2613         int ret, irq;
2614         u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2615
2616         /* Disable IRQs first */
2617         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2618                                       ARM_SMMU_IRQ_CTRLACK);
2619         if (ret) {
2620                 dev_err(smmu->dev, "failed to disable irqs\n");
2621                 return ret;
2622         }
2623
2624         irq = smmu->combined_irq;
2625         if (irq) {
2626                 /*
2627                  * Cavium ThunderX2 implementation doesn't support unique irq
2628                  * lines. Use a single irq line for all the SMMUv3 interrupts.
2629                  */
2630                 ret = devm_request_threaded_irq(smmu->dev, irq,
2631                                         arm_smmu_combined_irq_handler,
2632                                         arm_smmu_combined_irq_thread,
2633                                         IRQF_ONESHOT,
2634                                         "arm-smmu-v3-combined-irq", smmu);
2635                 if (ret < 0)
2636                         dev_warn(smmu->dev, "failed to enable combined irq\n");
2637         } else
2638                 arm_smmu_setup_unique_irqs(smmu);
2639
2640         if (smmu->features & ARM_SMMU_FEAT_PRI)
2641                 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2642
2643         /* Enable interrupt generation on the SMMU */
2644         ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2645                                       ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2646         if (ret)
2647                 dev_warn(smmu->dev, "failed to enable irqs\n");
2648
2649         return 0;
2650 }
2651
2652 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2653 {
2654         int ret;
2655
2656         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2657         if (ret)
2658                 dev_err(smmu->dev, "failed to clear cr0\n");
2659
2660         return ret;
2661 }
2662
2663 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2664 {
2665         int ret;
2666         u32 reg, enables;
2667         struct arm_smmu_cmdq_ent cmd;
2668
2669         /* Clear CR0 and sync (disables SMMU and queue processing) */
2670         reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2671         if (reg & CR0_SMMUEN) {
2672                 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2673                 WARN_ON(is_kdump_kernel() && !disable_bypass);
2674                 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
2675         }
2676
2677         ret = arm_smmu_device_disable(smmu);
2678         if (ret)
2679                 return ret;
2680
2681         /* CR1 (table and queue memory attributes) */
2682         reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
2683               FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
2684               FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
2685               FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
2686               FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
2687               FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
2688         writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2689
2690         /* CR2 (random crap) */
2691         reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2692         writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2693
2694         /* Stream table */
2695         writeq_relaxed(smmu->strtab_cfg.strtab_base,
2696                        smmu->base + ARM_SMMU_STRTAB_BASE);
2697         writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2698                        smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2699
2700         /* Command queue */
2701         writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2702         writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2703         writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2704
2705         enables = CR0_CMDQEN;
2706         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2707                                       ARM_SMMU_CR0ACK);
2708         if (ret) {
2709                 dev_err(smmu->dev, "failed to enable command queue\n");
2710                 return ret;
2711         }
2712
2713         /* Invalidate any cached configuration */
2714         cmd.opcode = CMDQ_OP_CFGI_ALL;
2715         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2716         arm_smmu_cmdq_issue_sync(smmu);
2717
2718         /* Invalidate any stale TLB entries */
2719         if (smmu->features & ARM_SMMU_FEAT_HYP) {
2720                 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2721                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2722         }
2723
2724         cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2725         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2726         arm_smmu_cmdq_issue_sync(smmu);
2727
2728         /* Event queue */
2729         writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2730         writel_relaxed(smmu->evtq.q.prod,
2731                        arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
2732         writel_relaxed(smmu->evtq.q.cons,
2733                        arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
2734
2735         enables |= CR0_EVTQEN;
2736         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2737                                       ARM_SMMU_CR0ACK);
2738         if (ret) {
2739                 dev_err(smmu->dev, "failed to enable event queue\n");
2740                 return ret;
2741         }
2742
2743         /* PRI queue */
2744         if (smmu->features & ARM_SMMU_FEAT_PRI) {
2745                 writeq_relaxed(smmu->priq.q.q_base,
2746                                smmu->base + ARM_SMMU_PRIQ_BASE);
2747                 writel_relaxed(smmu->priq.q.prod,
2748                                arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
2749                 writel_relaxed(smmu->priq.q.cons,
2750                                arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
2751
2752                 enables |= CR0_PRIQEN;
2753                 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2754                                               ARM_SMMU_CR0ACK);
2755                 if (ret) {
2756                         dev_err(smmu->dev, "failed to enable PRI queue\n");
2757                         return ret;
2758                 }
2759         }
2760
2761         if (smmu->features & ARM_SMMU_FEAT_ATS) {
2762                 enables |= CR0_ATSCHK;
2763                 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2764                                               ARM_SMMU_CR0ACK);
2765                 if (ret) {
2766                         dev_err(smmu->dev, "failed to enable ATS check\n");
2767                         return ret;
2768                 }
2769         }
2770
2771         ret = arm_smmu_setup_irqs(smmu);
2772         if (ret) {
2773                 dev_err(smmu->dev, "failed to setup irqs\n");
2774                 return ret;
2775         }
2776
2777         if (is_kdump_kernel())
2778                 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
2779
2780         /* Enable the SMMU interface, or ensure bypass */
2781         if (!bypass || disable_bypass) {
2782                 enables |= CR0_SMMUEN;
2783         } else {
2784                 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
2785                 if (ret)
2786                         return ret;
2787         }
2788         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2789                                       ARM_SMMU_CR0ACK);
2790         if (ret) {
2791                 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2792                 return ret;
2793         }
2794
2795         return 0;
2796 }
2797
2798 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
2799 {
2800         u32 reg;
2801         bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
2802
2803         /* IDR0 */
2804         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2805
2806         /* 2-level structures */
2807         if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
2808                 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2809
2810         if (reg & IDR0_CD2L)
2811                 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2812
2813         /*
2814          * Translation table endianness.
2815          * We currently require the same endianness as the CPU, but this
2816          * could be changed later by adding a new IO_PGTABLE_QUIRK.
2817          */
2818         switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
2819         case IDR0_TTENDIAN_MIXED:
2820                 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2821                 break;
2822 #ifdef __BIG_ENDIAN
2823         case IDR0_TTENDIAN_BE:
2824                 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2825                 break;
2826 #else
2827         case IDR0_TTENDIAN_LE:
2828                 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2829                 break;
2830 #endif
2831         default:
2832                 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2833                 return -ENXIO;
2834         }
2835
2836         /* Boolean feature flags */
2837         if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2838                 smmu->features |= ARM_SMMU_FEAT_PRI;
2839
2840         if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2841                 smmu->features |= ARM_SMMU_FEAT_ATS;
2842
2843         if (reg & IDR0_SEV)
2844                 smmu->features |= ARM_SMMU_FEAT_SEV;
2845
2846         if (reg & IDR0_MSI)
2847                 smmu->features |= ARM_SMMU_FEAT_MSI;
2848
2849         if (reg & IDR0_HYP)
2850                 smmu->features |= ARM_SMMU_FEAT_HYP;
2851
2852         /*
2853          * The coherency feature as set by FW is used in preference to the ID
2854          * register, but warn on mismatch.
2855          */
2856         if (!!(reg & IDR0_COHACC) != coherent)
2857                 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
2858                          coherent ? "true" : "false");
2859
2860         switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
2861         case IDR0_STALL_MODEL_FORCE:
2862                 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
2863                 /* Fallthrough */
2864         case IDR0_STALL_MODEL_STALL:
2865                 smmu->features |= ARM_SMMU_FEAT_STALLS;
2866         }
2867
2868         if (reg & IDR0_S1P)
2869                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2870
2871         if (reg & IDR0_S2P)
2872                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2873
2874         if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2875                 dev_err(smmu->dev, "no translation support!\n");
2876                 return -ENXIO;
2877         }
2878
2879         /* We only support the AArch64 table format at present */
2880         switch (FIELD_GET(IDR0_TTF, reg)) {
2881         case IDR0_TTF_AARCH32_64:
2882                 smmu->ias = 40;
2883                 /* Fallthrough */
2884         case IDR0_TTF_AARCH64:
2885                 break;
2886         default:
2887                 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2888                 return -ENXIO;
2889         }
2890
2891         /* ASID/VMID sizes */
2892         smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2893         smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2894
2895         /* IDR1 */
2896         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2897         if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2898                 dev_err(smmu->dev, "embedded implementation not supported\n");
2899                 return -ENXIO;
2900         }
2901
2902         /* Queue sizes, capped to ensure natural alignment */
2903         smmu->cmdq.q.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
2904                                          FIELD_GET(IDR1_CMDQS, reg));
2905         if (!smmu->cmdq.q.max_n_shift) {
2906                 /* Odd alignment restrictions on the base, so ignore for now */
2907                 dev_err(smmu->dev, "unit-length command queue not supported\n");
2908                 return -ENXIO;
2909         }
2910
2911         smmu->evtq.q.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
2912                                          FIELD_GET(IDR1_EVTQS, reg));
2913         smmu->priq.q.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
2914                                          FIELD_GET(IDR1_PRIQS, reg));
2915
2916         /* SID/SSID sizes */
2917         smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
2918         smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
2919
2920         /*
2921          * If the SMMU supports fewer bits than would fill a single L2 stream
2922          * table, use a linear table instead.
2923          */
2924         if (smmu->sid_bits <= STRTAB_SPLIT)
2925                 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
2926
2927         /* IDR5 */
2928         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2929
2930         /* Maximum number of outstanding stalls */
2931         smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
2932
2933         /* Page sizes */
2934         if (reg & IDR5_GRAN64K)
2935                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2936         if (reg & IDR5_GRAN16K)
2937                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2938         if (reg & IDR5_GRAN4K)
2939                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2940
2941         /* Input address size */
2942         if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
2943                 smmu->features |= ARM_SMMU_FEAT_VAX;
2944
2945         /* Output address size */
2946         switch (FIELD_GET(IDR5_OAS, reg)) {
2947         case IDR5_OAS_32_BIT:
2948                 smmu->oas = 32;
2949                 break;
2950         case IDR5_OAS_36_BIT:
2951                 smmu->oas = 36;
2952                 break;
2953         case IDR5_OAS_40_BIT:
2954                 smmu->oas = 40;
2955                 break;
2956         case IDR5_OAS_42_BIT:
2957                 smmu->oas = 42;
2958                 break;
2959         case IDR5_OAS_44_BIT:
2960                 smmu->oas = 44;
2961                 break;
2962         case IDR5_OAS_52_BIT:
2963                 smmu->oas = 52;
2964                 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
2965                 break;
2966         default:
2967                 dev_info(smmu->dev,
2968                         "unknown output address size. Truncating to 48-bit\n");
2969                 /* Fallthrough */
2970         case IDR5_OAS_48_BIT:
2971                 smmu->oas = 48;
2972         }
2973
2974         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2975                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2976         else
2977                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2978
2979         /* Set the DMA mask for our table walker */
2980         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2981                 dev_warn(smmu->dev,
2982                          "failed to set DMA mask for table walker\n");
2983
2984         smmu->ias = max(smmu->ias, smmu->oas);
2985
2986         dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2987                  smmu->ias, smmu->oas, smmu->features);
2988         return 0;
2989 }
2990
2991 #ifdef CONFIG_ACPI
2992 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
2993 {
2994         switch (model) {
2995         case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
2996                 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
2997                 break;
2998         case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
2999                 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3000                 break;
3001         }
3002
3003         dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3004 }
3005
3006 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3007                                       struct arm_smmu_device *smmu)
3008 {
3009         struct acpi_iort_smmu_v3 *iort_smmu;
3010         struct device *dev = smmu->dev;
3011         struct acpi_iort_node *node;
3012
3013         node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3014
3015         /* Retrieve SMMUv3 specific data */
3016         iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3017
3018         acpi_smmu_get_options(iort_smmu->model, smmu);
3019
3020         if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3021                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3022
3023         return 0;
3024 }
3025 #else
3026 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3027                                              struct arm_smmu_device *smmu)
3028 {
3029         return -ENODEV;
3030 }
3031 #endif
3032
3033 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3034                                     struct arm_smmu_device *smmu)
3035 {
3036         struct device *dev = &pdev->dev;
3037         u32 cells;
3038         int ret = -EINVAL;
3039
3040         if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3041                 dev_err(dev, "missing #iommu-cells property\n");
3042         else if (cells != 1)
3043                 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3044         else
3045                 ret = 0;
3046
3047         parse_driver_options(smmu);
3048
3049         if (of_dma_is_coherent(dev->of_node))
3050                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3051
3052         return ret;
3053 }
3054
3055 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3056 {
3057         if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3058                 return SZ_64K;
3059         else
3060                 return SZ_128K;
3061 }
3062
3063 static int arm_smmu_device_probe(struct platform_device *pdev)
3064 {
3065         int irq, ret;
3066         struct resource *res;
3067         resource_size_t ioaddr;
3068         struct arm_smmu_device *smmu;
3069         struct device *dev = &pdev->dev;
3070         bool bypass;
3071
3072         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3073         if (!smmu) {
3074                 dev_err(dev, "failed to allocate arm_smmu_device\n");
3075                 return -ENOMEM;
3076         }
3077         smmu->dev = dev;
3078
3079         if (dev->of_node) {
3080                 ret = arm_smmu_device_dt_probe(pdev, smmu);
3081         } else {
3082                 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3083                 if (ret == -ENODEV)
3084                         return ret;
3085         }
3086
3087         /* Set bypass mode according to firmware probing result */
3088         bypass = !!ret;
3089
3090         /* Base address */
3091         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3092         if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {
3093                 dev_err(dev, "MMIO region too small (%pr)\n", res);
3094                 return -EINVAL;
3095         }
3096         ioaddr = res->start;
3097
3098         smmu->base = devm_ioremap_resource(dev, res);
3099         if (IS_ERR(smmu->base))
3100                 return PTR_ERR(smmu->base);
3101
3102         /* Interrupt lines */
3103
3104         irq = platform_get_irq_byname(pdev, "combined");
3105         if (irq > 0)
3106                 smmu->combined_irq = irq;
3107         else {
3108                 irq = platform_get_irq_byname(pdev, "eventq");
3109                 if (irq > 0)
3110                         smmu->evtq.q.irq = irq;
3111
3112                 irq = platform_get_irq_byname(pdev, "priq");
3113                 if (irq > 0)
3114                         smmu->priq.q.irq = irq;
3115
3116                 irq = platform_get_irq_byname(pdev, "gerror");
3117                 if (irq > 0)
3118                         smmu->gerr_irq = irq;
3119         }
3120         /* Probe the h/w */
3121         ret = arm_smmu_device_hw_probe(smmu);
3122         if (ret)
3123                 return ret;
3124
3125         /* Initialise in-memory data structures */
3126         ret = arm_smmu_init_structures(smmu);
3127         if (ret)
3128                 return ret;
3129
3130         /* Record our private device structure */
3131         platform_set_drvdata(pdev, smmu);
3132
3133         /* Reset the device */
3134         ret = arm_smmu_device_reset(smmu, bypass);
3135         if (ret)
3136                 return ret;
3137
3138         /* And we're up. Go go go! */
3139         ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3140                                      "smmu3.%pa", &ioaddr);
3141         if (ret)
3142                 return ret;
3143
3144         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
3145         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
3146
3147         ret = iommu_device_register(&smmu->iommu);
3148         if (ret) {
3149                 dev_err(dev, "Failed to register iommu\n");
3150                 return ret;
3151         }
3152
3153 #ifdef CONFIG_PCI
3154         if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
3155                 pci_request_acs();
3156                 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
3157                 if (ret)
3158                         return ret;
3159         }
3160 #endif
3161 #ifdef CONFIG_ARM_AMBA
3162         if (amba_bustype.iommu_ops != &arm_smmu_ops) {
3163                 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
3164                 if (ret)
3165                         return ret;
3166         }
3167 #endif