65de2458999fd63428005dcdd5bd2355dc6bde3a
[sfrench/cifs-2.6.git] / drivers / iommu / arm-smmu-v3.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * IOMMU API for ARM architected SMMUv3 implementations.
4  *
5  * Copyright (C) 2015 ARM Limited
6  *
7  * Author: Will Deacon <will.deacon@arm.com>
8  *
9  * This driver is powered by bad coffee and bombay mix.
10  */
11
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/bitops.h>
16 #include <linux/crash_dump.h>
17 #include <linux/delay.h>
18 #include <linux/dma-iommu.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/init.h>
25 #include <linux/moduleparam.h>
26 #include <linux/msi.h>
27 #include <linux/of.h>
28 #include <linux/of_address.h>
29 #include <linux/of_iommu.h>
30 #include <linux/of_platform.h>
31 #include <linux/pci.h>
32 #include <linux/pci-ats.h>
33 #include <linux/platform_device.h>
34
35 #include <linux/amba/bus.h>
36
37 /* MMIO registers */
38 #define ARM_SMMU_IDR0                   0x0
39 #define IDR0_ST_LVL                     GENMASK(28, 27)
40 #define IDR0_ST_LVL_2LVL                1
41 #define IDR0_STALL_MODEL                GENMASK(25, 24)
42 #define IDR0_STALL_MODEL_STALL          0
43 #define IDR0_STALL_MODEL_FORCE          2
44 #define IDR0_TTENDIAN                   GENMASK(22, 21)
45 #define IDR0_TTENDIAN_MIXED             0
46 #define IDR0_TTENDIAN_LE                2
47 #define IDR0_TTENDIAN_BE                3
48 #define IDR0_CD2L                       (1 << 19)
49 #define IDR0_VMID16                     (1 << 18)
50 #define IDR0_PRI                        (1 << 16)
51 #define IDR0_SEV                        (1 << 14)
52 #define IDR0_MSI                        (1 << 13)
53 #define IDR0_ASID16                     (1 << 12)
54 #define IDR0_ATS                        (1 << 10)
55 #define IDR0_HYP                        (1 << 9)
56 #define IDR0_COHACC                     (1 << 4)
57 #define IDR0_TTF                        GENMASK(3, 2)
58 #define IDR0_TTF_AARCH64                2
59 #define IDR0_TTF_AARCH32_64             3
60 #define IDR0_S1P                        (1 << 1)
61 #define IDR0_S2P                        (1 << 0)
62
63 #define ARM_SMMU_IDR1                   0x4
64 #define IDR1_TABLES_PRESET              (1 << 30)
65 #define IDR1_QUEUES_PRESET              (1 << 29)
66 #define IDR1_REL                        (1 << 28)
67 #define IDR1_CMDQS                      GENMASK(25, 21)
68 #define IDR1_EVTQS                      GENMASK(20, 16)
69 #define IDR1_PRIQS                      GENMASK(15, 11)
70 #define IDR1_SSIDSIZE                   GENMASK(10, 6)
71 #define IDR1_SIDSIZE                    GENMASK(5, 0)
72
73 #define ARM_SMMU_IDR5                   0x14
74 #define IDR5_STALL_MAX                  GENMASK(31, 16)
75 #define IDR5_GRAN64K                    (1 << 6)
76 #define IDR5_GRAN16K                    (1 << 5)
77 #define IDR5_GRAN4K                     (1 << 4)
78 #define IDR5_OAS                        GENMASK(2, 0)
79 #define IDR5_OAS_32_BIT                 0
80 #define IDR5_OAS_36_BIT                 1
81 #define IDR5_OAS_40_BIT                 2
82 #define IDR5_OAS_42_BIT                 3
83 #define IDR5_OAS_44_BIT                 4
84 #define IDR5_OAS_48_BIT                 5
85 #define IDR5_OAS_52_BIT                 6
86 #define IDR5_VAX                        GENMASK(11, 10)
87 #define IDR5_VAX_52_BIT                 1
88
89 #define ARM_SMMU_CR0                    0x20
90 #define CR0_ATSCHK                      (1 << 4)
91 #define CR0_CMDQEN                      (1 << 3)
92 #define CR0_EVTQEN                      (1 << 2)
93 #define CR0_PRIQEN                      (1 << 1)
94 #define CR0_SMMUEN                      (1 << 0)
95
96 #define ARM_SMMU_CR0ACK                 0x24
97
98 #define ARM_SMMU_CR1                    0x28
99 #define CR1_TABLE_SH                    GENMASK(11, 10)
100 #define CR1_TABLE_OC                    GENMASK(9, 8)
101 #define CR1_TABLE_IC                    GENMASK(7, 6)
102 #define CR1_QUEUE_SH                    GENMASK(5, 4)
103 #define CR1_QUEUE_OC                    GENMASK(3, 2)
104 #define CR1_QUEUE_IC                    GENMASK(1, 0)
105 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
106 #define CR1_CACHE_NC                    0
107 #define CR1_CACHE_WB                    1
108 #define CR1_CACHE_WT                    2
109
110 #define ARM_SMMU_CR2                    0x2c
111 #define CR2_PTM                         (1 << 2)
112 #define CR2_RECINVSID                   (1 << 1)
113 #define CR2_E2H                         (1 << 0)
114
115 #define ARM_SMMU_GBPA                   0x44
116 #define GBPA_UPDATE                     (1 << 31)
117 #define GBPA_ABORT                      (1 << 20)
118
119 #define ARM_SMMU_IRQ_CTRL               0x50
120 #define IRQ_CTRL_EVTQ_IRQEN             (1 << 2)
121 #define IRQ_CTRL_PRIQ_IRQEN             (1 << 1)
122 #define IRQ_CTRL_GERROR_IRQEN           (1 << 0)
123
124 #define ARM_SMMU_IRQ_CTRLACK            0x54
125
126 #define ARM_SMMU_GERROR                 0x60
127 #define GERROR_SFM_ERR                  (1 << 8)
128 #define GERROR_MSI_GERROR_ABT_ERR       (1 << 7)
129 #define GERROR_MSI_PRIQ_ABT_ERR         (1 << 6)
130 #define GERROR_MSI_EVTQ_ABT_ERR         (1 << 5)
131 #define GERROR_MSI_CMDQ_ABT_ERR         (1 << 4)
132 #define GERROR_PRIQ_ABT_ERR             (1 << 3)
133 #define GERROR_EVTQ_ABT_ERR             (1 << 2)
134 #define GERROR_CMDQ_ERR                 (1 << 0)
135 #define GERROR_ERR_MASK                 0xfd
136
137 #define ARM_SMMU_GERRORN                0x64
138
139 #define ARM_SMMU_GERROR_IRQ_CFG0        0x68
140 #define ARM_SMMU_GERROR_IRQ_CFG1        0x70
141 #define ARM_SMMU_GERROR_IRQ_CFG2        0x74
142
143 #define ARM_SMMU_STRTAB_BASE            0x80
144 #define STRTAB_BASE_RA                  (1UL << 62)
145 #define STRTAB_BASE_ADDR_MASK           GENMASK_ULL(51, 6)
146
147 #define ARM_SMMU_STRTAB_BASE_CFG        0x88
148 #define STRTAB_BASE_CFG_FMT             GENMASK(17, 16)
149 #define STRTAB_BASE_CFG_FMT_LINEAR      0
150 #define STRTAB_BASE_CFG_FMT_2LVL        1
151 #define STRTAB_BASE_CFG_SPLIT           GENMASK(10, 6)
152 #define STRTAB_BASE_CFG_LOG2SIZE        GENMASK(5, 0)
153
154 #define ARM_SMMU_CMDQ_BASE              0x90
155 #define ARM_SMMU_CMDQ_PROD              0x98
156 #define ARM_SMMU_CMDQ_CONS              0x9c
157
158 #define ARM_SMMU_EVTQ_BASE              0xa0
159 #define ARM_SMMU_EVTQ_PROD              0x100a8
160 #define ARM_SMMU_EVTQ_CONS              0x100ac
161 #define ARM_SMMU_EVTQ_IRQ_CFG0          0xb0
162 #define ARM_SMMU_EVTQ_IRQ_CFG1          0xb8
163 #define ARM_SMMU_EVTQ_IRQ_CFG2          0xbc
164
165 #define ARM_SMMU_PRIQ_BASE              0xc0
166 #define ARM_SMMU_PRIQ_PROD              0x100c8
167 #define ARM_SMMU_PRIQ_CONS              0x100cc
168 #define ARM_SMMU_PRIQ_IRQ_CFG0          0xd0
169 #define ARM_SMMU_PRIQ_IRQ_CFG1          0xd8
170 #define ARM_SMMU_PRIQ_IRQ_CFG2          0xdc
171
172 /* Common MSI config fields */
173 #define MSI_CFG0_ADDR_MASK              GENMASK_ULL(51, 2)
174 #define MSI_CFG2_SH                     GENMASK(5, 4)
175 #define MSI_CFG2_MEMATTR                GENMASK(3, 0)
176
177 /* Common memory attribute values */
178 #define ARM_SMMU_SH_NSH                 0
179 #define ARM_SMMU_SH_OSH                 2
180 #define ARM_SMMU_SH_ISH                 3
181 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE   0x1
182 #define ARM_SMMU_MEMATTR_OIWB           0xf
183
184 #define Q_IDX(q, p)                     ((p) & ((1 << (q)->max_n_shift) - 1))
185 #define Q_WRP(q, p)                     ((p) & (1 << (q)->max_n_shift))
186 #define Q_OVERFLOW_FLAG                 (1 << 31)
187 #define Q_OVF(q, p)                     ((p) & Q_OVERFLOW_FLAG)
188 #define Q_ENT(q, p)                     ((q)->base +                    \
189                                          Q_IDX(q, p) * (q)->ent_dwords)
190
191 #define Q_BASE_RWA                      (1UL << 62)
192 #define Q_BASE_ADDR_MASK                GENMASK_ULL(51, 5)
193 #define Q_BASE_LOG2SIZE                 GENMASK(4, 0)
194 #define Q_MAX_SZ_SHIFT                  (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
195
196 /*
197  * Stream table.
198  *
199  * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
200  * 2lvl: 128k L1 entries,
201  *       256 lazy entries per table (each table covers a PCI bus)
202  */
203 #define STRTAB_L1_SZ_SHIFT              20
204 #define STRTAB_SPLIT                    8
205
206 #define STRTAB_L1_DESC_DWORDS           1
207 #define STRTAB_L1_DESC_SPAN             GENMASK_ULL(4, 0)
208 #define STRTAB_L1_DESC_L2PTR_MASK       GENMASK_ULL(51, 6)
209
210 #define STRTAB_STE_DWORDS               8
211 #define STRTAB_STE_0_V                  (1UL << 0)
212 #define STRTAB_STE_0_CFG                GENMASK_ULL(3, 1)
213 #define STRTAB_STE_0_CFG_ABORT          0
214 #define STRTAB_STE_0_CFG_BYPASS         4
215 #define STRTAB_STE_0_CFG_S1_TRANS       5
216 #define STRTAB_STE_0_CFG_S2_TRANS       6
217
218 #define STRTAB_STE_0_S1FMT              GENMASK_ULL(5, 4)
219 #define STRTAB_STE_0_S1FMT_LINEAR       0
220 #define STRTAB_STE_0_S1CTXPTR_MASK      GENMASK_ULL(51, 6)
221 #define STRTAB_STE_0_S1CDMAX            GENMASK_ULL(63, 59)
222
223 #define STRTAB_STE_1_S1C_CACHE_NC       0UL
224 #define STRTAB_STE_1_S1C_CACHE_WBRA     1UL
225 #define STRTAB_STE_1_S1C_CACHE_WT       2UL
226 #define STRTAB_STE_1_S1C_CACHE_WB       3UL
227 #define STRTAB_STE_1_S1CIR              GENMASK_ULL(3, 2)
228 #define STRTAB_STE_1_S1COR              GENMASK_ULL(5, 4)
229 #define STRTAB_STE_1_S1CSH              GENMASK_ULL(7, 6)
230
231 #define STRTAB_STE_1_S1STALLD           (1UL << 27)
232
233 #define STRTAB_STE_1_EATS               GENMASK_ULL(29, 28)
234 #define STRTAB_STE_1_EATS_ABT           0UL
235 #define STRTAB_STE_1_EATS_TRANS         1UL
236 #define STRTAB_STE_1_EATS_S1CHK         2UL
237
238 #define STRTAB_STE_1_STRW               GENMASK_ULL(31, 30)
239 #define STRTAB_STE_1_STRW_NSEL1         0UL
240 #define STRTAB_STE_1_STRW_EL2           2UL
241
242 #define STRTAB_STE_1_SHCFG              GENMASK_ULL(45, 44)
243 #define STRTAB_STE_1_SHCFG_INCOMING     1UL
244
245 #define STRTAB_STE_2_S2VMID             GENMASK_ULL(15, 0)
246 #define STRTAB_STE_2_VTCR               GENMASK_ULL(50, 32)
247 #define STRTAB_STE_2_S2AA64             (1UL << 51)
248 #define STRTAB_STE_2_S2ENDI             (1UL << 52)
249 #define STRTAB_STE_2_S2PTW              (1UL << 54)
250 #define STRTAB_STE_2_S2R                (1UL << 58)
251
252 #define STRTAB_STE_3_S2TTB_MASK         GENMASK_ULL(51, 4)
253
254 /* Context descriptor (stage-1 only) */
255 #define CTXDESC_CD_DWORDS               8
256 #define CTXDESC_CD_0_TCR_T0SZ           GENMASK_ULL(5, 0)
257 #define ARM64_TCR_T0SZ                  GENMASK_ULL(5, 0)
258 #define CTXDESC_CD_0_TCR_TG0            GENMASK_ULL(7, 6)
259 #define ARM64_TCR_TG0                   GENMASK_ULL(15, 14)
260 #define CTXDESC_CD_0_TCR_IRGN0          GENMASK_ULL(9, 8)
261 #define ARM64_TCR_IRGN0                 GENMASK_ULL(9, 8)
262 #define CTXDESC_CD_0_TCR_ORGN0          GENMASK_ULL(11, 10)
263 #define ARM64_TCR_ORGN0                 GENMASK_ULL(11, 10)
264 #define CTXDESC_CD_0_TCR_SH0            GENMASK_ULL(13, 12)
265 #define ARM64_TCR_SH0                   GENMASK_ULL(13, 12)
266 #define CTXDESC_CD_0_TCR_EPD0           (1ULL << 14)
267 #define ARM64_TCR_EPD0                  (1ULL << 7)
268 #define CTXDESC_CD_0_TCR_EPD1           (1ULL << 30)
269 #define ARM64_TCR_EPD1                  (1ULL << 23)
270
271 #define CTXDESC_CD_0_ENDI               (1UL << 15)
272 #define CTXDESC_CD_0_V                  (1UL << 31)
273
274 #define CTXDESC_CD_0_TCR_IPS            GENMASK_ULL(34, 32)
275 #define ARM64_TCR_IPS                   GENMASK_ULL(34, 32)
276 #define CTXDESC_CD_0_TCR_TBI0           (1ULL << 38)
277 #define ARM64_TCR_TBI0                  (1ULL << 37)
278
279 #define CTXDESC_CD_0_AA64               (1UL << 41)
280 #define CTXDESC_CD_0_S                  (1UL << 44)
281 #define CTXDESC_CD_0_R                  (1UL << 45)
282 #define CTXDESC_CD_0_A                  (1UL << 46)
283 #define CTXDESC_CD_0_ASET               (1UL << 47)
284 #define CTXDESC_CD_0_ASID               GENMASK_ULL(63, 48)
285
286 #define CTXDESC_CD_1_TTB0_MASK          GENMASK_ULL(51, 4)
287
288 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
289 #define ARM_SMMU_TCR2CD(tcr, fld)       FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \
290                                         FIELD_GET(ARM64_TCR_##fld, tcr))
291
292 /* Command queue */
293 #define CMDQ_ENT_SZ_SHIFT               4
294 #define CMDQ_ENT_DWORDS                 ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
295 #define CMDQ_MAX_SZ_SHIFT               (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
296
297 #define CMDQ_CONS_ERR                   GENMASK(30, 24)
298 #define CMDQ_ERR_CERROR_NONE_IDX        0
299 #define CMDQ_ERR_CERROR_ILL_IDX         1
300 #define CMDQ_ERR_CERROR_ABT_IDX         2
301 #define CMDQ_ERR_CERROR_ATC_INV_IDX     3
302
303 #define CMDQ_0_OP                       GENMASK_ULL(7, 0)
304 #define CMDQ_0_SSV                      (1UL << 11)
305
306 #define CMDQ_PREFETCH_0_SID             GENMASK_ULL(63, 32)
307 #define CMDQ_PREFETCH_1_SIZE            GENMASK_ULL(4, 0)
308 #define CMDQ_PREFETCH_1_ADDR_MASK       GENMASK_ULL(63, 12)
309
310 #define CMDQ_CFGI_0_SID                 GENMASK_ULL(63, 32)
311 #define CMDQ_CFGI_1_LEAF                (1UL << 0)
312 #define CMDQ_CFGI_1_RANGE               GENMASK_ULL(4, 0)
313
314 #define CMDQ_TLBI_0_VMID                GENMASK_ULL(47, 32)
315 #define CMDQ_TLBI_0_ASID                GENMASK_ULL(63, 48)
316 #define CMDQ_TLBI_1_LEAF                (1UL << 0)
317 #define CMDQ_TLBI_1_VA_MASK             GENMASK_ULL(63, 12)
318 #define CMDQ_TLBI_1_IPA_MASK            GENMASK_ULL(51, 12)
319
320 #define CMDQ_ATC_0_SSID                 GENMASK_ULL(31, 12)
321 #define CMDQ_ATC_0_SID                  GENMASK_ULL(63, 32)
322 #define CMDQ_ATC_0_GLOBAL               (1UL << 9)
323 #define CMDQ_ATC_1_SIZE                 GENMASK_ULL(5, 0)
324 #define CMDQ_ATC_1_ADDR_MASK            GENMASK_ULL(63, 12)
325
326 #define CMDQ_PRI_0_SSID                 GENMASK_ULL(31, 12)
327 #define CMDQ_PRI_0_SID                  GENMASK_ULL(63, 32)
328 #define CMDQ_PRI_1_GRPID                GENMASK_ULL(8, 0)
329 #define CMDQ_PRI_1_RESP                 GENMASK_ULL(13, 12)
330
331 #define CMDQ_SYNC_0_CS                  GENMASK_ULL(13, 12)
332 #define CMDQ_SYNC_0_CS_NONE             0
333 #define CMDQ_SYNC_0_CS_IRQ              1
334 #define CMDQ_SYNC_0_CS_SEV              2
335 #define CMDQ_SYNC_0_MSH                 GENMASK_ULL(23, 22)
336 #define CMDQ_SYNC_0_MSIATTR             GENMASK_ULL(27, 24)
337 #define CMDQ_SYNC_0_MSIDATA             GENMASK_ULL(63, 32)
338 #define CMDQ_SYNC_1_MSIADDR_MASK        GENMASK_ULL(51, 2)
339
340 /* Event queue */
341 #define EVTQ_ENT_SZ_SHIFT               5
342 #define EVTQ_ENT_DWORDS                 ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
343 #define EVTQ_MAX_SZ_SHIFT               (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
344
345 #define EVTQ_0_ID                       GENMASK_ULL(7, 0)
346
347 /* PRI queue */
348 #define PRIQ_ENT_SZ_SHIFT               4
349 #define PRIQ_ENT_DWORDS                 ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
350 #define PRIQ_MAX_SZ_SHIFT               (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
351
352 #define PRIQ_0_SID                      GENMASK_ULL(31, 0)
353 #define PRIQ_0_SSID                     GENMASK_ULL(51, 32)
354 #define PRIQ_0_PERM_PRIV                (1UL << 58)
355 #define PRIQ_0_PERM_EXEC                (1UL << 59)
356 #define PRIQ_0_PERM_READ                (1UL << 60)
357 #define PRIQ_0_PERM_WRITE               (1UL << 61)
358 #define PRIQ_0_PRG_LAST                 (1UL << 62)
359 #define PRIQ_0_SSID_V                   (1UL << 63)
360
361 #define PRIQ_1_PRG_IDX                  GENMASK_ULL(8, 0)
362 #define PRIQ_1_ADDR_MASK                GENMASK_ULL(63, 12)
363
364 /* High-level queue structures */
365 #define ARM_SMMU_POLL_TIMEOUT_US        100
366 #define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US   1000000 /* 1s! */
367 #define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT   10
368
369 #define MSI_IOVA_BASE                   0x8000000
370 #define MSI_IOVA_LENGTH                 0x100000
371
372 /*
373  * not really modular, but the easiest way to keep compat with existing
374  * bootargs behaviour is to continue using module_param_named here.
375  */
376 static bool disable_bypass = 1;
377 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
378 MODULE_PARM_DESC(disable_bypass,
379         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
380
381 enum pri_resp {
382         PRI_RESP_DENY = 0,
383         PRI_RESP_FAIL = 1,
384         PRI_RESP_SUCC = 2,
385 };
386
387 enum arm_smmu_msi_index {
388         EVTQ_MSI_INDEX,
389         GERROR_MSI_INDEX,
390         PRIQ_MSI_INDEX,
391         ARM_SMMU_MAX_MSIS,
392 };
393
394 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
395         [EVTQ_MSI_INDEX] = {
396                 ARM_SMMU_EVTQ_IRQ_CFG0,
397                 ARM_SMMU_EVTQ_IRQ_CFG1,
398                 ARM_SMMU_EVTQ_IRQ_CFG2,
399         },
400         [GERROR_MSI_INDEX] = {
401                 ARM_SMMU_GERROR_IRQ_CFG0,
402                 ARM_SMMU_GERROR_IRQ_CFG1,
403                 ARM_SMMU_GERROR_IRQ_CFG2,
404         },
405         [PRIQ_MSI_INDEX] = {
406                 ARM_SMMU_PRIQ_IRQ_CFG0,
407                 ARM_SMMU_PRIQ_IRQ_CFG1,
408                 ARM_SMMU_PRIQ_IRQ_CFG2,
409         },
410 };
411
412 struct arm_smmu_cmdq_ent {
413         /* Common fields */
414         u8                              opcode;
415         bool                            substream_valid;
416
417         /* Command-specific fields */
418         union {
419                 #define CMDQ_OP_PREFETCH_CFG    0x1
420                 struct {
421                         u32                     sid;
422                         u8                      size;
423                         u64                     addr;
424                 } prefetch;
425
426                 #define CMDQ_OP_CFGI_STE        0x3
427                 #define CMDQ_OP_CFGI_ALL        0x4
428                 struct {
429                         u32                     sid;
430                         union {
431                                 bool            leaf;
432                                 u8              span;
433                         };
434                 } cfgi;
435
436                 #define CMDQ_OP_TLBI_NH_ASID    0x11
437                 #define CMDQ_OP_TLBI_NH_VA      0x12
438                 #define CMDQ_OP_TLBI_EL2_ALL    0x20
439                 #define CMDQ_OP_TLBI_S12_VMALL  0x28
440                 #define CMDQ_OP_TLBI_S2_IPA     0x2a
441                 #define CMDQ_OP_TLBI_NSNH_ALL   0x30
442                 struct {
443                         u16                     asid;
444                         u16                     vmid;
445                         bool                    leaf;
446                         u64                     addr;
447                 } tlbi;
448
449                 #define CMDQ_OP_ATC_INV         0x40
450                 #define ATC_INV_SIZE_ALL        52
451                 struct {
452                         u32                     sid;
453                         u32                     ssid;
454                         u64                     addr;
455                         u8                      size;
456                         bool                    global;
457                 } atc;
458
459                 #define CMDQ_OP_PRI_RESP        0x41
460                 struct {
461                         u32                     sid;
462                         u32                     ssid;
463                         u16                     grpid;
464                         enum pri_resp           resp;
465                 } pri;
466
467                 #define CMDQ_OP_CMD_SYNC        0x46
468                 struct {
469                         u32                     msidata;
470                         u64                     msiaddr;
471                 } sync;
472         };
473 };
474
475 struct arm_smmu_queue {
476         int                             irq; /* Wired interrupt */
477
478         __le64                          *base;
479         dma_addr_t                      base_dma;
480         u64                             q_base;
481
482         size_t                          ent_dwords;
483         u32                             max_n_shift;
484         u32                             prod;
485         u32                             cons;
486
487         u32 __iomem                     *prod_reg;
488         u32 __iomem                     *cons_reg;
489 };
490
491 struct arm_smmu_cmdq {
492         struct arm_smmu_queue           q;
493         spinlock_t                      lock;
494 };
495
496 struct arm_smmu_evtq {
497         struct arm_smmu_queue           q;
498         u32                             max_stalls;
499 };
500
501 struct arm_smmu_priq {
502         struct arm_smmu_queue           q;
503 };
504
505 /* High-level stream table and context descriptor structures */
506 struct arm_smmu_strtab_l1_desc {
507         u8                              span;
508
509         __le64                          *l2ptr;
510         dma_addr_t                      l2ptr_dma;
511 };
512
513 struct arm_smmu_s1_cfg {
514         __le64                          *cdptr;
515         dma_addr_t                      cdptr_dma;
516
517         struct arm_smmu_ctx_desc {
518                 u16     asid;
519                 u64     ttbr;
520                 u64     tcr;
521                 u64     mair;
522         }                               cd;
523 };
524
525 struct arm_smmu_s2_cfg {
526         u16                             vmid;
527         u64                             vttbr;
528         u64                             vtcr;
529 };
530
531 struct arm_smmu_strtab_cfg {
532         __le64                          *strtab;
533         dma_addr_t                      strtab_dma;
534         struct arm_smmu_strtab_l1_desc  *l1_desc;
535         unsigned int                    num_l1_ents;
536
537         u64                             strtab_base;
538         u32                             strtab_base_cfg;
539 };
540
541 /* An SMMUv3 instance */
542 struct arm_smmu_device {
543         struct device                   *dev;
544         void __iomem                    *base;
545
546 #define ARM_SMMU_FEAT_2_LVL_STRTAB      (1 << 0)
547 #define ARM_SMMU_FEAT_2_LVL_CDTAB       (1 << 1)
548 #define ARM_SMMU_FEAT_TT_LE             (1 << 2)
549 #define ARM_SMMU_FEAT_TT_BE             (1 << 3)
550 #define ARM_SMMU_FEAT_PRI               (1 << 4)
551 #define ARM_SMMU_FEAT_ATS               (1 << 5)
552 #define ARM_SMMU_FEAT_SEV               (1 << 6)
553 #define ARM_SMMU_FEAT_MSI               (1 << 7)
554 #define ARM_SMMU_FEAT_COHERENCY         (1 << 8)
555 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 9)
556 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 10)
557 #define ARM_SMMU_FEAT_STALLS            (1 << 11)
558 #define ARM_SMMU_FEAT_HYP               (1 << 12)
559 #define ARM_SMMU_FEAT_STALL_FORCE       (1 << 13)
560 #define ARM_SMMU_FEAT_VAX               (1 << 14)
561         u32                             features;
562
563 #define ARM_SMMU_OPT_SKIP_PREFETCH      (1 << 0)
564 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY    (1 << 1)
565         u32                             options;
566
567         struct arm_smmu_cmdq            cmdq;
568         struct arm_smmu_evtq            evtq;
569         struct arm_smmu_priq            priq;
570
571         int                             gerr_irq;
572         int                             combined_irq;
573         u32                             sync_nr;
574         u8                              prev_cmd_opcode;
575
576         unsigned long                   ias; /* IPA */
577         unsigned long                   oas; /* PA */
578         unsigned long                   pgsize_bitmap;
579
580 #define ARM_SMMU_MAX_ASIDS              (1 << 16)
581         unsigned int                    asid_bits;
582         DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
583
584 #define ARM_SMMU_MAX_VMIDS              (1 << 16)
585         unsigned int                    vmid_bits;
586         DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
587
588         unsigned int                    ssid_bits;
589         unsigned int                    sid_bits;
590
591         struct arm_smmu_strtab_cfg      strtab_cfg;
592
593         /* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
594         union {
595                 u32                     sync_count;
596                 u64                     padding;
597         };
598
599         /* IOMMU core code handle */
600         struct iommu_device             iommu;
601 };
602
603 /* SMMU private data for each master */
604 struct arm_smmu_master {
605         struct arm_smmu_device          *smmu;
606         struct device                   *dev;
607         struct arm_smmu_domain          *domain;
608         struct list_head                domain_head;
609         u32                             *sids;
610         unsigned int                    num_sids;
611         bool                            ats_enabled             :1;
612 };
613
614 /* SMMU private data for an IOMMU domain */
615 enum arm_smmu_domain_stage {
616         ARM_SMMU_DOMAIN_S1 = 0,
617         ARM_SMMU_DOMAIN_S2,
618         ARM_SMMU_DOMAIN_NESTED,
619         ARM_SMMU_DOMAIN_BYPASS,
620 };
621
622 struct arm_smmu_domain {
623         struct arm_smmu_device          *smmu;
624         struct mutex                    init_mutex; /* Protects smmu pointer */
625
626         struct io_pgtable_ops           *pgtbl_ops;
627         bool                            non_strict;
628
629         enum arm_smmu_domain_stage      stage;
630         union {
631                 struct arm_smmu_s1_cfg  s1_cfg;
632                 struct arm_smmu_s2_cfg  s2_cfg;
633         };
634
635         struct iommu_domain             domain;
636
637         struct list_head                devices;
638         spinlock_t                      devices_lock;
639 };
640
641 struct arm_smmu_option_prop {
642         u32 opt;
643         const char *prop;
644 };
645
646 static struct arm_smmu_option_prop arm_smmu_options[] = {
647         { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
648         { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
649         { 0, NULL},
650 };
651
652 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
653                                                  struct arm_smmu_device *smmu)
654 {
655         if ((offset > SZ_64K) &&
656             (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
657                 offset -= SZ_64K;
658
659         return smmu->base + offset;
660 }
661
662 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
663 {
664         return container_of(dom, struct arm_smmu_domain, domain);
665 }
666
667 static void parse_driver_options(struct arm_smmu_device *smmu)
668 {
669         int i = 0;
670
671         do {
672                 if (of_property_read_bool(smmu->dev->of_node,
673                                                 arm_smmu_options[i].prop)) {
674                         smmu->options |= arm_smmu_options[i].opt;
675                         dev_notice(smmu->dev, "option %s\n",
676                                 arm_smmu_options[i].prop);
677                 }
678         } while (arm_smmu_options[++i].opt);
679 }
680
681 /* Low-level queue manipulation functions */
682 static bool queue_full(struct arm_smmu_queue *q)
683 {
684         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
685                Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
686 }
687
688 static bool queue_empty(struct arm_smmu_queue *q)
689 {
690         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
691                Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
692 }
693
694 static void queue_sync_cons(struct arm_smmu_queue *q)
695 {
696         q->cons = readl_relaxed(q->cons_reg);
697 }
698
699 static void queue_inc_cons(struct arm_smmu_queue *q)
700 {
701         u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
702
703         q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
704
705         /*
706          * Ensure that all CPU accesses (reads and writes) to the queue
707          * are complete before we update the cons pointer.
708          */
709         mb();
710         writel_relaxed(q->cons, q->cons_reg);
711 }
712
713 static int queue_sync_prod(struct arm_smmu_queue *q)
714 {
715         int ret = 0;
716         u32 prod = readl_relaxed(q->prod_reg);
717
718         if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
719                 ret = -EOVERFLOW;
720
721         q->prod = prod;
722         return ret;
723 }
724
725 static void queue_inc_prod(struct arm_smmu_queue *q)
726 {
727         u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
728
729         q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
730         writel(q->prod, q->prod_reg);
731 }
732
733 /*
734  * Wait for the SMMU to consume items. If sync is true, wait until the queue
735  * is empty. Otherwise, wait until there is at least one free slot.
736  */
737 static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
738 {
739         ktime_t timeout;
740         unsigned int delay = 1, spin_cnt = 0;
741
742         /* Wait longer if it's a CMD_SYNC */
743         timeout = ktime_add_us(ktime_get(), sync ?
744                                             ARM_SMMU_CMDQ_SYNC_TIMEOUT_US :
745                                             ARM_SMMU_POLL_TIMEOUT_US);
746
747         while (queue_sync_cons(q), (sync ? !queue_empty(q) : queue_full(q))) {
748                 if (ktime_compare(ktime_get(), timeout) > 0)
749                         return -ETIMEDOUT;
750
751                 if (wfe) {
752                         wfe();
753                 } else if (++spin_cnt < ARM_SMMU_CMDQ_SYNC_SPIN_COUNT) {
754                         cpu_relax();
755                         continue;
756                 } else {
757                         udelay(delay);
758                         delay *= 2;
759                         spin_cnt = 0;
760                 }
761         }
762
763         return 0;
764 }
765
766 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
767 {
768         int i;
769
770         for (i = 0; i < n_dwords; ++i)
771                 *dst++ = cpu_to_le64(*src++);
772 }
773
774 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
775 {
776         if (queue_full(q))
777                 return -ENOSPC;
778
779         queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
780         queue_inc_prod(q);
781         return 0;
782 }
783
784 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
785 {
786         int i;
787
788         for (i = 0; i < n_dwords; ++i)
789                 *dst++ = le64_to_cpu(*src++);
790 }
791
792 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
793 {
794         if (queue_empty(q))
795                 return -EAGAIN;
796
797         queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
798         queue_inc_cons(q);
799         return 0;
800 }
801
802 /* High-level queue accessors */
803 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
804 {
805         memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
806         cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
807
808         switch (ent->opcode) {
809         case CMDQ_OP_TLBI_EL2_ALL:
810         case CMDQ_OP_TLBI_NSNH_ALL:
811                 break;
812         case CMDQ_OP_PREFETCH_CFG:
813                 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
814                 cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
815                 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
816                 break;
817         case CMDQ_OP_CFGI_STE:
818                 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
819                 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
820                 break;
821         case CMDQ_OP_CFGI_ALL:
822                 /* Cover the entire SID range */
823                 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
824                 break;
825         case CMDQ_OP_TLBI_NH_VA:
826                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
827                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
828                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
829                 break;
830         case CMDQ_OP_TLBI_S2_IPA:
831                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
832                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
833                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
834                 break;
835         case CMDQ_OP_TLBI_NH_ASID:
836                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
837                 /* Fallthrough */
838         case CMDQ_OP_TLBI_S12_VMALL:
839                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
840                 break;
841         case CMDQ_OP_ATC_INV:
842                 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
843                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
844                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
845                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
846                 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
847                 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
848                 break;
849         case CMDQ_OP_PRI_RESP:
850                 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
851                 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
852                 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
853                 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
854                 switch (ent->pri.resp) {
855                 case PRI_RESP_DENY:
856                 case PRI_RESP_FAIL:
857                 case PRI_RESP_SUCC:
858                         break;
859                 default:
860                         return -EINVAL;
861                 }
862                 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
863                 break;
864         case CMDQ_OP_CMD_SYNC:
865                 if (ent->sync.msiaddr)
866                         cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
867                 else
868                         cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
869                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
870                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
871                 /*
872                  * Commands are written little-endian, but we want the SMMU to
873                  * receive MSIData, and thus write it back to memory, in CPU
874                  * byte order, so big-endian needs an extra byteswap here.
875                  */
876                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIDATA,
877                                      cpu_to_le32(ent->sync.msidata));
878                 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
879                 break;
880         default:
881                 return -ENOENT;
882         }
883
884         return 0;
885 }
886
887 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
888 {
889         static const char *cerror_str[] = {
890                 [CMDQ_ERR_CERROR_NONE_IDX]      = "No error",
891                 [CMDQ_ERR_CERROR_ILL_IDX]       = "Illegal command",
892                 [CMDQ_ERR_CERROR_ABT_IDX]       = "Abort on command fetch",
893                 [CMDQ_ERR_CERROR_ATC_INV_IDX]   = "ATC invalidate timeout",
894         };
895
896         int i;
897         u64 cmd[CMDQ_ENT_DWORDS];
898         struct arm_smmu_queue *q = &smmu->cmdq.q;
899         u32 cons = readl_relaxed(q->cons_reg);
900         u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
901         struct arm_smmu_cmdq_ent cmd_sync = {
902                 .opcode = CMDQ_OP_CMD_SYNC,
903         };
904
905         dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
906                 idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
907
908         switch (idx) {
909         case CMDQ_ERR_CERROR_ABT_IDX:
910                 dev_err(smmu->dev, "retrying command fetch\n");
911         case CMDQ_ERR_CERROR_NONE_IDX:
912                 return;
913         case CMDQ_ERR_CERROR_ATC_INV_IDX:
914                 /*
915                  * ATC Invalidation Completion timeout. CONS is still pointing
916                  * at the CMD_SYNC. Attempt to complete other pending commands
917                  * by repeating the CMD_SYNC, though we might well end up back
918                  * here since the ATC invalidation may still be pending.
919                  */
920                 return;
921         case CMDQ_ERR_CERROR_ILL_IDX:
922                 /* Fallthrough */
923         default:
924                 break;
925         }
926
927         /*
928          * We may have concurrent producers, so we need to be careful
929          * not to touch any of the shadow cmdq state.
930          */
931         queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
932         dev_err(smmu->dev, "skipping command in error state:\n");
933         for (i = 0; i < ARRAY_SIZE(cmd); ++i)
934                 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
935
936         /* Convert the erroneous command into a CMD_SYNC */
937         if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
938                 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
939                 return;
940         }
941
942         queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
943 }
944
945 static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device *smmu, u64 *cmd)
946 {
947         struct arm_smmu_queue *q = &smmu->cmdq.q;
948         bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
949
950         smmu->prev_cmd_opcode = FIELD_GET(CMDQ_0_OP, cmd[0]);
951
952         while (queue_insert_raw(q, cmd) == -ENOSPC) {
953                 if (queue_poll_cons(q, false, wfe))
954                         dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
955         }
956 }
957
958 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
959                                     struct arm_smmu_cmdq_ent *ent)
960 {
961         u64 cmd[CMDQ_ENT_DWORDS];
962         unsigned long flags;
963
964         if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
965                 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
966                          ent->opcode);
967                 return;
968         }
969
970         spin_lock_irqsave(&smmu->cmdq.lock, flags);
971         arm_smmu_cmdq_insert_cmd(smmu, cmd);
972         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
973 }
974
975 /*
976  * The difference between val and sync_idx is bounded by the maximum size of
977  * a queue at 2^20 entries, so 32 bits is plenty for wrap-safe arithmetic.
978  */
979 static int __arm_smmu_sync_poll_msi(struct arm_smmu_device *smmu, u32 sync_idx)
980 {
981         ktime_t timeout;
982         u32 val;
983
984         timeout = ktime_add_us(ktime_get(), ARM_SMMU_CMDQ_SYNC_TIMEOUT_US);
985         val = smp_cond_load_acquire(&smmu->sync_count,
986                                     (int)(VAL - sync_idx) >= 0 ||
987                                     !ktime_before(ktime_get(), timeout));
988
989         return (int)(val - sync_idx) < 0 ? -ETIMEDOUT : 0;
990 }
991
992 static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device *smmu)
993 {
994         u64 cmd[CMDQ_ENT_DWORDS];
995         unsigned long flags;
996         struct arm_smmu_cmdq_ent ent = {
997                 .opcode = CMDQ_OP_CMD_SYNC,
998                 .sync   = {
999                         .msiaddr = virt_to_phys(&smmu->sync_count),
1000                 },
1001         };
1002
1003         spin_lock_irqsave(&smmu->cmdq.lock, flags);
1004
1005         /* Piggy-back on the previous command if it's a SYNC */
1006         if (smmu->prev_cmd_opcode == CMDQ_OP_CMD_SYNC) {
1007                 ent.sync.msidata = smmu->sync_nr;
1008         } else {
1009                 ent.sync.msidata = ++smmu->sync_nr;
1010                 arm_smmu_cmdq_build_cmd(cmd, &ent);
1011                 arm_smmu_cmdq_insert_cmd(smmu, cmd);
1012         }
1013
1014         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1015
1016         return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata);
1017 }
1018
1019 static int __arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1020 {
1021         u64 cmd[CMDQ_ENT_DWORDS];
1022         unsigned long flags;
1023         bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
1024         struct arm_smmu_cmdq_ent ent = { .opcode = CMDQ_OP_CMD_SYNC };
1025         int ret;
1026
1027         arm_smmu_cmdq_build_cmd(cmd, &ent);
1028
1029         spin_lock_irqsave(&smmu->cmdq.lock, flags);
1030         arm_smmu_cmdq_insert_cmd(smmu, cmd);
1031         ret = queue_poll_cons(&smmu->cmdq.q, true, wfe);
1032         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1033
1034         return ret;
1035 }
1036
1037 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1038 {
1039         int ret;
1040         bool msi = (smmu->features & ARM_SMMU_FEAT_MSI) &&
1041                    (smmu->features & ARM_SMMU_FEAT_COHERENCY);
1042
1043         ret = msi ? __arm_smmu_cmdq_issue_sync_msi(smmu)
1044                   : __arm_smmu_cmdq_issue_sync(smmu);
1045         if (ret)
1046                 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
1047         return ret;
1048 }
1049
1050 /* Context descriptor manipulation functions */
1051 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
1052 {
1053         u64 val = 0;
1054
1055         /* Repack the TCR. Just care about TTBR0 for now */
1056         val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
1057         val |= ARM_SMMU_TCR2CD(tcr, TG0);
1058         val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
1059         val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
1060         val |= ARM_SMMU_TCR2CD(tcr, SH0);
1061         val |= ARM_SMMU_TCR2CD(tcr, EPD0);
1062         val |= ARM_SMMU_TCR2CD(tcr, EPD1);
1063         val |= ARM_SMMU_TCR2CD(tcr, IPS);
1064
1065         return val;
1066 }
1067
1068 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
1069                                     struct arm_smmu_s1_cfg *cfg)
1070 {
1071         u64 val;
1072
1073         /*
1074          * We don't need to issue any invalidation here, as we'll invalidate
1075          * the STE when installing the new entry anyway.
1076          */
1077         val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
1078 #ifdef __BIG_ENDIAN
1079               CTXDESC_CD_0_ENDI |
1080 #endif
1081               CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
1082               CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) |
1083               CTXDESC_CD_0_V;
1084
1085         /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1086         if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1087                 val |= CTXDESC_CD_0_S;
1088
1089         cfg->cdptr[0] = cpu_to_le64(val);
1090
1091         val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
1092         cfg->cdptr[1] = cpu_to_le64(val);
1093
1094         cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
1095 }
1096
1097 /* Stream table manipulation functions */
1098 static void
1099 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1100 {
1101         u64 val = 0;
1102
1103         val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1104         val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1105
1106         *dst = cpu_to_le64(val);
1107 }
1108
1109 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1110 {
1111         struct arm_smmu_cmdq_ent cmd = {
1112                 .opcode = CMDQ_OP_CFGI_STE,
1113                 .cfgi   = {
1114                         .sid    = sid,
1115                         .leaf   = true,
1116                 },
1117         };
1118
1119         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1120         arm_smmu_cmdq_issue_sync(smmu);
1121 }
1122
1123 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1124                                       __le64 *dst)
1125 {
1126         /*
1127          * This is hideously complicated, but we only really care about
1128          * three cases at the moment:
1129          *
1130          * 1. Invalid (all zero) -> bypass/fault (init)
1131          * 2. Bypass/fault -> translation/bypass (attach)
1132          * 3. Translation/bypass -> bypass/fault (detach)
1133          *
1134          * Given that we can't update the STE atomically and the SMMU
1135          * doesn't read the thing in a defined order, that leaves us
1136          * with the following maintenance requirements:
1137          *
1138          * 1. Update Config, return (init time STEs aren't live)
1139          * 2. Write everything apart from dword 0, sync, write dword 0, sync
1140          * 3. Update Config, sync
1141          */
1142         u64 val = le64_to_cpu(dst[0]);
1143         bool ste_live = false;
1144         struct arm_smmu_device *smmu = NULL;
1145         struct arm_smmu_s1_cfg *s1_cfg = NULL;
1146         struct arm_smmu_s2_cfg *s2_cfg = NULL;
1147         struct arm_smmu_domain *smmu_domain = NULL;
1148         struct arm_smmu_cmdq_ent prefetch_cmd = {
1149                 .opcode         = CMDQ_OP_PREFETCH_CFG,
1150                 .prefetch       = {
1151                         .sid    = sid,
1152                 },
1153         };
1154
1155         if (master) {
1156                 smmu_domain = master->domain;
1157                 smmu = master->smmu;
1158         }
1159
1160         if (smmu_domain) {
1161                 switch (smmu_domain->stage) {
1162                 case ARM_SMMU_DOMAIN_S1:
1163                         s1_cfg = &smmu_domain->s1_cfg;
1164                         break;
1165                 case ARM_SMMU_DOMAIN_S2:
1166                 case ARM_SMMU_DOMAIN_NESTED:
1167                         s2_cfg = &smmu_domain->s2_cfg;
1168                         break;
1169                 default:
1170                         break;
1171                 }
1172         }
1173
1174         if (val & STRTAB_STE_0_V) {
1175                 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1176                 case STRTAB_STE_0_CFG_BYPASS:
1177                         break;
1178                 case STRTAB_STE_0_CFG_S1_TRANS:
1179                 case STRTAB_STE_0_CFG_S2_TRANS:
1180                         ste_live = true;
1181                         break;
1182                 case STRTAB_STE_0_CFG_ABORT:
1183                         if (disable_bypass)
1184                                 break;
1185                 default:
1186                         BUG(); /* STE corruption */
1187                 }
1188         }
1189
1190         /* Nuke the existing STE_0 value, as we're going to rewrite it */
1191         val = STRTAB_STE_0_V;
1192
1193         /* Bypass/fault */
1194         if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1195                 if (!smmu_domain && disable_bypass)
1196                         val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1197                 else
1198                         val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1199
1200                 dst[0] = cpu_to_le64(val);
1201                 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1202                                                 STRTAB_STE_1_SHCFG_INCOMING));
1203                 dst[2] = 0; /* Nuke the VMID */
1204                 /*
1205                  * The SMMU can perform negative caching, so we must sync
1206                  * the STE regardless of whether the old value was live.
1207                  */
1208                 if (smmu)
1209                         arm_smmu_sync_ste_for_sid(smmu, sid);
1210                 return;
1211         }
1212
1213         if (s1_cfg) {
1214                 BUG_ON(ste_live);
1215                 dst[1] = cpu_to_le64(
1216                          FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1217                          FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1218                          FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1219                          FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1220
1221                 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1222                    !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1223                         dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1224
1225                 val |= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1226                         FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);
1227         }
1228
1229         if (s2_cfg) {
1230                 BUG_ON(ste_live);
1231                 dst[2] = cpu_to_le64(
1232                          FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1233                          FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1234 #ifdef __BIG_ENDIAN
1235                          STRTAB_STE_2_S2ENDI |
1236 #endif
1237                          STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1238                          STRTAB_STE_2_S2R);
1239
1240                 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1241
1242                 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1243         }
1244
1245         if (master->ats_enabled)
1246                 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1247                                                  STRTAB_STE_1_EATS_TRANS));
1248
1249         arm_smmu_sync_ste_for_sid(smmu, sid);
1250         dst[0] = cpu_to_le64(val);
1251         arm_smmu_sync_ste_for_sid(smmu, sid);
1252
1253         /* It's likely that we'll want to use the new STE soon */
1254         if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1255                 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1256 }
1257
1258 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1259 {
1260         unsigned int i;
1261
1262         for (i = 0; i < nent; ++i) {
1263                 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1264                 strtab += STRTAB_STE_DWORDS;
1265         }
1266 }
1267
1268 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1269 {
1270         size_t size;
1271         void *strtab;
1272         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1273         struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1274
1275         if (desc->l2ptr)
1276                 return 0;
1277
1278         size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1279         strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1280
1281         desc->span = STRTAB_SPLIT + 1;
1282         desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1283                                           GFP_KERNEL | __GFP_ZERO);
1284         if (!desc->l2ptr) {
1285                 dev_err(smmu->dev,
1286                         "failed to allocate l2 stream table for SID %u\n",
1287                         sid);
1288                 return -ENOMEM;
1289         }
1290
1291         arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1292         arm_smmu_write_strtab_l1_desc(strtab, desc);
1293         return 0;
1294 }
1295
1296 /* IRQ and event handlers */
1297 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1298 {
1299         int i;
1300         struct arm_smmu_device *smmu = dev;
1301         struct arm_smmu_queue *q = &smmu->evtq.q;
1302         u64 evt[EVTQ_ENT_DWORDS];
1303
1304         do {
1305                 while (!queue_remove_raw(q, evt)) {
1306                         u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1307
1308                         dev_info(smmu->dev, "event 0x%02x received:\n", id);
1309                         for (i = 0; i < ARRAY_SIZE(evt); ++i)
1310                                 dev_info(smmu->dev, "\t0x%016llx\n",
1311                                          (unsigned long long)evt[i]);
1312
1313                 }
1314
1315                 /*
1316                  * Not much we can do on overflow, so scream and pretend we're
1317                  * trying harder.
1318                  */
1319                 if (queue_sync_prod(q) == -EOVERFLOW)
1320                         dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1321         } while (!queue_empty(q));
1322
1323         /* Sync our overflow flag, as we believe we're up to speed */
1324         q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1325         return IRQ_HANDLED;
1326 }
1327
1328 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1329 {
1330         u32 sid, ssid;
1331         u16 grpid;
1332         bool ssv, last;
1333
1334         sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1335         ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1336         ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
1337         last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1338         grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1339
1340         dev_info(smmu->dev, "unexpected PRI request received:\n");
1341         dev_info(smmu->dev,
1342                  "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1343                  sid, ssid, grpid, last ? "L" : "",
1344                  evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1345                  evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1346                  evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1347                  evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1348                  evt[1] & PRIQ_1_ADDR_MASK);
1349
1350         if (last) {
1351                 struct arm_smmu_cmdq_ent cmd = {
1352                         .opcode                 = CMDQ_OP_PRI_RESP,
1353                         .substream_valid        = ssv,
1354                         .pri                    = {
1355                                 .sid    = sid,
1356                                 .ssid   = ssid,
1357                                 .grpid  = grpid,
1358                                 .resp   = PRI_RESP_DENY,
1359                         },
1360                 };
1361
1362                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1363         }
1364 }
1365
1366 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1367 {
1368         struct arm_smmu_device *smmu = dev;
1369         struct arm_smmu_queue *q = &smmu->priq.q;
1370         u64 evt[PRIQ_ENT_DWORDS];
1371
1372         do {
1373                 while (!queue_remove_raw(q, evt))
1374                         arm_smmu_handle_ppr(smmu, evt);
1375
1376                 if (queue_sync_prod(q) == -EOVERFLOW)
1377                         dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1378         } while (!queue_empty(q));
1379
1380         /* Sync our overflow flag, as we believe we're up to speed */
1381         q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1382         writel(q->cons, q->cons_reg);
1383         return IRQ_HANDLED;
1384 }
1385
1386 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1387
1388 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1389 {
1390         u32 gerror, gerrorn, active;
1391         struct arm_smmu_device *smmu = dev;
1392
1393         gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1394         gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1395
1396         active = gerror ^ gerrorn;
1397         if (!(active & GERROR_ERR_MASK))
1398                 return IRQ_NONE; /* No errors pending */
1399
1400         dev_warn(smmu->dev,
1401                  "unexpected global error reported (0x%08x), this could be serious\n",
1402                  active);
1403
1404         if (active & GERROR_SFM_ERR) {
1405                 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1406                 arm_smmu_device_disable(smmu);
1407         }
1408
1409         if (active & GERROR_MSI_GERROR_ABT_ERR)
1410                 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1411
1412         if (active & GERROR_MSI_PRIQ_ABT_ERR)
1413                 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1414
1415         if (active & GERROR_MSI_EVTQ_ABT_ERR)
1416                 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1417
1418         if (active & GERROR_MSI_CMDQ_ABT_ERR)
1419                 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1420
1421         if (active & GERROR_PRIQ_ABT_ERR)
1422                 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1423
1424         if (active & GERROR_EVTQ_ABT_ERR)
1425                 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1426
1427         if (active & GERROR_CMDQ_ERR)
1428                 arm_smmu_cmdq_skip_err(smmu);
1429
1430         writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1431         return IRQ_HANDLED;
1432 }
1433
1434 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1435 {
1436         struct arm_smmu_device *smmu = dev;
1437
1438         arm_smmu_evtq_thread(irq, dev);
1439         if (smmu->features & ARM_SMMU_FEAT_PRI)
1440                 arm_smmu_priq_thread(irq, dev);
1441
1442         return IRQ_HANDLED;
1443 }
1444
1445 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1446 {
1447         arm_smmu_gerror_handler(irq, dev);
1448         return IRQ_WAKE_THREAD;
1449 }
1450
1451 static void
1452 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1453                         struct arm_smmu_cmdq_ent *cmd)
1454 {
1455         size_t log2_span;
1456         size_t span_mask;
1457         /* ATC invalidates are always on 4096-bytes pages */
1458         size_t inval_grain_shift = 12;
1459         unsigned long page_start, page_end;
1460
1461         *cmd = (struct arm_smmu_cmdq_ent) {
1462                 .opcode                 = CMDQ_OP_ATC_INV,
1463                 .substream_valid        = !!ssid,
1464                 .atc.ssid               = ssid,
1465         };
1466
1467         if (!size) {
1468                 cmd->atc.size = ATC_INV_SIZE_ALL;
1469                 return;
1470         }
1471
1472         page_start      = iova >> inval_grain_shift;
1473         page_end        = (iova + size - 1) >> inval_grain_shift;
1474
1475         /*
1476          * In an ATS Invalidate Request, the address must be aligned on the
1477          * range size, which must be a power of two number of page sizes. We
1478          * thus have to choose between grossly over-invalidating the region, or
1479          * splitting the invalidation into multiple commands. For simplicity
1480          * we'll go with the first solution, but should refine it in the future
1481          * if multiple commands are shown to be more efficient.
1482          *
1483          * Find the smallest power of two that covers the range. The most
1484          * significant differing bit between the start and end addresses,
1485          * fls(start ^ end), indicates the required span. For example:
1486          *
1487          * We want to invalidate pages [8; 11]. This is already the ideal range:
1488          *              x = 0b1000 ^ 0b1011 = 0b11
1489          *              span = 1 << fls(x) = 4
1490          *
1491          * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1492          *              x = 0b0111 ^ 0b1010 = 0b1101
1493          *              span = 1 << fls(x) = 16
1494          */
1495         log2_span       = fls_long(page_start ^ page_end);
1496         span_mask       = (1ULL << log2_span) - 1;
1497
1498         page_start      &= ~span_mask;
1499
1500         cmd->atc.addr   = page_start << inval_grain_shift;
1501         cmd->atc.size   = log2_span;
1502 }
1503
1504 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
1505                                    struct arm_smmu_cmdq_ent *cmd)
1506 {
1507         int i;
1508
1509         if (!master->ats_enabled)
1510                 return 0;
1511
1512         for (i = 0; i < master->num_sids; i++) {
1513                 cmd->atc.sid = master->sids[i];
1514                 arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
1515         }
1516
1517         return arm_smmu_cmdq_issue_sync(master->smmu);
1518 }
1519
1520 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
1521                                    int ssid, unsigned long iova, size_t size)
1522 {
1523         int ret = 0;
1524         unsigned long flags;
1525         struct arm_smmu_cmdq_ent cmd;
1526         struct arm_smmu_master *master;
1527
1528         if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1529                 return 0;
1530
1531         arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1532
1533         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1534         list_for_each_entry(master, &smmu_domain->devices, domain_head)
1535                 ret |= arm_smmu_atc_inv_master(master, &cmd);
1536         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1537
1538         return ret ? -ETIMEDOUT : 0;
1539 }
1540
1541 /* IO_PGTABLE API */
1542 static void arm_smmu_tlb_sync(void *cookie)
1543 {
1544         struct arm_smmu_domain *smmu_domain = cookie;
1545
1546         arm_smmu_cmdq_issue_sync(smmu_domain->smmu);
1547 }
1548
1549 static void arm_smmu_tlb_inv_context(void *cookie)
1550 {
1551         struct arm_smmu_domain *smmu_domain = cookie;
1552         struct arm_smmu_device *smmu = smmu_domain->smmu;
1553         struct arm_smmu_cmdq_ent cmd;
1554
1555         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1556                 cmd.opcode      = CMDQ_OP_TLBI_NH_ASID;
1557                 cmd.tlbi.asid   = smmu_domain->s1_cfg.cd.asid;
1558                 cmd.tlbi.vmid   = 0;
1559         } else {
1560                 cmd.opcode      = CMDQ_OP_TLBI_S12_VMALL;
1561                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1562         }
1563
1564         /*
1565          * NOTE: when io-pgtable is in non-strict mode, we may get here with
1566          * PTEs previously cleared by unmaps on the current CPU not yet visible
1567          * to the SMMU. We are relying on the DSB implicit in queue_inc_prod()
1568          * to guarantee those are observed before the TLBI. Do be careful, 007.
1569          */
1570         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1571         arm_smmu_cmdq_issue_sync(smmu);
1572 }
1573
1574 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1575                                           size_t granule, bool leaf, void *cookie)
1576 {
1577         struct arm_smmu_domain *smmu_domain = cookie;
1578         struct arm_smmu_device *smmu = smmu_domain->smmu;
1579         struct arm_smmu_cmdq_ent cmd = {
1580                 .tlbi = {
1581                         .leaf   = leaf,
1582                         .addr   = iova,
1583                 },
1584         };
1585
1586         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1587                 cmd.opcode      = CMDQ_OP_TLBI_NH_VA;
1588                 cmd.tlbi.asid   = smmu_domain->s1_cfg.cd.asid;
1589         } else {
1590                 cmd.opcode      = CMDQ_OP_TLBI_S2_IPA;
1591                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1592         }
1593
1594         do {
1595                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1596                 cmd.tlbi.addr += granule;
1597         } while (size -= granule);
1598 }
1599
1600 static const struct iommu_gather_ops arm_smmu_gather_ops = {
1601         .tlb_flush_all  = arm_smmu_tlb_inv_context,
1602         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
1603         .tlb_sync       = arm_smmu_tlb_sync,
1604 };
1605
1606 /* IOMMU API */
1607 static bool arm_smmu_capable(enum iommu_cap cap)
1608 {
1609         switch (cap) {
1610         case IOMMU_CAP_CACHE_COHERENCY:
1611                 return true;
1612         case IOMMU_CAP_NOEXEC:
1613                 return true;
1614         default:
1615                 return false;
1616         }
1617 }
1618
1619 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1620 {
1621         struct arm_smmu_domain *smmu_domain;
1622
1623         if (type != IOMMU_DOMAIN_UNMANAGED &&
1624             type != IOMMU_DOMAIN_DMA &&
1625             type != IOMMU_DOMAIN_IDENTITY)
1626                 return NULL;
1627
1628         /*
1629          * Allocate the domain and initialise some of its data structures.
1630          * We can't really do anything meaningful until we've added a
1631          * master.
1632          */
1633         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1634         if (!smmu_domain)
1635                 return NULL;
1636
1637         if (type == IOMMU_DOMAIN_DMA &&
1638             iommu_get_dma_cookie(&smmu_domain->domain)) {
1639                 kfree(smmu_domain);
1640                 return NULL;
1641         }
1642
1643         mutex_init(&smmu_domain->init_mutex);
1644         INIT_LIST_HEAD(&smmu_domain->devices);
1645         spin_lock_init(&smmu_domain->devices_lock);
1646
1647         return &smmu_domain->domain;
1648 }
1649
1650 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1651 {
1652         int idx, size = 1 << span;
1653
1654         do {
1655                 idx = find_first_zero_bit(map, size);
1656                 if (idx == size)
1657                         return -ENOSPC;
1658         } while (test_and_set_bit(idx, map));
1659
1660         return idx;
1661 }
1662
1663 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1664 {
1665         clear_bit(idx, map);
1666 }
1667
1668 static void arm_smmu_domain_free(struct iommu_domain *domain)
1669 {
1670         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1671         struct arm_smmu_device *smmu = smmu_domain->smmu;
1672
1673         iommu_put_dma_cookie(domain);
1674         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1675
1676         /* Free the CD and ASID, if we allocated them */
1677         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1678                 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1679
1680                 if (cfg->cdptr) {
1681                         dmam_free_coherent(smmu_domain->smmu->dev,
1682                                            CTXDESC_CD_DWORDS << 3,
1683                                            cfg->cdptr,
1684                                            cfg->cdptr_dma);
1685
1686                         arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1687                 }
1688         } else {
1689                 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1690                 if (cfg->vmid)
1691                         arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1692         }
1693
1694         kfree(smmu_domain);
1695 }
1696
1697 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1698                                        struct io_pgtable_cfg *pgtbl_cfg)
1699 {
1700         int ret;
1701         int asid;
1702         struct arm_smmu_device *smmu = smmu_domain->smmu;
1703         struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1704
1705         asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1706         if (asid < 0)
1707                 return asid;
1708
1709         cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1710                                          &cfg->cdptr_dma,
1711                                          GFP_KERNEL | __GFP_ZERO);
1712         if (!cfg->cdptr) {
1713                 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1714                 ret = -ENOMEM;
1715                 goto out_free_asid;
1716         }
1717
1718         cfg->cd.asid    = (u16)asid;
1719         cfg->cd.ttbr    = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1720         cfg->cd.tcr     = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1721         cfg->cd.mair    = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1722         return 0;
1723
1724 out_free_asid:
1725         arm_smmu_bitmap_free(smmu->asid_map, asid);
1726         return ret;
1727 }
1728
1729 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1730                                        struct io_pgtable_cfg *pgtbl_cfg)
1731 {
1732         int vmid;
1733         struct arm_smmu_device *smmu = smmu_domain->smmu;
1734         struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1735
1736         vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1737         if (vmid < 0)
1738                 return vmid;
1739
1740         cfg->vmid       = (u16)vmid;
1741         cfg->vttbr      = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1742         cfg->vtcr       = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1743         return 0;
1744 }
1745
1746 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1747 {
1748         int ret;
1749         unsigned long ias, oas;
1750         enum io_pgtable_fmt fmt;
1751         struct io_pgtable_cfg pgtbl_cfg;
1752         struct io_pgtable_ops *pgtbl_ops;
1753         int (*finalise_stage_fn)(struct arm_smmu_domain *,
1754                                  struct io_pgtable_cfg *);
1755         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1756         struct arm_smmu_device *smmu = smmu_domain->smmu;
1757
1758         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
1759                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
1760                 return 0;
1761         }
1762
1763         /* Restrict the stage to what we can actually support */
1764         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1765                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1766         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1767                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1768
1769         switch (smmu_domain->stage) {
1770         case ARM_SMMU_DOMAIN_S1:
1771                 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
1772                 ias = min_t(unsigned long, ias, VA_BITS);
1773                 oas = smmu->ias;
1774                 fmt = ARM_64_LPAE_S1;
1775                 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1776                 break;
1777         case ARM_SMMU_DOMAIN_NESTED:
1778         case ARM_SMMU_DOMAIN_S2:
1779                 ias = smmu->ias;
1780                 oas = smmu->oas;
1781                 fmt = ARM_64_LPAE_S2;
1782                 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1783                 break;
1784         default:
1785                 return -EINVAL;
1786         }
1787
1788         pgtbl_cfg = (struct io_pgtable_cfg) {
1789                 .pgsize_bitmap  = smmu->pgsize_bitmap,
1790                 .ias            = ias,
1791                 .oas            = oas,
1792                 .tlb            = &arm_smmu_gather_ops,
1793                 .iommu_dev      = smmu->dev,
1794         };
1795
1796         if (smmu->features & ARM_SMMU_FEAT_COHERENCY)
1797                 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
1798
1799         if (smmu_domain->non_strict)
1800                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
1801
1802         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1803         if (!pgtbl_ops)
1804                 return -ENOMEM;
1805
1806         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1807         domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
1808         domain->geometry.force_aperture = true;
1809
1810         ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1811         if (ret < 0) {
1812                 free_io_pgtable_ops(pgtbl_ops);
1813                 return ret;
1814         }
1815
1816         smmu_domain->pgtbl_ops = pgtbl_ops;
1817         return 0;
1818 }
1819
1820 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1821 {
1822         __le64 *step;
1823         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1824
1825         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1826                 struct arm_smmu_strtab_l1_desc *l1_desc;
1827                 int idx;
1828
1829                 /* Two-level walk */
1830                 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1831                 l1_desc = &cfg->l1_desc[idx];
1832                 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1833                 step = &l1_desc->l2ptr[idx];
1834         } else {
1835                 /* Simple linear lookup */
1836                 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1837         }
1838
1839         return step;
1840 }
1841
1842 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
1843 {
1844         int i, j;
1845         struct arm_smmu_device *smmu = master->smmu;
1846
1847         for (i = 0; i < master->num_sids; ++i) {
1848                 u32 sid = master->sids[i];
1849                 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1850
1851                 /* Bridged PCI devices may end up with duplicated IDs */
1852                 for (j = 0; j < i; j++)
1853                         if (master->sids[j] == sid)
1854                                 break;
1855                 if (j < i)
1856                         continue;
1857
1858                 arm_smmu_write_strtab_ent(master, sid, step);
1859         }
1860 }
1861
1862 static int arm_smmu_enable_ats(struct arm_smmu_master *master)
1863 {
1864         int ret;
1865         size_t stu;
1866         struct pci_dev *pdev;
1867         struct arm_smmu_device *smmu = master->smmu;
1868         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
1869
1870         if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
1871             !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
1872                 return -ENXIO;
1873
1874         pdev = to_pci_dev(master->dev);
1875         if (pdev->untrusted)
1876                 return -EPERM;
1877
1878         /* Smallest Translation Unit: log2 of the smallest supported granule */
1879         stu = __ffs(smmu->pgsize_bitmap);
1880
1881         ret = pci_enable_ats(pdev, stu);
1882         if (ret)
1883                 return ret;
1884
1885         master->ats_enabled = true;
1886         return 0;
1887 }
1888
1889 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
1890 {
1891         if (!master->ats_enabled || !dev_is_pci(master->dev))
1892                 return;
1893
1894         pci_disable_ats(to_pci_dev(master->dev));
1895         master->ats_enabled = false;
1896 }
1897
1898 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
1899 {
1900         unsigned long flags;
1901         struct arm_smmu_domain *smmu_domain = master->domain;
1902
1903         if (!smmu_domain)
1904                 return;
1905
1906         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1907         list_del(&master->domain_head);
1908         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1909
1910         master->domain = NULL;
1911         arm_smmu_install_ste_for_dev(master);
1912
1913         /* Disabling ATS invalidates all ATC entries */
1914         arm_smmu_disable_ats(master);
1915 }
1916
1917 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1918 {
1919         int ret = 0;
1920         unsigned long flags;
1921         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1922         struct arm_smmu_device *smmu;
1923         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1924         struct arm_smmu_master *master;
1925
1926         if (!fwspec)
1927                 return -ENOENT;
1928
1929         master = fwspec->iommu_priv;
1930         smmu = master->smmu;
1931
1932         arm_smmu_detach_dev(master);
1933
1934         mutex_lock(&smmu_domain->init_mutex);
1935
1936         if (!smmu_domain->smmu) {
1937                 smmu_domain->smmu = smmu;
1938                 ret = arm_smmu_domain_finalise(domain);
1939                 if (ret) {
1940                         smmu_domain->smmu = NULL;
1941                         goto out_unlock;
1942                 }
1943         } else if (smmu_domain->smmu != smmu) {
1944                 dev_err(dev,
1945                         "cannot attach to SMMU %s (upstream of %s)\n",
1946                         dev_name(smmu_domain->smmu->dev),
1947                         dev_name(smmu->dev));
1948                 ret = -ENXIO;
1949                 goto out_unlock;
1950         }
1951
1952         master->domain = smmu_domain;
1953
1954         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1955         list_add(&master->domain_head, &smmu_domain->devices);
1956         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1957
1958         if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
1959                 arm_smmu_enable_ats(master);
1960
1961         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1962                 arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
1963
1964         arm_smmu_install_ste_for_dev(master);
1965 out_unlock:
1966         mutex_unlock(&smmu_domain->init_mutex);
1967         return ret;
1968 }
1969
1970 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1971                         phys_addr_t paddr, size_t size, int prot)
1972 {
1973         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1974
1975         if (!ops)
1976                 return -ENODEV;
1977
1978         return ops->map(ops, iova, paddr, size, prot);
1979 }
1980
1981 static size_t
1982 arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1983 {
1984         int ret;
1985         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1986         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1987
1988         if (!ops)
1989                 return 0;
1990
1991         ret = ops->unmap(ops, iova, size);
1992         if (ret && arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size))
1993                 return 0;
1994
1995         return ret;
1996 }
1997
1998 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1999 {
2000         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2001
2002         if (smmu_domain->smmu)
2003                 arm_smmu_tlb_inv_context(smmu_domain);
2004 }
2005
2006 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
2007 {
2008         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
2009
2010         if (smmu)
2011                 arm_smmu_cmdq_issue_sync(smmu);
2012 }
2013
2014 static phys_addr_t
2015 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2016 {
2017         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2018
2019         if (domain->type == IOMMU_DOMAIN_IDENTITY)
2020                 return iova;
2021
2022         if (!ops)
2023                 return 0;
2024
2025         return ops->iova_to_phys(ops, iova);
2026 }
2027
2028 static struct platform_driver arm_smmu_driver;
2029
2030 static int arm_smmu_match_node(struct device *dev, void *data)
2031 {
2032         return dev->fwnode == data;
2033 }
2034
2035 static
2036 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2037 {
2038         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
2039                                                 fwnode, arm_smmu_match_node);
2040         put_device(dev);
2041         return dev ? dev_get_drvdata(dev) : NULL;
2042 }
2043
2044 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2045 {
2046         unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2047
2048         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2049                 limit *= 1UL << STRTAB_SPLIT;
2050
2051         return sid < limit;
2052 }
2053
2054 static struct iommu_ops arm_smmu_ops;
2055
2056 static int arm_smmu_add_device(struct device *dev)
2057 {
2058         int i, ret;
2059         struct arm_smmu_device *smmu;
2060         struct arm_smmu_master *master;
2061         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2062         struct iommu_group *group;
2063
2064         if (!fwspec || fwspec->ops != &arm_smmu_ops)
2065                 return -ENODEV;
2066         /*
2067          * We _can_ actually withstand dodgy bus code re-calling add_device()
2068          * without an intervening remove_device()/of_xlate() sequence, but
2069          * we're not going to do so quietly...
2070          */
2071         if (WARN_ON_ONCE(fwspec->iommu_priv)) {
2072                 master = fwspec->iommu_priv;
2073                 smmu = master->smmu;
2074         } else {
2075                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2076                 if (!smmu)
2077                         return -ENODEV;
2078                 master = kzalloc(sizeof(*master), GFP_KERNEL);
2079                 if (!master)
2080                         return -ENOMEM;
2081
2082                 master->dev = dev;
2083                 master->smmu = smmu;
2084                 master->sids = fwspec->ids;
2085                 master->num_sids = fwspec->num_ids;
2086                 fwspec->iommu_priv = master;
2087         }
2088
2089         /* Check the SIDs are in range of the SMMU and our stream table */
2090         for (i = 0; i < master->num_sids; i++) {
2091                 u32 sid = master->sids[i];
2092
2093                 if (!arm_smmu_sid_in_range(smmu, sid))
2094                         return -ERANGE;
2095
2096                 /* Ensure l2 strtab is initialised */
2097                 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2098                         ret = arm_smmu_init_l2_strtab(smmu, sid);
2099                         if (ret)
2100                                 return ret;
2101                 }
2102         }
2103
2104         group = iommu_group_get_for_dev(dev);
2105         if (!IS_ERR(group)) {
2106                 iommu_group_put(group);
2107                 iommu_device_link(&smmu->iommu, dev);
2108         }
2109
2110         return PTR_ERR_OR_ZERO(group);
2111 }
2112
2113 static void arm_smmu_remove_device(struct device *dev)
2114 {
2115         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2116         struct arm_smmu_master *master;
2117         struct arm_smmu_device *smmu;
2118
2119         if (!fwspec || fwspec->ops != &arm_smmu_ops)
2120                 return;
2121
2122         master = fwspec->iommu_priv;
2123         smmu = master->smmu;
2124         arm_smmu_detach_dev(master);
2125         iommu_group_remove_device(dev);
2126         iommu_device_unlink(&smmu->iommu, dev);
2127         kfree(master);
2128         iommu_fwspec_free(dev);
2129 }
2130
2131 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2132 {
2133         struct iommu_group *group;
2134
2135         /*
2136          * We don't support devices sharing stream IDs other than PCI RID
2137          * aliases, since the necessary ID-to-device lookup becomes rather
2138          * impractical given a potential sparse 32-bit stream ID space.
2139          */
2140         if (dev_is_pci(dev))
2141                 group = pci_device_group(dev);
2142         else
2143                 group = generic_device_group(dev);
2144
2145         return group;
2146 }
2147
2148 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
2149                                     enum iommu_attr attr, void *data)
2150 {
2151         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2152
2153         switch (domain->type) {
2154         case IOMMU_DOMAIN_UNMANAGED:
2155                 switch (attr) {
2156                 case DOMAIN_ATTR_NESTING:
2157                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
2158                         return 0;
2159                 default:
2160                         return -ENODEV;
2161                 }
2162                 break;
2163         case IOMMU_DOMAIN_DMA:
2164                 switch (attr) {
2165                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2166                         *(int *)data = smmu_domain->non_strict;
2167                         return 0;
2168                 default:
2169                         return -ENODEV;
2170                 }
2171                 break;
2172         default:
2173                 return -EINVAL;
2174         }
2175 }
2176
2177 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
2178                                     enum iommu_attr attr, void *data)
2179 {
2180         int ret = 0;
2181         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2182
2183         mutex_lock(&smmu_domain->init_mutex);
2184
2185         switch (domain->type) {
2186         case IOMMU_DOMAIN_UNMANAGED:
2187                 switch (attr) {
2188                 case DOMAIN_ATTR_NESTING:
2189                         if (smmu_domain->smmu) {
2190                                 ret = -EPERM;
2191                                 goto out_unlock;
2192                         }
2193
2194                         if (*(int *)data)
2195                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2196                         else
2197                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2198                         break;
2199                 default:
2200                         ret = -ENODEV;
2201                 }
2202                 break;
2203         case IOMMU_DOMAIN_DMA:
2204                 switch(attr) {
2205                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2206                         smmu_domain->non_strict = *(int *)data;
2207                         break;
2208                 default:
2209                         ret = -ENODEV;
2210                 }
2211                 break;
2212         default:
2213                 ret = -EINVAL;
2214         }
2215
2216 out_unlock:
2217         mutex_unlock(&smmu_domain->init_mutex);
2218         return ret;
2219 }
2220
2221 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2222 {
2223         return iommu_fwspec_add_ids(dev, args->args, 1);
2224 }
2225
2226 static void arm_smmu_get_resv_regions(struct device *dev,
2227                                       struct list_head *head)
2228 {
2229         struct iommu_resv_region *region;
2230         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2231
2232         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2233                                          prot, IOMMU_RESV_SW_MSI);
2234         if (!region)
2235                 return;
2236
2237         list_add_tail(&region->list, head);
2238
2239         iommu_dma_get_resv_regions(dev, head);
2240 }
2241
2242 static void arm_smmu_put_resv_regions(struct device *dev,
2243                                       struct list_head *head)
2244 {
2245         struct iommu_resv_region *entry, *next;
2246
2247         list_for_each_entry_safe(entry, next, head, list)
2248                 kfree(entry);
2249 }
2250
2251 static struct iommu_ops arm_smmu_ops = {
2252         .capable                = arm_smmu_capable,
2253         .domain_alloc           = arm_smmu_domain_alloc,
2254         .domain_free            = arm_smmu_domain_free,
2255         .attach_dev             = arm_smmu_attach_dev,
2256         .map                    = arm_smmu_map,
2257         .unmap                  = arm_smmu_unmap,
2258         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
2259         .iotlb_sync             = arm_smmu_iotlb_sync,
2260         .iova_to_phys           = arm_smmu_iova_to_phys,
2261         .add_device             = arm_smmu_add_device,
2262         .remove_device          = arm_smmu_remove_device,
2263         .device_group           = arm_smmu_device_group,
2264         .domain_get_attr        = arm_smmu_domain_get_attr,
2265         .domain_set_attr        = arm_smmu_domain_set_attr,
2266         .of_xlate               = arm_smmu_of_xlate,
2267         .get_resv_regions       = arm_smmu_get_resv_regions,
2268         .put_resv_regions       = arm_smmu_put_resv_regions,
2269         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
2270 };
2271
2272 /* Probing and initialisation functions */
2273 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2274                                    struct arm_smmu_queue *q,
2275                                    unsigned long prod_off,
2276                                    unsigned long cons_off,
2277                                    size_t dwords, const char *name)
2278 {
2279         size_t qsz;
2280
2281         do {
2282                 qsz = ((1 << q->max_n_shift) * dwords) << 3;
2283                 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2284                                               GFP_KERNEL);
2285                 if (q->base || qsz < PAGE_SIZE)
2286                         break;
2287
2288                 q->max_n_shift--;
2289         } while (1);
2290
2291         if (!q->base) {
2292                 dev_err(smmu->dev,
2293                         "failed to allocate queue (0x%zx bytes) for %s\n",
2294                         qsz, name);
2295                 return -ENOMEM;
2296         }
2297
2298         if (!WARN_ON(q->base_dma & (qsz - 1))) {
2299                 dev_info(smmu->dev, "allocated %u entries for %s\n",
2300                          1 << q->max_n_shift, name);
2301         }
2302
2303         q->prod_reg     = arm_smmu_page1_fixup(prod_off, smmu);
2304         q->cons_reg     = arm_smmu_page1_fixup(cons_off, smmu);
2305         q->ent_dwords   = dwords;
2306
2307         q->q_base  = Q_BASE_RWA;
2308         q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2309         q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->max_n_shift);
2310
2311         q->prod = q->cons = 0;
2312         return 0;
2313 }
2314
2315 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2316 {
2317         int ret;
2318
2319         /* cmdq */
2320         spin_lock_init(&smmu->cmdq.lock);
2321         ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
2322                                       ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS,
2323                                       "cmdq");
2324         if (ret)
2325                 return ret;
2326
2327         /* evtq */
2328         ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
2329                                       ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS,
2330                                       "evtq");
2331         if (ret)
2332                 return ret;
2333
2334         /* priq */
2335         if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2336                 return 0;
2337
2338         return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2339                                        ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS,
2340                                        "priq");
2341 }
2342
2343 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2344 {
2345         unsigned int i;
2346         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2347         size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2348         void *strtab = smmu->strtab_cfg.strtab;
2349
2350         cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2351         if (!cfg->l1_desc) {
2352                 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2353                 return -ENOMEM;
2354         }
2355
2356         for (i = 0; i < cfg->num_l1_ents; ++i) {
2357                 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2358                 strtab += STRTAB_L1_DESC_DWORDS << 3;
2359         }
2360
2361         return 0;
2362 }
2363
2364 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2365 {
2366         void *strtab;
2367         u64 reg;
2368         u32 size, l1size;
2369         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2370
2371         /* Calculate the L1 size, capped to the SIDSIZE. */
2372         size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2373         size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2374         cfg->num_l1_ents = 1 << size;
2375
2376         size += STRTAB_SPLIT;
2377         if (size < smmu->sid_bits)
2378                 dev_warn(smmu->dev,
2379                          "2-level strtab only covers %u/%u bits of SID\n",
2380                          size, smmu->sid_bits);
2381
2382         l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2383         strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2384                                      GFP_KERNEL | __GFP_ZERO);
2385         if (!strtab) {
2386                 dev_err(smmu->dev,
2387                         "failed to allocate l1 stream table (%u bytes)\n",
2388                         size);
2389                 return -ENOMEM;
2390         }
2391         cfg->strtab = strtab;
2392
2393         /* Configure strtab_base_cfg for 2 levels */
2394         reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
2395         reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
2396         reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
2397         cfg->strtab_base_cfg = reg;
2398
2399         return arm_smmu_init_l1_strtab(smmu);
2400 }
2401
2402 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2403 {
2404         void *strtab;
2405         u64 reg;
2406         u32 size;
2407         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2408
2409         size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2410         strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2411                                      GFP_KERNEL | __GFP_ZERO);
2412         if (!strtab) {
2413                 dev_err(smmu->dev,
2414                         "failed to allocate linear stream table (%u bytes)\n",
2415                         size);
2416                 return -ENOMEM;
2417         }
2418         cfg->strtab = strtab;
2419         cfg->num_l1_ents = 1 << smmu->sid_bits;
2420
2421         /* Configure strtab_base_cfg for a linear table covering all SIDs */
2422         reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
2423         reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
2424         cfg->strtab_base_cfg = reg;
2425
2426         arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2427         return 0;
2428 }
2429
2430 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2431 {
2432         u64 reg;
2433         int ret;
2434
2435         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2436                 ret = arm_smmu_init_strtab_2lvl(smmu);
2437         else
2438                 ret = arm_smmu_init_strtab_linear(smmu);
2439
2440         if (ret)
2441                 return ret;
2442
2443         /* Set the strtab base address */
2444         reg  = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
2445         reg |= STRTAB_BASE_RA;
2446         smmu->strtab_cfg.strtab_base = reg;
2447
2448         /* Allocate the first VMID for stage-2 bypass STEs */
2449         set_bit(0, smmu->vmid_map);
2450         return 0;
2451 }
2452
2453 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2454 {
2455         int ret;
2456
2457         ret = arm_smmu_init_queues(smmu);
2458         if (ret)
2459                 return ret;
2460
2461         return arm_smmu_init_strtab(smmu);
2462 }
2463
2464 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2465                                    unsigned int reg_off, unsigned int ack_off)
2466 {
2467         u32 reg;
2468
2469         writel_relaxed(val, smmu->base + reg_off);
2470         return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2471                                           1, ARM_SMMU_POLL_TIMEOUT_US);
2472 }
2473
2474 /* GBPA is "special" */
2475 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2476 {
2477         int ret;
2478         u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2479
2480         ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2481                                          1, ARM_SMMU_POLL_TIMEOUT_US);
2482         if (ret)
2483                 return ret;
2484
2485         reg &= ~clr;
2486         reg |= set;
2487         writel_relaxed(reg | GBPA_UPDATE, gbpa);
2488         ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2489                                          1, ARM_SMMU_POLL_TIMEOUT_US);
2490
2491         if (ret)
2492                 dev_err(smmu->dev, "GBPA not responding to update\n");
2493         return ret;
2494 }
2495
2496 static void arm_smmu_free_msis(void *data)
2497 {
2498         struct device *dev = data;
2499         platform_msi_domain_free_irqs(dev);
2500 }
2501
2502 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2503 {
2504         phys_addr_t doorbell;
2505         struct device *dev = msi_desc_to_dev(desc);
2506         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2507         phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2508
2509         doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2510         doorbell &= MSI_CFG0_ADDR_MASK;
2511
2512         writeq_relaxed(doorbell, smmu->base + cfg[0]);
2513         writel_relaxed(msg->data, smmu->base + cfg[1]);
2514         writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2515 }
2516
2517 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2518 {
2519         struct msi_desc *desc;
2520         int ret, nvec = ARM_SMMU_MAX_MSIS;
2521         struct device *dev = smmu->dev;
2522
2523         /* Clear the MSI address regs */
2524         writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2525         writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2526
2527         if (smmu->features & ARM_SMMU_FEAT_PRI)
2528                 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2529         else
2530                 nvec--;
2531
2532         if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2533                 return;
2534
2535         if (!dev->msi_domain) {
2536                 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
2537                 return;
2538         }
2539
2540         /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2541         ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2542         if (ret) {
2543                 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
2544                 return;
2545         }
2546
2547         for_each_msi_entry(desc, dev) {
2548                 switch (desc->platform.msi_index) {
2549                 case EVTQ_MSI_INDEX:
2550                         smmu->evtq.q.irq = desc->irq;
2551                         break;
2552                 case GERROR_MSI_INDEX:
2553                         smmu->gerr_irq = desc->irq;
2554                         break;
2555                 case PRIQ_MSI_INDEX:
2556                         smmu->priq.q.irq = desc->irq;
2557                         break;
2558                 default:        /* Unknown */
2559                         continue;
2560                 }
2561         }
2562
2563         /* Add callback to free MSIs on teardown */
2564         devm_add_action(dev, arm_smmu_free_msis, dev);
2565 }
2566
2567 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
2568 {
2569         int irq, ret;
2570
2571         arm_smmu_setup_msis(smmu);
2572
2573         /* Request interrupt lines */
2574         irq = smmu->evtq.q.irq;
2575         if (irq) {
2576                 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2577                                                 arm_smmu_evtq_thread,
2578                                                 IRQF_ONESHOT,
2579                                                 "arm-smmu-v3-evtq", smmu);
2580                 if (ret < 0)
2581                         dev_warn(smmu->dev, "failed to enable evtq irq\n");
2582         } else {
2583                 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
2584         }
2585
2586         irq = smmu->gerr_irq;
2587         if (irq) {
2588                 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2589                                        0, "arm-smmu-v3-gerror", smmu);
2590                 if (ret < 0)
2591                         dev_warn(smmu->dev, "failed to enable gerror irq\n");
2592         } else {
2593                 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
2594         }
2595
2596         if (smmu->features & ARM_SMMU_FEAT_PRI) {
2597                 irq = smmu->priq.q.irq;
2598                 if (irq) {
2599                         ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2600                                                         arm_smmu_priq_thread,
2601                                                         IRQF_ONESHOT,
2602                                                         "arm-smmu-v3-priq",
2603                                                         smmu);
2604                         if (ret < 0)
2605                                 dev_warn(smmu->dev,
2606                                          "failed to enable priq irq\n");
2607                 } else {
2608                         dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
2609                 }
2610         }
2611 }
2612
2613 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2614 {
2615         int ret, irq;
2616         u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2617
2618         /* Disable IRQs first */
2619         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2620                                       ARM_SMMU_IRQ_CTRLACK);
2621         if (ret) {
2622                 dev_err(smmu->dev, "failed to disable irqs\n");
2623                 return ret;
2624         }
2625
2626         irq = smmu->combined_irq;
2627         if (irq) {
2628                 /*
2629                  * Cavium ThunderX2 implementation doesn't support unique irq
2630                  * lines. Use a single irq line for all the SMMUv3 interrupts.
2631                  */
2632                 ret = devm_request_threaded_irq(smmu->dev, irq,
2633                                         arm_smmu_combined_irq_handler,
2634                                         arm_smmu_combined_irq_thread,
2635                                         IRQF_ONESHOT,
2636                                         "arm-smmu-v3-combined-irq", smmu);
2637                 if (ret < 0)
2638                         dev_warn(smmu->dev, "failed to enable combined irq\n");
2639         } else
2640                 arm_smmu_setup_unique_irqs(smmu);
2641
2642         if (smmu->features & ARM_SMMU_FEAT_PRI)
2643                 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2644
2645         /* Enable interrupt generation on the SMMU */
2646         ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2647                                       ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2648         if (ret)
2649                 dev_warn(smmu->dev, "failed to enable irqs\n");
2650
2651         return 0;
2652 }
2653
2654 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2655 {
2656         int ret;
2657
2658         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2659         if (ret)
2660                 dev_err(smmu->dev, "failed to clear cr0\n");
2661
2662         return ret;
2663 }
2664
2665 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2666 {
2667         int ret;
2668         u32 reg, enables;
2669         struct arm_smmu_cmdq_ent cmd;
2670
2671         /* Clear CR0 and sync (disables SMMU and queue processing) */
2672         reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2673         if (reg & CR0_SMMUEN) {
2674                 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2675                 WARN_ON(is_kdump_kernel() && !disable_bypass);
2676                 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
2677         }
2678
2679         ret = arm_smmu_device_disable(smmu);
2680         if (ret)
2681                 return ret;
2682
2683         /* CR1 (table and queue memory attributes) */
2684         reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
2685               FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
2686               FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
2687               FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
2688               FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
2689               FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
2690         writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2691
2692         /* CR2 (random crap) */
2693         reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2694         writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2695
2696         /* Stream table */
2697         writeq_relaxed(smmu->strtab_cfg.strtab_base,
2698                        smmu->base + ARM_SMMU_STRTAB_BASE);
2699         writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2700                        smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2701
2702         /* Command queue */
2703         writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2704         writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2705         writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2706
2707         enables = CR0_CMDQEN;
2708         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2709                                       ARM_SMMU_CR0ACK);
2710         if (ret) {
2711                 dev_err(smmu->dev, "failed to enable command queue\n");
2712                 return ret;
2713         }
2714
2715         /* Invalidate any cached configuration */
2716         cmd.opcode = CMDQ_OP_CFGI_ALL;
2717         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2718         arm_smmu_cmdq_issue_sync(smmu);
2719
2720         /* Invalidate any stale TLB entries */
2721         if (smmu->features & ARM_SMMU_FEAT_HYP) {
2722                 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2723                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2724         }
2725
2726         cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2727         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2728         arm_smmu_cmdq_issue_sync(smmu);
2729
2730         /* Event queue */
2731         writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2732         writel_relaxed(smmu->evtq.q.prod,
2733                        arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
2734         writel_relaxed(smmu->evtq.q.cons,
2735                        arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
2736
2737         enables |= CR0_EVTQEN;
2738         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2739                                       ARM_SMMU_CR0ACK);
2740         if (ret) {
2741                 dev_err(smmu->dev, "failed to enable event queue\n");
2742                 return ret;
2743         }
2744
2745         /* PRI queue */
2746         if (smmu->features & ARM_SMMU_FEAT_PRI) {
2747                 writeq_relaxed(smmu->priq.q.q_base,
2748                                smmu->base + ARM_SMMU_PRIQ_BASE);
2749                 writel_relaxed(smmu->priq.q.prod,
2750                                arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
2751                 writel_relaxed(smmu->priq.q.cons,
2752                                arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
2753
2754                 enables |= CR0_PRIQEN;
2755                 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2756                                               ARM_SMMU_CR0ACK);
2757                 if (ret) {
2758                         dev_err(smmu->dev, "failed to enable PRI queue\n");
2759                         return ret;
2760                 }
2761         }
2762
2763         if (smmu->features & ARM_SMMU_FEAT_ATS) {
2764                 enables |= CR0_ATSCHK;
2765                 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2766                                               ARM_SMMU_CR0ACK);
2767                 if (ret) {
2768                         dev_err(smmu->dev, "failed to enable ATS check\n");
2769                         return ret;
2770                 }
2771         }
2772
2773         ret = arm_smmu_setup_irqs(smmu);
2774         if (ret) {
2775                 dev_err(smmu->dev, "failed to setup irqs\n");
2776                 return ret;
2777         }
2778
2779         if (is_kdump_kernel())
2780                 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
2781
2782         /* Enable the SMMU interface, or ensure bypass */
2783         if (!bypass || disable_bypass) {
2784                 enables |= CR0_SMMUEN;
2785         } else {
2786                 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
2787                 if (ret)
2788                         return ret;
2789         }
2790         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2791                                       ARM_SMMU_CR0ACK);
2792         if (ret) {
2793                 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2794                 return ret;
2795         }
2796
2797         return 0;
2798 }
2799
2800 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
2801 {
2802         u32 reg;
2803         bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
2804
2805         /* IDR0 */
2806         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2807
2808         /* 2-level structures */
2809         if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
2810                 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2811
2812         if (reg & IDR0_CD2L)
2813                 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2814
2815         /*
2816          * Translation table endianness.
2817          * We currently require the same endianness as the CPU, but this
2818          * could be changed later by adding a new IO_PGTABLE_QUIRK.
2819          */
2820         switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
2821         case IDR0_TTENDIAN_MIXED:
2822                 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2823                 break;
2824 #ifdef __BIG_ENDIAN
2825         case IDR0_TTENDIAN_BE:
2826                 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2827                 break;
2828 #else
2829         case IDR0_TTENDIAN_LE:
2830                 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2831                 break;
2832 #endif
2833         default:
2834                 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2835                 return -ENXIO;
2836         }
2837
2838         /* Boolean feature flags */
2839         if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2840                 smmu->features |= ARM_SMMU_FEAT_PRI;
2841
2842         if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2843                 smmu->features |= ARM_SMMU_FEAT_ATS;
2844
2845         if (reg & IDR0_SEV)
2846                 smmu->features |= ARM_SMMU_FEAT_SEV;
2847
2848         if (reg & IDR0_MSI)
2849                 smmu->features |= ARM_SMMU_FEAT_MSI;
2850
2851         if (reg & IDR0_HYP)
2852                 smmu->features |= ARM_SMMU_FEAT_HYP;
2853
2854         /*
2855          * The coherency feature as set by FW is used in preference to the ID
2856          * register, but warn on mismatch.
2857          */
2858         if (!!(reg & IDR0_COHACC) != coherent)
2859                 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
2860                          coherent ? "true" : "false");
2861
2862         switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
2863         case IDR0_STALL_MODEL_FORCE:
2864                 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
2865                 /* Fallthrough */
2866         case IDR0_STALL_MODEL_STALL:
2867                 smmu->features |= ARM_SMMU_FEAT_STALLS;
2868         }
2869
2870         if (reg & IDR0_S1P)
2871                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2872
2873         if (reg & IDR0_S2P)
2874                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2875
2876         if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2877                 dev_err(smmu->dev, "no translation support!\n");
2878                 return -ENXIO;
2879         }
2880
2881         /* We only support the AArch64 table format at present */
2882         switch (FIELD_GET(IDR0_TTF, reg)) {
2883         case IDR0_TTF_AARCH32_64:
2884                 smmu->ias = 40;
2885                 /* Fallthrough */
2886         case IDR0_TTF_AARCH64:
2887                 break;
2888         default:
2889                 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2890                 return -ENXIO;
2891         }
2892
2893         /* ASID/VMID sizes */
2894         smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2895         smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2896
2897         /* IDR1 */
2898         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2899         if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2900                 dev_err(smmu->dev, "embedded implementation not supported\n");
2901                 return -ENXIO;
2902         }
2903
2904         /* Queue sizes, capped to ensure natural alignment */
2905         smmu->cmdq.q.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
2906                                          FIELD_GET(IDR1_CMDQS, reg));
2907         if (!smmu->cmdq.q.max_n_shift) {
2908                 /* Odd alignment restrictions on the base, so ignore for now */
2909                 dev_err(smmu->dev, "unit-length command queue not supported\n");
2910                 return -ENXIO;
2911         }
2912
2913         smmu->evtq.q.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
2914                                          FIELD_GET(IDR1_EVTQS, reg));
2915         smmu->priq.q.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
2916                                          FIELD_GET(IDR1_PRIQS, reg));
2917
2918         /* SID/SSID sizes */
2919         smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
2920         smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
2921
2922         /*
2923          * If the SMMU supports fewer bits than would fill a single L2 stream
2924          * table, use a linear table instead.
2925          */
2926         if (smmu->sid_bits <= STRTAB_SPLIT)
2927                 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
2928
2929         /* IDR5 */
2930         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2931
2932         /* Maximum number of outstanding stalls */
2933         smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
2934
2935         /* Page sizes */
2936         if (reg & IDR5_GRAN64K)
2937                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2938         if (reg & IDR5_GRAN16K)
2939                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2940         if (reg & IDR5_GRAN4K)
2941                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2942
2943         /* Input address size */
2944         if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
2945                 smmu->features |= ARM_SMMU_FEAT_VAX;
2946
2947         /* Output address size */
2948         switch (FIELD_GET(IDR5_OAS, reg)) {
2949         case IDR5_OAS_32_BIT:
2950                 smmu->oas = 32;
2951                 break;
2952         case IDR5_OAS_36_BIT:
2953                 smmu->oas = 36;
2954                 break;
2955         case IDR5_OAS_40_BIT:
2956                 smmu->oas = 40;
2957                 break;
2958         case IDR5_OAS_42_BIT:
2959                 smmu->oas = 42;
2960                 break;
2961         case IDR5_OAS_44_BIT:
2962                 smmu->oas = 44;
2963                 break;
2964         case IDR5_OAS_52_BIT:
2965                 smmu->oas = 52;
2966                 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
2967                 break;
2968         default:
2969                 dev_info(smmu->dev,
2970                         "unknown output address size. Truncating to 48-bit\n");
2971                 /* Fallthrough */
2972         case IDR5_OAS_48_BIT:
2973                 smmu->oas = 48;
2974         }
2975
2976         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2977                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2978         else
2979                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2980
2981         /* Set the DMA mask for our table walker */
2982         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2983                 dev_warn(smmu->dev,
2984                          "failed to set DMA mask for table walker\n");
2985
2986         smmu->ias = max(smmu->ias, smmu->oas);
2987
2988         dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2989                  smmu->ias, smmu->oas, smmu->features);
2990         return 0;
2991 }
2992
2993 #ifdef CONFIG_ACPI
2994 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
2995 {
2996         switch (model) {
2997         case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
2998                 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
2999                 break;
3000         case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3001                 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3002                 break;
3003         }
3004
3005         dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3006 }
3007
3008 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3009                                       struct arm_smmu_device *smmu)
3010 {
3011         struct acpi_iort_smmu_v3 *iort_smmu;
3012         struct device *dev = smmu->dev;
3013         struct acpi_iort_node *node;
3014
3015         node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3016
3017         /* Retrieve SMMUv3 specific data */
3018         iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3019
3020         acpi_smmu_get_options(iort_smmu->model, smmu);
3021
3022         if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3023                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3024
3025         return 0;
3026 }
3027 #else
3028 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3029                                              struct arm_smmu_device *smmu)
3030 {
3031         return -ENODEV;
3032 }
3033 #endif
3034
3035 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3036                                     struct arm_smmu_device *smmu)
3037 {
3038         struct device *dev = &pdev->dev;
3039         u32 cells;
3040         int ret = -EINVAL;
3041
3042         if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3043                 dev_err(dev, "missing #iommu-cells property\n");
3044         else if (cells != 1)
3045                 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3046         else
3047                 ret = 0;
3048
3049         parse_driver_options(smmu);
3050
3051         if (of_dma_is_coherent(dev->of_node))
3052                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3053
3054         return ret;
3055 }
3056
3057 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3058 {
3059         if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3060                 return SZ_64K;
3061         else
3062                 return SZ_128K;
3063 }
3064
3065 static int arm_smmu_device_probe(struct platform_device *pdev)
3066 {
3067         int irq, ret;
3068         struct resource *res;
3069         resource_size_t ioaddr;
3070         struct arm_smmu_device *smmu;
3071         struct device *dev = &pdev->dev;
3072         bool bypass;
3073
3074         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3075         if (!smmu) {
3076                 dev_err(dev, "failed to allocate arm_smmu_device\n");
3077                 return -ENOMEM;
3078         }
3079         smmu->dev = dev;
3080
3081         if (dev->of_node) {
3082                 ret = arm_smmu_device_dt_probe(pdev, smmu);
3083         } else {
3084                 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3085                 if (ret == -ENODEV)
3086                         return ret;
3087         }
3088
3089         /* Set bypass mode according to firmware probing result */
3090         bypass = !!ret;
3091
3092         /* Base address */
3093         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3094         if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {
3095                 dev_err(dev, "MMIO region too small (%pr)\n", res);
3096                 return -EINVAL;
3097         }
3098         ioaddr = res->start;
3099
3100         smmu->base = devm_ioremap_resource(dev, res);
3101         if (IS_ERR(smmu->base))
3102                 return PTR_ERR(smmu->base);
3103
3104         /* Interrupt lines */
3105
3106         irq = platform_get_irq_byname(pdev, "combined");
3107         if (irq > 0)
3108                 smmu->combined_irq = irq;
3109         else {
3110                 irq = platform_get_irq_byname(pdev, "eventq");
3111                 if (irq > 0)
3112                         smmu->evtq.q.irq = irq;
3113
3114                 irq = platform_get_irq_byname(pdev, "priq");
3115                 if (irq > 0)
3116                         smmu->priq.q.irq = irq;
3117
3118                 irq = platform_get_irq_byname(pdev, "gerror");
3119                 if (irq > 0)
3120                         smmu->gerr_irq = irq;
3121         }
3122         /* Probe the h/w */
3123         ret = arm_smmu_device_hw_probe(smmu);
3124         if (ret)
3125                 return ret;
3126
3127         /* Initialise in-memory data structures */
3128         ret = arm_smmu_init_structures(smmu);
3129         if (ret)
3130                 return ret;
3131
3132         /* Record our private device structure */
3133         platform_set_drvdata(pdev, smmu);
3134
3135         /* Reset the device */
3136         ret = arm_smmu_device_reset(smmu, bypass);
3137         if (ret)
3138                 return ret;
3139
3140         /* And we're up. Go go go! */
3141         ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3142                                      "smmu3.%pa", &ioaddr);
3143         if (ret)
3144                 return ret;
3145
3146         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
3147         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
3148
3149         ret = iommu_device_register(&smmu->iommu);
3150         if (ret) {
3151                 dev_err(dev, "Failed to register iommu\n");
3152                 return ret;
3153         }
3154
3155 #ifdef CONFIG_PCI
3156         if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
3157                 pci_request_acs();
3158                 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
3159                 if (ret)
3160                         return ret;
3161         }
3162 #endif
3163 #ifdef CONFIG_ARM_AMBA
3164         if (amba_bustype.iommu_ops != &arm_smmu_ops) {
3165                 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
3166                 if (ret)
3167                         return ret;
3168         }
3169 #endif
3170         if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
3171                 ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
3172                 if (ret)
3173                         return ret;
3174         }
3175         return 0;
3176 }
3177
3178 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3179 {
3180         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3181
3182         arm_smmu_device_disable(smmu);
3183 }
3184
3185 static const struct of_device_id arm_smmu_of_match[] = {
3186         { .compatible = "arm,smmu-v3", },
3187         { },
3188 };
3189
3190 static struct platform_driver arm_smmu_driver = {
3191         .driver = {
3192                 .name           = "arm-smmu-v3",
3193                 .of_match_table = of_match_ptr(arm_smmu_of_match),
3194                 .suppress_bind_attrs = true,
3195         },
3196         .probe  = arm_smmu_device_probe,
3197         .shutdown = arm_smmu_device_shutdown,
3198 };
3199 builtin_platform_driver(arm_smmu_driver);