Merge tag 'iommu-updates-v4.16' of git://git.kernel.org/pub/scm/linux/kernel/git...
[sfrench/cifs-2.6.git] / drivers / iommu / arm-smmu-v3.c
1 /*
2  * IOMMU API for ARM architected SMMUv3 implementations.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Copyright (C) 2015 ARM Limited
17  *
18  * Author: Will Deacon <will.deacon@arm.com>
19  *
20  * This driver is powered by bad coffee and bombay mix.
21  */
22
23 #include <linux/acpi.h>
24 #include <linux/acpi_iort.h>
25 #include <linux/delay.h>
26 #include <linux/dma-iommu.h>
27 #include <linux/err.h>
28 #include <linux/interrupt.h>
29 #include <linux/iommu.h>
30 #include <linux/iopoll.h>
31 #include <linux/module.h>
32 #include <linux/msi.h>
33 #include <linux/of.h>
34 #include <linux/of_address.h>
35 #include <linux/of_iommu.h>
36 #include <linux/of_platform.h>
37 #include <linux/pci.h>
38 #include <linux/platform_device.h>
39
40 #include <linux/amba/bus.h>
41
42 #include "io-pgtable.h"
43
44 /* MMIO registers */
45 #define ARM_SMMU_IDR0                   0x0
46 #define IDR0_ST_LVL_SHIFT               27
47 #define IDR0_ST_LVL_MASK                0x3
48 #define IDR0_ST_LVL_2LVL                (1 << IDR0_ST_LVL_SHIFT)
49 #define IDR0_STALL_MODEL_SHIFT          24
50 #define IDR0_STALL_MODEL_MASK           0x3
51 #define IDR0_STALL_MODEL_STALL          (0 << IDR0_STALL_MODEL_SHIFT)
52 #define IDR0_STALL_MODEL_FORCE          (2 << IDR0_STALL_MODEL_SHIFT)
53 #define IDR0_TTENDIAN_SHIFT             21
54 #define IDR0_TTENDIAN_MASK              0x3
55 #define IDR0_TTENDIAN_LE                (2 << IDR0_TTENDIAN_SHIFT)
56 #define IDR0_TTENDIAN_BE                (3 << IDR0_TTENDIAN_SHIFT)
57 #define IDR0_TTENDIAN_MIXED             (0 << IDR0_TTENDIAN_SHIFT)
58 #define IDR0_CD2L                       (1 << 19)
59 #define IDR0_VMID16                     (1 << 18)
60 #define IDR0_PRI                        (1 << 16)
61 #define IDR0_SEV                        (1 << 14)
62 #define IDR0_MSI                        (1 << 13)
63 #define IDR0_ASID16                     (1 << 12)
64 #define IDR0_ATS                        (1 << 10)
65 #define IDR0_HYP                        (1 << 9)
66 #define IDR0_COHACC                     (1 << 4)
67 #define IDR0_TTF_SHIFT                  2
68 #define IDR0_TTF_MASK                   0x3
69 #define IDR0_TTF_AARCH64                (2 << IDR0_TTF_SHIFT)
70 #define IDR0_TTF_AARCH32_64             (3 << IDR0_TTF_SHIFT)
71 #define IDR0_S1P                        (1 << 1)
72 #define IDR0_S2P                        (1 << 0)
73
74 #define ARM_SMMU_IDR1                   0x4
75 #define IDR1_TABLES_PRESET              (1 << 30)
76 #define IDR1_QUEUES_PRESET              (1 << 29)
77 #define IDR1_REL                        (1 << 28)
78 #define IDR1_CMDQ_SHIFT                 21
79 #define IDR1_CMDQ_MASK                  0x1f
80 #define IDR1_EVTQ_SHIFT                 16
81 #define IDR1_EVTQ_MASK                  0x1f
82 #define IDR1_PRIQ_SHIFT                 11
83 #define IDR1_PRIQ_MASK                  0x1f
84 #define IDR1_SSID_SHIFT                 6
85 #define IDR1_SSID_MASK                  0x1f
86 #define IDR1_SID_SHIFT                  0
87 #define IDR1_SID_MASK                   0x3f
88
89 #define ARM_SMMU_IDR5                   0x14
90 #define IDR5_STALL_MAX_SHIFT            16
91 #define IDR5_STALL_MAX_MASK             0xffff
92 #define IDR5_GRAN64K                    (1 << 6)
93 #define IDR5_GRAN16K                    (1 << 5)
94 #define IDR5_GRAN4K                     (1 << 4)
95 #define IDR5_OAS_SHIFT                  0
96 #define IDR5_OAS_MASK                   0x7
97 #define IDR5_OAS_32_BIT                 (0 << IDR5_OAS_SHIFT)
98 #define IDR5_OAS_36_BIT                 (1 << IDR5_OAS_SHIFT)
99 #define IDR5_OAS_40_BIT                 (2 << IDR5_OAS_SHIFT)
100 #define IDR5_OAS_42_BIT                 (3 << IDR5_OAS_SHIFT)
101 #define IDR5_OAS_44_BIT                 (4 << IDR5_OAS_SHIFT)
102 #define IDR5_OAS_48_BIT                 (5 << IDR5_OAS_SHIFT)
103
104 #define ARM_SMMU_CR0                    0x20
105 #define CR0_CMDQEN                      (1 << 3)
106 #define CR0_EVTQEN                      (1 << 2)
107 #define CR0_PRIQEN                      (1 << 1)
108 #define CR0_SMMUEN                      (1 << 0)
109
110 #define ARM_SMMU_CR0ACK                 0x24
111
112 #define ARM_SMMU_CR1                    0x28
113 #define CR1_SH_NSH                      0
114 #define CR1_SH_OSH                      2
115 #define CR1_SH_ISH                      3
116 #define CR1_CACHE_NC                    0
117 #define CR1_CACHE_WB                    1
118 #define CR1_CACHE_WT                    2
119 #define CR1_TABLE_SH_SHIFT              10
120 #define CR1_TABLE_OC_SHIFT              8
121 #define CR1_TABLE_IC_SHIFT              6
122 #define CR1_QUEUE_SH_SHIFT              4
123 #define CR1_QUEUE_OC_SHIFT              2
124 #define CR1_QUEUE_IC_SHIFT              0
125
126 #define ARM_SMMU_CR2                    0x2c
127 #define CR2_PTM                         (1 << 2)
128 #define CR2_RECINVSID                   (1 << 1)
129 #define CR2_E2H                         (1 << 0)
130
131 #define ARM_SMMU_GBPA                   0x44
132 #define GBPA_ABORT                      (1 << 20)
133 #define GBPA_UPDATE                     (1 << 31)
134
135 #define ARM_SMMU_IRQ_CTRL               0x50
136 #define IRQ_CTRL_EVTQ_IRQEN             (1 << 2)
137 #define IRQ_CTRL_PRIQ_IRQEN             (1 << 1)
138 #define IRQ_CTRL_GERROR_IRQEN           (1 << 0)
139
140 #define ARM_SMMU_IRQ_CTRLACK            0x54
141
142 #define ARM_SMMU_GERROR                 0x60
143 #define GERROR_SFM_ERR                  (1 << 8)
144 #define GERROR_MSI_GERROR_ABT_ERR       (1 << 7)
145 #define GERROR_MSI_PRIQ_ABT_ERR         (1 << 6)
146 #define GERROR_MSI_EVTQ_ABT_ERR         (1 << 5)
147 #define GERROR_MSI_CMDQ_ABT_ERR         (1 << 4)
148 #define GERROR_PRIQ_ABT_ERR             (1 << 3)
149 #define GERROR_EVTQ_ABT_ERR             (1 << 2)
150 #define GERROR_CMDQ_ERR                 (1 << 0)
151 #define GERROR_ERR_MASK                 0xfd
152
153 #define ARM_SMMU_GERRORN                0x64
154
155 #define ARM_SMMU_GERROR_IRQ_CFG0        0x68
156 #define ARM_SMMU_GERROR_IRQ_CFG1        0x70
157 #define ARM_SMMU_GERROR_IRQ_CFG2        0x74
158
159 #define ARM_SMMU_STRTAB_BASE            0x80
160 #define STRTAB_BASE_RA                  (1UL << 62)
161 #define STRTAB_BASE_ADDR_SHIFT          6
162 #define STRTAB_BASE_ADDR_MASK           0x3ffffffffffUL
163
164 #define ARM_SMMU_STRTAB_BASE_CFG        0x88
165 #define STRTAB_BASE_CFG_LOG2SIZE_SHIFT  0
166 #define STRTAB_BASE_CFG_LOG2SIZE_MASK   0x3f
167 #define STRTAB_BASE_CFG_SPLIT_SHIFT     6
168 #define STRTAB_BASE_CFG_SPLIT_MASK      0x1f
169 #define STRTAB_BASE_CFG_FMT_SHIFT       16
170 #define STRTAB_BASE_CFG_FMT_MASK        0x3
171 #define STRTAB_BASE_CFG_FMT_LINEAR      (0 << STRTAB_BASE_CFG_FMT_SHIFT)
172 #define STRTAB_BASE_CFG_FMT_2LVL        (1 << STRTAB_BASE_CFG_FMT_SHIFT)
173
174 #define ARM_SMMU_CMDQ_BASE              0x90
175 #define ARM_SMMU_CMDQ_PROD              0x98
176 #define ARM_SMMU_CMDQ_CONS              0x9c
177
178 #define ARM_SMMU_EVTQ_BASE              0xa0
179 #define ARM_SMMU_EVTQ_PROD              0x100a8
180 #define ARM_SMMU_EVTQ_CONS              0x100ac
181 #define ARM_SMMU_EVTQ_IRQ_CFG0          0xb0
182 #define ARM_SMMU_EVTQ_IRQ_CFG1          0xb8
183 #define ARM_SMMU_EVTQ_IRQ_CFG2          0xbc
184
185 #define ARM_SMMU_PRIQ_BASE              0xc0
186 #define ARM_SMMU_PRIQ_PROD              0x100c8
187 #define ARM_SMMU_PRIQ_CONS              0x100cc
188 #define ARM_SMMU_PRIQ_IRQ_CFG0          0xd0
189 #define ARM_SMMU_PRIQ_IRQ_CFG1          0xd8
190 #define ARM_SMMU_PRIQ_IRQ_CFG2          0xdc
191
192 /* Common MSI config fields */
193 #define MSI_CFG0_ADDR_SHIFT             2
194 #define MSI_CFG0_ADDR_MASK              0x3fffffffffffUL
195 #define MSI_CFG2_SH_SHIFT               4
196 #define MSI_CFG2_SH_NSH                 (0UL << MSI_CFG2_SH_SHIFT)
197 #define MSI_CFG2_SH_OSH                 (2UL << MSI_CFG2_SH_SHIFT)
198 #define MSI_CFG2_SH_ISH                 (3UL << MSI_CFG2_SH_SHIFT)
199 #define MSI_CFG2_MEMATTR_SHIFT          0
200 #define MSI_CFG2_MEMATTR_DEVICE_nGnRE   (0x1 << MSI_CFG2_MEMATTR_SHIFT)
201
202 #define Q_IDX(q, p)                     ((p) & ((1 << (q)->max_n_shift) - 1))
203 #define Q_WRP(q, p)                     ((p) & (1 << (q)->max_n_shift))
204 #define Q_OVERFLOW_FLAG                 (1 << 31)
205 #define Q_OVF(q, p)                     ((p) & Q_OVERFLOW_FLAG)
206 #define Q_ENT(q, p)                     ((q)->base +                    \
207                                          Q_IDX(q, p) * (q)->ent_dwords)
208
209 #define Q_BASE_RWA                      (1UL << 62)
210 #define Q_BASE_ADDR_SHIFT               5
211 #define Q_BASE_ADDR_MASK                0xfffffffffffUL
212 #define Q_BASE_LOG2SIZE_SHIFT           0
213 #define Q_BASE_LOG2SIZE_MASK            0x1fUL
214
215 /*
216  * Stream table.
217  *
218  * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
219  * 2lvl: 128k L1 entries,
220  *       256 lazy entries per table (each table covers a PCI bus)
221  */
222 #define STRTAB_L1_SZ_SHIFT              20
223 #define STRTAB_SPLIT                    8
224
225 #define STRTAB_L1_DESC_DWORDS           1
226 #define STRTAB_L1_DESC_SPAN_SHIFT       0
227 #define STRTAB_L1_DESC_SPAN_MASK        0x1fUL
228 #define STRTAB_L1_DESC_L2PTR_SHIFT      6
229 #define STRTAB_L1_DESC_L2PTR_MASK       0x3ffffffffffUL
230
231 #define STRTAB_STE_DWORDS               8
232 #define STRTAB_STE_0_V                  (1UL << 0)
233 #define STRTAB_STE_0_CFG_SHIFT          1
234 #define STRTAB_STE_0_CFG_MASK           0x7UL
235 #define STRTAB_STE_0_CFG_ABORT          (0UL << STRTAB_STE_0_CFG_SHIFT)
236 #define STRTAB_STE_0_CFG_BYPASS         (4UL << STRTAB_STE_0_CFG_SHIFT)
237 #define STRTAB_STE_0_CFG_S1_TRANS       (5UL << STRTAB_STE_0_CFG_SHIFT)
238 #define STRTAB_STE_0_CFG_S2_TRANS       (6UL << STRTAB_STE_0_CFG_SHIFT)
239
240 #define STRTAB_STE_0_S1FMT_SHIFT        4
241 #define STRTAB_STE_0_S1FMT_LINEAR       (0UL << STRTAB_STE_0_S1FMT_SHIFT)
242 #define STRTAB_STE_0_S1CTXPTR_SHIFT     6
243 #define STRTAB_STE_0_S1CTXPTR_MASK      0x3ffffffffffUL
244 #define STRTAB_STE_0_S1CDMAX_SHIFT      59
245 #define STRTAB_STE_0_S1CDMAX_MASK       0x1fUL
246
247 #define STRTAB_STE_1_S1C_CACHE_NC       0UL
248 #define STRTAB_STE_1_S1C_CACHE_WBRA     1UL
249 #define STRTAB_STE_1_S1C_CACHE_WT       2UL
250 #define STRTAB_STE_1_S1C_CACHE_WB       3UL
251 #define STRTAB_STE_1_S1C_SH_NSH         0UL
252 #define STRTAB_STE_1_S1C_SH_OSH         2UL
253 #define STRTAB_STE_1_S1C_SH_ISH         3UL
254 #define STRTAB_STE_1_S1CIR_SHIFT        2
255 #define STRTAB_STE_1_S1COR_SHIFT        4
256 #define STRTAB_STE_1_S1CSH_SHIFT        6
257
258 #define STRTAB_STE_1_S1STALLD           (1UL << 27)
259
260 #define STRTAB_STE_1_EATS_ABT           0UL
261 #define STRTAB_STE_1_EATS_TRANS         1UL
262 #define STRTAB_STE_1_EATS_S1CHK         2UL
263 #define STRTAB_STE_1_EATS_SHIFT         28
264
265 #define STRTAB_STE_1_STRW_NSEL1         0UL
266 #define STRTAB_STE_1_STRW_EL2           2UL
267 #define STRTAB_STE_1_STRW_SHIFT         30
268
269 #define STRTAB_STE_1_SHCFG_INCOMING     1UL
270 #define STRTAB_STE_1_SHCFG_SHIFT        44
271
272 #define STRTAB_STE_2_S2VMID_SHIFT       0
273 #define STRTAB_STE_2_S2VMID_MASK        0xffffUL
274 #define STRTAB_STE_2_VTCR_SHIFT         32
275 #define STRTAB_STE_2_VTCR_MASK          0x7ffffUL
276 #define STRTAB_STE_2_S2AA64             (1UL << 51)
277 #define STRTAB_STE_2_S2ENDI             (1UL << 52)
278 #define STRTAB_STE_2_S2PTW              (1UL << 54)
279 #define STRTAB_STE_2_S2R                (1UL << 58)
280
281 #define STRTAB_STE_3_S2TTB_SHIFT        4
282 #define STRTAB_STE_3_S2TTB_MASK         0xfffffffffffUL
283
284 /* Context descriptor (stage-1 only) */
285 #define CTXDESC_CD_DWORDS               8
286 #define CTXDESC_CD_0_TCR_T0SZ_SHIFT     0
287 #define ARM64_TCR_T0SZ_SHIFT            0
288 #define ARM64_TCR_T0SZ_MASK             0x1fUL
289 #define CTXDESC_CD_0_TCR_TG0_SHIFT      6
290 #define ARM64_TCR_TG0_SHIFT             14
291 #define ARM64_TCR_TG0_MASK              0x3UL
292 #define CTXDESC_CD_0_TCR_IRGN0_SHIFT    8
293 #define ARM64_TCR_IRGN0_SHIFT           8
294 #define ARM64_TCR_IRGN0_MASK            0x3UL
295 #define CTXDESC_CD_0_TCR_ORGN0_SHIFT    10
296 #define ARM64_TCR_ORGN0_SHIFT           10
297 #define ARM64_TCR_ORGN0_MASK            0x3UL
298 #define CTXDESC_CD_0_TCR_SH0_SHIFT      12
299 #define ARM64_TCR_SH0_SHIFT             12
300 #define ARM64_TCR_SH0_MASK              0x3UL
301 #define CTXDESC_CD_0_TCR_EPD0_SHIFT     14
302 #define ARM64_TCR_EPD0_SHIFT            7
303 #define ARM64_TCR_EPD0_MASK             0x1UL
304 #define CTXDESC_CD_0_TCR_EPD1_SHIFT     30
305 #define ARM64_TCR_EPD1_SHIFT            23
306 #define ARM64_TCR_EPD1_MASK             0x1UL
307
308 #define CTXDESC_CD_0_ENDI               (1UL << 15)
309 #define CTXDESC_CD_0_V                  (1UL << 31)
310
311 #define CTXDESC_CD_0_TCR_IPS_SHIFT      32
312 #define ARM64_TCR_IPS_SHIFT             32
313 #define ARM64_TCR_IPS_MASK              0x7UL
314 #define CTXDESC_CD_0_TCR_TBI0_SHIFT     38
315 #define ARM64_TCR_TBI0_SHIFT            37
316 #define ARM64_TCR_TBI0_MASK             0x1UL
317
318 #define CTXDESC_CD_0_AA64               (1UL << 41)
319 #define CTXDESC_CD_0_S                  (1UL << 44)
320 #define CTXDESC_CD_0_R                  (1UL << 45)
321 #define CTXDESC_CD_0_A                  (1UL << 46)
322 #define CTXDESC_CD_0_ASET_SHIFT         47
323 #define CTXDESC_CD_0_ASET_SHARED        (0UL << CTXDESC_CD_0_ASET_SHIFT)
324 #define CTXDESC_CD_0_ASET_PRIVATE       (1UL << CTXDESC_CD_0_ASET_SHIFT)
325 #define CTXDESC_CD_0_ASID_SHIFT         48
326 #define CTXDESC_CD_0_ASID_MASK          0xffffUL
327
328 #define CTXDESC_CD_1_TTB0_SHIFT         4
329 #define CTXDESC_CD_1_TTB0_MASK          0xfffffffffffUL
330
331 #define CTXDESC_CD_3_MAIR_SHIFT         0
332
333 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
334 #define ARM_SMMU_TCR2CD(tcr, fld)                                       \
335         (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK)    \
336          << CTXDESC_CD_0_TCR_##fld##_SHIFT)
337
338 /* Command queue */
339 #define CMDQ_ENT_DWORDS                 2
340 #define CMDQ_MAX_SZ_SHIFT               8
341
342 #define CMDQ_ERR_SHIFT                  24
343 #define CMDQ_ERR_MASK                   0x7f
344 #define CMDQ_ERR_CERROR_NONE_IDX        0
345 #define CMDQ_ERR_CERROR_ILL_IDX         1
346 #define CMDQ_ERR_CERROR_ABT_IDX         2
347
348 #define CMDQ_0_OP_SHIFT                 0
349 #define CMDQ_0_OP_MASK                  0xffUL
350 #define CMDQ_0_SSV                      (1UL << 11)
351
352 #define CMDQ_PREFETCH_0_SID_SHIFT       32
353 #define CMDQ_PREFETCH_1_SIZE_SHIFT      0
354 #define CMDQ_PREFETCH_1_ADDR_MASK       ~0xfffUL
355
356 #define CMDQ_CFGI_0_SID_SHIFT           32
357 #define CMDQ_CFGI_0_SID_MASK            0xffffffffUL
358 #define CMDQ_CFGI_1_LEAF                (1UL << 0)
359 #define CMDQ_CFGI_1_RANGE_SHIFT         0
360 #define CMDQ_CFGI_1_RANGE_MASK          0x1fUL
361
362 #define CMDQ_TLBI_0_VMID_SHIFT          32
363 #define CMDQ_TLBI_0_ASID_SHIFT          48
364 #define CMDQ_TLBI_1_LEAF                (1UL << 0)
365 #define CMDQ_TLBI_1_VA_MASK             ~0xfffUL
366 #define CMDQ_TLBI_1_IPA_MASK            0xfffffffff000UL
367
368 #define CMDQ_PRI_0_SSID_SHIFT           12
369 #define CMDQ_PRI_0_SSID_MASK            0xfffffUL
370 #define CMDQ_PRI_0_SID_SHIFT            32
371 #define CMDQ_PRI_0_SID_MASK             0xffffffffUL
372 #define CMDQ_PRI_1_GRPID_SHIFT          0
373 #define CMDQ_PRI_1_GRPID_MASK           0x1ffUL
374 #define CMDQ_PRI_1_RESP_SHIFT           12
375 #define CMDQ_PRI_1_RESP_DENY            (0UL << CMDQ_PRI_1_RESP_SHIFT)
376 #define CMDQ_PRI_1_RESP_FAIL            (1UL << CMDQ_PRI_1_RESP_SHIFT)
377 #define CMDQ_PRI_1_RESP_SUCC            (2UL << CMDQ_PRI_1_RESP_SHIFT)
378
379 #define CMDQ_SYNC_0_CS_SHIFT            12
380 #define CMDQ_SYNC_0_CS_NONE             (0UL << CMDQ_SYNC_0_CS_SHIFT)
381 #define CMDQ_SYNC_0_CS_IRQ              (1UL << CMDQ_SYNC_0_CS_SHIFT)
382 #define CMDQ_SYNC_0_CS_SEV              (2UL << CMDQ_SYNC_0_CS_SHIFT)
383 #define CMDQ_SYNC_0_MSH_SHIFT           22
384 #define CMDQ_SYNC_0_MSH_ISH             (3UL << CMDQ_SYNC_0_MSH_SHIFT)
385 #define CMDQ_SYNC_0_MSIATTR_SHIFT       24
386 #define CMDQ_SYNC_0_MSIATTR_OIWB        (0xfUL << CMDQ_SYNC_0_MSIATTR_SHIFT)
387 #define CMDQ_SYNC_0_MSIDATA_SHIFT       32
388 #define CMDQ_SYNC_0_MSIDATA_MASK        0xffffffffUL
389 #define CMDQ_SYNC_1_MSIADDR_SHIFT       0
390 #define CMDQ_SYNC_1_MSIADDR_MASK        0xffffffffffffcUL
391
392 /* Event queue */
393 #define EVTQ_ENT_DWORDS                 4
394 #define EVTQ_MAX_SZ_SHIFT               7
395
396 #define EVTQ_0_ID_SHIFT                 0
397 #define EVTQ_0_ID_MASK                  0xffUL
398
399 /* PRI queue */
400 #define PRIQ_ENT_DWORDS                 2
401 #define PRIQ_MAX_SZ_SHIFT               8
402
403 #define PRIQ_0_SID_SHIFT                0
404 #define PRIQ_0_SID_MASK                 0xffffffffUL
405 #define PRIQ_0_SSID_SHIFT               32
406 #define PRIQ_0_SSID_MASK                0xfffffUL
407 #define PRIQ_0_PERM_PRIV                (1UL << 58)
408 #define PRIQ_0_PERM_EXEC                (1UL << 59)
409 #define PRIQ_0_PERM_READ                (1UL << 60)
410 #define PRIQ_0_PERM_WRITE               (1UL << 61)
411 #define PRIQ_0_PRG_LAST                 (1UL << 62)
412 #define PRIQ_0_SSID_V                   (1UL << 63)
413
414 #define PRIQ_1_PRG_IDX_SHIFT            0
415 #define PRIQ_1_PRG_IDX_MASK             0x1ffUL
416 #define PRIQ_1_ADDR_SHIFT               12
417 #define PRIQ_1_ADDR_MASK                0xfffffffffffffUL
418
419 /* High-level queue structures */
420 #define ARM_SMMU_POLL_TIMEOUT_US        100
421 #define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US   1000000 /* 1s! */
422 #define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT   10
423
424 #define MSI_IOVA_BASE                   0x8000000
425 #define MSI_IOVA_LENGTH                 0x100000
426
427 static bool disable_bypass;
428 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
429 MODULE_PARM_DESC(disable_bypass,
430         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
431
432 enum pri_resp {
433         PRI_RESP_DENY,
434         PRI_RESP_FAIL,
435         PRI_RESP_SUCC,
436 };
437
438 enum arm_smmu_msi_index {
439         EVTQ_MSI_INDEX,
440         GERROR_MSI_INDEX,
441         PRIQ_MSI_INDEX,
442         ARM_SMMU_MAX_MSIS,
443 };
444
445 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
446         [EVTQ_MSI_INDEX] = {
447                 ARM_SMMU_EVTQ_IRQ_CFG0,
448                 ARM_SMMU_EVTQ_IRQ_CFG1,
449                 ARM_SMMU_EVTQ_IRQ_CFG2,
450         },
451         [GERROR_MSI_INDEX] = {
452                 ARM_SMMU_GERROR_IRQ_CFG0,
453                 ARM_SMMU_GERROR_IRQ_CFG1,
454                 ARM_SMMU_GERROR_IRQ_CFG2,
455         },
456         [PRIQ_MSI_INDEX] = {
457                 ARM_SMMU_PRIQ_IRQ_CFG0,
458                 ARM_SMMU_PRIQ_IRQ_CFG1,
459                 ARM_SMMU_PRIQ_IRQ_CFG2,
460         },
461 };
462
463 struct arm_smmu_cmdq_ent {
464         /* Common fields */
465         u8                              opcode;
466         bool                            substream_valid;
467
468         /* Command-specific fields */
469         union {
470                 #define CMDQ_OP_PREFETCH_CFG    0x1
471                 struct {
472                         u32                     sid;
473                         u8                      size;
474                         u64                     addr;
475                 } prefetch;
476
477                 #define CMDQ_OP_CFGI_STE        0x3
478                 #define CMDQ_OP_CFGI_ALL        0x4
479                 struct {
480                         u32                     sid;
481                         union {
482                                 bool            leaf;
483                                 u8              span;
484                         };
485                 } cfgi;
486
487                 #define CMDQ_OP_TLBI_NH_ASID    0x11
488                 #define CMDQ_OP_TLBI_NH_VA      0x12
489                 #define CMDQ_OP_TLBI_EL2_ALL    0x20
490                 #define CMDQ_OP_TLBI_S12_VMALL  0x28
491                 #define CMDQ_OP_TLBI_S2_IPA     0x2a
492                 #define CMDQ_OP_TLBI_NSNH_ALL   0x30
493                 struct {
494                         u16                     asid;
495                         u16                     vmid;
496                         bool                    leaf;
497                         u64                     addr;
498                 } tlbi;
499
500                 #define CMDQ_OP_PRI_RESP        0x41
501                 struct {
502                         u32                     sid;
503                         u32                     ssid;
504                         u16                     grpid;
505                         enum pri_resp           resp;
506                 } pri;
507
508                 #define CMDQ_OP_CMD_SYNC        0x46
509                 struct {
510                         u32                     msidata;
511                         u64                     msiaddr;
512                 } sync;
513         };
514 };
515
516 struct arm_smmu_queue {
517         int                             irq; /* Wired interrupt */
518
519         __le64                          *base;
520         dma_addr_t                      base_dma;
521         u64                             q_base;
522
523         size_t                          ent_dwords;
524         u32                             max_n_shift;
525         u32                             prod;
526         u32                             cons;
527
528         u32 __iomem                     *prod_reg;
529         u32 __iomem                     *cons_reg;
530 };
531
532 struct arm_smmu_cmdq {
533         struct arm_smmu_queue           q;
534         spinlock_t                      lock;
535 };
536
537 struct arm_smmu_evtq {
538         struct arm_smmu_queue           q;
539         u32                             max_stalls;
540 };
541
542 struct arm_smmu_priq {
543         struct arm_smmu_queue           q;
544 };
545
546 /* High-level stream table and context descriptor structures */
547 struct arm_smmu_strtab_l1_desc {
548         u8                              span;
549
550         __le64                          *l2ptr;
551         dma_addr_t                      l2ptr_dma;
552 };
553
554 struct arm_smmu_s1_cfg {
555         __le64                          *cdptr;
556         dma_addr_t                      cdptr_dma;
557
558         struct arm_smmu_ctx_desc {
559                 u16     asid;
560                 u64     ttbr;
561                 u64     tcr;
562                 u64     mair;
563         }                               cd;
564 };
565
566 struct arm_smmu_s2_cfg {
567         u16                             vmid;
568         u64                             vttbr;
569         u64                             vtcr;
570 };
571
572 struct arm_smmu_strtab_ent {
573         /*
574          * An STE is "assigned" if the master emitting the corresponding SID
575          * is attached to a domain. The behaviour of an unassigned STE is
576          * determined by the disable_bypass parameter, whereas an assigned
577          * STE behaves according to s1_cfg/s2_cfg, which themselves are
578          * configured according to the domain type.
579          */
580         bool                            assigned;
581         struct arm_smmu_s1_cfg          *s1_cfg;
582         struct arm_smmu_s2_cfg          *s2_cfg;
583 };
584
585 struct arm_smmu_strtab_cfg {
586         __le64                          *strtab;
587         dma_addr_t                      strtab_dma;
588         struct arm_smmu_strtab_l1_desc  *l1_desc;
589         unsigned int                    num_l1_ents;
590
591         u64                             strtab_base;
592         u32                             strtab_base_cfg;
593 };
594
595 /* An SMMUv3 instance */
596 struct arm_smmu_device {
597         struct device                   *dev;
598         void __iomem                    *base;
599
600 #define ARM_SMMU_FEAT_2_LVL_STRTAB      (1 << 0)
601 #define ARM_SMMU_FEAT_2_LVL_CDTAB       (1 << 1)
602 #define ARM_SMMU_FEAT_TT_LE             (1 << 2)
603 #define ARM_SMMU_FEAT_TT_BE             (1 << 3)
604 #define ARM_SMMU_FEAT_PRI               (1 << 4)
605 #define ARM_SMMU_FEAT_ATS               (1 << 5)
606 #define ARM_SMMU_FEAT_SEV               (1 << 6)
607 #define ARM_SMMU_FEAT_MSI               (1 << 7)
608 #define ARM_SMMU_FEAT_COHERENCY         (1 << 8)
609 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 9)
610 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 10)
611 #define ARM_SMMU_FEAT_STALLS            (1 << 11)
612 #define ARM_SMMU_FEAT_HYP               (1 << 12)
613 #define ARM_SMMU_FEAT_STALL_FORCE       (1 << 13)
614         u32                             features;
615
616 #define ARM_SMMU_OPT_SKIP_PREFETCH      (1 << 0)
617 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY    (1 << 1)
618         u32                             options;
619
620         struct arm_smmu_cmdq            cmdq;
621         struct arm_smmu_evtq            evtq;
622         struct arm_smmu_priq            priq;
623
624         int                             gerr_irq;
625         int                             combined_irq;
626         atomic_t                        sync_nr;
627
628         unsigned long                   ias; /* IPA */
629         unsigned long                   oas; /* PA */
630         unsigned long                   pgsize_bitmap;
631
632 #define ARM_SMMU_MAX_ASIDS              (1 << 16)
633         unsigned int                    asid_bits;
634         DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
635
636 #define ARM_SMMU_MAX_VMIDS              (1 << 16)
637         unsigned int                    vmid_bits;
638         DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
639
640         unsigned int                    ssid_bits;
641         unsigned int                    sid_bits;
642
643         struct arm_smmu_strtab_cfg      strtab_cfg;
644
645         u32                             sync_count;
646
647         /* IOMMU core code handle */
648         struct iommu_device             iommu;
649 };
650
651 /* SMMU private data for each master */
652 struct arm_smmu_master_data {
653         struct arm_smmu_device          *smmu;
654         struct arm_smmu_strtab_ent      ste;
655 };
656
657 /* SMMU private data for an IOMMU domain */
658 enum arm_smmu_domain_stage {
659         ARM_SMMU_DOMAIN_S1 = 0,
660         ARM_SMMU_DOMAIN_S2,
661         ARM_SMMU_DOMAIN_NESTED,
662         ARM_SMMU_DOMAIN_BYPASS,
663 };
664
665 struct arm_smmu_domain {
666         struct arm_smmu_device          *smmu;
667         struct mutex                    init_mutex; /* Protects smmu pointer */
668
669         struct io_pgtable_ops           *pgtbl_ops;
670
671         enum arm_smmu_domain_stage      stage;
672         union {
673                 struct arm_smmu_s1_cfg  s1_cfg;
674                 struct arm_smmu_s2_cfg  s2_cfg;
675         };
676
677         struct iommu_domain             domain;
678 };
679
680 struct arm_smmu_option_prop {
681         u32 opt;
682         const char *prop;
683 };
684
685 static struct arm_smmu_option_prop arm_smmu_options[] = {
686         { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
687         { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
688         { 0, NULL},
689 };
690
691 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
692                                                  struct arm_smmu_device *smmu)
693 {
694         if ((offset > SZ_64K) &&
695             (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
696                 offset -= SZ_64K;
697
698         return smmu->base + offset;
699 }
700
701 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
702 {
703         return container_of(dom, struct arm_smmu_domain, domain);
704 }
705
706 static void parse_driver_options(struct arm_smmu_device *smmu)
707 {
708         int i = 0;
709
710         do {
711                 if (of_property_read_bool(smmu->dev->of_node,
712                                                 arm_smmu_options[i].prop)) {
713                         smmu->options |= arm_smmu_options[i].opt;
714                         dev_notice(smmu->dev, "option %s\n",
715                                 arm_smmu_options[i].prop);
716                 }
717         } while (arm_smmu_options[++i].opt);
718 }
719
720 /* Low-level queue manipulation functions */
721 static bool queue_full(struct arm_smmu_queue *q)
722 {
723         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
724                Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
725 }
726
727 static bool queue_empty(struct arm_smmu_queue *q)
728 {
729         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
730                Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
731 }
732
733 static void queue_sync_cons(struct arm_smmu_queue *q)
734 {
735         q->cons = readl_relaxed(q->cons_reg);
736 }
737
738 static void queue_inc_cons(struct arm_smmu_queue *q)
739 {
740         u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
741
742         q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
743         writel(q->cons, q->cons_reg);
744 }
745
746 static int queue_sync_prod(struct arm_smmu_queue *q)
747 {
748         int ret = 0;
749         u32 prod = readl_relaxed(q->prod_reg);
750
751         if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
752                 ret = -EOVERFLOW;
753
754         q->prod = prod;
755         return ret;
756 }
757
758 static void queue_inc_prod(struct arm_smmu_queue *q)
759 {
760         u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
761
762         q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
763         writel(q->prod, q->prod_reg);
764 }
765
766 /*
767  * Wait for the SMMU to consume items. If drain is true, wait until the queue
768  * is empty. Otherwise, wait until there is at least one free slot.
769  */
770 static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
771 {
772         ktime_t timeout;
773         unsigned int delay = 1, spin_cnt = 0;
774
775         /* Wait longer if it's a CMD_SYNC */
776         timeout = ktime_add_us(ktime_get(), sync ?
777                                             ARM_SMMU_CMDQ_SYNC_TIMEOUT_US :
778                                             ARM_SMMU_POLL_TIMEOUT_US);
779
780         while (queue_sync_cons(q), (sync ? !queue_empty(q) : queue_full(q))) {
781                 if (ktime_compare(ktime_get(), timeout) > 0)
782                         return -ETIMEDOUT;
783
784                 if (wfe) {
785                         wfe();
786                 } else if (++spin_cnt < ARM_SMMU_CMDQ_SYNC_SPIN_COUNT) {
787                         cpu_relax();
788                         continue;
789                 } else {
790                         udelay(delay);
791                         delay *= 2;
792                         spin_cnt = 0;
793                 }
794         }
795
796         return 0;
797 }
798
799 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
800 {
801         int i;
802
803         for (i = 0; i < n_dwords; ++i)
804                 *dst++ = cpu_to_le64(*src++);
805 }
806
807 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
808 {
809         if (queue_full(q))
810                 return -ENOSPC;
811
812         queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
813         queue_inc_prod(q);
814         return 0;
815 }
816
817 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
818 {
819         int i;
820
821         for (i = 0; i < n_dwords; ++i)
822                 *dst++ = le64_to_cpu(*src++);
823 }
824
825 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
826 {
827         if (queue_empty(q))
828                 return -EAGAIN;
829
830         queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
831         queue_inc_cons(q);
832         return 0;
833 }
834
835 /* High-level queue accessors */
836 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
837 {
838         memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
839         cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;
840
841         switch (ent->opcode) {
842         case CMDQ_OP_TLBI_EL2_ALL:
843         case CMDQ_OP_TLBI_NSNH_ALL:
844                 break;
845         case CMDQ_OP_PREFETCH_CFG:
846                 cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;
847                 cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;
848                 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
849                 break;
850         case CMDQ_OP_CFGI_STE:
851                 cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;
852                 cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;
853                 break;
854         case CMDQ_OP_CFGI_ALL:
855                 /* Cover the entire SID range */
856                 cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT;
857                 break;
858         case CMDQ_OP_TLBI_NH_VA:
859                 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
860                 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
861                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
862                 break;
863         case CMDQ_OP_TLBI_S2_IPA:
864                 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
865                 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
866                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
867                 break;
868         case CMDQ_OP_TLBI_NH_ASID:
869                 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
870                 /* Fallthrough */
871         case CMDQ_OP_TLBI_S12_VMALL:
872                 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
873                 break;
874         case CMDQ_OP_PRI_RESP:
875                 cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;
876                 cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;
877                 cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;
878                 cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;
879                 switch (ent->pri.resp) {
880                 case PRI_RESP_DENY:
881                         cmd[1] |= CMDQ_PRI_1_RESP_DENY;
882                         break;
883                 case PRI_RESP_FAIL:
884                         cmd[1] |= CMDQ_PRI_1_RESP_FAIL;
885                         break;
886                 case PRI_RESP_SUCC:
887                         cmd[1] |= CMDQ_PRI_1_RESP_SUCC;
888                         break;
889                 default:
890                         return -EINVAL;
891                 }
892                 break;
893         case CMDQ_OP_CMD_SYNC:
894                 if (ent->sync.msiaddr)
895                         cmd[0] |= CMDQ_SYNC_0_CS_IRQ;
896                 else
897                         cmd[0] |= CMDQ_SYNC_0_CS_SEV;
898                 cmd[0] |= CMDQ_SYNC_0_MSH_ISH | CMDQ_SYNC_0_MSIATTR_OIWB;
899                 cmd[0] |= (u64)ent->sync.msidata << CMDQ_SYNC_0_MSIDATA_SHIFT;
900                 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
901                 break;
902         default:
903                 return -ENOENT;
904         }
905
906         return 0;
907 }
908
909 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
910 {
911         static const char *cerror_str[] = {
912                 [CMDQ_ERR_CERROR_NONE_IDX]      = "No error",
913                 [CMDQ_ERR_CERROR_ILL_IDX]       = "Illegal command",
914                 [CMDQ_ERR_CERROR_ABT_IDX]       = "Abort on command fetch",
915         };
916
917         int i;
918         u64 cmd[CMDQ_ENT_DWORDS];
919         struct arm_smmu_queue *q = &smmu->cmdq.q;
920         u32 cons = readl_relaxed(q->cons_reg);
921         u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK;
922         struct arm_smmu_cmdq_ent cmd_sync = {
923                 .opcode = CMDQ_OP_CMD_SYNC,
924         };
925
926         dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
927                 idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
928
929         switch (idx) {
930         case CMDQ_ERR_CERROR_ABT_IDX:
931                 dev_err(smmu->dev, "retrying command fetch\n");
932         case CMDQ_ERR_CERROR_NONE_IDX:
933                 return;
934         case CMDQ_ERR_CERROR_ILL_IDX:
935                 /* Fallthrough */
936         default:
937                 break;
938         }
939
940         /*
941          * We may have concurrent producers, so we need to be careful
942          * not to touch any of the shadow cmdq state.
943          */
944         queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
945         dev_err(smmu->dev, "skipping command in error state:\n");
946         for (i = 0; i < ARRAY_SIZE(cmd); ++i)
947                 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
948
949         /* Convert the erroneous command into a CMD_SYNC */
950         if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
951                 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
952                 return;
953         }
954
955         queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
956 }
957
958 static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device *smmu, u64 *cmd)
959 {
960         struct arm_smmu_queue *q = &smmu->cmdq.q;
961         bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
962
963         while (queue_insert_raw(q, cmd) == -ENOSPC) {
964                 if (queue_poll_cons(q, false, wfe))
965                         dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
966         }
967 }
968
969 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
970                                     struct arm_smmu_cmdq_ent *ent)
971 {
972         u64 cmd[CMDQ_ENT_DWORDS];
973         unsigned long flags;
974
975         if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
976                 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
977                          ent->opcode);
978                 return;
979         }
980
981         spin_lock_irqsave(&smmu->cmdq.lock, flags);
982         arm_smmu_cmdq_insert_cmd(smmu, cmd);
983         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
984 }
985
986 /*
987  * The difference between val and sync_idx is bounded by the maximum size of
988  * a queue at 2^20 entries, so 32 bits is plenty for wrap-safe arithmetic.
989  */
990 static int __arm_smmu_sync_poll_msi(struct arm_smmu_device *smmu, u32 sync_idx)
991 {
992         ktime_t timeout;
993         u32 val;
994
995         timeout = ktime_add_us(ktime_get(), ARM_SMMU_CMDQ_SYNC_TIMEOUT_US);
996         val = smp_cond_load_acquire(&smmu->sync_count,
997                                     (int)(VAL - sync_idx) >= 0 ||
998                                     !ktime_before(ktime_get(), timeout));
999
1000         return (int)(val - sync_idx) < 0 ? -ETIMEDOUT : 0;
1001 }
1002
1003 static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device *smmu)
1004 {
1005         u64 cmd[CMDQ_ENT_DWORDS];
1006         unsigned long flags;
1007         struct arm_smmu_cmdq_ent ent = {
1008                 .opcode = CMDQ_OP_CMD_SYNC,
1009                 .sync   = {
1010                         .msidata = atomic_inc_return_relaxed(&smmu->sync_nr),
1011                         .msiaddr = virt_to_phys(&smmu->sync_count),
1012                 },
1013         };
1014
1015         arm_smmu_cmdq_build_cmd(cmd, &ent);
1016
1017         spin_lock_irqsave(&smmu->cmdq.lock, flags);
1018         arm_smmu_cmdq_insert_cmd(smmu, cmd);
1019         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1020
1021         return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata);
1022 }
1023
1024 static int __arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1025 {
1026         u64 cmd[CMDQ_ENT_DWORDS];
1027         unsigned long flags;
1028         bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
1029         struct arm_smmu_cmdq_ent ent = { .opcode = CMDQ_OP_CMD_SYNC };
1030         int ret;
1031
1032         arm_smmu_cmdq_build_cmd(cmd, &ent);
1033
1034         spin_lock_irqsave(&smmu->cmdq.lock, flags);
1035         arm_smmu_cmdq_insert_cmd(smmu, cmd);
1036         ret = queue_poll_cons(&smmu->cmdq.q, true, wfe);
1037         spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
1038
1039         return ret;
1040 }
1041
1042 static void arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1043 {
1044         int ret;
1045         bool msi = (smmu->features & ARM_SMMU_FEAT_MSI) &&
1046                    (smmu->features & ARM_SMMU_FEAT_COHERENCY);
1047
1048         ret = msi ? __arm_smmu_cmdq_issue_sync_msi(smmu)
1049                   : __arm_smmu_cmdq_issue_sync(smmu);
1050         if (ret)
1051                 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
1052 }
1053
1054 /* Context descriptor manipulation functions */
1055 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
1056 {
1057         u64 val = 0;
1058
1059         /* Repack the TCR. Just care about TTBR0 for now */
1060         val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
1061         val |= ARM_SMMU_TCR2CD(tcr, TG0);
1062         val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
1063         val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
1064         val |= ARM_SMMU_TCR2CD(tcr, SH0);
1065         val |= ARM_SMMU_TCR2CD(tcr, EPD0);
1066         val |= ARM_SMMU_TCR2CD(tcr, EPD1);
1067         val |= ARM_SMMU_TCR2CD(tcr, IPS);
1068         val |= ARM_SMMU_TCR2CD(tcr, TBI0);
1069
1070         return val;
1071 }
1072
1073 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
1074                                     struct arm_smmu_s1_cfg *cfg)
1075 {
1076         u64 val;
1077
1078         /*
1079          * We don't need to issue any invalidation here, as we'll invalidate
1080          * the STE when installing the new entry anyway.
1081          */
1082         val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
1083 #ifdef __BIG_ENDIAN
1084               CTXDESC_CD_0_ENDI |
1085 #endif
1086               CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE |
1087               CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT |
1088               CTXDESC_CD_0_V;
1089
1090         /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1091         if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1092                 val |= CTXDESC_CD_0_S;
1093
1094         cfg->cdptr[0] = cpu_to_le64(val);
1095
1096         val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT;
1097         cfg->cdptr[1] = cpu_to_le64(val);
1098
1099         cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT);
1100 }
1101
1102 /* Stream table manipulation functions */
1103 static void
1104 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1105 {
1106         u64 val = 0;
1107
1108         val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK)
1109                 << STRTAB_L1_DESC_SPAN_SHIFT;
1110         val |= desc->l2ptr_dma &
1111                STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT;
1112
1113         *dst = cpu_to_le64(val);
1114 }
1115
1116 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1117 {
1118         struct arm_smmu_cmdq_ent cmd = {
1119                 .opcode = CMDQ_OP_CFGI_STE,
1120                 .cfgi   = {
1121                         .sid    = sid,
1122                         .leaf   = true,
1123                 },
1124         };
1125
1126         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1127         arm_smmu_cmdq_issue_sync(smmu);
1128 }
1129
1130 static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
1131                                       __le64 *dst, struct arm_smmu_strtab_ent *ste)
1132 {
1133         /*
1134          * This is hideously complicated, but we only really care about
1135          * three cases at the moment:
1136          *
1137          * 1. Invalid (all zero) -> bypass/fault (init)
1138          * 2. Bypass/fault -> translation/bypass (attach)
1139          * 3. Translation/bypass -> bypass/fault (detach)
1140          *
1141          * Given that we can't update the STE atomically and the SMMU
1142          * doesn't read the thing in a defined order, that leaves us
1143          * with the following maintenance requirements:
1144          *
1145          * 1. Update Config, return (init time STEs aren't live)
1146          * 2. Write everything apart from dword 0, sync, write dword 0, sync
1147          * 3. Update Config, sync
1148          */
1149         u64 val = le64_to_cpu(dst[0]);
1150         bool ste_live = false;
1151         struct arm_smmu_cmdq_ent prefetch_cmd = {
1152                 .opcode         = CMDQ_OP_PREFETCH_CFG,
1153                 .prefetch       = {
1154                         .sid    = sid,
1155                 },
1156         };
1157
1158         if (val & STRTAB_STE_0_V) {
1159                 u64 cfg;
1160
1161                 cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT;
1162                 switch (cfg) {
1163                 case STRTAB_STE_0_CFG_BYPASS:
1164                         break;
1165                 case STRTAB_STE_0_CFG_S1_TRANS:
1166                 case STRTAB_STE_0_CFG_S2_TRANS:
1167                         ste_live = true;
1168                         break;
1169                 case STRTAB_STE_0_CFG_ABORT:
1170                         if (disable_bypass)
1171                                 break;
1172                 default:
1173                         BUG(); /* STE corruption */
1174                 }
1175         }
1176
1177         /* Nuke the existing STE_0 value, as we're going to rewrite it */
1178         val = STRTAB_STE_0_V;
1179
1180         /* Bypass/fault */
1181         if (!ste->assigned || !(ste->s1_cfg || ste->s2_cfg)) {
1182                 if (!ste->assigned && disable_bypass)
1183                         val |= STRTAB_STE_0_CFG_ABORT;
1184                 else
1185                         val |= STRTAB_STE_0_CFG_BYPASS;
1186
1187                 dst[0] = cpu_to_le64(val);
1188                 dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING
1189                          << STRTAB_STE_1_SHCFG_SHIFT);
1190                 dst[2] = 0; /* Nuke the VMID */
1191                 /*
1192                  * The SMMU can perform negative caching, so we must sync
1193                  * the STE regardless of whether the old value was live.
1194                  */
1195                 if (smmu)
1196                         arm_smmu_sync_ste_for_sid(smmu, sid);
1197                 return;
1198         }
1199
1200         if (ste->s1_cfg) {
1201                 BUG_ON(ste_live);
1202                 dst[1] = cpu_to_le64(
1203                          STRTAB_STE_1_S1C_CACHE_WBRA
1204                          << STRTAB_STE_1_S1CIR_SHIFT |
1205                          STRTAB_STE_1_S1C_CACHE_WBRA
1206                          << STRTAB_STE_1_S1COR_SHIFT |
1207                          STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
1208 #ifdef CONFIG_PCI_ATS
1209                          STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
1210 #endif
1211                          STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
1212
1213                 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1214                    !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1215                         dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1216
1217                 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
1218                         << STRTAB_STE_0_S1CTXPTR_SHIFT) |
1219                         STRTAB_STE_0_CFG_S1_TRANS;
1220         }
1221
1222         if (ste->s2_cfg) {
1223                 BUG_ON(ste_live);
1224                 dst[2] = cpu_to_le64(
1225                          ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT |
1226                          (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK)
1227                           << STRTAB_STE_2_VTCR_SHIFT |
1228 #ifdef __BIG_ENDIAN
1229                          STRTAB_STE_2_S2ENDI |
1230 #endif
1231                          STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1232                          STRTAB_STE_2_S2R);
1233
1234                 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr &
1235                          STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT);
1236
1237                 val |= STRTAB_STE_0_CFG_S2_TRANS;
1238         }
1239
1240         arm_smmu_sync_ste_for_sid(smmu, sid);
1241         dst[0] = cpu_to_le64(val);
1242         arm_smmu_sync_ste_for_sid(smmu, sid);
1243
1244         /* It's likely that we'll want to use the new STE soon */
1245         if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1246                 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1247 }
1248
1249 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1250 {
1251         unsigned int i;
1252         struct arm_smmu_strtab_ent ste = { .assigned = false };
1253
1254         for (i = 0; i < nent; ++i) {
1255                 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
1256                 strtab += STRTAB_STE_DWORDS;
1257         }
1258 }
1259
1260 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1261 {
1262         size_t size;
1263         void *strtab;
1264         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1265         struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1266
1267         if (desc->l2ptr)
1268                 return 0;
1269
1270         size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1271         strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1272
1273         desc->span = STRTAB_SPLIT + 1;
1274         desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1275                                           GFP_KERNEL | __GFP_ZERO);
1276         if (!desc->l2ptr) {
1277                 dev_err(smmu->dev,
1278                         "failed to allocate l2 stream table for SID %u\n",
1279                         sid);
1280                 return -ENOMEM;
1281         }
1282
1283         arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1284         arm_smmu_write_strtab_l1_desc(strtab, desc);
1285         return 0;
1286 }
1287
1288 /* IRQ and event handlers */
1289 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1290 {
1291         int i;
1292         struct arm_smmu_device *smmu = dev;
1293         struct arm_smmu_queue *q = &smmu->evtq.q;
1294         u64 evt[EVTQ_ENT_DWORDS];
1295
1296         do {
1297                 while (!queue_remove_raw(q, evt)) {
1298                         u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
1299
1300                         dev_info(smmu->dev, "event 0x%02x received:\n", id);
1301                         for (i = 0; i < ARRAY_SIZE(evt); ++i)
1302                                 dev_info(smmu->dev, "\t0x%016llx\n",
1303                                          (unsigned long long)evt[i]);
1304
1305                 }
1306
1307                 /*
1308                  * Not much we can do on overflow, so scream and pretend we're
1309                  * trying harder.
1310                  */
1311                 if (queue_sync_prod(q) == -EOVERFLOW)
1312                         dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1313         } while (!queue_empty(q));
1314
1315         /* Sync our overflow flag, as we believe we're up to speed */
1316         q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1317         return IRQ_HANDLED;
1318 }
1319
1320 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1321 {
1322         u32 sid, ssid;
1323         u16 grpid;
1324         bool ssv, last;
1325
1326         sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
1327         ssv = evt[0] & PRIQ_0_SSID_V;
1328         ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
1329         last = evt[0] & PRIQ_0_PRG_LAST;
1330         grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
1331
1332         dev_info(smmu->dev, "unexpected PRI request received:\n");
1333         dev_info(smmu->dev,
1334                  "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1335                  sid, ssid, grpid, last ? "L" : "",
1336                  evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1337                  evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1338                  evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1339                  evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1340                  evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
1341
1342         if (last) {
1343                 struct arm_smmu_cmdq_ent cmd = {
1344                         .opcode                 = CMDQ_OP_PRI_RESP,
1345                         .substream_valid        = ssv,
1346                         .pri                    = {
1347                                 .sid    = sid,
1348                                 .ssid   = ssid,
1349                                 .grpid  = grpid,
1350                                 .resp   = PRI_RESP_DENY,
1351                         },
1352                 };
1353
1354                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1355         }
1356 }
1357
1358 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1359 {
1360         struct arm_smmu_device *smmu = dev;
1361         struct arm_smmu_queue *q = &smmu->priq.q;
1362         u64 evt[PRIQ_ENT_DWORDS];
1363
1364         do {
1365                 while (!queue_remove_raw(q, evt))
1366                         arm_smmu_handle_ppr(smmu, evt);
1367
1368                 if (queue_sync_prod(q) == -EOVERFLOW)
1369                         dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1370         } while (!queue_empty(q));
1371
1372         /* Sync our overflow flag, as we believe we're up to speed */
1373         q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1374         return IRQ_HANDLED;
1375 }
1376
1377 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1378
1379 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1380 {
1381         u32 gerror, gerrorn, active;
1382         struct arm_smmu_device *smmu = dev;
1383
1384         gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1385         gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1386
1387         active = gerror ^ gerrorn;
1388         if (!(active & GERROR_ERR_MASK))
1389                 return IRQ_NONE; /* No errors pending */
1390
1391         dev_warn(smmu->dev,
1392                  "unexpected global error reported (0x%08x), this could be serious\n",
1393                  active);
1394
1395         if (active & GERROR_SFM_ERR) {
1396                 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1397                 arm_smmu_device_disable(smmu);
1398         }
1399
1400         if (active & GERROR_MSI_GERROR_ABT_ERR)
1401                 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1402
1403         if (active & GERROR_MSI_PRIQ_ABT_ERR)
1404                 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1405
1406         if (active & GERROR_MSI_EVTQ_ABT_ERR)
1407                 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1408
1409         if (active & GERROR_MSI_CMDQ_ABT_ERR)
1410                 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1411
1412         if (active & GERROR_PRIQ_ABT_ERR)
1413                 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1414
1415         if (active & GERROR_EVTQ_ABT_ERR)
1416                 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1417
1418         if (active & GERROR_CMDQ_ERR)
1419                 arm_smmu_cmdq_skip_err(smmu);
1420
1421         writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1422         return IRQ_HANDLED;
1423 }
1424
1425 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1426 {
1427         struct arm_smmu_device *smmu = dev;
1428
1429         arm_smmu_evtq_thread(irq, dev);
1430         if (smmu->features & ARM_SMMU_FEAT_PRI)
1431                 arm_smmu_priq_thread(irq, dev);
1432
1433         return IRQ_HANDLED;
1434 }
1435
1436 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1437 {
1438         arm_smmu_gerror_handler(irq, dev);
1439         return IRQ_WAKE_THREAD;
1440 }
1441
1442 /* IO_PGTABLE API */
1443 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1444 {
1445         arm_smmu_cmdq_issue_sync(smmu);
1446 }
1447
1448 static void arm_smmu_tlb_sync(void *cookie)
1449 {
1450         struct arm_smmu_domain *smmu_domain = cookie;
1451         __arm_smmu_tlb_sync(smmu_domain->smmu);
1452 }
1453
1454 static void arm_smmu_tlb_inv_context(void *cookie)
1455 {
1456         struct arm_smmu_domain *smmu_domain = cookie;
1457         struct arm_smmu_device *smmu = smmu_domain->smmu;
1458         struct arm_smmu_cmdq_ent cmd;
1459
1460         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1461                 cmd.opcode      = CMDQ_OP_TLBI_NH_ASID;
1462                 cmd.tlbi.asid   = smmu_domain->s1_cfg.cd.asid;
1463                 cmd.tlbi.vmid   = 0;
1464         } else {
1465                 cmd.opcode      = CMDQ_OP_TLBI_S12_VMALL;
1466                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1467         }
1468
1469         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1470         __arm_smmu_tlb_sync(smmu);
1471 }
1472
1473 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1474                                           size_t granule, bool leaf, void *cookie)
1475 {
1476         struct arm_smmu_domain *smmu_domain = cookie;
1477         struct arm_smmu_device *smmu = smmu_domain->smmu;
1478         struct arm_smmu_cmdq_ent cmd = {
1479                 .tlbi = {
1480                         .leaf   = leaf,
1481                         .addr   = iova,
1482                 },
1483         };
1484
1485         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1486                 cmd.opcode      = CMDQ_OP_TLBI_NH_VA;
1487                 cmd.tlbi.asid   = smmu_domain->s1_cfg.cd.asid;
1488         } else {
1489                 cmd.opcode      = CMDQ_OP_TLBI_S2_IPA;
1490                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1491         }
1492
1493         do {
1494                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1495                 cmd.tlbi.addr += granule;
1496         } while (size -= granule);
1497 }
1498
1499 static const struct iommu_gather_ops arm_smmu_gather_ops = {
1500         .tlb_flush_all  = arm_smmu_tlb_inv_context,
1501         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
1502         .tlb_sync       = arm_smmu_tlb_sync,
1503 };
1504
1505 /* IOMMU API */
1506 static bool arm_smmu_capable(enum iommu_cap cap)
1507 {
1508         switch (cap) {
1509         case IOMMU_CAP_CACHE_COHERENCY:
1510                 return true;
1511         case IOMMU_CAP_NOEXEC:
1512                 return true;
1513         default:
1514                 return false;
1515         }
1516 }
1517
1518 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1519 {
1520         struct arm_smmu_domain *smmu_domain;
1521
1522         if (type != IOMMU_DOMAIN_UNMANAGED &&
1523             type != IOMMU_DOMAIN_DMA &&
1524             type != IOMMU_DOMAIN_IDENTITY)
1525                 return NULL;
1526
1527         /*
1528          * Allocate the domain and initialise some of its data structures.
1529          * We can't really do anything meaningful until we've added a
1530          * master.
1531          */
1532         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1533         if (!smmu_domain)
1534                 return NULL;
1535
1536         if (type == IOMMU_DOMAIN_DMA &&
1537             iommu_get_dma_cookie(&smmu_domain->domain)) {
1538                 kfree(smmu_domain);
1539                 return NULL;
1540         }
1541
1542         mutex_init(&smmu_domain->init_mutex);
1543         return &smmu_domain->domain;
1544 }
1545
1546 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1547 {
1548         int idx, size = 1 << span;
1549
1550         do {
1551                 idx = find_first_zero_bit(map, size);
1552                 if (idx == size)
1553                         return -ENOSPC;
1554         } while (test_and_set_bit(idx, map));
1555
1556         return idx;
1557 }
1558
1559 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1560 {
1561         clear_bit(idx, map);
1562 }
1563
1564 static void arm_smmu_domain_free(struct iommu_domain *domain)
1565 {
1566         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1567         struct arm_smmu_device *smmu = smmu_domain->smmu;
1568
1569         iommu_put_dma_cookie(domain);
1570         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1571
1572         /* Free the CD and ASID, if we allocated them */
1573         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1574                 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1575
1576                 if (cfg->cdptr) {
1577                         dmam_free_coherent(smmu_domain->smmu->dev,
1578                                            CTXDESC_CD_DWORDS << 3,
1579                                            cfg->cdptr,
1580                                            cfg->cdptr_dma);
1581
1582                         arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1583                 }
1584         } else {
1585                 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1586                 if (cfg->vmid)
1587                         arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1588         }
1589
1590         kfree(smmu_domain);
1591 }
1592
1593 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1594                                        struct io_pgtable_cfg *pgtbl_cfg)
1595 {
1596         int ret;
1597         int asid;
1598         struct arm_smmu_device *smmu = smmu_domain->smmu;
1599         struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1600
1601         asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1602         if (asid < 0)
1603                 return asid;
1604
1605         cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1606                                          &cfg->cdptr_dma,
1607                                          GFP_KERNEL | __GFP_ZERO);
1608         if (!cfg->cdptr) {
1609                 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1610                 ret = -ENOMEM;
1611                 goto out_free_asid;
1612         }
1613
1614         cfg->cd.asid    = (u16)asid;
1615         cfg->cd.ttbr    = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1616         cfg->cd.tcr     = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1617         cfg->cd.mair    = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1618         return 0;
1619
1620 out_free_asid:
1621         arm_smmu_bitmap_free(smmu->asid_map, asid);
1622         return ret;
1623 }
1624
1625 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1626                                        struct io_pgtable_cfg *pgtbl_cfg)
1627 {
1628         int vmid;
1629         struct arm_smmu_device *smmu = smmu_domain->smmu;
1630         struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1631
1632         vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1633         if (vmid < 0)
1634                 return vmid;
1635
1636         cfg->vmid       = (u16)vmid;
1637         cfg->vttbr      = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1638         cfg->vtcr       = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1639         return 0;
1640 }
1641
1642 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1643 {
1644         int ret;
1645         unsigned long ias, oas;
1646         enum io_pgtable_fmt fmt;
1647         struct io_pgtable_cfg pgtbl_cfg;
1648         struct io_pgtable_ops *pgtbl_ops;
1649         int (*finalise_stage_fn)(struct arm_smmu_domain *,
1650                                  struct io_pgtable_cfg *);
1651         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1652         struct arm_smmu_device *smmu = smmu_domain->smmu;
1653
1654         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
1655                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
1656                 return 0;
1657         }
1658
1659         /* Restrict the stage to what we can actually support */
1660         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1661                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1662         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1663                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1664
1665         switch (smmu_domain->stage) {
1666         case ARM_SMMU_DOMAIN_S1:
1667                 ias = VA_BITS;
1668                 oas = smmu->ias;
1669                 fmt = ARM_64_LPAE_S1;
1670                 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1671                 break;
1672         case ARM_SMMU_DOMAIN_NESTED:
1673         case ARM_SMMU_DOMAIN_S2:
1674                 ias = smmu->ias;
1675                 oas = smmu->oas;
1676                 fmt = ARM_64_LPAE_S2;
1677                 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1678                 break;
1679         default:
1680                 return -EINVAL;
1681         }
1682
1683         pgtbl_cfg = (struct io_pgtable_cfg) {
1684                 .pgsize_bitmap  = smmu->pgsize_bitmap,
1685                 .ias            = ias,
1686                 .oas            = oas,
1687                 .tlb            = &arm_smmu_gather_ops,
1688                 .iommu_dev      = smmu->dev,
1689         };
1690
1691         if (smmu->features & ARM_SMMU_FEAT_COHERENCY)
1692                 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
1693
1694         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1695         if (!pgtbl_ops)
1696                 return -ENOMEM;
1697
1698         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1699         domain->geometry.aperture_end = (1UL << ias) - 1;
1700         domain->geometry.force_aperture = true;
1701
1702         ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1703         if (ret < 0) {
1704                 free_io_pgtable_ops(pgtbl_ops);
1705                 return ret;
1706         }
1707
1708         smmu_domain->pgtbl_ops = pgtbl_ops;
1709         return 0;
1710 }
1711
1712 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1713 {
1714         __le64 *step;
1715         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1716
1717         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1718                 struct arm_smmu_strtab_l1_desc *l1_desc;
1719                 int idx;
1720
1721                 /* Two-level walk */
1722                 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1723                 l1_desc = &cfg->l1_desc[idx];
1724                 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1725                 step = &l1_desc->l2ptr[idx];
1726         } else {
1727                 /* Simple linear lookup */
1728                 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1729         }
1730
1731         return step;
1732 }
1733
1734 static void arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec)
1735 {
1736         int i, j;
1737         struct arm_smmu_master_data *master = fwspec->iommu_priv;
1738         struct arm_smmu_device *smmu = master->smmu;
1739
1740         for (i = 0; i < fwspec->num_ids; ++i) {
1741                 u32 sid = fwspec->ids[i];
1742                 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1743
1744                 /* Bridged PCI devices may end up with duplicated IDs */
1745                 for (j = 0; j < i; j++)
1746                         if (fwspec->ids[j] == sid)
1747                                 break;
1748                 if (j < i)
1749                         continue;
1750
1751                 arm_smmu_write_strtab_ent(smmu, sid, step, &master->ste);
1752         }
1753 }
1754
1755 static void arm_smmu_detach_dev(struct device *dev)
1756 {
1757         struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv;
1758
1759         master->ste.assigned = false;
1760         arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
1761 }
1762
1763 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1764 {
1765         int ret = 0;
1766         struct arm_smmu_device *smmu;
1767         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1768         struct arm_smmu_master_data *master;
1769         struct arm_smmu_strtab_ent *ste;
1770
1771         if (!dev->iommu_fwspec)
1772                 return -ENOENT;
1773
1774         master = dev->iommu_fwspec->iommu_priv;
1775         smmu = master->smmu;
1776         ste = &master->ste;
1777
1778         /* Already attached to a different domain? */
1779         if (ste->assigned)
1780                 arm_smmu_detach_dev(dev);
1781
1782         mutex_lock(&smmu_domain->init_mutex);
1783
1784         if (!smmu_domain->smmu) {
1785                 smmu_domain->smmu = smmu;
1786                 ret = arm_smmu_domain_finalise(domain);
1787                 if (ret) {
1788                         smmu_domain->smmu = NULL;
1789                         goto out_unlock;
1790                 }
1791         } else if (smmu_domain->smmu != smmu) {
1792                 dev_err(dev,
1793                         "cannot attach to SMMU %s (upstream of %s)\n",
1794                         dev_name(smmu_domain->smmu->dev),
1795                         dev_name(smmu->dev));
1796                 ret = -ENXIO;
1797                 goto out_unlock;
1798         }
1799
1800         ste->assigned = true;
1801
1802         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) {
1803                 ste->s1_cfg = NULL;
1804                 ste->s2_cfg = NULL;
1805         } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1806                 ste->s1_cfg = &smmu_domain->s1_cfg;
1807                 ste->s2_cfg = NULL;
1808                 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1809         } else {
1810                 ste->s1_cfg = NULL;
1811                 ste->s2_cfg = &smmu_domain->s2_cfg;
1812         }
1813
1814         arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
1815 out_unlock:
1816         mutex_unlock(&smmu_domain->init_mutex);
1817         return ret;
1818 }
1819
1820 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1821                         phys_addr_t paddr, size_t size, int prot)
1822 {
1823         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1824
1825         if (!ops)
1826                 return -ENODEV;
1827
1828         return ops->map(ops, iova, paddr, size, prot);
1829 }
1830
1831 static size_t
1832 arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1833 {
1834         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1835
1836         if (!ops)
1837                 return 0;
1838
1839         return ops->unmap(ops, iova, size);
1840 }
1841
1842 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1843 {
1844         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1845
1846         if (smmu)
1847                 __arm_smmu_tlb_sync(smmu);
1848 }
1849
1850 static phys_addr_t
1851 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1852 {
1853         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1854
1855         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1856                 return iova;
1857
1858         if (!ops)
1859                 return 0;
1860
1861         return ops->iova_to_phys(ops, iova);
1862 }
1863
1864 static struct platform_driver arm_smmu_driver;
1865
1866 static int arm_smmu_match_node(struct device *dev, void *data)
1867 {
1868         return dev->fwnode == data;
1869 }
1870
1871 static
1872 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1873 {
1874         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1875                                                 fwnode, arm_smmu_match_node);
1876         put_device(dev);
1877         return dev ? dev_get_drvdata(dev) : NULL;
1878 }
1879
1880 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
1881 {
1882         unsigned long limit = smmu->strtab_cfg.num_l1_ents;
1883
1884         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
1885                 limit *= 1UL << STRTAB_SPLIT;
1886
1887         return sid < limit;
1888 }
1889
1890 static struct iommu_ops arm_smmu_ops;
1891
1892 static int arm_smmu_add_device(struct device *dev)
1893 {
1894         int i, ret;
1895         struct arm_smmu_device *smmu;
1896         struct arm_smmu_master_data *master;
1897         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1898         struct iommu_group *group;
1899
1900         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1901                 return -ENODEV;
1902         /*
1903          * We _can_ actually withstand dodgy bus code re-calling add_device()
1904          * without an intervening remove_device()/of_xlate() sequence, but
1905          * we're not going to do so quietly...
1906          */
1907         if (WARN_ON_ONCE(fwspec->iommu_priv)) {
1908                 master = fwspec->iommu_priv;
1909                 smmu = master->smmu;
1910         } else {
1911                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1912                 if (!smmu)
1913                         return -ENODEV;
1914                 master = kzalloc(sizeof(*master), GFP_KERNEL);
1915                 if (!master)
1916                         return -ENOMEM;
1917
1918                 master->smmu = smmu;
1919                 fwspec->iommu_priv = master;
1920         }
1921
1922         /* Check the SIDs are in range of the SMMU and our stream table */
1923         for (i = 0; i < fwspec->num_ids; i++) {
1924                 u32 sid = fwspec->ids[i];
1925
1926                 if (!arm_smmu_sid_in_range(smmu, sid))
1927                         return -ERANGE;
1928
1929                 /* Ensure l2 strtab is initialised */
1930                 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1931                         ret = arm_smmu_init_l2_strtab(smmu, sid);
1932                         if (ret)
1933                                 return ret;
1934                 }
1935         }
1936
1937         group = iommu_group_get_for_dev(dev);
1938         if (!IS_ERR(group)) {
1939                 iommu_group_put(group);
1940                 iommu_device_link(&smmu->iommu, dev);
1941         }
1942
1943         return PTR_ERR_OR_ZERO(group);
1944 }
1945
1946 static void arm_smmu_remove_device(struct device *dev)
1947 {
1948         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1949         struct arm_smmu_master_data *master;
1950         struct arm_smmu_device *smmu;
1951
1952         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1953                 return;
1954
1955         master = fwspec->iommu_priv;
1956         smmu = master->smmu;
1957         if (master && master->ste.assigned)
1958                 arm_smmu_detach_dev(dev);
1959         iommu_group_remove_device(dev);
1960         iommu_device_unlink(&smmu->iommu, dev);
1961         kfree(master);
1962         iommu_fwspec_free(dev);
1963 }
1964
1965 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1966 {
1967         struct iommu_group *group;
1968
1969         /*
1970          * We don't support devices sharing stream IDs other than PCI RID
1971          * aliases, since the necessary ID-to-device lookup becomes rather
1972          * impractical given a potential sparse 32-bit stream ID space.
1973          */
1974         if (dev_is_pci(dev))
1975                 group = pci_device_group(dev);
1976         else
1977                 group = generic_device_group(dev);
1978
1979         return group;
1980 }
1981
1982 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1983                                     enum iommu_attr attr, void *data)
1984 {
1985         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1986
1987         if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1988                 return -EINVAL;
1989
1990         switch (attr) {
1991         case DOMAIN_ATTR_NESTING:
1992                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1993                 return 0;
1994         default:
1995                 return -ENODEV;
1996         }
1997 }
1998
1999 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
2000                                     enum iommu_attr attr, void *data)
2001 {
2002         int ret = 0;
2003         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2004
2005         if (domain->type != IOMMU_DOMAIN_UNMANAGED)
2006                 return -EINVAL;
2007
2008         mutex_lock(&smmu_domain->init_mutex);
2009
2010         switch (attr) {
2011         case DOMAIN_ATTR_NESTING:
2012                 if (smmu_domain->smmu) {
2013                         ret = -EPERM;
2014                         goto out_unlock;
2015                 }
2016
2017                 if (*(int *)data)
2018                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2019                 else
2020                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2021
2022                 break;
2023         default:
2024                 ret = -ENODEV;
2025         }
2026
2027 out_unlock:
2028         mutex_unlock(&smmu_domain->init_mutex);
2029         return ret;
2030 }
2031
2032 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2033 {
2034         return iommu_fwspec_add_ids(dev, args->args, 1);
2035 }
2036
2037 static void arm_smmu_get_resv_regions(struct device *dev,
2038                                       struct list_head *head)
2039 {
2040         struct iommu_resv_region *region;
2041         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2042
2043         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2044                                          prot, IOMMU_RESV_SW_MSI);
2045         if (!region)
2046                 return;
2047
2048         list_add_tail(&region->list, head);
2049
2050         iommu_dma_get_resv_regions(dev, head);
2051 }
2052
2053 static void arm_smmu_put_resv_regions(struct device *dev,
2054                                       struct list_head *head)
2055 {
2056         struct iommu_resv_region *entry, *next;
2057
2058         list_for_each_entry_safe(entry, next, head, list)
2059                 kfree(entry);
2060 }
2061
2062 static struct iommu_ops arm_smmu_ops = {
2063         .capable                = arm_smmu_capable,
2064         .domain_alloc           = arm_smmu_domain_alloc,
2065         .domain_free            = arm_smmu_domain_free,
2066         .attach_dev             = arm_smmu_attach_dev,
2067         .map                    = arm_smmu_map,
2068         .unmap                  = arm_smmu_unmap,
2069         .map_sg                 = default_iommu_map_sg,
2070         .flush_iotlb_all        = arm_smmu_iotlb_sync,
2071         .iotlb_sync             = arm_smmu_iotlb_sync,
2072         .iova_to_phys           = arm_smmu_iova_to_phys,
2073         .add_device             = arm_smmu_add_device,
2074         .remove_device          = arm_smmu_remove_device,
2075         .device_group           = arm_smmu_device_group,
2076         .domain_get_attr        = arm_smmu_domain_get_attr,
2077         .domain_set_attr        = arm_smmu_domain_set_attr,
2078         .of_xlate               = arm_smmu_of_xlate,
2079         .get_resv_regions       = arm_smmu_get_resv_regions,
2080         .put_resv_regions       = arm_smmu_put_resv_regions,
2081         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
2082 };
2083
2084 /* Probing and initialisation functions */
2085 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2086                                    struct arm_smmu_queue *q,
2087                                    unsigned long prod_off,
2088                                    unsigned long cons_off,
2089                                    size_t dwords)
2090 {
2091         size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
2092
2093         q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
2094         if (!q->base) {
2095                 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
2096                         qsz);
2097                 return -ENOMEM;
2098         }
2099
2100         q->prod_reg     = arm_smmu_page1_fixup(prod_off, smmu);
2101         q->cons_reg     = arm_smmu_page1_fixup(cons_off, smmu);
2102         q->ent_dwords   = dwords;
2103
2104         q->q_base  = Q_BASE_RWA;
2105         q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;
2106         q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)
2107                      << Q_BASE_LOG2SIZE_SHIFT;
2108
2109         q->prod = q->cons = 0;
2110         return 0;
2111 }
2112
2113 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2114 {
2115         int ret;
2116
2117         /* cmdq */
2118         spin_lock_init(&smmu->cmdq.lock);
2119         ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
2120                                       ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
2121         if (ret)
2122                 return ret;
2123
2124         /* evtq */
2125         ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
2126                                       ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
2127         if (ret)
2128                 return ret;
2129
2130         /* priq */
2131         if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2132                 return 0;
2133
2134         return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2135                                        ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
2136 }
2137
2138 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2139 {
2140         unsigned int i;
2141         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2142         size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2143         void *strtab = smmu->strtab_cfg.strtab;
2144
2145         cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2146         if (!cfg->l1_desc) {
2147                 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2148                 return -ENOMEM;
2149         }
2150
2151         for (i = 0; i < cfg->num_l1_ents; ++i) {
2152                 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2153                 strtab += STRTAB_L1_DESC_DWORDS << 3;
2154         }
2155
2156         return 0;
2157 }
2158
2159 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2160 {
2161         void *strtab;
2162         u64 reg;
2163         u32 size, l1size;
2164         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2165
2166         /* Calculate the L1 size, capped to the SIDSIZE. */
2167         size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2168         size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2169         cfg->num_l1_ents = 1 << size;
2170
2171         size += STRTAB_SPLIT;
2172         if (size < smmu->sid_bits)
2173                 dev_warn(smmu->dev,
2174                          "2-level strtab only covers %u/%u bits of SID\n",
2175                          size, smmu->sid_bits);
2176
2177         l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2178         strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2179                                      GFP_KERNEL | __GFP_ZERO);
2180         if (!strtab) {
2181                 dev_err(smmu->dev,
2182                         "failed to allocate l1 stream table (%u bytes)\n",
2183                         size);
2184                 return -ENOMEM;
2185         }
2186         cfg->strtab = strtab;
2187
2188         /* Configure strtab_base_cfg for 2 levels */
2189         reg  = STRTAB_BASE_CFG_FMT_2LVL;
2190         reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2191                 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2192         reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK)
2193                 << STRTAB_BASE_CFG_SPLIT_SHIFT;
2194         cfg->strtab_base_cfg = reg;
2195
2196         return arm_smmu_init_l1_strtab(smmu);
2197 }
2198
2199 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2200 {
2201         void *strtab;
2202         u64 reg;
2203         u32 size;
2204         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2205
2206         size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2207         strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2208                                      GFP_KERNEL | __GFP_ZERO);
2209         if (!strtab) {
2210                 dev_err(smmu->dev,
2211                         "failed to allocate linear stream table (%u bytes)\n",
2212                         size);
2213                 return -ENOMEM;
2214         }
2215         cfg->strtab = strtab;
2216         cfg->num_l1_ents = 1 << smmu->sid_bits;
2217
2218         /* Configure strtab_base_cfg for a linear table covering all SIDs */
2219         reg  = STRTAB_BASE_CFG_FMT_LINEAR;
2220         reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2221                 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2222         cfg->strtab_base_cfg = reg;
2223
2224         arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2225         return 0;
2226 }
2227
2228 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2229 {
2230         u64 reg;
2231         int ret;
2232
2233         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2234                 ret = arm_smmu_init_strtab_2lvl(smmu);
2235         else
2236                 ret = arm_smmu_init_strtab_linear(smmu);
2237
2238         if (ret)
2239                 return ret;
2240
2241         /* Set the strtab base address */
2242         reg  = smmu->strtab_cfg.strtab_dma &
2243                STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT;
2244         reg |= STRTAB_BASE_RA;
2245         smmu->strtab_cfg.strtab_base = reg;
2246
2247         /* Allocate the first VMID for stage-2 bypass STEs */
2248         set_bit(0, smmu->vmid_map);
2249         return 0;
2250 }
2251
2252 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2253 {
2254         int ret;
2255
2256         atomic_set(&smmu->sync_nr, 0);
2257         ret = arm_smmu_init_queues(smmu);
2258         if (ret)
2259                 return ret;
2260
2261         return arm_smmu_init_strtab(smmu);
2262 }
2263
2264 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2265                                    unsigned int reg_off, unsigned int ack_off)
2266 {
2267         u32 reg;
2268
2269         writel_relaxed(val, smmu->base + reg_off);
2270         return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2271                                           1, ARM_SMMU_POLL_TIMEOUT_US);
2272 }
2273
2274 /* GBPA is "special" */
2275 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2276 {
2277         int ret;
2278         u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2279
2280         ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2281                                          1, ARM_SMMU_POLL_TIMEOUT_US);
2282         if (ret)
2283                 return ret;
2284
2285         reg &= ~clr;
2286         reg |= set;
2287         writel_relaxed(reg | GBPA_UPDATE, gbpa);
2288         return readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2289                                           1, ARM_SMMU_POLL_TIMEOUT_US);
2290 }
2291
2292 static void arm_smmu_free_msis(void *data)
2293 {
2294         struct device *dev = data;
2295         platform_msi_domain_free_irqs(dev);
2296 }
2297
2298 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
2299 {
2300         phys_addr_t doorbell;
2301         struct device *dev = msi_desc_to_dev(desc);
2302         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2303         phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
2304
2305         doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
2306         doorbell &= MSI_CFG0_ADDR_MASK << MSI_CFG0_ADDR_SHIFT;
2307
2308         writeq_relaxed(doorbell, smmu->base + cfg[0]);
2309         writel_relaxed(msg->data, smmu->base + cfg[1]);
2310         writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
2311 }
2312
2313 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
2314 {
2315         struct msi_desc *desc;
2316         int ret, nvec = ARM_SMMU_MAX_MSIS;
2317         struct device *dev = smmu->dev;
2318
2319         /* Clear the MSI address regs */
2320         writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2321         writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2322
2323         if (smmu->features & ARM_SMMU_FEAT_PRI)
2324                 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2325         else
2326                 nvec--;
2327
2328         if (!(smmu->features & ARM_SMMU_FEAT_MSI))
2329                 return;
2330
2331         /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
2332         ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
2333         if (ret) {
2334                 dev_warn(dev, "failed to allocate MSIs\n");
2335                 return;
2336         }
2337
2338         for_each_msi_entry(desc, dev) {
2339                 switch (desc->platform.msi_index) {
2340                 case EVTQ_MSI_INDEX:
2341                         smmu->evtq.q.irq = desc->irq;
2342                         break;
2343                 case GERROR_MSI_INDEX:
2344                         smmu->gerr_irq = desc->irq;
2345                         break;
2346                 case PRIQ_MSI_INDEX:
2347                         smmu->priq.q.irq = desc->irq;
2348                         break;
2349                 default:        /* Unknown */
2350                         continue;
2351                 }
2352         }
2353
2354         /* Add callback to free MSIs on teardown */
2355         devm_add_action(dev, arm_smmu_free_msis, dev);
2356 }
2357
2358 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
2359 {
2360         int irq, ret;
2361
2362         arm_smmu_setup_msis(smmu);
2363
2364         /* Request interrupt lines */
2365         irq = smmu->evtq.q.irq;
2366         if (irq) {
2367                 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2368                                                 arm_smmu_evtq_thread,
2369                                                 IRQF_ONESHOT,
2370                                                 "arm-smmu-v3-evtq", smmu);
2371                 if (ret < 0)
2372                         dev_warn(smmu->dev, "failed to enable evtq irq\n");
2373         }
2374
2375         irq = smmu->gerr_irq;
2376         if (irq) {
2377                 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2378                                        0, "arm-smmu-v3-gerror", smmu);
2379                 if (ret < 0)
2380                         dev_warn(smmu->dev, "failed to enable gerror irq\n");
2381         }
2382
2383         if (smmu->features & ARM_SMMU_FEAT_PRI) {
2384                 irq = smmu->priq.q.irq;
2385                 if (irq) {
2386                         ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
2387                                                         arm_smmu_priq_thread,
2388                                                         IRQF_ONESHOT,
2389                                                         "arm-smmu-v3-priq",
2390                                                         smmu);
2391                         if (ret < 0)
2392                                 dev_warn(smmu->dev,
2393                                          "failed to enable priq irq\n");
2394                 }
2395         }
2396 }
2397
2398 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2399 {
2400         int ret, irq;
2401         u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2402
2403         /* Disable IRQs first */
2404         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2405                                       ARM_SMMU_IRQ_CTRLACK);
2406         if (ret) {
2407                 dev_err(smmu->dev, "failed to disable irqs\n");
2408                 return ret;
2409         }
2410
2411         irq = smmu->combined_irq;
2412         if (irq) {
2413                 /*
2414                  * Cavium ThunderX2 implementation doesn't not support unique
2415                  * irq lines. Use single irq line for all the SMMUv3 interrupts.
2416                  */
2417                 ret = devm_request_threaded_irq(smmu->dev, irq,
2418                                         arm_smmu_combined_irq_handler,
2419                                         arm_smmu_combined_irq_thread,
2420                                         IRQF_ONESHOT,
2421                                         "arm-smmu-v3-combined-irq", smmu);
2422                 if (ret < 0)
2423                         dev_warn(smmu->dev, "failed to enable combined irq\n");
2424         } else
2425                 arm_smmu_setup_unique_irqs(smmu);
2426
2427         if (smmu->features & ARM_SMMU_FEAT_PRI)
2428                 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2429
2430         /* Enable interrupt generation on the SMMU */
2431         ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2432                                       ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2433         if (ret)
2434                 dev_warn(smmu->dev, "failed to enable irqs\n");
2435
2436         return 0;
2437 }
2438
2439 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2440 {
2441         int ret;
2442
2443         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2444         if (ret)
2445                 dev_err(smmu->dev, "failed to clear cr0\n");
2446
2447         return ret;
2448 }
2449
2450 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
2451 {
2452         int ret;
2453         u32 reg, enables;
2454         struct arm_smmu_cmdq_ent cmd;
2455
2456         /* Clear CR0 and sync (disables SMMU and queue processing) */
2457         reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2458         if (reg & CR0_SMMUEN)
2459                 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2460
2461         ret = arm_smmu_device_disable(smmu);
2462         if (ret)
2463                 return ret;
2464
2465         /* CR1 (table and queue memory attributes) */
2466         reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) |
2467               (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) |
2468               (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) |
2469               (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) |
2470               (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) |
2471               (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT);
2472         writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2473
2474         /* CR2 (random crap) */
2475         reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2476         writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2477
2478         /* Stream table */
2479         writeq_relaxed(smmu->strtab_cfg.strtab_base,
2480                        smmu->base + ARM_SMMU_STRTAB_BASE);
2481         writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2482                        smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2483
2484         /* Command queue */
2485         writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2486         writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2487         writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2488
2489         enables = CR0_CMDQEN;
2490         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2491                                       ARM_SMMU_CR0ACK);
2492         if (ret) {
2493                 dev_err(smmu->dev, "failed to enable command queue\n");
2494                 return ret;
2495         }
2496
2497         /* Invalidate any cached configuration */
2498         cmd.opcode = CMDQ_OP_CFGI_ALL;
2499         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2500         arm_smmu_cmdq_issue_sync(smmu);
2501
2502         /* Invalidate any stale TLB entries */
2503         if (smmu->features & ARM_SMMU_FEAT_HYP) {
2504                 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2505                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2506         }
2507
2508         cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2509         arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2510         arm_smmu_cmdq_issue_sync(smmu);
2511
2512         /* Event queue */
2513         writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2514         writel_relaxed(smmu->evtq.q.prod,
2515                        arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
2516         writel_relaxed(smmu->evtq.q.cons,
2517                        arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
2518
2519         enables |= CR0_EVTQEN;
2520         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2521                                       ARM_SMMU_CR0ACK);
2522         if (ret) {
2523                 dev_err(smmu->dev, "failed to enable event queue\n");
2524                 return ret;
2525         }
2526
2527         /* PRI queue */
2528         if (smmu->features & ARM_SMMU_FEAT_PRI) {
2529                 writeq_relaxed(smmu->priq.q.q_base,
2530                                smmu->base + ARM_SMMU_PRIQ_BASE);
2531                 writel_relaxed(smmu->priq.q.prod,
2532                                arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
2533                 writel_relaxed(smmu->priq.q.cons,
2534                                arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
2535
2536                 enables |= CR0_PRIQEN;
2537                 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2538                                               ARM_SMMU_CR0ACK);
2539                 if (ret) {
2540                         dev_err(smmu->dev, "failed to enable PRI queue\n");
2541                         return ret;
2542                 }
2543         }
2544
2545         ret = arm_smmu_setup_irqs(smmu);
2546         if (ret) {
2547                 dev_err(smmu->dev, "failed to setup irqs\n");
2548                 return ret;
2549         }
2550
2551
2552         /* Enable the SMMU interface, or ensure bypass */
2553         if (!bypass || disable_bypass) {
2554                 enables |= CR0_SMMUEN;
2555         } else {
2556                 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
2557                 if (ret) {
2558                         dev_err(smmu->dev, "GBPA not responding to update\n");
2559                         return ret;
2560                 }
2561         }
2562         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2563                                       ARM_SMMU_CR0ACK);
2564         if (ret) {
2565                 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2566                 return ret;
2567         }
2568
2569         return 0;
2570 }
2571
2572 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
2573 {
2574         u32 reg;
2575         bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
2576
2577         /* IDR0 */
2578         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2579
2580         /* 2-level structures */
2581         if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL)
2582                 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2583
2584         if (reg & IDR0_CD2L)
2585                 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2586
2587         /*
2588          * Translation table endianness.
2589          * We currently require the same endianness as the CPU, but this
2590          * could be changed later by adding a new IO_PGTABLE_QUIRK.
2591          */
2592         switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) {
2593         case IDR0_TTENDIAN_MIXED:
2594                 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2595                 break;
2596 #ifdef __BIG_ENDIAN
2597         case IDR0_TTENDIAN_BE:
2598                 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2599                 break;
2600 #else
2601         case IDR0_TTENDIAN_LE:
2602                 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2603                 break;
2604 #endif
2605         default:
2606                 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2607                 return -ENXIO;
2608         }
2609
2610         /* Boolean feature flags */
2611         if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2612                 smmu->features |= ARM_SMMU_FEAT_PRI;
2613
2614         if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2615                 smmu->features |= ARM_SMMU_FEAT_ATS;
2616
2617         if (reg & IDR0_SEV)
2618                 smmu->features |= ARM_SMMU_FEAT_SEV;
2619
2620         if (reg & IDR0_MSI)
2621                 smmu->features |= ARM_SMMU_FEAT_MSI;
2622
2623         if (reg & IDR0_HYP)
2624                 smmu->features |= ARM_SMMU_FEAT_HYP;
2625
2626         /*
2627          * The coherency feature as set by FW is used in preference to the ID
2628          * register, but warn on mismatch.
2629          */
2630         if (!!(reg & IDR0_COHACC) != coherent)
2631                 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
2632                          coherent ? "true" : "false");
2633
2634         switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) {
2635         case IDR0_STALL_MODEL_FORCE:
2636                 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
2637                 /* Fallthrough */
2638         case IDR0_STALL_MODEL_STALL:
2639                 smmu->features |= ARM_SMMU_FEAT_STALLS;
2640         }
2641
2642         if (reg & IDR0_S1P)
2643                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2644
2645         if (reg & IDR0_S2P)
2646                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2647
2648         if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2649                 dev_err(smmu->dev, "no translation support!\n");
2650                 return -ENXIO;
2651         }
2652
2653         /* We only support the AArch64 table format at present */
2654         switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) {
2655         case IDR0_TTF_AARCH32_64:
2656                 smmu->ias = 40;
2657                 /* Fallthrough */
2658         case IDR0_TTF_AARCH64:
2659                 break;
2660         default:
2661                 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2662                 return -ENXIO;
2663         }
2664
2665         /* ASID/VMID sizes */
2666         smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2667         smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2668
2669         /* IDR1 */
2670         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2671         if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2672                 dev_err(smmu->dev, "embedded implementation not supported\n");
2673                 return -ENXIO;
2674         }
2675
2676         /* Queue sizes, capped at 4k */
2677         smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,
2678                                        reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK);
2679         if (!smmu->cmdq.q.max_n_shift) {
2680                 /* Odd alignment restrictions on the base, so ignore for now */
2681                 dev_err(smmu->dev, "unit-length command queue not supported\n");
2682                 return -ENXIO;
2683         }
2684
2685         smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,
2686                                        reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK);
2687         smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,
2688                                        reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK);
2689
2690         /* SID/SSID sizes */
2691         smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK;
2692         smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK;
2693
2694         /*
2695          * If the SMMU supports fewer bits than would fill a single L2 stream
2696          * table, use a linear table instead.
2697          */
2698         if (smmu->sid_bits <= STRTAB_SPLIT)
2699                 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
2700
2701         /* IDR5 */
2702         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2703
2704         /* Maximum number of outstanding stalls */
2705         smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT
2706                                 & IDR5_STALL_MAX_MASK;
2707
2708         /* Page sizes */
2709         if (reg & IDR5_GRAN64K)
2710                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2711         if (reg & IDR5_GRAN16K)
2712                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2713         if (reg & IDR5_GRAN4K)
2714                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2715
2716         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2717                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2718         else
2719                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2720
2721         /* Output address size */
2722         switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) {
2723         case IDR5_OAS_32_BIT:
2724                 smmu->oas = 32;
2725                 break;
2726         case IDR5_OAS_36_BIT:
2727                 smmu->oas = 36;
2728                 break;
2729         case IDR5_OAS_40_BIT:
2730                 smmu->oas = 40;
2731                 break;
2732         case IDR5_OAS_42_BIT:
2733                 smmu->oas = 42;
2734                 break;
2735         case IDR5_OAS_44_BIT:
2736                 smmu->oas = 44;
2737                 break;
2738         default:
2739                 dev_info(smmu->dev,
2740                         "unknown output address size. Truncating to 48-bit\n");
2741                 /* Fallthrough */
2742         case IDR5_OAS_48_BIT:
2743                 smmu->oas = 48;
2744         }
2745
2746         /* Set the DMA mask for our table walker */
2747         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2748                 dev_warn(smmu->dev,
2749                          "failed to set DMA mask for table walker\n");
2750
2751         smmu->ias = max(smmu->ias, smmu->oas);
2752
2753         dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2754                  smmu->ias, smmu->oas, smmu->features);
2755         return 0;
2756 }
2757
2758 #ifdef CONFIG_ACPI
2759 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
2760 {
2761         switch (model) {
2762         case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
2763                 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
2764                 break;
2765         case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
2766                 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
2767                 break;
2768         }
2769
2770         dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
2771 }
2772
2773 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2774                                       struct arm_smmu_device *smmu)
2775 {
2776         struct acpi_iort_smmu_v3 *iort_smmu;
2777         struct device *dev = smmu->dev;
2778         struct acpi_iort_node *node;
2779
2780         node = *(struct acpi_iort_node **)dev_get_platdata(dev);
2781
2782         /* Retrieve SMMUv3 specific data */
2783         iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
2784
2785         acpi_smmu_get_options(iort_smmu->model, smmu);
2786
2787         if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
2788                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2789
2790         return 0;
2791 }
2792 #else
2793 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2794                                              struct arm_smmu_device *smmu)
2795 {
2796         return -ENODEV;
2797 }
2798 #endif
2799
2800 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2801                                     struct arm_smmu_device *smmu)
2802 {
2803         struct device *dev = &pdev->dev;
2804         u32 cells;
2805         int ret = -EINVAL;
2806
2807         if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
2808                 dev_err(dev, "missing #iommu-cells property\n");
2809         else if (cells != 1)
2810                 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
2811         else
2812                 ret = 0;
2813
2814         parse_driver_options(smmu);
2815
2816         if (of_dma_is_coherent(dev->of_node))
2817                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2818
2819         return ret;
2820 }
2821
2822 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
2823 {
2824         if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
2825                 return SZ_64K;
2826         else
2827                 return SZ_128K;
2828 }
2829
2830 static int arm_smmu_device_probe(struct platform_device *pdev)
2831 {
2832         int irq, ret;
2833         struct resource *res;
2834         resource_size_t ioaddr;
2835         struct arm_smmu_device *smmu;
2836         struct device *dev = &pdev->dev;
2837         bool bypass;
2838
2839         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2840         if (!smmu) {
2841                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2842                 return -ENOMEM;
2843         }
2844         smmu->dev = dev;
2845
2846         if (dev->of_node) {
2847                 ret = arm_smmu_device_dt_probe(pdev, smmu);
2848         } else {
2849                 ret = arm_smmu_device_acpi_probe(pdev, smmu);
2850                 if (ret == -ENODEV)
2851                         return ret;
2852         }
2853
2854         /* Set bypass mode according to firmware probing result */
2855         bypass = !!ret;
2856
2857         /* Base address */
2858         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2859         if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {
2860                 dev_err(dev, "MMIO region too small (%pr)\n", res);
2861                 return -EINVAL;
2862         }
2863         ioaddr = res->start;
2864
2865         smmu->base = devm_ioremap_resource(dev, res);
2866         if (IS_ERR(smmu->base))
2867                 return PTR_ERR(smmu->base);
2868
2869         /* Interrupt lines */
2870
2871         irq = platform_get_irq_byname(pdev, "combined");
2872         if (irq > 0)
2873                 smmu->combined_irq = irq;
2874         else {
2875                 irq = platform_get_irq_byname(pdev, "eventq");
2876                 if (irq > 0)
2877                         smmu->evtq.q.irq = irq;
2878
2879                 irq = platform_get_irq_byname(pdev, "priq");
2880                 if (irq > 0)
2881                         smmu->priq.q.irq = irq;
2882
2883                 irq = platform_get_irq_byname(pdev, "gerror");
2884                 if (irq > 0)
2885                         smmu->gerr_irq = irq;
2886         }
2887         /* Probe the h/w */
2888         ret = arm_smmu_device_hw_probe(smmu);
2889         if (ret)
2890                 return ret;
2891
2892         /* Initialise in-memory data structures */
2893         ret = arm_smmu_init_structures(smmu);
2894         if (ret)
2895                 return ret;
2896
2897         /* Record our private device structure */
2898         platform_set_drvdata(pdev, smmu);
2899
2900         /* Reset the device */
2901         ret = arm_smmu_device_reset(smmu, bypass);
2902         if (ret)
2903                 return ret;
2904
2905         /* And we're up. Go go go! */
2906         ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
2907                                      "smmu3.%pa", &ioaddr);
2908         if (ret)
2909                 return ret;
2910
2911         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2912         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2913
2914         ret = iommu_device_register(&smmu->iommu);
2915         if (ret) {
2916                 dev_err(dev, "Failed to register iommu\n");
2917                 return ret;
2918         }
2919
2920 #ifdef CONFIG_PCI
2921         if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
2922                 pci_request_acs();
2923                 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2924                 if (ret)
2925                         return ret;
2926         }
2927 #endif
2928 #ifdef CONFIG_ARM_AMBA
2929         if (amba_bustype.iommu_ops != &arm_smmu_ops) {
2930                 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2931                 if (ret)
2932                         return ret;
2933         }
2934 #endif
2935         if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
2936                 ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2937                 if (ret)
2938                         return ret;
2939         }
2940         return 0;
2941 }
2942
2943 static int arm_smmu_device_remove(struct platform_device *pdev)
2944 {
2945         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2946
2947         arm_smmu_device_disable(smmu);
2948
2949         return 0;
2950 }
2951
2952 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2953 {
2954         arm_smmu_device_remove(pdev);
2955 }
2956
2957 static const struct of_device_id arm_smmu_of_match[] = {
2958         { .compatible = "arm,smmu-v3", },
2959         { },
2960 };
2961 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2962
2963 static struct platform_driver arm_smmu_driver = {
2964         .driver = {
2965                 .name           = "arm-smmu-v3",
2966                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2967         },
2968         .probe  = arm_smmu_device_probe,
2969         .remove = arm_smmu_device_remove,
2970         .shutdown = arm_smmu_device_shutdown,
2971 };
2972 module_platform_driver(arm_smmu_driver);
2973
2974 IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3");
2975
2976 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2977 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2978 MODULE_LICENSE("GPL v2");