arm64: mm: Support Common Not Private translations
[sfrench/cifs-2.6.git] / arch / arm64 / kernel / cpufeature.c
1 /*
2  * Contains CPU feature definitions
3  *
4  * Copyright (C) 2015 ARM Ltd.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  */
18
19 #define pr_fmt(fmt) "CPU features: " fmt
20
21 #include <linux/bsearch.h>
22 #include <linux/cpumask.h>
23 #include <linux/crash_dump.h>
24 #include <linux/sort.h>
25 #include <linux/stop_machine.h>
26 #include <linux/types.h>
27 #include <linux/mm.h>
28 #include <asm/cpu.h>
29 #include <asm/cpufeature.h>
30 #include <asm/cpu_ops.h>
31 #include <asm/fpsimd.h>
32 #include <asm/mmu_context.h>
33 #include <asm/processor.h>
34 #include <asm/sysreg.h>
35 #include <asm/traps.h>
36 #include <asm/virt.h>
37
38 unsigned long elf_hwcap __read_mostly;
39 EXPORT_SYMBOL_GPL(elf_hwcap);
40
41 #ifdef CONFIG_COMPAT
42 #define COMPAT_ELF_HWCAP_DEFAULT        \
43                                 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
44                                  COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
45                                  COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
46                                  COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
47                                  COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
48                                  COMPAT_HWCAP_LPAE)
49 unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
50 unsigned int compat_elf_hwcap2 __read_mostly;
51 #endif
52
53 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
54 EXPORT_SYMBOL(cpu_hwcaps);
55
56 /*
57  * Flag to indicate if we have computed the system wide
58  * capabilities based on the boot time active CPUs. This
59  * will be used to determine if a new booting CPU should
60  * go through the verification process to make sure that it
61  * supports the system capabilities, without using a hotplug
62  * notifier.
63  */
64 static bool sys_caps_initialised;
65
66 static inline void set_sys_caps_initialised(void)
67 {
68         sys_caps_initialised = true;
69 }
70
71 static int dump_cpu_hwcaps(struct notifier_block *self, unsigned long v, void *p)
72 {
73         /* file-wide pr_fmt adds "CPU features: " prefix */
74         pr_emerg("0x%*pb\n", ARM64_NCAPS, &cpu_hwcaps);
75         return 0;
76 }
77
78 static struct notifier_block cpu_hwcaps_notifier = {
79         .notifier_call = dump_cpu_hwcaps
80 };
81
82 static int __init register_cpu_hwcaps_dumper(void)
83 {
84         atomic_notifier_chain_register(&panic_notifier_list,
85                                        &cpu_hwcaps_notifier);
86         return 0;
87 }
88 __initcall(register_cpu_hwcaps_dumper);
89
90 DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
91 EXPORT_SYMBOL(cpu_hwcap_keys);
92
93 #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
94         {                                               \
95                 .sign = SIGNED,                         \
96                 .visible = VISIBLE,                     \
97                 .strict = STRICT,                       \
98                 .type = TYPE,                           \
99                 .shift = SHIFT,                         \
100                 .width = WIDTH,                         \
101                 .safe_val = SAFE_VAL,                   \
102         }
103
104 /* Define a feature with unsigned values */
105 #define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
106         __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
107
108 /* Define a feature with a signed value */
109 #define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
110         __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
111
112 #define ARM64_FTR_END                                   \
113         {                                               \
114                 .width = 0,                             \
115         }
116
117 /* meta feature for alternatives */
118 static bool __maybe_unused
119 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused);
120
121 static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
122
123 /*
124  * NOTE: Any changes to the visibility of features should be kept in
125  * sync with the documentation of the CPU feature register ABI.
126  */
127 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
128         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_TS_SHIFT, 4, 0),
129         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_FHM_SHIFT, 4, 0),
130         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0),
131         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM4_SHIFT, 4, 0),
132         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM3_SHIFT, 4, 0),
133         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA3_SHIFT, 4, 0),
134         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
135         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
136         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
137         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
138         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0),
139         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0),
140         ARM64_FTR_END,
141 };
142
143 static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
144         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
145         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
146         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
147         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
148         ARM64_FTR_END,
149 };
150
151 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
152         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
153         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
154         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_DIT_SHIFT, 4, 0),
155         ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
156                                    FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
157         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0),
158         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
159         S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
160         S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
161         /* Linux doesn't care about the EL3 */
162         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0),
163         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0),
164         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
165         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
166         ARM64_FTR_END,
167 };
168
169 static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
170         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
171         ARM64_FTR_END,
172 };
173
174 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
175         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
176         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
177         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
178         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
179         /* Linux shouldn't care about secure memory */
180         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
181         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
182         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
183         /*
184          * Differing PARange is fine as long as all peripherals and memory are mapped
185          * within the minimum PARange of all CPUs
186          */
187         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
188         ARM64_FTR_END,
189 };
190
191 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
192         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
193         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
194         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
195         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
196         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
197         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
198         ARM64_FTR_END,
199 };
200
201 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
202         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_FWB_SHIFT, 4, 0),
203         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_AT_SHIFT, 4, 0),
204         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
205         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
206         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
207         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
208         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
209         ARM64_FTR_END,
210 };
211
212 static const struct arm64_ftr_bits ftr_ctr[] = {
213         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
214         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DIC_SHIFT, 1, 1),
215         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IDC_SHIFT, 1, 1),
216         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_CWG_SHIFT, 4, 0),
217         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_ERG_SHIFT, 4, 0),
218         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DMINLINE_SHIFT, 4, 1),
219         /*
220          * Linux can handle differing I-cache policies. Userspace JITs will
221          * make use of *minLine.
222          * If we have differing I-cache policies, report it as the weakest - VIPT.
223          */
224         ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_VIPT),       /* L1Ip */
225         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IMINLINE_SHIFT, 4, 0),
226         ARM64_FTR_END,
227 };
228
229 struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
230         .name           = "SYS_CTR_EL0",
231         .ftr_bits       = ftr_ctr
232 };
233
234 static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
235         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0xf),   /* InnerShr */
236         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),       /* FCSE */
237         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0),    /* AuxReg */
238         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),       /* TCM */
239         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),       /* ShareLvl */
240         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0xf),    /* OuterShr */
241         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),        /* PMSA */
242         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),        /* VMSA */
243         ARM64_FTR_END,
244 };
245
246 static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
247         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 36, 28, 0),
248         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_PMSVER_SHIFT, 4, 0),
249         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
250         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
251         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
252         /*
253          * We can instantiate multiple PMU instances with different levels
254          * of support.
255          */
256         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
257         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_TRACEVER_SHIFT, 4, 0),
258         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
259         ARM64_FTR_END,
260 };
261
262 static const struct arm64_ftr_bits ftr_mvfr2[] = {
263         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),                /* FPMisc */
264         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),                /* SIMDMisc */
265         ARM64_FTR_END,
266 };
267
268 static const struct arm64_ftr_bits ftr_dczid[] = {
269         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 4, 1, 1),            /* DZP */
270         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),       /* BS */
271         ARM64_FTR_END,
272 };
273
274
275 static const struct arm64_ftr_bits ftr_id_isar5[] = {
276         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
277         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
278         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
279         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
280         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
281         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
282         ARM64_FTR_END,
283 };
284
285 static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
286         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),        /* ac2 */
287         ARM64_FTR_END,
288 };
289
290 static const struct arm64_ftr_bits ftr_id_pfr0[] = {
291         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),               /* State3 */
292         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),                /* State2 */
293         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),                /* State1 */
294         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),                /* State0 */
295         ARM64_FTR_END,
296 };
297
298 static const struct arm64_ftr_bits ftr_id_dfr0[] = {
299         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
300         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0xf),   /* PerfMon */
301         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
302         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
303         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
304         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
305         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
306         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
307         ARM64_FTR_END,
308 };
309
310 static const struct arm64_ftr_bits ftr_zcr[] = {
311         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
312                 ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0),        /* LEN */
313         ARM64_FTR_END,
314 };
315
316 /*
317  * Common ftr bits for a 32bit register with all hidden, strict
318  * attributes, with 4bit feature fields and a default safe value of
319  * 0. Covers the following 32bit registers:
320  * id_isar[0-4], id_mmfr[1-3], id_pfr1, mvfr[0-1]
321  */
322 static const struct arm64_ftr_bits ftr_generic_32bits[] = {
323         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
324         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
325         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
326         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
327         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
328         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
329         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
330         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
331         ARM64_FTR_END,
332 };
333
334 /* Table for a single 32bit feature value */
335 static const struct arm64_ftr_bits ftr_single32[] = {
336         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0),
337         ARM64_FTR_END,
338 };
339
340 static const struct arm64_ftr_bits ftr_raz[] = {
341         ARM64_FTR_END,
342 };
343
344 #define ARM64_FTR_REG(id, table) {              \
345         .sys_id = id,                           \
346         .reg =  &(struct arm64_ftr_reg){        \
347                 .name = #id,                    \
348                 .ftr_bits = &((table)[0]),      \
349         }}
350
351 static const struct __ftr_reg_entry {
352         u32                     sys_id;
353         struct arm64_ftr_reg    *reg;
354 } arm64_ftr_regs[] = {
355
356         /* Op1 = 0, CRn = 0, CRm = 1 */
357         ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
358         ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_generic_32bits),
359         ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0),
360         ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
361         ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
362         ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
363         ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
364
365         /* Op1 = 0, CRn = 0, CRm = 2 */
366         ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_generic_32bits),
367         ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
368         ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
369         ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
370         ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_generic_32bits),
371         ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
372         ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
373
374         /* Op1 = 0, CRn = 0, CRm = 3 */
375         ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits),
376         ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits),
377         ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
378
379         /* Op1 = 0, CRn = 0, CRm = 4 */
380         ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
381         ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1),
382         ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),
383
384         /* Op1 = 0, CRn = 0, CRm = 5 */
385         ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
386         ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz),
387
388         /* Op1 = 0, CRn = 0, CRm = 6 */
389         ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
390         ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1),
391
392         /* Op1 = 0, CRn = 0, CRm = 7 */
393         ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
394         ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
395         ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
396
397         /* Op1 = 0, CRn = 1, CRm = 2 */
398         ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
399
400         /* Op1 = 3, CRn = 0, CRm = 0 */
401         { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
402         ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
403
404         /* Op1 = 3, CRn = 14, CRm = 0 */
405         ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32),
406 };
407
408 static int search_cmp_ftr_reg(const void *id, const void *regp)
409 {
410         return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id;
411 }
412
413 /*
414  * get_arm64_ftr_reg - Lookup a feature register entry using its
415  * sys_reg() encoding. With the array arm64_ftr_regs sorted in the
416  * ascending order of sys_id , we use binary search to find a matching
417  * entry.
418  *
419  * returns - Upon success,  matching ftr_reg entry for id.
420  *         - NULL on failure. It is upto the caller to decide
421  *           the impact of a failure.
422  */
423 static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
424 {
425         const struct __ftr_reg_entry *ret;
426
427         ret = bsearch((const void *)(unsigned long)sys_id,
428                         arm64_ftr_regs,
429                         ARRAY_SIZE(arm64_ftr_regs),
430                         sizeof(arm64_ftr_regs[0]),
431                         search_cmp_ftr_reg);
432         if (ret)
433                 return ret->reg;
434         return NULL;
435 }
436
437 static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg,
438                                s64 ftr_val)
439 {
440         u64 mask = arm64_ftr_mask(ftrp);
441
442         reg &= ~mask;
443         reg |= (ftr_val << ftrp->shift) & mask;
444         return reg;
445 }
446
447 static s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new,
448                                 s64 cur)
449 {
450         s64 ret = 0;
451
452         switch (ftrp->type) {
453         case FTR_EXACT:
454                 ret = ftrp->safe_val;
455                 break;
456         case FTR_LOWER_SAFE:
457                 ret = new < cur ? new : cur;
458                 break;
459         case FTR_HIGHER_SAFE:
460                 ret = new > cur ? new : cur;
461                 break;
462         default:
463                 BUG();
464         }
465
466         return ret;
467 }
468
469 static void __init sort_ftr_regs(void)
470 {
471         int i;
472
473         /* Check that the array is sorted so that we can do the binary search */
474         for (i = 1; i < ARRAY_SIZE(arm64_ftr_regs); i++)
475                 BUG_ON(arm64_ftr_regs[i].sys_id < arm64_ftr_regs[i - 1].sys_id);
476 }
477
478 /*
479  * Initialise the CPU feature register from Boot CPU values.
480  * Also initiliases the strict_mask for the register.
481  * Any bits that are not covered by an arm64_ftr_bits entry are considered
482  * RES0 for the system-wide value, and must strictly match.
483  */
484 static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
485 {
486         u64 val = 0;
487         u64 strict_mask = ~0x0ULL;
488         u64 user_mask = 0;
489         u64 valid_mask = 0;
490
491         const struct arm64_ftr_bits *ftrp;
492         struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
493
494         BUG_ON(!reg);
495
496         for (ftrp  = reg->ftr_bits; ftrp->width; ftrp++) {
497                 u64 ftr_mask = arm64_ftr_mask(ftrp);
498                 s64 ftr_new = arm64_ftr_value(ftrp, new);
499
500                 val = arm64_ftr_set_value(ftrp, val, ftr_new);
501
502                 valid_mask |= ftr_mask;
503                 if (!ftrp->strict)
504                         strict_mask &= ~ftr_mask;
505                 if (ftrp->visible)
506                         user_mask |= ftr_mask;
507                 else
508                         reg->user_val = arm64_ftr_set_value(ftrp,
509                                                             reg->user_val,
510                                                             ftrp->safe_val);
511         }
512
513         val &= valid_mask;
514
515         reg->sys_val = val;
516         reg->strict_mask = strict_mask;
517         reg->user_mask = user_mask;
518 }
519
520 extern const struct arm64_cpu_capabilities arm64_errata[];
521 static void __init setup_boot_cpu_capabilities(void);
522
523 void __init init_cpu_features(struct cpuinfo_arm64 *info)
524 {
525         /* Before we start using the tables, make sure it is sorted */
526         sort_ftr_regs();
527
528         init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
529         init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
530         init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
531         init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
532         init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
533         init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
534         init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
535         init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
536         init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
537         init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
538         init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
539         init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
540         init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
541
542         if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
543                 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
544                 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
545                 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
546                 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
547                 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
548                 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
549                 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
550                 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
551                 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
552                 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
553                 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
554                 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
555                 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
556                 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
557                 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
558                 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
559         }
560
561         if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
562                 init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
563                 sve_init_vq_map();
564         }
565
566         /*
567          * Detect and enable early CPU capabilities based on the boot CPU,
568          * after we have initialised the CPU feature infrastructure.
569          */
570         setup_boot_cpu_capabilities();
571 }
572
573 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
574 {
575         const struct arm64_ftr_bits *ftrp;
576
577         for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
578                 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
579                 s64 ftr_new = arm64_ftr_value(ftrp, new);
580
581                 if (ftr_cur == ftr_new)
582                         continue;
583                 /* Find a safe value */
584                 ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
585                 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
586         }
587
588 }
589
590 static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
591 {
592         struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
593
594         BUG_ON(!regp);
595         update_cpu_ftr_reg(regp, val);
596         if ((boot & regp->strict_mask) == (val & regp->strict_mask))
597                 return 0;
598         pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
599                         regp->name, boot, cpu, val);
600         return 1;
601 }
602
603 /*
604  * Update system wide CPU feature registers with the values from a
605  * non-boot CPU. Also performs SANITY checks to make sure that there
606  * aren't any insane variations from that of the boot CPU.
607  */
608 void update_cpu_features(int cpu,
609                          struct cpuinfo_arm64 *info,
610                          struct cpuinfo_arm64 *boot)
611 {
612         int taint = 0;
613
614         /*
615          * The kernel can handle differing I-cache policies, but otherwise
616          * caches should look identical. Userspace JITs will make use of
617          * *minLine.
618          */
619         taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
620                                       info->reg_ctr, boot->reg_ctr);
621
622         /*
623          * Userspace may perform DC ZVA instructions. Mismatched block sizes
624          * could result in too much or too little memory being zeroed if a
625          * process is preempted and migrated between CPUs.
626          */
627         taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
628                                       info->reg_dczid, boot->reg_dczid);
629
630         /* If different, timekeeping will be broken (especially with KVM) */
631         taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
632                                       info->reg_cntfrq, boot->reg_cntfrq);
633
634         /*
635          * The kernel uses self-hosted debug features and expects CPUs to
636          * support identical debug features. We presently need CTX_CMPs, WRPs,
637          * and BRPs to be identical.
638          * ID_AA64DFR1 is currently RES0.
639          */
640         taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
641                                       info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
642         taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
643                                       info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
644         /*
645          * Even in big.LITTLE, processors should be identical instruction-set
646          * wise.
647          */
648         taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
649                                       info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
650         taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
651                                       info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
652
653         /*
654          * Differing PARange support is fine as long as all peripherals and
655          * memory are mapped within the minimum PARange of all CPUs.
656          * Linux should not care about secure memory.
657          */
658         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
659                                       info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
660         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
661                                       info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
662         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu,
663                                       info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2);
664
665         /*
666          * EL3 is not our concern.
667          */
668         taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
669                                       info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
670         taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
671                                       info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
672
673         taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
674                                       info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
675
676         /*
677          * If we have AArch32, we care about 32-bit features for compat.
678          * If the system doesn't support AArch32, don't update them.
679          */
680         if (id_aa64pfr0_32bit_el0(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
681                 id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
682
683                 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
684                                         info->reg_id_dfr0, boot->reg_id_dfr0);
685                 taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
686                                         info->reg_id_isar0, boot->reg_id_isar0);
687                 taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
688                                         info->reg_id_isar1, boot->reg_id_isar1);
689                 taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
690                                         info->reg_id_isar2, boot->reg_id_isar2);
691                 taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
692                                         info->reg_id_isar3, boot->reg_id_isar3);
693                 taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
694                                         info->reg_id_isar4, boot->reg_id_isar4);
695                 taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
696                                         info->reg_id_isar5, boot->reg_id_isar5);
697
698                 /*
699                  * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
700                  * ACTLR formats could differ across CPUs and therefore would have to
701                  * be trapped for virtualization anyway.
702                  */
703                 taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
704                                         info->reg_id_mmfr0, boot->reg_id_mmfr0);
705                 taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
706                                         info->reg_id_mmfr1, boot->reg_id_mmfr1);
707                 taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
708                                         info->reg_id_mmfr2, boot->reg_id_mmfr2);
709                 taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
710                                         info->reg_id_mmfr3, boot->reg_id_mmfr3);
711                 taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
712                                         info->reg_id_pfr0, boot->reg_id_pfr0);
713                 taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
714                                         info->reg_id_pfr1, boot->reg_id_pfr1);
715                 taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
716                                         info->reg_mvfr0, boot->reg_mvfr0);
717                 taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
718                                         info->reg_mvfr1, boot->reg_mvfr1);
719                 taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
720                                         info->reg_mvfr2, boot->reg_mvfr2);
721         }
722
723         if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
724                 taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
725                                         info->reg_zcr, boot->reg_zcr);
726
727                 /* Probe vector lengths, unless we already gave up on SVE */
728                 if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
729                     !sys_caps_initialised)
730                         sve_update_vq_map();
731         }
732
733         /*
734          * Mismatched CPU features are a recipe for disaster. Don't even
735          * pretend to support them.
736          */
737         if (taint) {
738                 pr_warn_once("Unsupported CPU feature variation detected.\n");
739                 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
740         }
741 }
742
743 u64 read_sanitised_ftr_reg(u32 id)
744 {
745         struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
746
747         /* We shouldn't get a request for an unsupported register */
748         BUG_ON(!regp);
749         return regp->sys_val;
750 }
751
752 #define read_sysreg_case(r)     \
753         case r:         return read_sysreg_s(r)
754
755 /*
756  * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
757  * Read the system register on the current CPU
758  */
759 static u64 __read_sysreg_by_encoding(u32 sys_id)
760 {
761         switch (sys_id) {
762         read_sysreg_case(SYS_ID_PFR0_EL1);
763         read_sysreg_case(SYS_ID_PFR1_EL1);
764         read_sysreg_case(SYS_ID_DFR0_EL1);
765         read_sysreg_case(SYS_ID_MMFR0_EL1);
766         read_sysreg_case(SYS_ID_MMFR1_EL1);
767         read_sysreg_case(SYS_ID_MMFR2_EL1);
768         read_sysreg_case(SYS_ID_MMFR3_EL1);
769         read_sysreg_case(SYS_ID_ISAR0_EL1);
770         read_sysreg_case(SYS_ID_ISAR1_EL1);
771         read_sysreg_case(SYS_ID_ISAR2_EL1);
772         read_sysreg_case(SYS_ID_ISAR3_EL1);
773         read_sysreg_case(SYS_ID_ISAR4_EL1);
774         read_sysreg_case(SYS_ID_ISAR5_EL1);
775         read_sysreg_case(SYS_MVFR0_EL1);
776         read_sysreg_case(SYS_MVFR1_EL1);
777         read_sysreg_case(SYS_MVFR2_EL1);
778
779         read_sysreg_case(SYS_ID_AA64PFR0_EL1);
780         read_sysreg_case(SYS_ID_AA64PFR1_EL1);
781         read_sysreg_case(SYS_ID_AA64DFR0_EL1);
782         read_sysreg_case(SYS_ID_AA64DFR1_EL1);
783         read_sysreg_case(SYS_ID_AA64MMFR0_EL1);
784         read_sysreg_case(SYS_ID_AA64MMFR1_EL1);
785         read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
786         read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
787         read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
788
789         read_sysreg_case(SYS_CNTFRQ_EL0);
790         read_sysreg_case(SYS_CTR_EL0);
791         read_sysreg_case(SYS_DCZID_EL0);
792
793         default:
794                 BUG();
795                 return 0;
796         }
797 }
798
799 #include <linux/irqchip/arm-gic-v3.h>
800
801 static bool
802 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
803 {
804         int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign);
805
806         return val >= entry->min_field_value;
807 }
808
809 static bool
810 has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
811 {
812         u64 val;
813
814         WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
815         if (scope == SCOPE_SYSTEM)
816                 val = read_sanitised_ftr_reg(entry->sys_reg);
817         else
818                 val = __read_sysreg_by_encoding(entry->sys_reg);
819
820         return feature_matches(val, entry);
821 }
822
823 static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
824 {
825         bool has_sre;
826
827         if (!has_cpuid_feature(entry, scope))
828                 return false;
829
830         has_sre = gic_enable_sre();
831         if (!has_sre)
832                 pr_warn_once("%s present but disabled by higher exception level\n",
833                              entry->desc);
834
835         return has_sre;
836 }
837
838 static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
839 {
840         u32 midr = read_cpuid_id();
841
842         /* Cavium ThunderX pass 1.x and 2.x */
843         return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX,
844                 MIDR_CPU_VAR_REV(0, 0),
845                 MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
846 }
847
848 static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
849 {
850         u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
851
852         return cpuid_feature_extract_signed_field(pfr0,
853                                         ID_AA64PFR0_FP_SHIFT) < 0;
854 }
855
856 static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
857                           int __unused)
858 {
859         return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_IDC_SHIFT);
860 }
861
862 static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
863                           int __unused)
864 {
865         return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_DIC_SHIFT);
866 }
867
868 static bool __maybe_unused
869 has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope)
870 {
871         /*
872          * Kdump isn't guaranteed to power-off all secondary CPUs, CNP
873          * may share TLB entries with a CPU stuck in the crashed
874          * kernel.
875          */
876          if (is_kdump_kernel())
877                 return false;
878
879         return has_cpuid_feature(entry, scope);
880 }
881
882 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
883 static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
884
885 static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
886                                 int scope)
887 {
888         /* List of CPUs that are not vulnerable and don't need KPTI */
889         static const struct midr_range kpti_safe_list[] = {
890                 MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
891                 MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
892                 { /* sentinel */ }
893         };
894         char const *str = "command line option";
895
896         /*
897          * For reasons that aren't entirely clear, enabling KPTI on Cavium
898          * ThunderX leads to apparent I-cache corruption of kernel text, which
899          * ends as well as you might imagine. Don't even try.
900          */
901         if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
902                 str = "ARM64_WORKAROUND_CAVIUM_27456";
903                 __kpti_forced = -1;
904         }
905
906         /* Forced? */
907         if (__kpti_forced) {
908                 pr_info_once("kernel page table isolation forced %s by %s\n",
909                              __kpti_forced > 0 ? "ON" : "OFF", str);
910                 return __kpti_forced > 0;
911         }
912
913         /* Useful for KASLR robustness */
914         if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
915                 return true;
916
917         /* Don't force KPTI for CPUs that are not vulnerable */
918         if (is_midr_in_range_list(read_cpuid_id(), kpti_safe_list))
919                 return false;
920
921         /* Defer to CPU feature registers */
922         return !has_cpuid_feature(entry, scope);
923 }
924
925 static void
926 kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
927 {
928         typedef void (kpti_remap_fn)(int, int, phys_addr_t);
929         extern kpti_remap_fn idmap_kpti_install_ng_mappings;
930         kpti_remap_fn *remap_fn;
931
932         static bool kpti_applied = false;
933         int cpu = smp_processor_id();
934
935         if (kpti_applied)
936                 return;
937
938         remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
939
940         cpu_install_idmap();
941         remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
942         cpu_uninstall_idmap();
943
944         if (!cpu)
945                 kpti_applied = true;
946
947         return;
948 }
949
950 static int __init parse_kpti(char *str)
951 {
952         bool enabled;
953         int ret = strtobool(str, &enabled);
954
955         if (ret)
956                 return ret;
957
958         __kpti_forced = enabled ? 1 : -1;
959         return 0;
960 }
961 early_param("kpti", parse_kpti);
962 #endif  /* CONFIG_UNMAP_KERNEL_AT_EL0 */
963
964 #ifdef CONFIG_ARM64_HW_AFDBM
965 static inline void __cpu_enable_hw_dbm(void)
966 {
967         u64 tcr = read_sysreg(tcr_el1) | TCR_HD;
968
969         write_sysreg(tcr, tcr_el1);
970         isb();
971 }
972
973 static bool cpu_has_broken_dbm(void)
974 {
975         /* List of CPUs which have broken DBM support. */
976         static const struct midr_range cpus[] = {
977 #ifdef CONFIG_ARM64_ERRATUM_1024718
978                 MIDR_RANGE(MIDR_CORTEX_A55, 0, 0, 1, 0),  // A55 r0p0 -r1p0
979 #endif
980                 {},
981         };
982
983         return is_midr_in_range_list(read_cpuid_id(), cpus);
984 }
985
986 static bool cpu_can_use_dbm(const struct arm64_cpu_capabilities *cap)
987 {
988         return has_cpuid_feature(cap, SCOPE_LOCAL_CPU) &&
989                !cpu_has_broken_dbm();
990 }
991
992 static void cpu_enable_hw_dbm(struct arm64_cpu_capabilities const *cap)
993 {
994         if (cpu_can_use_dbm(cap))
995                 __cpu_enable_hw_dbm();
996 }
997
998 static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
999                        int __unused)
1000 {
1001         static bool detected = false;
1002         /*
1003          * DBM is a non-conflicting feature. i.e, the kernel can safely
1004          * run a mix of CPUs with and without the feature. So, we
1005          * unconditionally enable the capability to allow any late CPU
1006          * to use the feature. We only enable the control bits on the
1007          * CPU, if it actually supports.
1008          *
1009          * We have to make sure we print the "feature" detection only
1010          * when at least one CPU actually uses it. So check if this CPU
1011          * can actually use it and print the message exactly once.
1012          *
1013          * This is safe as all CPUs (including secondary CPUs - due to the
1014          * LOCAL_CPU scope - and the hotplugged CPUs - via verification)
1015          * goes through the "matches" check exactly once. Also if a CPU
1016          * matches the criteria, it is guaranteed that the CPU will turn
1017          * the DBM on, as the capability is unconditionally enabled.
1018          */
1019         if (!detected && cpu_can_use_dbm(cap)) {
1020                 detected = true;
1021                 pr_info("detected: Hardware dirty bit management\n");
1022         }
1023
1024         return true;
1025 }
1026
1027 #endif
1028
1029 #ifdef CONFIG_ARM64_VHE
1030 static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
1031 {
1032         return is_kernel_in_hyp_mode();
1033 }
1034
1035 static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
1036 {
1037         /*
1038          * Copy register values that aren't redirected by hardware.
1039          *
1040          * Before code patching, we only set tpidr_el1, all CPUs need to copy
1041          * this value to tpidr_el2 before we patch the code. Once we've done
1042          * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
1043          * do anything here.
1044          */
1045         if (!alternatives_applied)
1046                 write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
1047 }
1048 #endif
1049
1050 static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused)
1051 {
1052         u64 val = read_sysreg_s(SYS_CLIDR_EL1);
1053
1054         /* Check that CLIDR_EL1.LOU{U,IS} are both 0 */
1055         WARN_ON(val & (7 << 27 | 7 << 21));
1056 }
1057
1058 #ifdef CONFIG_ARM64_SSBD
1059 static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
1060 {
1061         if (user_mode(regs))
1062                 return 1;
1063
1064         if (instr & BIT(PSTATE_Imm_shift))
1065                 regs->pstate |= PSR_SSBS_BIT;
1066         else
1067                 regs->pstate &= ~PSR_SSBS_BIT;
1068
1069         arm64_skip_faulting_instruction(regs, 4);
1070         return 0;
1071 }
1072
1073 static struct undef_hook ssbs_emulation_hook = {
1074         .instr_mask     = ~(1U << PSTATE_Imm_shift),
1075         .instr_val      = 0xd500401f | PSTATE_SSBS,
1076         .fn             = ssbs_emulation_handler,
1077 };
1078
1079 static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused)
1080 {
1081         static bool undef_hook_registered = false;
1082         static DEFINE_SPINLOCK(hook_lock);
1083
1084         spin_lock(&hook_lock);
1085         if (!undef_hook_registered) {
1086                 register_undef_hook(&ssbs_emulation_hook);
1087                 undef_hook_registered = true;
1088         }
1089         spin_unlock(&hook_lock);
1090
1091         if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
1092                 sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
1093                 arm64_set_ssbd_mitigation(false);
1094         } else {
1095                 arm64_set_ssbd_mitigation(true);
1096         }
1097 }
1098 #endif /* CONFIG_ARM64_SSBD */
1099
1100 #ifdef CONFIG_ARM64_PAN
1101 static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
1102 {
1103         /*
1104          * We modify PSTATE. This won't work from irq context as the PSTATE
1105          * is discarded once we return from the exception.
1106          */
1107         WARN_ON_ONCE(in_interrupt());
1108
1109         sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
1110         asm(SET_PSTATE_PAN(1));
1111 }
1112 #endif /* CONFIG_ARM64_PAN */
1113
1114 #ifdef CONFIG_ARM64_RAS_EXTN
1115 static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
1116 {
1117         /* Firmware may have left a deferred SError in this register. */
1118         write_sysreg_s(0, SYS_DISR_EL1);
1119 }
1120 #endif /* CONFIG_ARM64_RAS_EXTN */
1121
1122 static const struct arm64_cpu_capabilities arm64_features[] = {
1123         {
1124                 .desc = "GIC system register CPU interface",
1125                 .capability = ARM64_HAS_SYSREG_GIC_CPUIF,
1126                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1127                 .matches = has_useable_gicv3_cpuif,
1128                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1129                 .field_pos = ID_AA64PFR0_GIC_SHIFT,
1130                 .sign = FTR_UNSIGNED,
1131                 .min_field_value = 1,
1132         },
1133 #ifdef CONFIG_ARM64_PAN
1134         {
1135                 .desc = "Privileged Access Never",
1136                 .capability = ARM64_HAS_PAN,
1137                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1138                 .matches = has_cpuid_feature,
1139                 .sys_reg = SYS_ID_AA64MMFR1_EL1,
1140                 .field_pos = ID_AA64MMFR1_PAN_SHIFT,
1141                 .sign = FTR_UNSIGNED,
1142                 .min_field_value = 1,
1143                 .cpu_enable = cpu_enable_pan,
1144         },
1145 #endif /* CONFIG_ARM64_PAN */
1146 #if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
1147         {
1148                 .desc = "LSE atomic instructions",
1149                 .capability = ARM64_HAS_LSE_ATOMICS,
1150                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1151                 .matches = has_cpuid_feature,
1152                 .sys_reg = SYS_ID_AA64ISAR0_EL1,
1153                 .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
1154                 .sign = FTR_UNSIGNED,
1155                 .min_field_value = 2,
1156         },
1157 #endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
1158         {
1159                 .desc = "Software prefetching using PRFM",
1160                 .capability = ARM64_HAS_NO_HW_PREFETCH,
1161                 .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
1162                 .matches = has_no_hw_prefetch,
1163         },
1164 #ifdef CONFIG_ARM64_UAO
1165         {
1166                 .desc = "User Access Override",
1167                 .capability = ARM64_HAS_UAO,
1168                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1169                 .matches = has_cpuid_feature,
1170                 .sys_reg = SYS_ID_AA64MMFR2_EL1,
1171                 .field_pos = ID_AA64MMFR2_UAO_SHIFT,
1172                 .min_field_value = 1,
1173                 /*
1174                  * We rely on stop_machine() calling uao_thread_switch() to set
1175                  * UAO immediately after patching.
1176                  */
1177         },
1178 #endif /* CONFIG_ARM64_UAO */
1179 #ifdef CONFIG_ARM64_PAN
1180         {
1181                 .capability = ARM64_ALT_PAN_NOT_UAO,
1182                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1183                 .matches = cpufeature_pan_not_uao,
1184         },
1185 #endif /* CONFIG_ARM64_PAN */
1186 #ifdef CONFIG_ARM64_VHE
1187         {
1188                 .desc = "Virtualization Host Extensions",
1189                 .capability = ARM64_HAS_VIRT_HOST_EXTN,
1190                 .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
1191                 .matches = runs_at_el2,
1192                 .cpu_enable = cpu_copy_el2regs,
1193         },
1194 #endif  /* CONFIG_ARM64_VHE */
1195         {
1196                 .desc = "32-bit EL0 Support",
1197                 .capability = ARM64_HAS_32BIT_EL0,
1198                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1199                 .matches = has_cpuid_feature,
1200                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1201                 .sign = FTR_UNSIGNED,
1202                 .field_pos = ID_AA64PFR0_EL0_SHIFT,
1203                 .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT,
1204         },
1205 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1206         {
1207                 .desc = "Kernel page table isolation (KPTI)",
1208                 .capability = ARM64_UNMAP_KERNEL_AT_EL0,
1209                 .type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
1210                 /*
1211                  * The ID feature fields below are used to indicate that
1212                  * the CPU doesn't need KPTI. See unmap_kernel_at_el0 for
1213                  * more details.
1214                  */
1215                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1216                 .field_pos = ID_AA64PFR0_CSV3_SHIFT,
1217                 .min_field_value = 1,
1218                 .matches = unmap_kernel_at_el0,
1219                 .cpu_enable = kpti_install_ng_mappings,
1220         },
1221 #endif
1222         {
1223                 /* FP/SIMD is not implemented */
1224                 .capability = ARM64_HAS_NO_FPSIMD,
1225                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1226                 .min_field_value = 0,
1227                 .matches = has_no_fpsimd,
1228         },
1229 #ifdef CONFIG_ARM64_PMEM
1230         {
1231                 .desc = "Data cache clean to Point of Persistence",
1232                 .capability = ARM64_HAS_DCPOP,
1233                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1234                 .matches = has_cpuid_feature,
1235                 .sys_reg = SYS_ID_AA64ISAR1_EL1,
1236                 .field_pos = ID_AA64ISAR1_DPB_SHIFT,
1237                 .min_field_value = 1,
1238         },
1239 #endif
1240 #ifdef CONFIG_ARM64_SVE
1241         {
1242                 .desc = "Scalable Vector Extension",
1243                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1244                 .capability = ARM64_SVE,
1245                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1246                 .sign = FTR_UNSIGNED,
1247                 .field_pos = ID_AA64PFR0_SVE_SHIFT,
1248                 .min_field_value = ID_AA64PFR0_SVE,
1249                 .matches = has_cpuid_feature,
1250                 .cpu_enable = sve_kernel_enable,
1251         },
1252 #endif /* CONFIG_ARM64_SVE */
1253 #ifdef CONFIG_ARM64_RAS_EXTN
1254         {
1255                 .desc = "RAS Extension Support",
1256                 .capability = ARM64_HAS_RAS_EXTN,
1257                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1258                 .matches = has_cpuid_feature,
1259                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1260                 .sign = FTR_UNSIGNED,
1261                 .field_pos = ID_AA64PFR0_RAS_SHIFT,
1262                 .min_field_value = ID_AA64PFR0_RAS_V1,
1263                 .cpu_enable = cpu_clear_disr,
1264         },
1265 #endif /* CONFIG_ARM64_RAS_EXTN */
1266         {
1267                 .desc = "Data cache clean to the PoU not required for I/D coherence",
1268                 .capability = ARM64_HAS_CACHE_IDC,
1269                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1270                 .matches = has_cache_idc,
1271         },
1272         {
1273                 .desc = "Instruction cache invalidation not required for I/D coherence",
1274                 .capability = ARM64_HAS_CACHE_DIC,
1275                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1276                 .matches = has_cache_dic,
1277         },
1278         {
1279                 .desc = "Stage-2 Force Write-Back",
1280                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1281                 .capability = ARM64_HAS_STAGE2_FWB,
1282                 .sys_reg = SYS_ID_AA64MMFR2_EL1,
1283                 .sign = FTR_UNSIGNED,
1284                 .field_pos = ID_AA64MMFR2_FWB_SHIFT,
1285                 .min_field_value = 1,
1286                 .matches = has_cpuid_feature,
1287                 .cpu_enable = cpu_has_fwb,
1288         },
1289 #ifdef CONFIG_ARM64_HW_AFDBM
1290         {
1291                 /*
1292                  * Since we turn this on always, we don't want the user to
1293                  * think that the feature is available when it may not be.
1294                  * So hide the description.
1295                  *
1296                  * .desc = "Hardware pagetable Dirty Bit Management",
1297                  *
1298                  */
1299                 .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
1300                 .capability = ARM64_HW_DBM,
1301                 .sys_reg = SYS_ID_AA64MMFR1_EL1,
1302                 .sign = FTR_UNSIGNED,
1303                 .field_pos = ID_AA64MMFR1_HADBS_SHIFT,
1304                 .min_field_value = 2,
1305                 .matches = has_hw_dbm,
1306                 .cpu_enable = cpu_enable_hw_dbm,
1307         },
1308 #endif
1309 #ifdef CONFIG_ARM64_SSBD
1310         {
1311                 .desc = "CRC32 instructions",
1312                 .capability = ARM64_HAS_CRC32,
1313                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1314                 .matches = has_cpuid_feature,
1315                 .sys_reg = SYS_ID_AA64ISAR0_EL1,
1316                 .field_pos = ID_AA64ISAR0_CRC32_SHIFT,
1317                 .min_field_value = 1,
1318         },
1319         {
1320                 .desc = "Speculative Store Bypassing Safe (SSBS)",
1321                 .capability = ARM64_SSBS,
1322                 .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
1323                 .matches = has_cpuid_feature,
1324                 .sys_reg = SYS_ID_AA64PFR1_EL1,
1325                 .field_pos = ID_AA64PFR1_SSBS_SHIFT,
1326                 .sign = FTR_UNSIGNED,
1327                 .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,
1328                 .cpu_enable = cpu_enable_ssbs,
1329         },
1330 #endif
1331 #ifdef CONFIG_ARM64_CNP
1332         {
1333                 .desc = "Common not Private translations",
1334                 .capability = ARM64_HAS_CNP,
1335                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1336                 .matches = has_useable_cnp,
1337                 .sys_reg = SYS_ID_AA64MMFR2_EL1,
1338                 .sign = FTR_UNSIGNED,
1339                 .field_pos = ID_AA64MMFR2_CNP_SHIFT,
1340                 .min_field_value = 1,
1341                 .cpu_enable = cpu_enable_cnp,
1342         },
1343 #endif
1344         {},
1345 };
1346
1347 #define HWCAP_CAP(reg, field, s, min_value, cap_type, cap)      \
1348         {                                                       \
1349                 .desc = #cap,                                   \
1350                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,            \
1351                 .matches = has_cpuid_feature,                   \
1352                 .sys_reg = reg,                                 \
1353                 .field_pos = field,                             \
1354                 .sign = s,                                      \
1355                 .min_field_value = min_value,                   \
1356                 .hwcap_type = cap_type,                         \
1357                 .hwcap = cap,                                   \
1358         }
1359
1360 static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
1361         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_PMULL),
1362         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES),
1363         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA1),
1364         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA2),
1365         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_SHA512),
1366         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_CRC32),
1367         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ATOMICS),
1368         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDRDM),
1369         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA3),
1370         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM3),
1371         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM4),
1372         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDDP),
1373         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_FHM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDFHM),
1374         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FLAGM),
1375         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP),
1376         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
1377         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
1378         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
1379         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_DIT_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_DIT),
1380         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_DCPOP),
1381         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
1382         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
1383         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
1384         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ILRCPC),
1385         HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_USCAT),
1386 #ifdef CONFIG_ARM64_SVE
1387         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),
1388 #endif
1389         HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, HWCAP_SSBS),
1390         {},
1391 };
1392
1393 static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
1394 #ifdef CONFIG_COMPAT
1395         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
1396         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
1397         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
1398         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
1399         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
1400 #endif
1401         {},
1402 };
1403
1404 static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
1405 {
1406         switch (cap->hwcap_type) {
1407         case CAP_HWCAP:
1408                 elf_hwcap |= cap->hwcap;
1409                 break;
1410 #ifdef CONFIG_COMPAT
1411         case CAP_COMPAT_HWCAP:
1412                 compat_elf_hwcap |= (u32)cap->hwcap;
1413                 break;
1414         case CAP_COMPAT_HWCAP2:
1415                 compat_elf_hwcap2 |= (u32)cap->hwcap;
1416                 break;
1417 #endif
1418         default:
1419                 WARN_ON(1);
1420                 break;
1421         }
1422 }
1423
1424 /* Check if we have a particular HWCAP enabled */
1425 static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
1426 {
1427         bool rc;
1428
1429         switch (cap->hwcap_type) {
1430         case CAP_HWCAP:
1431                 rc = (elf_hwcap & cap->hwcap) != 0;
1432                 break;
1433 #ifdef CONFIG_COMPAT
1434         case CAP_COMPAT_HWCAP:
1435                 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
1436                 break;
1437         case CAP_COMPAT_HWCAP2:
1438                 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
1439                 break;
1440 #endif
1441         default:
1442                 WARN_ON(1);
1443                 rc = false;
1444         }
1445
1446         return rc;
1447 }
1448
1449 static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
1450 {
1451         /* We support emulation of accesses to CPU ID feature registers */
1452         elf_hwcap |= HWCAP_CPUID;
1453         for (; hwcaps->matches; hwcaps++)
1454                 if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps)))
1455                         cap_set_elf_hwcap(hwcaps);
1456 }
1457
1458 /*
1459  * Check if the current CPU has a given feature capability.
1460  * Should be called from non-preemptible context.
1461  */
1462 static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array,
1463                                unsigned int cap)
1464 {
1465         const struct arm64_cpu_capabilities *caps;
1466
1467         if (WARN_ON(preemptible()))
1468                 return false;
1469
1470         for (caps = cap_array; caps->matches; caps++)
1471                 if (caps->capability == cap)
1472                         return caps->matches(caps, SCOPE_LOCAL_CPU);
1473
1474         return false;
1475 }
1476
1477 static void __update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
1478                                       u16 scope_mask, const char *info)
1479 {
1480         scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
1481         for (; caps->matches; caps++) {
1482                 if (!(caps->type & scope_mask) ||
1483                     !caps->matches(caps, cpucap_default_scope(caps)))
1484                         continue;
1485
1486                 if (!cpus_have_cap(caps->capability) && caps->desc)
1487                         pr_info("%s %s\n", info, caps->desc);
1488                 cpus_set_cap(caps->capability);
1489         }
1490 }
1491
1492 static void update_cpu_capabilities(u16 scope_mask)
1493 {
1494         __update_cpu_capabilities(arm64_errata, scope_mask,
1495                                   "enabling workaround for");
1496         __update_cpu_capabilities(arm64_features, scope_mask, "detected:");
1497 }
1498
1499 static int __enable_cpu_capability(void *arg)
1500 {
1501         const struct arm64_cpu_capabilities *cap = arg;
1502
1503         cap->cpu_enable(cap);
1504         return 0;
1505 }
1506
1507 /*
1508  * Run through the enabled capabilities and enable() it on all active
1509  * CPUs
1510  */
1511 static void __init
1512 __enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
1513                           u16 scope_mask)
1514 {
1515         scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
1516         for (; caps->matches; caps++) {
1517                 unsigned int num = caps->capability;
1518
1519                 if (!(caps->type & scope_mask) || !cpus_have_cap(num))
1520                         continue;
1521
1522                 /* Ensure cpus_have_const_cap(num) works */
1523                 static_branch_enable(&cpu_hwcap_keys[num]);
1524
1525                 if (caps->cpu_enable) {
1526                         /*
1527                          * Capabilities with SCOPE_BOOT_CPU scope are finalised
1528                          * before any secondary CPU boots. Thus, each secondary
1529                          * will enable the capability as appropriate via
1530                          * check_local_cpu_capabilities(). The only exception is
1531                          * the boot CPU, for which the capability must be
1532                          * enabled here. This approach avoids costly
1533                          * stop_machine() calls for this case.
1534                          *
1535                          * Otherwise, use stop_machine() as it schedules the
1536                          * work allowing us to modify PSTATE, instead of
1537                          * on_each_cpu() which uses an IPI, giving us a PSTATE
1538                          * that disappears when we return.
1539                          */
1540                         if (scope_mask & SCOPE_BOOT_CPU)
1541                                 caps->cpu_enable(caps);
1542                         else
1543                                 stop_machine(__enable_cpu_capability,
1544                                              (void *)caps, cpu_online_mask);
1545                 }
1546         }
1547 }
1548
1549 static void __init enable_cpu_capabilities(u16 scope_mask)
1550 {
1551         __enable_cpu_capabilities(arm64_errata, scope_mask);
1552         __enable_cpu_capabilities(arm64_features, scope_mask);
1553 }
1554
1555 /*
1556  * Run through the list of capabilities to check for conflicts.
1557  * If the system has already detected a capability, take necessary
1558  * action on this CPU.
1559  *
1560  * Returns "false" on conflicts.
1561  */
1562 static bool
1563 __verify_local_cpu_caps(const struct arm64_cpu_capabilities *caps,
1564                         u16 scope_mask)
1565 {
1566         bool cpu_has_cap, system_has_cap;
1567
1568         scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
1569
1570         for (; caps->matches; caps++) {
1571                 if (!(caps->type & scope_mask))
1572                         continue;
1573
1574                 cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU);
1575                 system_has_cap = cpus_have_cap(caps->capability);
1576
1577                 if (system_has_cap) {
1578                         /*
1579                          * Check if the new CPU misses an advertised feature,
1580                          * which is not safe to miss.
1581                          */
1582                         if (!cpu_has_cap && !cpucap_late_cpu_optional(caps))
1583                                 break;
1584                         /*
1585                          * We have to issue cpu_enable() irrespective of
1586                          * whether the CPU has it or not, as it is enabeld
1587                          * system wide. It is upto the call back to take
1588                          * appropriate action on this CPU.
1589                          */
1590                         if (caps->cpu_enable)
1591                                 caps->cpu_enable(caps);
1592                 } else {
1593                         /*
1594                          * Check if the CPU has this capability if it isn't
1595                          * safe to have when the system doesn't.
1596                          */
1597                         if (cpu_has_cap && !cpucap_late_cpu_permitted(caps))
1598                                 break;
1599                 }
1600         }
1601
1602         if (caps->matches) {
1603                 pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n",
1604                         smp_processor_id(), caps->capability,
1605                         caps->desc, system_has_cap, cpu_has_cap);
1606                 return false;
1607         }
1608
1609         return true;
1610 }
1611
1612 static bool verify_local_cpu_caps(u16 scope_mask)
1613 {
1614         return __verify_local_cpu_caps(arm64_errata, scope_mask) &&
1615                __verify_local_cpu_caps(arm64_features, scope_mask);
1616 }
1617
1618 /*
1619  * Check for CPU features that are used in early boot
1620  * based on the Boot CPU value.
1621  */
1622 static void check_early_cpu_features(void)
1623 {
1624         verify_cpu_asid_bits();
1625         /*
1626          * Early features are used by the kernel already. If there
1627          * is a conflict, we cannot proceed further.
1628          */
1629         if (!verify_local_cpu_caps(SCOPE_BOOT_CPU))
1630                 cpu_panic_kernel();
1631 }
1632
1633 static void
1634 verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
1635 {
1636
1637         for (; caps->matches; caps++)
1638                 if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) {
1639                         pr_crit("CPU%d: missing HWCAP: %s\n",
1640                                         smp_processor_id(), caps->desc);
1641                         cpu_die_early();
1642                 }
1643 }
1644
1645 static void verify_sve_features(void)
1646 {
1647         u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
1648         u64 zcr = read_zcr_features();
1649
1650         unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
1651         unsigned int len = zcr & ZCR_ELx_LEN_MASK;
1652
1653         if (len < safe_len || sve_verify_vq_map()) {
1654                 pr_crit("CPU%d: SVE: required vector length(s) missing\n",
1655                         smp_processor_id());
1656                 cpu_die_early();
1657         }
1658
1659         /* Add checks on other ZCR bits here if necessary */
1660 }
1661
1662
1663 /*
1664  * Run through the enabled system capabilities and enable() it on this CPU.
1665  * The capabilities were decided based on the available CPUs at the boot time.
1666  * Any new CPU should match the system wide status of the capability. If the
1667  * new CPU doesn't have a capability which the system now has enabled, we
1668  * cannot do anything to fix it up and could cause unexpected failures. So
1669  * we park the CPU.
1670  */
1671 static void verify_local_cpu_capabilities(void)
1672 {
1673         /*
1674          * The capabilities with SCOPE_BOOT_CPU are checked from
1675          * check_early_cpu_features(), as they need to be verified
1676          * on all secondary CPUs.
1677          */
1678         if (!verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU))
1679                 cpu_die_early();
1680
1681         verify_local_elf_hwcaps(arm64_elf_hwcaps);
1682
1683         if (system_supports_32bit_el0())
1684                 verify_local_elf_hwcaps(compat_elf_hwcaps);
1685
1686         if (system_supports_sve())
1687                 verify_sve_features();
1688 }
1689
1690 void check_local_cpu_capabilities(void)
1691 {
1692         /*
1693          * All secondary CPUs should conform to the early CPU features
1694          * in use by the kernel based on boot CPU.
1695          */
1696         check_early_cpu_features();
1697
1698         /*
1699          * If we haven't finalised the system capabilities, this CPU gets
1700          * a chance to update the errata work arounds and local features.
1701          * Otherwise, this CPU should verify that it has all the system
1702          * advertised capabilities.
1703          */
1704         if (!sys_caps_initialised)
1705                 update_cpu_capabilities(SCOPE_LOCAL_CPU);
1706         else
1707                 verify_local_cpu_capabilities();
1708 }
1709
1710 static void __init setup_boot_cpu_capabilities(void)
1711 {
1712         /* Detect capabilities with either SCOPE_BOOT_CPU or SCOPE_LOCAL_CPU */
1713         update_cpu_capabilities(SCOPE_BOOT_CPU | SCOPE_LOCAL_CPU);
1714         /* Enable the SCOPE_BOOT_CPU capabilities alone right away */
1715         enable_cpu_capabilities(SCOPE_BOOT_CPU);
1716 }
1717
1718 DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready);
1719 EXPORT_SYMBOL(arm64_const_caps_ready);
1720
1721 static void __init mark_const_caps_ready(void)
1722 {
1723         static_branch_enable(&arm64_const_caps_ready);
1724 }
1725
1726 extern const struct arm64_cpu_capabilities arm64_errata[];
1727
1728 bool this_cpu_has_cap(unsigned int cap)
1729 {
1730         return (__this_cpu_has_cap(arm64_features, cap) ||
1731                 __this_cpu_has_cap(arm64_errata, cap));
1732 }
1733
1734 static void __init setup_system_capabilities(void)
1735 {
1736         /*
1737          * We have finalised the system-wide safe feature
1738          * registers, finalise the capabilities that depend
1739          * on it. Also enable all the available capabilities,
1740          * that are not enabled already.
1741          */
1742         update_cpu_capabilities(SCOPE_SYSTEM);
1743         enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);
1744 }
1745
1746 void __init setup_cpu_features(void)
1747 {
1748         u32 cwg;
1749
1750         setup_system_capabilities();
1751         mark_const_caps_ready();
1752         setup_elf_hwcaps(arm64_elf_hwcaps);
1753
1754         if (system_supports_32bit_el0())
1755                 setup_elf_hwcaps(compat_elf_hwcaps);
1756
1757         if (system_uses_ttbr0_pan())
1758                 pr_info("emulated: Privileged Access Never (PAN) using TTBR0_EL1 switching\n");
1759
1760         sve_setup();
1761         minsigstksz_setup();
1762
1763         /* Advertise that we have computed the system capabilities */
1764         set_sys_caps_initialised();
1765
1766         /*
1767          * Check for sane CTR_EL0.CWG value.
1768          */
1769         cwg = cache_type_cwg();
1770         if (!cwg)
1771                 pr_warn("No Cache Writeback Granule information, assuming %d\n",
1772                         ARCH_DMA_MINALIGN);
1773 }
1774
1775 static bool __maybe_unused
1776 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
1777 {
1778         return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO));
1779 }
1780
1781 static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
1782 {
1783         cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
1784 }
1785
1786 /*
1787  * We emulate only the following system register space.
1788  * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 4 - 7]
1789  * See Table C5-6 System instruction encodings for System register accesses,
1790  * ARMv8 ARM(ARM DDI 0487A.f) for more details.
1791  */
1792 static inline bool __attribute_const__ is_emulated(u32 id)
1793 {
1794         return (sys_reg_Op0(id) == 0x3 &&
1795                 sys_reg_CRn(id) == 0x0 &&
1796                 sys_reg_Op1(id) == 0x0 &&
1797                 (sys_reg_CRm(id) == 0 ||
1798                  ((sys_reg_CRm(id) >= 4) && (sys_reg_CRm(id) <= 7))));
1799 }
1800
1801 /*
1802  * With CRm == 0, reg should be one of :
1803  * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1.
1804  */
1805 static inline int emulate_id_reg(u32 id, u64 *valp)
1806 {
1807         switch (id) {
1808         case SYS_MIDR_EL1:
1809                 *valp = read_cpuid_id();
1810                 break;
1811         case SYS_MPIDR_EL1:
1812                 *valp = SYS_MPIDR_SAFE_VAL;
1813                 break;
1814         case SYS_REVIDR_EL1:
1815                 /* IMPLEMENTATION DEFINED values are emulated with 0 */
1816                 *valp = 0;
1817                 break;
1818         default:
1819                 return -EINVAL;
1820         }
1821
1822         return 0;
1823 }
1824
1825 static int emulate_sys_reg(u32 id, u64 *valp)
1826 {
1827         struct arm64_ftr_reg *regp;
1828
1829         if (!is_emulated(id))
1830                 return -EINVAL;
1831
1832         if (sys_reg_CRm(id) == 0)
1833                 return emulate_id_reg(id, valp);
1834
1835         regp = get_arm64_ftr_reg(id);
1836         if (regp)
1837                 *valp = arm64_ftr_reg_user_value(regp);
1838         else
1839                 /*
1840                  * The untracked registers are either IMPLEMENTATION DEFINED
1841                  * (e.g, ID_AFR0_EL1) or reserved RAZ.
1842                  */
1843                 *valp = 0;
1844         return 0;
1845 }
1846
1847 static int emulate_mrs(struct pt_regs *regs, u32 insn)
1848 {
1849         int rc;
1850         u32 sys_reg, dst;
1851         u64 val;
1852
1853         /*
1854          * sys_reg values are defined as used in mrs/msr instruction.
1855          * shift the imm value to get the encoding.
1856          */
1857         sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
1858         rc = emulate_sys_reg(sys_reg, &val);
1859         if (!rc) {
1860                 dst = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
1861                 pt_regs_write_reg(regs, dst, val);
1862                 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
1863         }
1864
1865         return rc;
1866 }
1867
1868 static struct undef_hook mrs_hook = {
1869         .instr_mask = 0xfff00000,
1870         .instr_val  = 0xd5300000,
1871         .pstate_mask = PSR_AA32_MODE_MASK,
1872         .pstate_val = PSR_MODE_EL0t,
1873         .fn = emulate_mrs,
1874 };
1875
1876 static int __init enable_mrs_emulation(void)
1877 {
1878         register_undef_hook(&mrs_hook);
1879         return 0;
1880 }
1881
1882 core_initcall(enable_mrs_emulation);