arch/arm64/kernel/cpu_errata.c

   1 /*
   2  * Contains CPU specific errata definitions
   3  *
   4  * Copyright (C) 2014 ARM Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/types.h>
  20 #include <asm/cpu.h>
  21 #include <asm/cputype.h>
  22 #include <asm/cpufeature.h>
  23
  24 static bool __maybe_unused
  25 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
  26 {
  27         const struct arm64_midr_revidr *fix;
  28         u32 midr = read_cpuid_id(), revidr;
  29
  30         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  31         if (!is_midr_in_range(midr, &entry->midr_range))
  32                 return false;
  33
  34         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
  35         revidr = read_cpuid(REVIDR_EL1);
  36         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
  37                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
  38                         return false;
  39
  40         return true;
  41 }
  42
  43 static bool __maybe_unused
  44 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
  45                             int scope)
  46 {
  47         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  48         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
  49 }
  50
  51 static bool __maybe_unused
  52 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
  53 {
  54         u32 model;
  55
  56         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  57
  58         model = read_cpuid_id();
  59         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
  60                  MIDR_ARCHITECTURE_MASK;
  61
  62         return model == entry->midr_range.model;
  63 }
  64
  65 static bool
  66 has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
  67                                 int scope)
  68 {
  69         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  70         return (read_cpuid_cachetype() & arm64_ftr_reg_ctrel0.strict_mask) !=
  71                 (arm64_ftr_reg_ctrel0.sys_val & arm64_ftr_reg_ctrel0.strict_mask);
  72 }
  73
  74 static void
  75 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
  76 {
  77         /* Clear SCTLR_EL1.UCT */
  78         config_sctlr_el1(SCTLR_EL1_UCT, 0);
  79 }
  80
  81 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
  82 #include <asm/mmu_context.h>
  83 #include <asm/cacheflush.h>
  84
  85 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
  86
  87 #ifdef CONFIG_KVM
  88 extern char __qcom_hyp_sanitize_link_stack_start[];
  89 extern char __qcom_hyp_sanitize_link_stack_end[];
  90 extern char __smccc_workaround_1_smc_start[];
  91 extern char __smccc_workaround_1_smc_end[];
  92 extern char __smccc_workaround_1_hvc_start[];
  93 extern char __smccc_workaround_1_hvc_end[];
  94
  95 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
  96                                 const char *hyp_vecs_end)
  97 {
  98         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
  99         int i;
 100
 101         for (i = 0; i < SZ_2K; i += 0x80)
 102                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
 103
 104         flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
 105 }
 106
 107 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
 108                                       const char *hyp_vecs_start,
 109                                       const char *hyp_vecs_end)
 110 {
 111         static int last_slot = -1;
 112         static DEFINE_SPINLOCK(bp_lock);
 113         int cpu, slot = -1;
 114
 115         spin_lock(&bp_lock);
 116         for_each_possible_cpu(cpu) {
 117                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
 118                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
 119                         break;
 120                 }
 121         }
 122
 123         if (slot == -1) {
 124                 last_slot++;
 125                 BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start)
 126                         / SZ_2K) <= last_slot);
 127                 slot = last_slot;
 128                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
 129         }
 130
 131         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
 132         __this_cpu_write(bp_hardening_data.fn, fn);
 133         spin_unlock(&bp_lock);
 134 }
 135 #else
 136 #define __qcom_hyp_sanitize_link_stack_start    NULL
 137 #define __qcom_hyp_sanitize_link_stack_end      NULL
 138 #define __smccc_workaround_1_smc_start          NULL
 139 #define __smccc_workaround_1_smc_end            NULL
 140 #define __smccc_workaround_1_hvc_start          NULL
 141 #define __smccc_workaround_1_hvc_end            NULL
 142
 143 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
 144                                       const char *hyp_vecs_start,
 145                                       const char *hyp_vecs_end)
 146 {
 147         __this_cpu_write(bp_hardening_data.fn, fn);
 148 }
 149 #endif  /* CONFIG_KVM */
 150
 151 static void  install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
 152                                      bp_hardening_cb_t fn,
 153                                      const char *hyp_vecs_start,
 154                                      const char *hyp_vecs_end)
 155 {
 156         u64 pfr0;
 157
 158         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
 159                 return;
 160
 161         pfr0 = read_cpuid(ID_AA64PFR0_EL1);
 162         if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
 163                 return;
 164
 165         __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
 166 }
 167
 168 #include <uapi/linux/psci.h>
 169 #include <linux/arm-smccc.h>
 170 #include <linux/psci.h>
 171
 172 static void call_smc_arch_workaround_1(void)
 173 {
 174         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 175 }
 176
 177 static void call_hvc_arch_workaround_1(void)
 178 {
 179         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 180 }
 181
 182 static void
 183 enable_smccc_arch_workaround_1(const struct arm64_cpu_capabilities *entry)
 184 {
 185         bp_hardening_cb_t cb;
 186         void *smccc_start, *smccc_end;
 187         struct arm_smccc_res res;
 188
 189         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
 190                 return;
 191
 192         if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
 193                 return;
 194
 195         switch (psci_ops.conduit) {
 196         case PSCI_CONDUIT_HVC:
 197                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 198                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 199                 if ((int)res.a0 < 0)
 200                         return;
 201                 cb = call_hvc_arch_workaround_1;
 202                 smccc_start = __smccc_workaround_1_hvc_start;
 203                 smccc_end = __smccc_workaround_1_hvc_end;
 204                 break;
 205
 206         case PSCI_CONDUIT_SMC:
 207                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 208                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 209                 if ((int)res.a0 < 0)
 210                         return;
 211                 cb = call_smc_arch_workaround_1;
 212                 smccc_start = __smccc_workaround_1_smc_start;
 213                 smccc_end = __smccc_workaround_1_smc_end;
 214                 break;
 215
 216         default:
 217                 return;
 218         }
 219
 220         install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
 221
 222         return;
 223 }
 224
 225 static void qcom_link_stack_sanitization(void)
 226 {
 227         u64 tmp;
 228
 229         asm volatile("mov       %0, x30         \n"
 230                      ".rept     16              \n"
 231                      "bl        . + 4           \n"
 232                      ".endr                     \n"
 233                      "mov       x30, %0         \n"
 234                      : "=&r" (tmp));
 235 }
 236
 237 static void
 238 qcom_enable_link_stack_sanitization(const struct arm64_cpu_capabilities *entry)
 239 {
 240         install_bp_hardening_cb(entry, qcom_link_stack_sanitization,
 241                                 __qcom_hyp_sanitize_link_stack_start,
 242                                 __qcom_hyp_sanitize_link_stack_end);
 243 }
 244 #endif  /* CONFIG_HARDEN_BRANCH_PREDICTOR */
 245
 246 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
 247         .matches = is_affected_midr_range,                      \
 248         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 249
 250 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
 251         .matches = is_affected_midr_range,                              \
 252         .midr_range = MIDR_ALL_VERSIONS(model)
 253
 254 #define MIDR_FIXED(rev, revidr_mask) \
 255         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
 256
 257 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
 258         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
 259         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 260
 261 #define CAP_MIDR_RANGE_LIST(list)                               \
 262         .matches = is_affected_midr_range_list,                 \
 263         .midr_range_list = list
 264
 265 /* Errata affecting a range of revisions of  given model variant */
 266 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
 267         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
 268
 269 /* Errata affecting a single variant/revision of a model */
 270 #define ERRATA_MIDR_REV(model, var, rev)        \
 271         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
 272
 273 /* Errata affecting all variants/revisions of a given a model */
 274 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
 275         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 276         CAP_MIDR_ALL_VERSIONS(model)
 277
 278 /* Errata affecting a list of midr ranges, with same work around */
 279 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
 280         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 281         CAP_MIDR_RANGE_LIST(midr_list)
 282
 283 /*
 284  * Generic helper for handling capabilties with multiple (match,enable) pairs
 285  * of call backs, sharing the same capability bit.
 286  * Iterate over each entry to see if at least one matches.
 287  */
 288 static bool __maybe_unused
 289 multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry, int scope)
 290 {
 291         const struct arm64_cpu_capabilities *caps;
 292
 293         for (caps = entry->match_list; caps->matches; caps++)
 294                 if (caps->matches(caps, scope))
 295                         return true;
 296
 297         return false;
 298 }
 299
 300 /*
 301  * Take appropriate action for all matching entries in the shared capability
 302  * entry.
 303  */
 304 static void __maybe_unused
 305 multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
 306 {
 307         const struct arm64_cpu_capabilities *caps;
 308
 309         for (caps = entry->match_list; caps->matches; caps++)
 310                 if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
 311                     caps->cpu_enable)
 312                         caps->cpu_enable(caps);
 313 }
 314
 315 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 316
 317 /*
 318  * List of CPUs where we need to issue a psci call to
 319  * harden the branch predictor.
 320  */
 321 static const struct midr_range arm64_bp_harden_smccc_cpus[] = {
 322         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 323         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 324         MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 325         MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
 326         MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
 327         MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
 328         {},
 329 };
 330
 331 static const struct midr_range qcom_bp_harden_cpus[] = {
 332         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
 333         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR),
 334         {},
 335 };
 336
 337 static const struct arm64_cpu_capabilities arm64_bp_harden_list[] = {
 338         {
 339                 CAP_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus),
 340                 .cpu_enable = enable_smccc_arch_workaround_1,
 341         },
 342         {
 343                 CAP_MIDR_RANGE_LIST(qcom_bp_harden_cpus),
 344                 .cpu_enable = qcom_enable_link_stack_sanitization,
 345         },
 346         {},
 347 };
 348
 349 #endif
 350
 351 const struct arm64_cpu_capabilities arm64_errata[] = {
 352 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
 353         defined(CONFIG_ARM64_ERRATUM_827319) || \
 354         defined(CONFIG_ARM64_ERRATUM_824069)
 355         {
 356         /* Cortex-A53 r0p[012] */
 357                 .desc = "ARM errata 826319, 827319, 824069",
 358                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 359                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
 360                 .cpu_enable = cpu_enable_cache_maint_trap,
 361         },
 362 #endif
 363 #ifdef CONFIG_ARM64_ERRATUM_819472
 364         {
 365         /* Cortex-A53 r0p[01] */
 366                 .desc = "ARM errata 819472",
 367                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 368                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
 369                 .cpu_enable = cpu_enable_cache_maint_trap,
 370         },
 371 #endif
 372 #ifdef CONFIG_ARM64_ERRATUM_832075
 373         {
 374         /* Cortex-A57 r0p0 - r1p2 */
 375                 .desc = "ARM erratum 832075",
 376                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
 377                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 378                                   0, 0,
 379                                   1, 2),
 380         },
 381 #endif
 382 #ifdef CONFIG_ARM64_ERRATUM_834220
 383         {
 384         /* Cortex-A57 r0p0 - r1p2 */
 385                 .desc = "ARM erratum 834220",
 386                 .capability = ARM64_WORKAROUND_834220,
 387                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 388                                   0, 0,
 389                                   1, 2),
 390         },
 391 #endif
 392 #ifdef CONFIG_ARM64_ERRATUM_843419
 393         {
 394         /* Cortex-A53 r0p[01234] */
 395                 .desc = "ARM erratum 843419",
 396                 .capability = ARM64_WORKAROUND_843419,
 397                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 398                 MIDR_FIXED(0x4, BIT(8)),
 399         },
 400 #endif
 401 #ifdef CONFIG_ARM64_ERRATUM_845719
 402         {
 403         /* Cortex-A53 r0p[01234] */
 404                 .desc = "ARM erratum 845719",
 405                 .capability = ARM64_WORKAROUND_845719,
 406                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 407         },
 408 #endif
 409 #ifdef CONFIG_CAVIUM_ERRATUM_23154
 410         {
 411         /* Cavium ThunderX, pass 1.x */
 412                 .desc = "Cavium erratum 23154",
 413                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
 414                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
 415         },
 416 #endif
 417 #ifdef CONFIG_CAVIUM_ERRATUM_27456
 418         {
 419         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
 420                 .desc = "Cavium erratum 27456",
 421                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 422                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
 423                                   0, 0,
 424                                   1, 1),
 425         },
 426         {
 427         /* Cavium ThunderX, T81 pass 1.0 */
 428                 .desc = "Cavium erratum 27456",
 429                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 430                 ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
 431         },
 432 #endif
 433 #ifdef CONFIG_CAVIUM_ERRATUM_30115
 434         {
 435         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
 436                 .desc = "Cavium erratum 30115",
 437                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 438                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
 439                                       0, 0,
 440                                       1, 2),
 441         },
 442         {
 443         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
 444                 .desc = "Cavium erratum 30115",
 445                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 446                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
 447         },
 448         {
 449         /* Cavium ThunderX, T83 pass 1.0 */
 450                 .desc = "Cavium erratum 30115",
 451                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 452                 ERRATA_MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
 453         },
 454 #endif
 455         {
 456                 .desc = "Mismatched cache line size",
 457                 .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
 458                 .matches = has_mismatched_cache_line_size,
 459                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 460                 .cpu_enable = cpu_enable_trap_ctr_access,
 461         },
 462 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
 463         {
 464                 .desc = "Qualcomm Technologies Falkor erratum 1003",
 465                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 466                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 467         },
 468         {
 469                 .desc = "Qualcomm Technologies Kryo erratum 1003",
 470                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 471                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 472                 .midr_range.model = MIDR_QCOM_KRYO,
 473                 .matches = is_kryo_midr,
 474         },
 475 #endif
 476 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
 477         {
 478                 .desc = "Qualcomm Technologies Falkor erratum 1009",
 479                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
 480                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 481         },
 482 #endif
 483 #ifdef CONFIG_ARM64_ERRATUM_858921
 484         {
 485         /* Cortex-A73 all versions */
 486                 .desc = "ARM erratum 858921",
 487                 .capability = ARM64_WORKAROUND_858921,
 488                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 489         },
 490 #endif
 491 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 492         {
 493                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
 494                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 495                 .matches = multi_entry_cap_matches,
 496                 .cpu_enable = multi_entry_cap_cpu_enable,
 497                 .match_list = arm64_bp_harden_list,
 498         },
 499         {
 500                 .capability = ARM64_HARDEN_BP_POST_GUEST_EXIT,
 501                 ERRATA_MIDR_RANGE_LIST(qcom_bp_harden_cpus),
 502         },
 503 #endif
 504         {
 505         }
 506 };