arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17 #include <linux/sched/smt.h>
  18
  19 #include <asm/spec-ctrl.h>
  20 #include <asm/cmdline.h>
  21 #include <asm/bugs.h>
  22 #include <asm/processor.h>
  23 #include <asm/processor-flags.h>
  24 #include <asm/fpu/internal.h>
  25 #include <asm/msr.h>
  26 #include <asm/vmx.h>
  27 #include <asm/paravirt.h>
  28 #include <asm/alternative.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/set_memory.h>
  31 #include <asm/intel-family.h>
  32 #include <asm/e820/api.h>
  33 #include <asm/hypervisor.h>
  34
  35 static void __init spectre_v2_select_mitigation(void);
  36 static void __init ssb_select_mitigation(void);
  37 static void __init l1tf_select_mitigation(void);
  38
  39 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  40 u64 x86_spec_ctrl_base;
  41 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  42 static DEFINE_MUTEX(spec_ctrl_mutex);
  43
  44 /*
  45  * The vendor and possibly platform specific bits which can be modified in
  46  * x86_spec_ctrl_base.
  47  */
  48 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  49
  50 /*
  51  * AMD specific MSR info for Speculative Store Bypass control.
  52  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  53  */
  54 u64 __ro_after_init x86_amd_ls_cfg_base;
  55 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  56
  57 /* Control conditional STIPB in switch_to() */
  58 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
  59 /* Control conditional IBPB in switch_mm() */
  60 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
  61 /* Control unconditional IBPB in switch_mm() */
  62 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  63
  64 void __init check_bugs(void)
  65 {
  66         identify_boot_cpu();
  67
  68         /*
  69          * identify_boot_cpu() initialized SMT support information, let the
  70          * core code know.
  71          */
  72         cpu_smt_check_topology_early();
  73
  74         if (!IS_ENABLED(CONFIG_SMP)) {
  75                 pr_info("CPU: ");
  76                 print_cpu_info(&boot_cpu_data);
  77         }
  78
  79         /*
  80          * Read the SPEC_CTRL MSR to account for reserved bits which may
  81          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  82          * init code as it is not enumerated and depends on the family.
  83          */
  84         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  85                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  86
  87         /* Allow STIBP in MSR_SPEC_CTRL if supported */
  88         if (boot_cpu_has(X86_FEATURE_STIBP))
  89                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
  90
  91         /* Select the proper spectre mitigation before patching alternatives */
  92         spectre_v2_select_mitigation();
  93
  94         /*
  95          * Select proper mitigation for any exposure to the Speculative Store
  96          * Bypass vulnerability.
  97          */
  98         ssb_select_mitigation();
  99
 100         l1tf_select_mitigation();
 101
 102 #ifdef CONFIG_X86_32
 103         /*
 104          * Check whether we are able to run this kernel safely on SMP.
 105          *
 106          * - i386 is no longer supported.
 107          * - In order to run on anything without a TSC, we need to be
 108          *   compiled for a i486.
 109          */
 110         if (boot_cpu_data.x86 < 4)
 111                 panic("Kernel requires i486+ for 'invlpg' and other features");
 112
 113         init_utsname()->machine[1] =
 114                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 115         alternative_instructions();
 116
 117         fpu__init_check_bugs();
 118 #else /* CONFIG_X86_64 */
 119         alternative_instructions();
 120
 121         /*
 122          * Make sure the first 2MB area is not mapped by huge pages
 123          * There are typically fixed size MTRRs in there and overlapping
 124          * MTRRs into large pages causes slow downs.
 125          *
 126          * Right now we don't do that with gbpages because there seems
 127          * very little benefit for that case.
 128          */
 129         if (!direct_gbpages)
 130                 set_memory_4k((unsigned long)__va(0), 1);
 131 #endif
 132 }
 133
 134 void
 135 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 136 {
 137         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 138         struct thread_info *ti = current_thread_info();
 139
 140         /* Is MSR_SPEC_CTRL implemented ? */
 141         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 142                 /*
 143                  * Restrict guest_spec_ctrl to supported values. Clear the
 144                  * modifiable bits in the host base value and or the
 145                  * modifiable bits from the guest value.
 146                  */
 147                 guestval = hostval & ~x86_spec_ctrl_mask;
 148                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 149
 150                 /* SSBD controlled in MSR_SPEC_CTRL */
 151                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 152                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 153                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 154
 155                 /* Conditional STIBP enabled? */
 156                 if (static_branch_unlikely(&switch_to_cond_stibp))
 157                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
 158
 159                 if (hostval != guestval) {
 160                         msrval = setguest ? guestval : hostval;
 161                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 162                 }
 163         }
 164
 165         /*
 166          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 167          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 168          */
 169         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 170             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 171                 return;
 172
 173         /*
 174          * If the host has SSBD mitigation enabled, force it in the host's
 175          * virtual MSR value. If its not permanently enabled, evaluate
 176          * current's TIF_SSBD thread flag.
 177          */
 178         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 179                 hostval = SPEC_CTRL_SSBD;
 180         else
 181                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 182
 183         /* Sanitize the guest value */
 184         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 185
 186         if (hostval != guestval) {
 187                 unsigned long tif;
 188
 189                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 190                                  ssbd_spec_ctrl_to_tif(hostval);
 191
 192                 speculation_ctrl_update(tif);
 193         }
 194 }
 195 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 196
 197 static void x86_amd_ssb_disable(void)
 198 {
 199         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 200
 201         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 202                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 203         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 204                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 205 }
 206
 207 #undef pr_fmt
 208 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 209
 210 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 211         SPECTRE_V2_NONE;
 212
 213 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
 214         SPECTRE_V2_USER_NONE;
 215
 216 #ifdef RETPOLINE
 217 static bool spectre_v2_bad_module;
 218
 219 bool retpoline_module_ok(bool has_retpoline)
 220 {
 221         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 222                 return true;
 223
 224         pr_err("System may be vulnerable to spectre v2\n");
 225         spectre_v2_bad_module = true;
 226         return false;
 227 }
 228
 229 static inline const char *spectre_v2_module_string(void)
 230 {
 231         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 232 }
 233 #else
 234 static inline const char *spectre_v2_module_string(void) { return ""; }
 235 #endif
 236
 237 static inline bool match_option(const char *arg, int arglen, const char *opt)
 238 {
 239         int len = strlen(opt);
 240
 241         return len == arglen && !strncmp(arg, opt, len);
 242 }
 243
 244 /* The kernel command line selection for spectre v2 */
 245 enum spectre_v2_mitigation_cmd {
 246         SPECTRE_V2_CMD_NONE,
 247         SPECTRE_V2_CMD_AUTO,
 248         SPECTRE_V2_CMD_FORCE,
 249         SPECTRE_V2_CMD_RETPOLINE,
 250         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 251         SPECTRE_V2_CMD_RETPOLINE_AMD,
 252 };
 253
 254 enum spectre_v2_user_cmd {
 255         SPECTRE_V2_USER_CMD_NONE,
 256         SPECTRE_V2_USER_CMD_AUTO,
 257         SPECTRE_V2_USER_CMD_FORCE,
 258 };
 259
 260 static const char * const spectre_v2_user_strings[] = {
 261         [SPECTRE_V2_USER_NONE]          = "User space: Vulnerable",
 262         [SPECTRE_V2_USER_STRICT]        = "User space: Mitigation: STIBP protection",
 263 };
 264
 265 static const struct {
 266         const char                      *option;
 267         enum spectre_v2_user_cmd        cmd;
 268         bool                            secure;
 269 } v2_user_options[] __initdata = {
 270         { "auto",       SPECTRE_V2_USER_CMD_AUTO,       false },
 271         { "off",        SPECTRE_V2_USER_CMD_NONE,       false },
 272         { "on",         SPECTRE_V2_USER_CMD_FORCE,      true  },
 273 };
 274
 275 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
 276 {
 277         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 278                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 279 }
 280
 281 static enum spectre_v2_user_cmd __init
 282 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
 283 {
 284         char arg[20];
 285         int ret, i;
 286
 287         switch (v2_cmd) {
 288         case SPECTRE_V2_CMD_NONE:
 289                 return SPECTRE_V2_USER_CMD_NONE;
 290         case SPECTRE_V2_CMD_FORCE:
 291                 return SPECTRE_V2_USER_CMD_FORCE;
 292         default:
 293                 break;
 294         }
 295
 296         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
 297                                   arg, sizeof(arg));
 298         if (ret < 0)
 299                 return SPECTRE_V2_USER_CMD_AUTO;
 300
 301         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
 302                 if (match_option(arg, ret, v2_user_options[i].option)) {
 303                         spec_v2_user_print_cond(v2_user_options[i].option,
 304                                                 v2_user_options[i].secure);
 305                         return v2_user_options[i].cmd;
 306                 }
 307         }
 308
 309         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
 310         return SPECTRE_V2_USER_CMD_AUTO;
 311 }
 312
 313 static void __init
 314 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 315 {
 316         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
 317         bool smt_possible = IS_ENABLED(CONFIG_SMP);
 318
 319         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
 320                 return;
 321
 322         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 323             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 324                 smt_possible = false;
 325
 326         switch (spectre_v2_parse_user_cmdline(v2_cmd)) {
 327         case SPECTRE_V2_USER_CMD_AUTO:
 328         case SPECTRE_V2_USER_CMD_NONE:
 329                 goto set_mode;
 330         case SPECTRE_V2_USER_CMD_FORCE:
 331                 mode = SPECTRE_V2_USER_STRICT;
 332                 break;
 333         }
 334
 335         /* Initialize Indirect Branch Prediction Barrier */
 336         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 337                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 338
 339                 switch (mode) {
 340                 case SPECTRE_V2_USER_STRICT:
 341                         static_branch_enable(&switch_mm_always_ibpb);
 342                         break;
 343                 default:
 344                         break;
 345                 }
 346
 347                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
 348                         mode == SPECTRE_V2_USER_STRICT ? "always-on" : "conditional");
 349         }
 350
 351         /* If enhanced IBRS is enabled no STIPB required */
 352         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 353                 return;
 354
 355 set_mode:
 356         spectre_v2_user = mode;
 357         /* Only print the STIBP mode when SMT possible */
 358         if (smt_possible)
 359                 pr_info("%s\n", spectre_v2_user_strings[mode]);
 360 }
 361
 362 static const char * const spectre_v2_strings[] = {
 363         [SPECTRE_V2_NONE]                       = "Vulnerable",
 364         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 365         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 366         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 367 };
 368
 369 static const struct {
 370         const char *option;
 371         enum spectre_v2_mitigation_cmd cmd;
 372         bool secure;
 373 } mitigation_options[] __initdata = {
 374         { "off",                SPECTRE_V2_CMD_NONE,              false },
 375         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 376         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 377         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 378         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 379         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 380 };
 381
 382 static void __init spec_v2_print_cond(const char *reason, bool secure)
 383 {
 384         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 385                 pr_info("%s selected on command line.\n", reason);
 386 }
 387
 388 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 389 {
 390         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 391         char arg[20];
 392         int ret, i;
 393
 394         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 395                 return SPECTRE_V2_CMD_NONE;
 396
 397         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 398         if (ret < 0)
 399                 return SPECTRE_V2_CMD_AUTO;
 400
 401         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 402                 if (!match_option(arg, ret, mitigation_options[i].option))
 403                         continue;
 404                 cmd = mitigation_options[i].cmd;
 405                 break;
 406         }
 407
 408         if (i >= ARRAY_SIZE(mitigation_options)) {
 409                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 410                 return SPECTRE_V2_CMD_AUTO;
 411         }
 412
 413         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 414              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 415              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 416             !IS_ENABLED(CONFIG_RETPOLINE)) {
 417                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 418                 return SPECTRE_V2_CMD_AUTO;
 419         }
 420
 421         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 422             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON &&
 423             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 424                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 425                 return SPECTRE_V2_CMD_AUTO;
 426         }
 427
 428         spec_v2_print_cond(mitigation_options[i].option,
 429                            mitigation_options[i].secure);
 430         return cmd;
 431 }
 432
 433 static void __init spectre_v2_select_mitigation(void)
 434 {
 435         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 436         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 437
 438         /*
 439          * If the CPU is not affected and the command line mode is NONE or AUTO
 440          * then nothing to do.
 441          */
 442         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 443             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 444                 return;
 445
 446         switch (cmd) {
 447         case SPECTRE_V2_CMD_NONE:
 448                 return;
 449
 450         case SPECTRE_V2_CMD_FORCE:
 451         case SPECTRE_V2_CMD_AUTO:
 452                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 453                         mode = SPECTRE_V2_IBRS_ENHANCED;
 454                         /* Force it so VMEXIT will restore correctly */
 455                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 456                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 457                         goto specv2_set_mode;
 458                 }
 459                 if (IS_ENABLED(CONFIG_RETPOLINE))
 460                         goto retpoline_auto;
 461                 break;
 462         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 463                 if (IS_ENABLED(CONFIG_RETPOLINE))
 464                         goto retpoline_amd;
 465                 break;
 466         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 467                 if (IS_ENABLED(CONFIG_RETPOLINE))
 468                         goto retpoline_generic;
 469                 break;
 470         case SPECTRE_V2_CMD_RETPOLINE:
 471                 if (IS_ENABLED(CONFIG_RETPOLINE))
 472                         goto retpoline_auto;
 473                 break;
 474         }
 475         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 476         return;
 477
 478 retpoline_auto:
 479         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
 480             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 481         retpoline_amd:
 482                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 483                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 484                         goto retpoline_generic;
 485                 }
 486                 mode = SPECTRE_V2_RETPOLINE_AMD;
 487                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 488                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 489         } else {
 490         retpoline_generic:
 491                 mode = SPECTRE_V2_RETPOLINE_GENERIC;
 492                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 493         }
 494
 495 specv2_set_mode:
 496         spectre_v2_enabled = mode;
 497         pr_info("%s\n", spectre_v2_strings[mode]);
 498
 499         /*
 500          * If spectre v2 protection has been enabled, unconditionally fill
 501          * RSB during a context switch; this protects against two independent
 502          * issues:
 503          *
 504          *      - RSB underflow (and switch to BTB) on Skylake+
 505          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 506          */
 507         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 508         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 509
 510         /*
 511          * Retpoline means the kernel is safe because it has no indirect
 512          * branches. Enhanced IBRS protects firmware too, so, enable restricted
 513          * speculation around firmware calls only when Enhanced IBRS isn't
 514          * supported.
 515          *
 516          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
 517          * the user might select retpoline on the kernel command line and if
 518          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
 519          * enable IBRS around firmware calls.
 520          */
 521         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
 522                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 523                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 524         }
 525
 526         /* Set up IBPB and STIBP depending on the general spectre V2 command */
 527         spectre_v2_user_select_mitigation(cmd);
 528
 529         /* Enable STIBP if appropriate */
 530         arch_smt_update();
 531 }
 532
 533 static bool stibp_needed(void)
 534 {
 535         /* Enhanced IBRS makes using STIBP unnecessary. */
 536         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 537                 return false;
 538
 539         /* Check for strict user mitigation mode */
 540         return spectre_v2_user == SPECTRE_V2_USER_STRICT;
 541 }
 542
 543 static void update_stibp_msr(void *info)
 544 {
 545         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 546 }
 547
 548 void arch_smt_update(void)
 549 {
 550         u64 mask;
 551
 552         if (!stibp_needed())
 553                 return;
 554
 555         mutex_lock(&spec_ctrl_mutex);
 556
 557         mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
 558         if (sched_smt_active())
 559                 mask |= SPEC_CTRL_STIBP;
 560
 561         if (mask != x86_spec_ctrl_base) {
 562                 pr_info("Spectre v2 cross-process SMT mitigation: %s STIBP\n",
 563                         mask & SPEC_CTRL_STIBP ? "Enabling" : "Disabling");
 564                 x86_spec_ctrl_base = mask;
 565                 on_each_cpu(update_stibp_msr, NULL, 1);
 566         }
 567         mutex_unlock(&spec_ctrl_mutex);
 568 }
 569
 570 #undef pr_fmt
 571 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 572
 573 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 574
 575 /* The kernel command line selection */
 576 enum ssb_mitigation_cmd {
 577         SPEC_STORE_BYPASS_CMD_NONE,
 578         SPEC_STORE_BYPASS_CMD_AUTO,
 579         SPEC_STORE_BYPASS_CMD_ON,
 580         SPEC_STORE_BYPASS_CMD_PRCTL,
 581         SPEC_STORE_BYPASS_CMD_SECCOMP,
 582 };
 583
 584 static const char * const ssb_strings[] = {
 585         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 586         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 587         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 588         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 589 };
 590
 591 static const struct {
 592         const char *option;
 593         enum ssb_mitigation_cmd cmd;
 594 } ssb_mitigation_options[]  __initdata = {
 595         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 596         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 597         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 598         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 599         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 600 };
 601
 602 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 603 {
 604         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 605         char arg[20];
 606         int ret, i;
 607
 608         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 609                 return SPEC_STORE_BYPASS_CMD_NONE;
 610         } else {
 611                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 612                                           arg, sizeof(arg));
 613                 if (ret < 0)
 614                         return SPEC_STORE_BYPASS_CMD_AUTO;
 615
 616                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 617                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 618                                 continue;
 619
 620                         cmd = ssb_mitigation_options[i].cmd;
 621                         break;
 622                 }
 623
 624                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 625                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 626                         return SPEC_STORE_BYPASS_CMD_AUTO;
 627                 }
 628         }
 629
 630         return cmd;
 631 }
 632
 633 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 634 {
 635         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 636         enum ssb_mitigation_cmd cmd;
 637
 638         if (!boot_cpu_has(X86_FEATURE_SSBD))
 639                 return mode;
 640
 641         cmd = ssb_parse_cmdline();
 642         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 643             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 644              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 645                 return mode;
 646
 647         switch (cmd) {
 648         case SPEC_STORE_BYPASS_CMD_AUTO:
 649         case SPEC_STORE_BYPASS_CMD_SECCOMP:
 650                 /*
 651                  * Choose prctl+seccomp as the default mode if seccomp is
 652                  * enabled.
 653                  */
 654                 if (IS_ENABLED(CONFIG_SECCOMP))
 655                         mode = SPEC_STORE_BYPASS_SECCOMP;
 656                 else
 657                         mode = SPEC_STORE_BYPASS_PRCTL;
 658                 break;
 659         case SPEC_STORE_BYPASS_CMD_ON:
 660                 mode = SPEC_STORE_BYPASS_DISABLE;
 661                 break;
 662         case SPEC_STORE_BYPASS_CMD_PRCTL:
 663                 mode = SPEC_STORE_BYPASS_PRCTL;
 664                 break;
 665         case SPEC_STORE_BYPASS_CMD_NONE:
 666                 break;
 667         }
 668
 669         /*
 670          * We have three CPU feature flags that are in play here:
 671          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 672          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
 673          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 674          */
 675         if (mode == SPEC_STORE_BYPASS_DISABLE) {
 676                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 677                 /*
 678                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
 679                  * use a completely different MSR and bit dependent on family.
 680                  */
 681                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
 682                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
 683                         x86_amd_ssb_disable();
 684                 } else {
 685                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
 686                         x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
 687                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 688                 }
 689         }
 690
 691         return mode;
 692 }
 693
 694 static void ssb_select_mitigation(void)
 695 {
 696         ssb_mode = __ssb_select_mitigation();
 697
 698         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 699                 pr_info("%s\n", ssb_strings[ssb_mode]);
 700 }
 701
 702 #undef pr_fmt
 703 #define pr_fmt(fmt)     "Speculation prctl: " fmt
 704
 705 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
 706 {
 707         bool update;
 708
 709         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
 710             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
 711                 return -ENXIO;
 712
 713         switch (ctrl) {
 714         case PR_SPEC_ENABLE:
 715                 /* If speculation is force disabled, enable is not allowed */
 716                 if (task_spec_ssb_force_disable(task))
 717                         return -EPERM;
 718                 task_clear_spec_ssb_disable(task);
 719                 update = test_and_clear_tsk_thread_flag(task, TIF_SSBD);
 720                 break;
 721         case PR_SPEC_DISABLE:
 722                 task_set_spec_ssb_disable(task);
 723                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 724                 break;
 725         case PR_SPEC_FORCE_DISABLE:
 726                 task_set_spec_ssb_disable(task);
 727                 task_set_spec_ssb_force_disable(task);
 728                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 729                 break;
 730         default:
 731                 return -ERANGE;
 732         }
 733
 734         /*
 735          * If being set on non-current task, delay setting the CPU
 736          * mitigation until it is next scheduled.
 737          */
 738         if (task == current && update)
 739                 speculation_ctrl_update_current();
 740
 741         return 0;
 742 }
 743
 744 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 745                              unsigned long ctrl)
 746 {
 747         switch (which) {
 748         case PR_SPEC_STORE_BYPASS:
 749                 return ssb_prctl_set(task, ctrl);
 750         default:
 751                 return -ENODEV;
 752         }
 753 }
 754
 755 #ifdef CONFIG_SECCOMP
 756 void arch_seccomp_spec_mitigate(struct task_struct *task)
 757 {
 758         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
 759                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 760 }
 761 #endif
 762
 763 static int ssb_prctl_get(struct task_struct *task)
 764 {
 765         switch (ssb_mode) {
 766         case SPEC_STORE_BYPASS_DISABLE:
 767                 return PR_SPEC_DISABLE;
 768         case SPEC_STORE_BYPASS_SECCOMP:
 769         case SPEC_STORE_BYPASS_PRCTL:
 770                 if (task_spec_ssb_force_disable(task))
 771                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 772                 if (task_spec_ssb_disable(task))
 773                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 774                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 775         default:
 776                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 777                         return PR_SPEC_ENABLE;
 778                 return PR_SPEC_NOT_AFFECTED;
 779         }
 780 }
 781
 782 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 783 {
 784         switch (which) {
 785         case PR_SPEC_STORE_BYPASS:
 786                 return ssb_prctl_get(task);
 787         default:
 788                 return -ENODEV;
 789         }
 790 }
 791
 792 void x86_spec_ctrl_setup_ap(void)
 793 {
 794         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
 795                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 796
 797         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
 798                 x86_amd_ssb_disable();
 799 }
 800
 801 #undef pr_fmt
 802 #define pr_fmt(fmt)     "L1TF: " fmt
 803
 804 /* Default mitigation for L1TF-affected CPUs */
 805 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
 806 #if IS_ENABLED(CONFIG_KVM_INTEL)
 807 EXPORT_SYMBOL_GPL(l1tf_mitigation);
 808 #endif
 809 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
 810 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
 811
 812 /*
 813  * These CPUs all support 44bits physical address space internally in the
 814  * cache but CPUID can report a smaller number of physical address bits.
 815  *
 816  * The L1TF mitigation uses the top most address bit for the inversion of
 817  * non present PTEs. When the installed memory reaches into the top most
 818  * address bit due to memory holes, which has been observed on machines
 819  * which report 36bits physical address bits and have 32G RAM installed,
 820  * then the mitigation range check in l1tf_select_mitigation() triggers.
 821  * This is a false positive because the mitigation is still possible due to
 822  * the fact that the cache uses 44bit internally. Use the cache bits
 823  * instead of the reported physical bits and adjust them on the affected
 824  * machines to 44bit if the reported bits are less than 44.
 825  */
 826 static void override_cache_bits(struct cpuinfo_x86 *c)
 827 {
 828         if (c->x86 != 6)
 829                 return;
 830
 831         switch (c->x86_model) {
 832         case INTEL_FAM6_NEHALEM:
 833         case INTEL_FAM6_WESTMERE:
 834         case INTEL_FAM6_SANDYBRIDGE:
 835         case INTEL_FAM6_IVYBRIDGE:
 836         case INTEL_FAM6_HASWELL_CORE:
 837         case INTEL_FAM6_HASWELL_ULT:
 838         case INTEL_FAM6_HASWELL_GT3E:
 839         case INTEL_FAM6_BROADWELL_CORE:
 840         case INTEL_FAM6_BROADWELL_GT3E:
 841         case INTEL_FAM6_SKYLAKE_MOBILE:
 842         case INTEL_FAM6_SKYLAKE_DESKTOP:
 843         case INTEL_FAM6_KABYLAKE_MOBILE:
 844         case INTEL_FAM6_KABYLAKE_DESKTOP:
 845                 if (c->x86_cache_bits < 44)
 846                         c->x86_cache_bits = 44;
 847                 break;
 848         }
 849 }
 850
 851 static void __init l1tf_select_mitigation(void)
 852 {
 853         u64 half_pa;
 854
 855         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 856                 return;
 857
 858         override_cache_bits(&boot_cpu_data);
 859
 860         switch (l1tf_mitigation) {
 861         case L1TF_MITIGATION_OFF:
 862         case L1TF_MITIGATION_FLUSH_NOWARN:
 863         case L1TF_MITIGATION_FLUSH:
 864                 break;
 865         case L1TF_MITIGATION_FLUSH_NOSMT:
 866         case L1TF_MITIGATION_FULL:
 867                 cpu_smt_disable(false);
 868                 break;
 869         case L1TF_MITIGATION_FULL_FORCE:
 870                 cpu_smt_disable(true);
 871                 break;
 872         }
 873
 874 #if CONFIG_PGTABLE_LEVELS == 2
 875         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
 876         return;
 877 #endif
 878
 879         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
 880         if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
 881                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
 882                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
 883                                 half_pa);
 884                 pr_info("However, doing so will make a part of your RAM unusable.\n");
 885                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
 886                 return;
 887         }
 888
 889         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
 890 }
 891
 892 static int __init l1tf_cmdline(char *str)
 893 {
 894         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 895                 return 0;
 896
 897         if (!str)
 898                 return -EINVAL;
 899
 900         if (!strcmp(str, "off"))
 901                 l1tf_mitigation = L1TF_MITIGATION_OFF;
 902         else if (!strcmp(str, "flush,nowarn"))
 903                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
 904         else if (!strcmp(str, "flush"))
 905                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
 906         else if (!strcmp(str, "flush,nosmt"))
 907                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
 908         else if (!strcmp(str, "full"))
 909                 l1tf_mitigation = L1TF_MITIGATION_FULL;
 910         else if (!strcmp(str, "full,force"))
 911                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
 912
 913         return 0;
 914 }
 915 early_param("l1tf", l1tf_cmdline);
 916
 917 #undef pr_fmt
 918
 919 #ifdef CONFIG_SYSFS
 920
 921 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
 922
 923 #if IS_ENABLED(CONFIG_KVM_INTEL)
 924 static const char * const l1tf_vmx_states[] = {
 925         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
 926         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
 927         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
 928         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
 929         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
 930         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
 931 };
 932
 933 static ssize_t l1tf_show_state(char *buf)
 934 {
 935         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
 936                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 937
 938         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
 939             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
 940              sched_smt_active())) {
 941                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
 942                                l1tf_vmx_states[l1tf_vmx_mitigation]);
 943         }
 944
 945         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
 946                        l1tf_vmx_states[l1tf_vmx_mitigation],
 947                        sched_smt_active() ? "vulnerable" : "disabled");
 948 }
 949 #else
 950 static ssize_t l1tf_show_state(char *buf)
 951 {
 952         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 953 }
 954 #endif
 955
 956 static char *stibp_state(void)
 957 {
 958         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 959                 return "";
 960
 961         switch (spectre_v2_user) {
 962         case SPECTRE_V2_USER_NONE:
 963                 return ", STIBP: disabled";
 964         case SPECTRE_V2_USER_STRICT:
 965                 return ", STIBP: forced";
 966         }
 967         return "";
 968 }
 969
 970 static char *ibpb_state(void)
 971 {
 972         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 973                 switch (spectre_v2_user) {
 974                 case SPECTRE_V2_USER_NONE:
 975                         return ", IBPB: disabled";
 976                 case SPECTRE_V2_USER_STRICT:
 977                         return ", IBPB: always-on";
 978                 }
 979         }
 980         return "";
 981 }
 982
 983 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 984                                char *buf, unsigned int bug)
 985 {
 986         if (!boot_cpu_has_bug(bug))
 987                 return sprintf(buf, "Not affected\n");
 988
 989         switch (bug) {
 990         case X86_BUG_CPU_MELTDOWN:
 991                 if (boot_cpu_has(X86_FEATURE_PTI))
 992                         return sprintf(buf, "Mitigation: PTI\n");
 993
 994                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
 995                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
 996
 997                 break;
 998
 999         case X86_BUG_SPECTRE_V1:
1000                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
1001
1002         case X86_BUG_SPECTRE_V2:
1003                 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
1004                                ibpb_state(),
1005                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1006                                stibp_state(),
1007                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1008                                spectre_v2_module_string());
1009
1010         case X86_BUG_SPEC_STORE_BYPASS:
1011                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1012
1013         case X86_BUG_L1TF:
1014                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1015                         return l1tf_show_state(buf);
1016                 break;
1017         default:
1018                 break;
1019         }
1020
1021         return sprintf(buf, "Vulnerable\n");
1022 }
1023
1024 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1025 {
1026         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1027 }
1028
1029 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1030 {
1031         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1032 }
1033
1034 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1035 {
1036         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1037 }
1038
1039 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1040 {
1041         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1042 }
1043
1044 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1045 {
1046         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1047 }
1048 #endif