arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17 #include <linux/sched/smt.h>
  18 #include <linux/pgtable.h>
  19 #include <linux/bpf.h>
  20
  21 #include <asm/spec-ctrl.h>
  22 #include <asm/cmdline.h>
  23 #include <asm/bugs.h>
  24 #include <asm/processor.h>
  25 #include <asm/processor-flags.h>
  26 #include <asm/fpu/api.h>
  27 #include <asm/msr.h>
  28 #include <asm/vmx.h>
  29 #include <asm/paravirt.h>
  30 #include <asm/alternative.h>
  31 #include <asm/set_memory.h>
  32 #include <asm/intel-family.h>
  33 #include <asm/e820/api.h>
  34 #include <asm/hypervisor.h>
  35 #include <asm/tlbflush.h>
  36
  37 #include "cpu.h"
  38
  39 static void __init spectre_v1_select_mitigation(void);
  40 static void __init spectre_v2_select_mitigation(void);
  41 static void __init ssb_select_mitigation(void);
  42 static void __init l1tf_select_mitigation(void);
  43 static void __init mds_select_mitigation(void);
  44 static void __init mds_print_mitigation(void);
  45 static void __init taa_select_mitigation(void);
  46 static void __init srbds_select_mitigation(void);
  47 static void __init l1d_flush_select_mitigation(void);
  48
  49 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  50 u64 x86_spec_ctrl_base;
  51 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  52 static DEFINE_MUTEX(spec_ctrl_mutex);
  53
  54 /*
  55  * The vendor and possibly platform specific bits which can be modified in
  56  * x86_spec_ctrl_base.
  57  */
  58 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  59
  60 /*
  61  * AMD specific MSR info for Speculative Store Bypass control.
  62  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  63  */
  64 u64 __ro_after_init x86_amd_ls_cfg_base;
  65 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  66
  67 /* Control conditional STIBP in switch_to() */
  68 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
  69 /* Control conditional IBPB in switch_mm() */
  70 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
  71 /* Control unconditional IBPB in switch_mm() */
  72 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  73
  74 /* Control MDS CPU buffer clear before returning to user space */
  75 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
  76 EXPORT_SYMBOL_GPL(mds_user_clear);
  77 /* Control MDS CPU buffer clear before idling (halt, mwait) */
  78 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
  79 EXPORT_SYMBOL_GPL(mds_idle_clear);
  80
  81 /*
  82  * Controls whether l1d flush based mitigations are enabled,
  83  * based on hw features and admin setting via boot parameter
  84  * defaults to false
  85  */
  86 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
  87
  88 void __init check_bugs(void)
  89 {
  90         identify_boot_cpu();
  91
  92         /*
  93          * identify_boot_cpu() initialized SMT support information, let the
  94          * core code know.
  95          */
  96         cpu_smt_check_topology();
  97
  98         if (!IS_ENABLED(CONFIG_SMP)) {
  99                 pr_info("CPU: ");
 100                 print_cpu_info(&boot_cpu_data);
 101         }
 102
 103         /*
 104          * Read the SPEC_CTRL MSR to account for reserved bits which may
 105          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
 106          * init code as it is not enumerated and depends on the family.
 107          */
 108         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
 109                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 110
 111         /* Allow STIBP in MSR_SPEC_CTRL if supported */
 112         if (boot_cpu_has(X86_FEATURE_STIBP))
 113                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
 114
 115         /* Select the proper CPU mitigations before patching alternatives: */
 116         spectre_v1_select_mitigation();
 117         spectre_v2_select_mitigation();
 118         ssb_select_mitigation();
 119         l1tf_select_mitigation();
 120         mds_select_mitigation();
 121         taa_select_mitigation();
 122         srbds_select_mitigation();
 123         l1d_flush_select_mitigation();
 124
 125         /*
 126          * As MDS and TAA mitigations are inter-related, print MDS
 127          * mitigation until after TAA mitigation selection is done.
 128          */
 129         mds_print_mitigation();
 130
 131         arch_smt_update();
 132
 133 #ifdef CONFIG_X86_32
 134         /*
 135          * Check whether we are able to run this kernel safely on SMP.
 136          *
 137          * - i386 is no longer supported.
 138          * - In order to run on anything without a TSC, we need to be
 139          *   compiled for a i486.
 140          */
 141         if (boot_cpu_data.x86 < 4)
 142                 panic("Kernel requires i486+ for 'invlpg' and other features");
 143
 144         init_utsname()->machine[1] =
 145                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 146         alternative_instructions();
 147
 148         fpu__init_check_bugs();
 149 #else /* CONFIG_X86_64 */
 150         alternative_instructions();
 151
 152         /*
 153          * Make sure the first 2MB area is not mapped by huge pages
 154          * There are typically fixed size MTRRs in there and overlapping
 155          * MTRRs into large pages causes slow downs.
 156          *
 157          * Right now we don't do that with gbpages because there seems
 158          * very little benefit for that case.
 159          */
 160         if (!direct_gbpages)
 161                 set_memory_4k((unsigned long)__va(0), 1);
 162 #endif
 163 }
 164
 165 void
 166 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 167 {
 168         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 169         struct thread_info *ti = current_thread_info();
 170
 171         /* Is MSR_SPEC_CTRL implemented ? */
 172         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 173                 /*
 174                  * Restrict guest_spec_ctrl to supported values. Clear the
 175                  * modifiable bits in the host base value and or the
 176                  * modifiable bits from the guest value.
 177                  */
 178                 guestval = hostval & ~x86_spec_ctrl_mask;
 179                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 180
 181                 /* SSBD controlled in MSR_SPEC_CTRL */
 182                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 183                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 184                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 185
 186                 /* Conditional STIBP enabled? */
 187                 if (static_branch_unlikely(&switch_to_cond_stibp))
 188                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
 189
 190                 if (hostval != guestval) {
 191                         msrval = setguest ? guestval : hostval;
 192                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 193                 }
 194         }
 195
 196         /*
 197          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 198          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 199          */
 200         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 201             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 202                 return;
 203
 204         /*
 205          * If the host has SSBD mitigation enabled, force it in the host's
 206          * virtual MSR value. If its not permanently enabled, evaluate
 207          * current's TIF_SSBD thread flag.
 208          */
 209         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 210                 hostval = SPEC_CTRL_SSBD;
 211         else
 212                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 213
 214         /* Sanitize the guest value */
 215         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 216
 217         if (hostval != guestval) {
 218                 unsigned long tif;
 219
 220                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 221                                  ssbd_spec_ctrl_to_tif(hostval);
 222
 223                 speculation_ctrl_update(tif);
 224         }
 225 }
 226 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 227
 228 static void x86_amd_ssb_disable(void)
 229 {
 230         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 231
 232         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 233                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 234         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 235                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 236 }
 237
 238 #undef pr_fmt
 239 #define pr_fmt(fmt)     "MDS: " fmt
 240
 241 /* Default mitigation for MDS-affected CPUs */
 242 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 243 static bool mds_nosmt __ro_after_init = false;
 244
 245 static const char * const mds_strings[] = {
 246         [MDS_MITIGATION_OFF]    = "Vulnerable",
 247         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 248         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 249 };
 250
 251 static void __init mds_select_mitigation(void)
 252 {
 253         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 254                 mds_mitigation = MDS_MITIGATION_OFF;
 255                 return;
 256         }
 257
 258         if (mds_mitigation == MDS_MITIGATION_FULL) {
 259                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 260                         mds_mitigation = MDS_MITIGATION_VMWERV;
 261
 262                 static_branch_enable(&mds_user_clear);
 263
 264                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 265                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 266                         cpu_smt_disable(false);
 267         }
 268 }
 269
 270 static void __init mds_print_mitigation(void)
 271 {
 272         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off())
 273                 return;
 274
 275         pr_info("%s\n", mds_strings[mds_mitigation]);
 276 }
 277
 278 static int __init mds_cmdline(char *str)
 279 {
 280         if (!boot_cpu_has_bug(X86_BUG_MDS))
 281                 return 0;
 282
 283         if (!str)
 284                 return -EINVAL;
 285
 286         if (!strcmp(str, "off"))
 287                 mds_mitigation = MDS_MITIGATION_OFF;
 288         else if (!strcmp(str, "full"))
 289                 mds_mitigation = MDS_MITIGATION_FULL;
 290         else if (!strcmp(str, "full,nosmt")) {
 291                 mds_mitigation = MDS_MITIGATION_FULL;
 292                 mds_nosmt = true;
 293         }
 294
 295         return 0;
 296 }
 297 early_param("mds", mds_cmdline);
 298
 299 #undef pr_fmt
 300 #define pr_fmt(fmt)     "TAA: " fmt
 301
 302 enum taa_mitigations {
 303         TAA_MITIGATION_OFF,
 304         TAA_MITIGATION_UCODE_NEEDED,
 305         TAA_MITIGATION_VERW,
 306         TAA_MITIGATION_TSX_DISABLED,
 307 };
 308
 309 /* Default mitigation for TAA-affected CPUs */
 310 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
 311 static bool taa_nosmt __ro_after_init;
 312
 313 static const char * const taa_strings[] = {
 314         [TAA_MITIGATION_OFF]            = "Vulnerable",
 315         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 316         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 317         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 318 };
 319
 320 static void __init taa_select_mitigation(void)
 321 {
 322         u64 ia32_cap;
 323
 324         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 325                 taa_mitigation = TAA_MITIGATION_OFF;
 326                 return;
 327         }
 328
 329         /* TSX previously disabled by tsx=off */
 330         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 331                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 332                 goto out;
 333         }
 334
 335         if (cpu_mitigations_off()) {
 336                 taa_mitigation = TAA_MITIGATION_OFF;
 337                 return;
 338         }
 339
 340         /*
 341          * TAA mitigation via VERW is turned off if both
 342          * tsx_async_abort=off and mds=off are specified.
 343          */
 344         if (taa_mitigation == TAA_MITIGATION_OFF &&
 345             mds_mitigation == MDS_MITIGATION_OFF)
 346                 goto out;
 347
 348         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 349                 taa_mitigation = TAA_MITIGATION_VERW;
 350         else
 351                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 352
 353         /*
 354          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 355          * A microcode update fixes this behavior to clear CPU buffers. It also
 356          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 357          * ARCH_CAP_TSX_CTRL_MSR bit.
 358          *
 359          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 360          * update is required.
 361          */
 362         ia32_cap = x86_read_arch_cap_msr();
 363         if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
 364             !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
 365                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 366
 367         /*
 368          * TSX is enabled, select alternate mitigation for TAA which is
 369          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 370          *
 371          * For guests that can't determine whether the correct microcode is
 372          * present on host, enable the mitigation for UCODE_NEEDED as well.
 373          */
 374         static_branch_enable(&mds_user_clear);
 375
 376         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 377                 cpu_smt_disable(false);
 378
 379         /*
 380          * Update MDS mitigation, if necessary, as the mds_user_clear is
 381          * now enabled for TAA mitigation.
 382          */
 383         if (mds_mitigation == MDS_MITIGATION_OFF &&
 384             boot_cpu_has_bug(X86_BUG_MDS)) {
 385                 mds_mitigation = MDS_MITIGATION_FULL;
 386                 mds_select_mitigation();
 387         }
 388 out:
 389         pr_info("%s\n", taa_strings[taa_mitigation]);
 390 }
 391
 392 static int __init tsx_async_abort_parse_cmdline(char *str)
 393 {
 394         if (!boot_cpu_has_bug(X86_BUG_TAA))
 395                 return 0;
 396
 397         if (!str)
 398                 return -EINVAL;
 399
 400         if (!strcmp(str, "off")) {
 401                 taa_mitigation = TAA_MITIGATION_OFF;
 402         } else if (!strcmp(str, "full")) {
 403                 taa_mitigation = TAA_MITIGATION_VERW;
 404         } else if (!strcmp(str, "full,nosmt")) {
 405                 taa_mitigation = TAA_MITIGATION_VERW;
 406                 taa_nosmt = true;
 407         }
 408
 409         return 0;
 410 }
 411 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 412
 413 #undef pr_fmt
 414 #define pr_fmt(fmt)     "SRBDS: " fmt
 415
 416 enum srbds_mitigations {
 417         SRBDS_MITIGATION_OFF,
 418         SRBDS_MITIGATION_UCODE_NEEDED,
 419         SRBDS_MITIGATION_FULL,
 420         SRBDS_MITIGATION_TSX_OFF,
 421         SRBDS_MITIGATION_HYPERVISOR,
 422 };
 423
 424 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL;
 425
 426 static const char * const srbds_strings[] = {
 427         [SRBDS_MITIGATION_OFF]          = "Vulnerable",
 428         [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
 429         [SRBDS_MITIGATION_FULL]         = "Mitigation: Microcode",
 430         [SRBDS_MITIGATION_TSX_OFF]      = "Mitigation: TSX disabled",
 431         [SRBDS_MITIGATION_HYPERVISOR]   = "Unknown: Dependent on hypervisor status",
 432 };
 433
 434 static bool srbds_off;
 435
 436 void update_srbds_msr(void)
 437 {
 438         u64 mcu_ctrl;
 439
 440         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 441                 return;
 442
 443         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 444                 return;
 445
 446         if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
 447                 return;
 448
 449         /*
 450          * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX
 451          * being disabled and it hasn't received the SRBDS MSR microcode.
 452          */
 453         if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 454                 return;
 455
 456         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 457
 458         switch (srbds_mitigation) {
 459         case SRBDS_MITIGATION_OFF:
 460         case SRBDS_MITIGATION_TSX_OFF:
 461                 mcu_ctrl |= RNGDS_MITG_DIS;
 462                 break;
 463         case SRBDS_MITIGATION_FULL:
 464                 mcu_ctrl &= ~RNGDS_MITG_DIS;
 465                 break;
 466         default:
 467                 break;
 468         }
 469
 470         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 471 }
 472
 473 static void __init srbds_select_mitigation(void)
 474 {
 475         u64 ia32_cap;
 476
 477         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 478                 return;
 479
 480         /*
 481          * Check to see if this is one of the MDS_NO systems supporting
 482          * TSX that are only exposed to SRBDS when TSX is enabled.
 483          */
 484         ia32_cap = x86_read_arch_cap_msr();
 485         if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM))
 486                 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
 487         else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 488                 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
 489         else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 490                 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
 491         else if (cpu_mitigations_off() || srbds_off)
 492                 srbds_mitigation = SRBDS_MITIGATION_OFF;
 493
 494         update_srbds_msr();
 495         pr_info("%s\n", srbds_strings[srbds_mitigation]);
 496 }
 497
 498 static int __init srbds_parse_cmdline(char *str)
 499 {
 500         if (!str)
 501                 return -EINVAL;
 502
 503         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 504                 return 0;
 505
 506         srbds_off = !strcmp(str, "off");
 507         return 0;
 508 }
 509 early_param("srbds", srbds_parse_cmdline);
 510
 511 #undef pr_fmt
 512 #define pr_fmt(fmt)     "L1D Flush : " fmt
 513
 514 enum l1d_flush_mitigations {
 515         L1D_FLUSH_OFF = 0,
 516         L1D_FLUSH_ON,
 517 };
 518
 519 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF;
 520
 521 static void __init l1d_flush_select_mitigation(void)
 522 {
 523         if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D))
 524                 return;
 525
 526         static_branch_enable(&switch_mm_cond_l1d_flush);
 527         pr_info("Conditional flush on switch_mm() enabled\n");
 528 }
 529
 530 static int __init l1d_flush_parse_cmdline(char *str)
 531 {
 532         if (!strcmp(str, "on"))
 533                 l1d_flush_mitigation = L1D_FLUSH_ON;
 534
 535         return 0;
 536 }
 537 early_param("l1d_flush", l1d_flush_parse_cmdline);
 538
 539 #undef pr_fmt
 540 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 541
 542 enum spectre_v1_mitigation {
 543         SPECTRE_V1_MITIGATION_NONE,
 544         SPECTRE_V1_MITIGATION_AUTO,
 545 };
 546
 547 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 548         SPECTRE_V1_MITIGATION_AUTO;
 549
 550 static const char * const spectre_v1_strings[] = {
 551         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 552         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 553 };
 554
 555 /*
 556  * Does SMAP provide full mitigation against speculative kernel access to
 557  * userspace?
 558  */
 559 static bool smap_works_speculatively(void)
 560 {
 561         if (!boot_cpu_has(X86_FEATURE_SMAP))
 562                 return false;
 563
 564         /*
 565          * On CPUs which are vulnerable to Meltdown, SMAP does not
 566          * prevent speculative access to user data in the L1 cache.
 567          * Consider SMAP to be non-functional as a mitigation on these
 568          * CPUs.
 569          */
 570         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 571                 return false;
 572
 573         return true;
 574 }
 575
 576 static void __init spectre_v1_select_mitigation(void)
 577 {
 578         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 579                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 580                 return;
 581         }
 582
 583         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 584                 /*
 585                  * With Spectre v1, a user can speculatively control either
 586                  * path of a conditional swapgs with a user-controlled GS
 587                  * value.  The mitigation is to add lfences to both code paths.
 588                  *
 589                  * If FSGSBASE is enabled, the user can put a kernel address in
 590                  * GS, in which case SMAP provides no protection.
 591                  *
 592                  * If FSGSBASE is disabled, the user can only put a user space
 593                  * address in GS.  That makes an attack harder, but still
 594                  * possible if there's no SMAP protection.
 595                  */
 596                 if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
 597                     !smap_works_speculatively()) {
 598                         /*
 599                          * Mitigation can be provided from SWAPGS itself or
 600                          * PTI as the CR3 write in the Meltdown mitigation
 601                          * is serializing.
 602                          *
 603                          * If neither is there, mitigate with an LFENCE to
 604                          * stop speculation through swapgs.
 605                          */
 606                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 607                             !boot_cpu_has(X86_FEATURE_PTI))
 608                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 609
 610                         /*
 611                          * Enable lfences in the kernel entry (non-swapgs)
 612                          * paths, to prevent user entry from speculatively
 613                          * skipping swapgs.
 614                          */
 615                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 616                 }
 617         }
 618
 619         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 620 }
 621
 622 static int __init nospectre_v1_cmdline(char *str)
 623 {
 624         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 625         return 0;
 626 }
 627 early_param("nospectre_v1", nospectre_v1_cmdline);
 628
 629 #undef pr_fmt
 630 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 631
 632 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 633         SPECTRE_V2_NONE;
 634
 635 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
 636         SPECTRE_V2_USER_NONE;
 637 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
 638         SPECTRE_V2_USER_NONE;
 639
 640 #ifdef CONFIG_RETPOLINE
 641 static bool spectre_v2_bad_module;
 642
 643 bool retpoline_module_ok(bool has_retpoline)
 644 {
 645         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 646                 return true;
 647
 648         pr_err("System may be vulnerable to spectre v2\n");
 649         spectre_v2_bad_module = true;
 650         return false;
 651 }
 652
 653 static inline const char *spectre_v2_module_string(void)
 654 {
 655         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 656 }
 657 #else
 658 static inline const char *spectre_v2_module_string(void) { return ""; }
 659 #endif
 660
 661 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
 662 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
 663 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
 664
 665 #ifdef CONFIG_BPF_SYSCALL
 666 void unpriv_ebpf_notify(int new_state)
 667 {
 668         if (new_state)
 669                 return;
 670
 671         /* Unprivileged eBPF is enabled */
 672
 673         switch (spectre_v2_enabled) {
 674         case SPECTRE_V2_EIBRS:
 675                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
 676                 break;
 677         case SPECTRE_V2_EIBRS_LFENCE:
 678                 if (sched_smt_active())
 679                         pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
 680                 break;
 681         default:
 682                 break;
 683         }
 684 }
 685 #endif
 686
 687 static inline bool match_option(const char *arg, int arglen, const char *opt)
 688 {
 689         int len = strlen(opt);
 690
 691         return len == arglen && !strncmp(arg, opt, len);
 692 }
 693
 694 /* The kernel command line selection for spectre v2 */
 695 enum spectre_v2_mitigation_cmd {
 696         SPECTRE_V2_CMD_NONE,
 697         SPECTRE_V2_CMD_AUTO,
 698         SPECTRE_V2_CMD_FORCE,
 699         SPECTRE_V2_CMD_RETPOLINE,
 700         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 701         SPECTRE_V2_CMD_RETPOLINE_LFENCE,
 702         SPECTRE_V2_CMD_EIBRS,
 703         SPECTRE_V2_CMD_EIBRS_RETPOLINE,
 704         SPECTRE_V2_CMD_EIBRS_LFENCE,
 705 };
 706
 707 enum spectre_v2_user_cmd {
 708         SPECTRE_V2_USER_CMD_NONE,
 709         SPECTRE_V2_USER_CMD_AUTO,
 710         SPECTRE_V2_USER_CMD_FORCE,
 711         SPECTRE_V2_USER_CMD_PRCTL,
 712         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
 713         SPECTRE_V2_USER_CMD_SECCOMP,
 714         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
 715 };
 716
 717 static const char * const spectre_v2_user_strings[] = {
 718         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
 719         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
 720         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
 721         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
 722         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
 723 };
 724
 725 static const struct {
 726         const char                      *option;
 727         enum spectre_v2_user_cmd        cmd;
 728         bool                            secure;
 729 } v2_user_options[] __initconst = {
 730         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
 731         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
 732         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
 733         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
 734         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
 735         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
 736         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
 737 };
 738
 739 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
 740 {
 741         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 742                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 743 }
 744
 745 static enum spectre_v2_user_cmd __init
 746 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
 747 {
 748         char arg[20];
 749         int ret, i;
 750
 751         switch (v2_cmd) {
 752         case SPECTRE_V2_CMD_NONE:
 753                 return SPECTRE_V2_USER_CMD_NONE;
 754         case SPECTRE_V2_CMD_FORCE:
 755                 return SPECTRE_V2_USER_CMD_FORCE;
 756         default:
 757                 break;
 758         }
 759
 760         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
 761                                   arg, sizeof(arg));
 762         if (ret < 0)
 763                 return SPECTRE_V2_USER_CMD_AUTO;
 764
 765         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
 766                 if (match_option(arg, ret, v2_user_options[i].option)) {
 767                         spec_v2_user_print_cond(v2_user_options[i].option,
 768                                                 v2_user_options[i].secure);
 769                         return v2_user_options[i].cmd;
 770                 }
 771         }
 772
 773         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
 774         return SPECTRE_V2_USER_CMD_AUTO;
 775 }
 776
 777 static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
 778 {
 779         return (mode == SPECTRE_V2_EIBRS ||
 780                 mode == SPECTRE_V2_EIBRS_RETPOLINE ||
 781                 mode == SPECTRE_V2_EIBRS_LFENCE);
 782 }
 783
 784 static void __init
 785 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 786 {
 787         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
 788         bool smt_possible = IS_ENABLED(CONFIG_SMP);
 789         enum spectre_v2_user_cmd cmd;
 790
 791         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
 792                 return;
 793
 794         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 795             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 796                 smt_possible = false;
 797
 798         cmd = spectre_v2_parse_user_cmdline(v2_cmd);
 799         switch (cmd) {
 800         case SPECTRE_V2_USER_CMD_NONE:
 801                 goto set_mode;
 802         case SPECTRE_V2_USER_CMD_FORCE:
 803                 mode = SPECTRE_V2_USER_STRICT;
 804                 break;
 805         case SPECTRE_V2_USER_CMD_AUTO:
 806         case SPECTRE_V2_USER_CMD_PRCTL:
 807         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 808                 mode = SPECTRE_V2_USER_PRCTL;
 809                 break;
 810         case SPECTRE_V2_USER_CMD_SECCOMP:
 811         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 812                 if (IS_ENABLED(CONFIG_SECCOMP))
 813                         mode = SPECTRE_V2_USER_SECCOMP;
 814                 else
 815                         mode = SPECTRE_V2_USER_PRCTL;
 816                 break;
 817         }
 818
 819         /* Initialize Indirect Branch Prediction Barrier */
 820         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 821                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 822
 823                 spectre_v2_user_ibpb = mode;
 824                 switch (cmd) {
 825                 case SPECTRE_V2_USER_CMD_FORCE:
 826                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 827                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 828                         static_branch_enable(&switch_mm_always_ibpb);
 829                         spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
 830                         break;
 831                 case SPECTRE_V2_USER_CMD_PRCTL:
 832                 case SPECTRE_V2_USER_CMD_AUTO:
 833                 case SPECTRE_V2_USER_CMD_SECCOMP:
 834                         static_branch_enable(&switch_mm_cond_ibpb);
 835                         break;
 836                 default:
 837                         break;
 838                 }
 839
 840                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
 841                         static_key_enabled(&switch_mm_always_ibpb) ?
 842                         "always-on" : "conditional");
 843         }
 844
 845         /*
 846          * If no STIBP, enhanced IBRS is enabled or SMT impossible, STIBP is not
 847          * required.
 848          */
 849         if (!boot_cpu_has(X86_FEATURE_STIBP) ||
 850             !smt_possible ||
 851             spectre_v2_in_eibrs_mode(spectre_v2_enabled))
 852                 return;
 853
 854         /*
 855          * At this point, an STIBP mode other than "off" has been set.
 856          * If STIBP support is not being forced, check if STIBP always-on
 857          * is preferred.
 858          */
 859         if (mode != SPECTRE_V2_USER_STRICT &&
 860             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
 861                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
 862
 863         spectre_v2_user_stibp = mode;
 864
 865 set_mode:
 866         pr_info("%s\n", spectre_v2_user_strings[mode]);
 867 }
 868
 869 static const char * const spectre_v2_strings[] = {
 870         [SPECTRE_V2_NONE]                       = "Vulnerable",
 871         [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
 872         [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
 873         [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced IBRS",
 874         [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced IBRS + LFENCE",
 875         [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced IBRS + Retpolines",
 876 };
 877
 878 static const struct {
 879         const char *option;
 880         enum spectre_v2_mitigation_cmd cmd;
 881         bool secure;
 882 } mitigation_options[] __initconst = {
 883         { "off",                SPECTRE_V2_CMD_NONE,              false },
 884         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 885         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 886         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
 887         { "retpoline,lfence",   SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
 888         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 889         { "eibrs",              SPECTRE_V2_CMD_EIBRS,             false },
 890         { "eibrs,lfence",       SPECTRE_V2_CMD_EIBRS_LFENCE,      false },
 891         { "eibrs,retpoline",    SPECTRE_V2_CMD_EIBRS_RETPOLINE,   false },
 892         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 893 };
 894
 895 static void __init spec_v2_print_cond(const char *reason, bool secure)
 896 {
 897         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 898                 pr_info("%s selected on command line.\n", reason);
 899 }
 900
 901 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 902 {
 903         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 904         char arg[20];
 905         int ret, i;
 906
 907         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
 908             cpu_mitigations_off())
 909                 return SPECTRE_V2_CMD_NONE;
 910
 911         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 912         if (ret < 0)
 913                 return SPECTRE_V2_CMD_AUTO;
 914
 915         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 916                 if (!match_option(arg, ret, mitigation_options[i].option))
 917                         continue;
 918                 cmd = mitigation_options[i].cmd;
 919                 break;
 920         }
 921
 922         if (i >= ARRAY_SIZE(mitigation_options)) {
 923                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 924                 return SPECTRE_V2_CMD_AUTO;
 925         }
 926
 927         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 928              cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
 929              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
 930              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
 931              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
 932             !IS_ENABLED(CONFIG_RETPOLINE)) {
 933                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
 934                        mitigation_options[i].option);
 935                 return SPECTRE_V2_CMD_AUTO;
 936         }
 937
 938         if ((cmd == SPECTRE_V2_CMD_EIBRS ||
 939              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
 940              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
 941             !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 942                 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
 943                        mitigation_options[i].option);
 944                 return SPECTRE_V2_CMD_AUTO;
 945         }
 946
 947         if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
 948              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
 949             !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 950                 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
 951                        mitigation_options[i].option);
 952                 return SPECTRE_V2_CMD_AUTO;
 953         }
 954
 955         spec_v2_print_cond(mitigation_options[i].option,
 956                            mitigation_options[i].secure);
 957         return cmd;
 958 }
 959
 960 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
 961 {
 962         if (!IS_ENABLED(CONFIG_RETPOLINE)) {
 963                 pr_err("Kernel not compiled with retpoline; no mitigation available!");
 964                 return SPECTRE_V2_NONE;
 965         }
 966
 967         return SPECTRE_V2_RETPOLINE;
 968 }
 969
 970 static void __init spectre_v2_select_mitigation(void)
 971 {
 972         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 973         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 974
 975         /*
 976          * If the CPU is not affected and the command line mode is NONE or AUTO
 977          * then nothing to do.
 978          */
 979         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 980             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 981                 return;
 982
 983         switch (cmd) {
 984         case SPECTRE_V2_CMD_NONE:
 985                 return;
 986
 987         case SPECTRE_V2_CMD_FORCE:
 988         case SPECTRE_V2_CMD_AUTO:
 989                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 990                         mode = SPECTRE_V2_EIBRS;
 991                         break;
 992                 }
 993
 994                 mode = spectre_v2_select_retpoline();
 995                 break;
 996
 997         case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
 998                 pr_err(SPECTRE_V2_LFENCE_MSG);
 999                 mode = SPECTRE_V2_LFENCE;
1000                 break;
1001
1002         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
1003                 mode = SPECTRE_V2_RETPOLINE;
1004                 break;
1005
1006         case SPECTRE_V2_CMD_RETPOLINE:
1007                 mode = spectre_v2_select_retpoline();
1008                 break;
1009
1010         case SPECTRE_V2_CMD_EIBRS:
1011                 mode = SPECTRE_V2_EIBRS;
1012                 break;
1013
1014         case SPECTRE_V2_CMD_EIBRS_LFENCE:
1015                 mode = SPECTRE_V2_EIBRS_LFENCE;
1016                 break;
1017
1018         case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
1019                 mode = SPECTRE_V2_EIBRS_RETPOLINE;
1020                 break;
1021         }
1022
1023         if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1024                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1025
1026         if (spectre_v2_in_eibrs_mode(mode)) {
1027                 /* Force it so VMEXIT will restore correctly */
1028                 x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
1029                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1030         }
1031
1032         switch (mode) {
1033         case SPECTRE_V2_NONE:
1034         case SPECTRE_V2_EIBRS:
1035                 break;
1036
1037         case SPECTRE_V2_LFENCE:
1038         case SPECTRE_V2_EIBRS_LFENCE:
1039                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
1040                 fallthrough;
1041
1042         case SPECTRE_V2_RETPOLINE:
1043         case SPECTRE_V2_EIBRS_RETPOLINE:
1044                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
1045                 break;
1046         }
1047
1048         spectre_v2_enabled = mode;
1049         pr_info("%s\n", spectre_v2_strings[mode]);
1050
1051         /*
1052          * If spectre v2 protection has been enabled, unconditionally fill
1053          * RSB during a context switch; this protects against two independent
1054          * issues:
1055          *
1056          *      - RSB underflow (and switch to BTB) on Skylake+
1057          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
1058          */
1059         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
1060         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1061
1062         /*
1063          * Retpoline means the kernel is safe because it has no indirect
1064          * branches. Enhanced IBRS protects firmware too, so, enable restricted
1065          * speculation around firmware calls only when Enhanced IBRS isn't
1066          * supported.
1067          *
1068          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1069          * the user might select retpoline on the kernel command line and if
1070          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1071          * enable IBRS around firmware calls.
1072          */
1073         if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) {
1074                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
1075                 pr_info("Enabling Restricted Speculation for firmware calls\n");
1076         }
1077
1078         /* Set up IBPB and STIBP depending on the general spectre V2 command */
1079         spectre_v2_user_select_mitigation(cmd);
1080 }
1081
1082 static void update_stibp_msr(void * __unused)
1083 {
1084         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1085 }
1086
1087 /* Update x86_spec_ctrl_base in case SMT state changed. */
1088 static void update_stibp_strict(void)
1089 {
1090         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
1091
1092         if (sched_smt_active())
1093                 mask |= SPEC_CTRL_STIBP;
1094
1095         if (mask == x86_spec_ctrl_base)
1096                 return;
1097
1098         pr_info("Update user space SMT mitigation: STIBP %s\n",
1099                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
1100         x86_spec_ctrl_base = mask;
1101         on_each_cpu(update_stibp_msr, NULL, 1);
1102 }
1103
1104 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1105 static void update_indir_branch_cond(void)
1106 {
1107         if (sched_smt_active())
1108                 static_branch_enable(&switch_to_cond_stibp);
1109         else
1110                 static_branch_disable(&switch_to_cond_stibp);
1111 }
1112
1113 #undef pr_fmt
1114 #define pr_fmt(fmt) fmt
1115
1116 /* Update the static key controlling the MDS CPU buffer clear in idle */
1117 static void update_mds_branch_idle(void)
1118 {
1119         /*
1120          * Enable the idle clearing if SMT is active on CPUs which are
1121          * affected only by MSBDS and not any other MDS variant.
1122          *
1123          * The other variants cannot be mitigated when SMT is enabled, so
1124          * clearing the buffers on idle just to prevent the Store Buffer
1125          * repartitioning leak would be a window dressing exercise.
1126          */
1127         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
1128                 return;
1129
1130         if (sched_smt_active())
1131                 static_branch_enable(&mds_idle_clear);
1132         else
1133                 static_branch_disable(&mds_idle_clear);
1134 }
1135
1136 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1137 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1138
1139 void cpu_bugs_smt_update(void)
1140 {
1141         mutex_lock(&spec_ctrl_mutex);
1142
1143         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1144             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1145                 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1146
1147         switch (spectre_v2_user_stibp) {
1148         case SPECTRE_V2_USER_NONE:
1149                 break;
1150         case SPECTRE_V2_USER_STRICT:
1151         case SPECTRE_V2_USER_STRICT_PREFERRED:
1152                 update_stibp_strict();
1153                 break;
1154         case SPECTRE_V2_USER_PRCTL:
1155         case SPECTRE_V2_USER_SECCOMP:
1156                 update_indir_branch_cond();
1157                 break;
1158         }
1159
1160         switch (mds_mitigation) {
1161         case MDS_MITIGATION_FULL:
1162         case MDS_MITIGATION_VMWERV:
1163                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
1164                         pr_warn_once(MDS_MSG_SMT);
1165                 update_mds_branch_idle();
1166                 break;
1167         case MDS_MITIGATION_OFF:
1168                 break;
1169         }
1170
1171         switch (taa_mitigation) {
1172         case TAA_MITIGATION_VERW:
1173         case TAA_MITIGATION_UCODE_NEEDED:
1174                 if (sched_smt_active())
1175                         pr_warn_once(TAA_MSG_SMT);
1176                 break;
1177         case TAA_MITIGATION_TSX_DISABLED:
1178         case TAA_MITIGATION_OFF:
1179                 break;
1180         }
1181
1182         mutex_unlock(&spec_ctrl_mutex);
1183 }
1184
1185 #undef pr_fmt
1186 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
1187
1188 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
1189
1190 /* The kernel command line selection */
1191 enum ssb_mitigation_cmd {
1192         SPEC_STORE_BYPASS_CMD_NONE,
1193         SPEC_STORE_BYPASS_CMD_AUTO,
1194         SPEC_STORE_BYPASS_CMD_ON,
1195         SPEC_STORE_BYPASS_CMD_PRCTL,
1196         SPEC_STORE_BYPASS_CMD_SECCOMP,
1197 };
1198
1199 static const char * const ssb_strings[] = {
1200         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
1201         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
1202         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
1203         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
1204 };
1205
1206 static const struct {
1207         const char *option;
1208         enum ssb_mitigation_cmd cmd;
1209 } ssb_mitigation_options[]  __initconst = {
1210         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
1211         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
1212         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
1213         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
1214         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
1215 };
1216
1217 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
1218 {
1219         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
1220         char arg[20];
1221         int ret, i;
1222
1223         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
1224             cpu_mitigations_off()) {
1225                 return SPEC_STORE_BYPASS_CMD_NONE;
1226         } else {
1227                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
1228                                           arg, sizeof(arg));
1229                 if (ret < 0)
1230                         return SPEC_STORE_BYPASS_CMD_AUTO;
1231
1232                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
1233                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
1234                                 continue;
1235
1236                         cmd = ssb_mitigation_options[i].cmd;
1237                         break;
1238                 }
1239
1240                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
1241                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1242                         return SPEC_STORE_BYPASS_CMD_AUTO;
1243                 }
1244         }
1245
1246         return cmd;
1247 }
1248
1249 static enum ssb_mitigation __init __ssb_select_mitigation(void)
1250 {
1251         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1252         enum ssb_mitigation_cmd cmd;
1253
1254         if (!boot_cpu_has(X86_FEATURE_SSBD))
1255                 return mode;
1256
1257         cmd = ssb_parse_cmdline();
1258         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1259             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1260              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1261                 return mode;
1262
1263         switch (cmd) {
1264         case SPEC_STORE_BYPASS_CMD_SECCOMP:
1265                 /*
1266                  * Choose prctl+seccomp as the default mode if seccomp is
1267                  * enabled.
1268                  */
1269                 if (IS_ENABLED(CONFIG_SECCOMP))
1270                         mode = SPEC_STORE_BYPASS_SECCOMP;
1271                 else
1272                         mode = SPEC_STORE_BYPASS_PRCTL;
1273                 break;
1274         case SPEC_STORE_BYPASS_CMD_ON:
1275                 mode = SPEC_STORE_BYPASS_DISABLE;
1276                 break;
1277         case SPEC_STORE_BYPASS_CMD_AUTO:
1278         case SPEC_STORE_BYPASS_CMD_PRCTL:
1279                 mode = SPEC_STORE_BYPASS_PRCTL;
1280                 break;
1281         case SPEC_STORE_BYPASS_CMD_NONE:
1282                 break;
1283         }
1284
1285         /*
1286          * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
1287          * bit in the mask to allow guests to use the mitigation even in the
1288          * case where the host does not enable it.
1289          */
1290         if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
1291             static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1292                 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
1293         }
1294
1295         /*
1296          * We have three CPU feature flags that are in play here:
1297          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1298          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1299          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1300          */
1301         if (mode == SPEC_STORE_BYPASS_DISABLE) {
1302                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1303                 /*
1304                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1305                  * use a completely different MSR and bit dependent on family.
1306                  */
1307                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1308                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1309                         x86_amd_ssb_disable();
1310                 } else {
1311                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1312                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1313                 }
1314         }
1315
1316         return mode;
1317 }
1318
1319 static void ssb_select_mitigation(void)
1320 {
1321         ssb_mode = __ssb_select_mitigation();
1322
1323         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1324                 pr_info("%s\n", ssb_strings[ssb_mode]);
1325 }
1326
1327 #undef pr_fmt
1328 #define pr_fmt(fmt)     "Speculation prctl: " fmt
1329
1330 static void task_update_spec_tif(struct task_struct *tsk)
1331 {
1332         /* Force the update of the real TIF bits */
1333         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1334
1335         /*
1336          * Immediately update the speculation control MSRs for the current
1337          * task, but for a non-current task delay setting the CPU
1338          * mitigation until it is scheduled next.
1339          *
1340          * This can only happen for SECCOMP mitigation. For PRCTL it's
1341          * always the current task.
1342          */
1343         if (tsk == current)
1344                 speculation_ctrl_update_current();
1345 }
1346
1347 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl)
1348 {
1349
1350         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
1351                 return -EPERM;
1352
1353         switch (ctrl) {
1354         case PR_SPEC_ENABLE:
1355                 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1356                 return 0;
1357         case PR_SPEC_DISABLE:
1358                 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
1359                 return 0;
1360         default:
1361                 return -ERANGE;
1362         }
1363 }
1364
1365 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1366 {
1367         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1368             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1369                 return -ENXIO;
1370
1371         switch (ctrl) {
1372         case PR_SPEC_ENABLE:
1373                 /* If speculation is force disabled, enable is not allowed */
1374                 if (task_spec_ssb_force_disable(task))
1375                         return -EPERM;
1376                 task_clear_spec_ssb_disable(task);
1377                 task_clear_spec_ssb_noexec(task);
1378                 task_update_spec_tif(task);
1379                 break;
1380         case PR_SPEC_DISABLE:
1381                 task_set_spec_ssb_disable(task);
1382                 task_clear_spec_ssb_noexec(task);
1383                 task_update_spec_tif(task);
1384                 break;
1385         case PR_SPEC_FORCE_DISABLE:
1386                 task_set_spec_ssb_disable(task);
1387                 task_set_spec_ssb_force_disable(task);
1388                 task_clear_spec_ssb_noexec(task);
1389                 task_update_spec_tif(task);
1390                 break;
1391         case PR_SPEC_DISABLE_NOEXEC:
1392                 if (task_spec_ssb_force_disable(task))
1393                         return -EPERM;
1394                 task_set_spec_ssb_disable(task);
1395                 task_set_spec_ssb_noexec(task);
1396                 task_update_spec_tif(task);
1397                 break;
1398         default:
1399                 return -ERANGE;
1400         }
1401         return 0;
1402 }
1403
1404 static bool is_spec_ib_user_controlled(void)
1405 {
1406         return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
1407                 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
1408                 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
1409                 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
1410 }
1411
1412 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
1413 {
1414         switch (ctrl) {
1415         case PR_SPEC_ENABLE:
1416                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1417                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1418                         return 0;
1419
1420                 /*
1421                  * With strict mode for both IBPB and STIBP, the instruction
1422                  * code paths avoid checking this task flag and instead,
1423                  * unconditionally run the instruction. However, STIBP and IBPB
1424                  * are independent and either can be set to conditionally
1425                  * enabled regardless of the mode of the other.
1426                  *
1427                  * If either is set to conditional, allow the task flag to be
1428                  * updated, unless it was force-disabled by a previous prctl
1429                  * call. Currently, this is possible on an AMD CPU which has the
1430                  * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
1431                  * kernel is booted with 'spectre_v2_user=seccomp', then
1432                  * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
1433                  * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
1434                  */
1435                 if (!is_spec_ib_user_controlled() ||
1436                     task_spec_ib_force_disable(task))
1437                         return -EPERM;
1438
1439                 task_clear_spec_ib_disable(task);
1440                 task_update_spec_tif(task);
1441                 break;
1442         case PR_SPEC_DISABLE:
1443         case PR_SPEC_FORCE_DISABLE:
1444                 /*
1445                  * Indirect branch speculation is always allowed when
1446                  * mitigation is force disabled.
1447                  */
1448                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1449                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1450                         return -EPERM;
1451
1452                 if (!is_spec_ib_user_controlled())
1453                         return 0;
1454
1455                 task_set_spec_ib_disable(task);
1456                 if (ctrl == PR_SPEC_FORCE_DISABLE)
1457                         task_set_spec_ib_force_disable(task);
1458                 task_update_spec_tif(task);
1459                 break;
1460         default:
1461                 return -ERANGE;
1462         }
1463         return 0;
1464 }
1465
1466 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
1467                              unsigned long ctrl)
1468 {
1469         switch (which) {
1470         case PR_SPEC_STORE_BYPASS:
1471                 return ssb_prctl_set(task, ctrl);
1472         case PR_SPEC_INDIRECT_BRANCH:
1473                 return ib_prctl_set(task, ctrl);
1474         case PR_SPEC_L1D_FLUSH:
1475                 return l1d_flush_prctl_set(task, ctrl);
1476         default:
1477                 return -ENODEV;
1478         }
1479 }
1480
1481 #ifdef CONFIG_SECCOMP
1482 void arch_seccomp_spec_mitigate(struct task_struct *task)
1483 {
1484         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
1485                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1486         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
1487             spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
1488                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1489 }
1490 #endif
1491
1492 static int l1d_flush_prctl_get(struct task_struct *task)
1493 {
1494         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
1495                 return PR_SPEC_FORCE_DISABLE;
1496
1497         if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))
1498                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1499         else
1500                 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1501 }
1502
1503 static int ssb_prctl_get(struct task_struct *task)
1504 {
1505         switch (ssb_mode) {
1506         case SPEC_STORE_BYPASS_DISABLE:
1507                 return PR_SPEC_DISABLE;
1508         case SPEC_STORE_BYPASS_SECCOMP:
1509         case SPEC_STORE_BYPASS_PRCTL:
1510                 if (task_spec_ssb_force_disable(task))
1511                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1512                 if (task_spec_ssb_noexec(task))
1513                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
1514                 if (task_spec_ssb_disable(task))
1515                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1516                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1517         default:
1518                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1519                         return PR_SPEC_ENABLE;
1520                 return PR_SPEC_NOT_AFFECTED;
1521         }
1522 }
1523
1524 static int ib_prctl_get(struct task_struct *task)
1525 {
1526         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
1527                 return PR_SPEC_NOT_AFFECTED;
1528
1529         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
1530             spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
1531                 return PR_SPEC_ENABLE;
1532         else if (is_spec_ib_user_controlled()) {
1533                 if (task_spec_ib_force_disable(task))
1534                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1535                 if (task_spec_ib_disable(task))
1536                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1537                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1538         } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
1539             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
1540             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
1541                 return PR_SPEC_DISABLE;
1542         else
1543                 return PR_SPEC_NOT_AFFECTED;
1544 }
1545
1546 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
1547 {
1548         switch (which) {
1549         case PR_SPEC_STORE_BYPASS:
1550                 return ssb_prctl_get(task);
1551         case PR_SPEC_INDIRECT_BRANCH:
1552                 return ib_prctl_get(task);
1553         case PR_SPEC_L1D_FLUSH:
1554                 return l1d_flush_prctl_get(task);
1555         default:
1556                 return -ENODEV;
1557         }
1558 }
1559
1560 void x86_spec_ctrl_setup_ap(void)
1561 {
1562         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1563                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1564
1565         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1566                 x86_amd_ssb_disable();
1567 }
1568
1569 bool itlb_multihit_kvm_mitigation;
1570 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
1571
1572 #undef pr_fmt
1573 #define pr_fmt(fmt)     "L1TF: " fmt
1574
1575 /* Default mitigation for L1TF-affected CPUs */
1576 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1577 #if IS_ENABLED(CONFIG_KVM_INTEL)
1578 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1579 #endif
1580 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1581 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1582
1583 /*
1584  * These CPUs all support 44bits physical address space internally in the
1585  * cache but CPUID can report a smaller number of physical address bits.
1586  *
1587  * The L1TF mitigation uses the top most address bit for the inversion of
1588  * non present PTEs. When the installed memory reaches into the top most
1589  * address bit due to memory holes, which has been observed on machines
1590  * which report 36bits physical address bits and have 32G RAM installed,
1591  * then the mitigation range check in l1tf_select_mitigation() triggers.
1592  * This is a false positive because the mitigation is still possible due to
1593  * the fact that the cache uses 44bit internally. Use the cache bits
1594  * instead of the reported physical bits and adjust them on the affected
1595  * machines to 44bit if the reported bits are less than 44.
1596  */
1597 static void override_cache_bits(struct cpuinfo_x86 *c)
1598 {
1599         if (c->x86 != 6)
1600                 return;
1601
1602         switch (c->x86_model) {
1603         case INTEL_FAM6_NEHALEM:
1604         case INTEL_FAM6_WESTMERE:
1605         case INTEL_FAM6_SANDYBRIDGE:
1606         case INTEL_FAM6_IVYBRIDGE:
1607         case INTEL_FAM6_HASWELL:
1608         case INTEL_FAM6_HASWELL_L:
1609         case INTEL_FAM6_HASWELL_G:
1610         case INTEL_FAM6_BROADWELL:
1611         case INTEL_FAM6_BROADWELL_G:
1612         case INTEL_FAM6_SKYLAKE_L:
1613         case INTEL_FAM6_SKYLAKE:
1614         case INTEL_FAM6_KABYLAKE_L:
1615         case INTEL_FAM6_KABYLAKE:
1616                 if (c->x86_cache_bits < 44)
1617                         c->x86_cache_bits = 44;
1618                 break;
1619         }
1620 }
1621
1622 static void __init l1tf_select_mitigation(void)
1623 {
1624         u64 half_pa;
1625
1626         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1627                 return;
1628
1629         if (cpu_mitigations_off())
1630                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1631         else if (cpu_mitigations_auto_nosmt())
1632                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1633
1634         override_cache_bits(&boot_cpu_data);
1635
1636         switch (l1tf_mitigation) {
1637         case L1TF_MITIGATION_OFF:
1638         case L1TF_MITIGATION_FLUSH_NOWARN:
1639         case L1TF_MITIGATION_FLUSH:
1640                 break;
1641         case L1TF_MITIGATION_FLUSH_NOSMT:
1642         case L1TF_MITIGATION_FULL:
1643                 cpu_smt_disable(false);
1644                 break;
1645         case L1TF_MITIGATION_FULL_FORCE:
1646                 cpu_smt_disable(true);
1647                 break;
1648         }
1649
1650 #if CONFIG_PGTABLE_LEVELS == 2
1651         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1652         return;
1653 #endif
1654
1655         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1656         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
1657                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
1658                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1659                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1660                                 half_pa);
1661                 pr_info("However, doing so will make a part of your RAM unusable.\n");
1662                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
1663                 return;
1664         }
1665
1666         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1667 }
1668
1669 static int __init l1tf_cmdline(char *str)
1670 {
1671         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1672                 return 0;
1673
1674         if (!str)
1675                 return -EINVAL;
1676
1677         if (!strcmp(str, "off"))
1678                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1679         else if (!strcmp(str, "flush,nowarn"))
1680                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1681         else if (!strcmp(str, "flush"))
1682                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1683         else if (!strcmp(str, "flush,nosmt"))
1684                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1685         else if (!strcmp(str, "full"))
1686                 l1tf_mitigation = L1TF_MITIGATION_FULL;
1687         else if (!strcmp(str, "full,force"))
1688                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1689
1690         return 0;
1691 }
1692 early_param("l1tf", l1tf_cmdline);
1693
1694 #undef pr_fmt
1695 #define pr_fmt(fmt) fmt
1696
1697 #ifdef CONFIG_SYSFS
1698
1699 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1700
1701 #if IS_ENABLED(CONFIG_KVM_INTEL)
1702 static const char * const l1tf_vmx_states[] = {
1703         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
1704         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
1705         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
1706         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
1707         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
1708         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
1709 };
1710
1711 static ssize_t l1tf_show_state(char *buf)
1712 {
1713         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1714                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1715
1716         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1717             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
1718              sched_smt_active())) {
1719                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1720                                l1tf_vmx_states[l1tf_vmx_mitigation]);
1721         }
1722
1723         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1724                        l1tf_vmx_states[l1tf_vmx_mitigation],
1725                        sched_smt_active() ? "vulnerable" : "disabled");
1726 }
1727
1728 static ssize_t itlb_multihit_show_state(char *buf)
1729 {
1730         if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
1731             !boot_cpu_has(X86_FEATURE_VMX))
1732                 return sprintf(buf, "KVM: Mitigation: VMX unsupported\n");
1733         else if (!(cr4_read_shadow() & X86_CR4_VMXE))
1734                 return sprintf(buf, "KVM: Mitigation: VMX disabled\n");
1735         else if (itlb_multihit_kvm_mitigation)
1736                 return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
1737         else
1738                 return sprintf(buf, "KVM: Vulnerable\n");
1739 }
1740 #else
1741 static ssize_t l1tf_show_state(char *buf)
1742 {
1743         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1744 }
1745
1746 static ssize_t itlb_multihit_show_state(char *buf)
1747 {
1748         return sprintf(buf, "Processor vulnerable\n");
1749 }
1750 #endif
1751
1752 static ssize_t mds_show_state(char *buf)
1753 {
1754         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1755                 return sprintf(buf, "%s; SMT Host state unknown\n",
1756                                mds_strings[mds_mitigation]);
1757         }
1758
1759         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1760                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1761                                (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
1762                                 sched_smt_active() ? "mitigated" : "disabled"));
1763         }
1764
1765         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1766                        sched_smt_active() ? "vulnerable" : "disabled");
1767 }
1768
1769 static ssize_t tsx_async_abort_show_state(char *buf)
1770 {
1771         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
1772             (taa_mitigation == TAA_MITIGATION_OFF))
1773                 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
1774
1775         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1776                 return sprintf(buf, "%s; SMT Host state unknown\n",
1777                                taa_strings[taa_mitigation]);
1778         }
1779
1780         return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
1781                        sched_smt_active() ? "vulnerable" : "disabled");
1782 }
1783
1784 static char *stibp_state(void)
1785 {
1786         if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
1787                 return "";
1788
1789         switch (spectre_v2_user_stibp) {
1790         case SPECTRE_V2_USER_NONE:
1791                 return ", STIBP: disabled";
1792         case SPECTRE_V2_USER_STRICT:
1793                 return ", STIBP: forced";
1794         case SPECTRE_V2_USER_STRICT_PREFERRED:
1795                 return ", STIBP: always-on";
1796         case SPECTRE_V2_USER_PRCTL:
1797         case SPECTRE_V2_USER_SECCOMP:
1798                 if (static_key_enabled(&switch_to_cond_stibp))
1799                         return ", STIBP: conditional";
1800         }
1801         return "";
1802 }
1803
1804 static char *ibpb_state(void)
1805 {
1806         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1807                 if (static_key_enabled(&switch_mm_always_ibpb))
1808                         return ", IBPB: always-on";
1809                 if (static_key_enabled(&switch_mm_cond_ibpb))
1810                         return ", IBPB: conditional";
1811                 return ", IBPB: disabled";
1812         }
1813         return "";
1814 }
1815
1816 static ssize_t spectre_v2_show_state(char *buf)
1817 {
1818         if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
1819                 return sprintf(buf, "Vulnerable: LFENCE\n");
1820
1821         if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1822                 return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
1823
1824         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1825             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1826                 return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
1827
1828         return sprintf(buf, "%s%s%s%s%s%s\n",
1829                        spectre_v2_strings[spectre_v2_enabled],
1830                        ibpb_state(),
1831                        boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1832                        stibp_state(),
1833                        boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1834                        spectre_v2_module_string());
1835 }
1836
1837 static ssize_t srbds_show_state(char *buf)
1838 {
1839         return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
1840 }
1841
1842 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
1843                                char *buf, unsigned int bug)
1844 {
1845         if (!boot_cpu_has_bug(bug))
1846                 return sprintf(buf, "Not affected\n");
1847
1848         switch (bug) {
1849         case X86_BUG_CPU_MELTDOWN:
1850                 if (boot_cpu_has(X86_FEATURE_PTI))
1851                         return sprintf(buf, "Mitigation: PTI\n");
1852
1853                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
1854                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
1855
1856                 break;
1857
1858         case X86_BUG_SPECTRE_V1:
1859                 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
1860
1861         case X86_BUG_SPECTRE_V2:
1862                 return spectre_v2_show_state(buf);
1863
1864         case X86_BUG_SPEC_STORE_BYPASS:
1865                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1866
1867         case X86_BUG_L1TF:
1868                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1869                         return l1tf_show_state(buf);
1870                 break;
1871
1872         case X86_BUG_MDS:
1873                 return mds_show_state(buf);
1874
1875         case X86_BUG_TAA:
1876                 return tsx_async_abort_show_state(buf);
1877
1878         case X86_BUG_ITLB_MULTIHIT:
1879                 return itlb_multihit_show_state(buf);
1880
1881         case X86_BUG_SRBDS:
1882                 return srbds_show_state(buf);
1883
1884         default:
1885                 break;
1886         }
1887
1888         return sprintf(buf, "Vulnerable\n");
1889 }
1890
1891 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1892 {
1893         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1894 }
1895
1896 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1897 {
1898         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1899 }
1900
1901 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1902 {
1903         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1904 }
1905
1906 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1907 {
1908         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1909 }
1910
1911 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1912 {
1913         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1914 }
1915
1916 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1917 {
1918         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1919 }
1920
1921 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
1922 {
1923         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
1924 }
1925
1926 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
1927 {
1928         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
1929 }
1930
1931 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
1932 {
1933         return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
1934 }
1935 #endif