config ARM_MODULE_PLTS
bool "Use PLTs to allow module memory to spill over into vmalloc area"
depends on MODULES
+ default y
help
Allocate PLTs when loading modules so that jumps and calls whose
targets are too far away for their relative offsets to be encoded
rounding up to page size, the actual memory footprint is usually
the same.
- Say y if you are getting out of memory errors while loading modules
+ Disabling this is usually safe for small single-platform
+ configurations. If unsure, say y.
source "mm/Kconfig"
tune-y := $(tune-y)
ifeq ($(CONFIG_AEABI),y)
-CFLAGS_ABI :=-mabi=aapcs-linux -mno-thumb-interwork -mfpu=vfp
+CFLAGS_ABI :=-mabi=aapcs-linux -mfpu=vfp
else
CFLAGS_ABI :=$(call cc-option,-mapcs-32,-mabi=apcs-gnu) $(call cc-option,-mno-thumb-interwork,)
endif
CFLAGS_fdt_rw.o := $(nossp_flags)
CFLAGS_fdt_wip.o := $(nossp_flags)
-ccflags-y := -fpic -mno-single-pic-base -fno-builtin -I$(obj)
+ccflags-y := -fpic $(call cc-option,-mno-single-pic-base,) -fno-builtin -I$(obj)
asflags-y := -DZIMAGE
# Supply kernel BSS size to the decompressor via a linker symbol.
.size \name , . - \name
.endm
+ .macro csdb
+#ifdef CONFIG_THUMB2_KERNEL
+ .inst.w 0xf3af8014
+#else
+ .inst 0xe320f014
+#endif
+ .endm
+
.macro check_uaccess, addr:req, size:req, limit:req, tmp:req, bad:req
#ifndef CONFIG_CPU_USE_DOMAINS
adds \tmp, \addr, #\size - 1
#define isb(option) __asm__ __volatile__ ("isb " #option : : : "memory")
#define dsb(option) __asm__ __volatile__ ("dsb " #option : : : "memory")
#define dmb(option) __asm__ __volatile__ ("dmb " #option : : : "memory")
+#ifdef CONFIG_THUMB2_KERNEL
+#define CSDB ".inst.w 0xf3af8014"
+#else
+#define CSDB ".inst 0xe320f014"
+#endif
+#define csdb() __asm__ __volatile__(CSDB : : : "memory")
#elif defined(CONFIG_CPU_XSC3) || __LINUX_ARM_ARCH__ == 6
#define isb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \
: : "r" (0) : "memory")
#define dmb(x) __asm__ __volatile__ ("" : : : "memory")
#endif
+#ifndef CSDB
+#define CSDB
+#endif
+#ifndef csdb
+#define csdb()
+#endif
+
#ifdef CONFIG_ARM_HEAVY_MB
extern void (*soc_mb)(void);
extern void arm_heavy_mb(void);
#define __smp_rmb() __smp_mb()
#define __smp_wmb() dmb(ishst)
+#ifdef CONFIG_CPU_SPECTRE
+static inline unsigned long array_index_mask_nospec(unsigned long idx,
+ unsigned long sz)
+{
+ unsigned long mask;
+
+ asm volatile(
+ "cmp %1, %2\n"
+ " sbc %0, %1, %1\n"
+ CSDB
+ : "=r" (mask)
+ : "r" (idx), "Ir" (sz)
+ : "cc");
+
+ return mask;
+}
+#define array_index_mask_nospec array_index_mask_nospec
+#endif
+
#include <asm-generic/barrier.h>
#endif /* !__ASSEMBLY__ */
#ifndef __ASM_BUGS_H
#define __ASM_BUGS_H
-#ifdef CONFIG_MMU
extern void check_writebuffer_bugs(void);
-#define check_bugs() check_writebuffer_bugs()
+#ifdef CONFIG_MMU
+extern void check_bugs(void);
+extern void check_other_bugs(void);
#else
#define check_bugs() do { } while (0)
+#define check_other_bugs() do { } while (0)
#endif
#endif
#define __write_sysreg(v, r, w, c, t) asm volatile(w " " c : : "r" ((t)(v)))
#define write_sysreg(v, ...) __write_sysreg(v, __VA_ARGS__)
+#define BPIALL __ACCESS_CP15(c7, 0, c5, 6)
+#define ICIALLU __ACCESS_CP15(c7, 0, c5, 0)
+
extern unsigned long cr_alignment; /* defined in entry-armv.S */
static inline unsigned long get_cr(void)
#define ARM_CPU_PART_CORTEX_A12 0x4100c0d0
#define ARM_CPU_PART_CORTEX_A17 0x4100c0e0
#define ARM_CPU_PART_CORTEX_A15 0x4100c0f0
+#define ARM_CPU_PART_CORTEX_A53 0x4100d030
+#define ARM_CPU_PART_CORTEX_A57 0x4100d070
+#define ARM_CPU_PART_CORTEX_A72 0x4100d080
+#define ARM_CPU_PART_CORTEX_A73 0x4100d090
+#define ARM_CPU_PART_CORTEX_A75 0x4100d0a0
#define ARM_CPU_PART_MASK 0xff00fff0
+/* Broadcom cores */
+#define ARM_CPU_PART_BRAHMA_B15 0x420000f0
+
/* DEC implemented cores */
#define ARM_CPU_PART_SA1100 0x4400a110
#define KGDB_MAX_NO_CPUS 1
#define BUFMAX 400
-#define NUMREGBYTES (DBG_MAX_REG_NUM << 2)
+#define NUMREGBYTES (GDB_MAX_REGS << 2)
#define NUMCRITREGBYTES (32 << 2)
#define _R0 0
extern char __kvm_hyp_init[];
extern char __kvm_hyp_init_end[];
-extern char __kvm_hyp_vector[];
-
extern void __kvm_flush_vm_context(void);
extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
#include <linux/types.h>
#include <linux/kvm_types.h>
+#include <asm/cputype.h>
#include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_mmio.h>
static inline bool kvm_arm_harden_branch_predictor(void)
{
- /* No way to detect it yet, pretend it is not there. */
- return false;
+ switch(read_cpuid_part()) {
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+ case ARM_CPU_PART_BRAHMA_B15:
+ case ARM_CPU_PART_CORTEX_A12:
+ case ARM_CPU_PART_CORTEX_A15:
+ case ARM_CPU_PART_CORTEX_A17:
+ return true;
+#endif
+ default:
+ return false;
+ }
}
static inline void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu) {}
static inline void *kvm_get_hyp_vector(void)
{
- return kvm_ksym_ref(__kvm_hyp_vector);
+ switch(read_cpuid_part()) {
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+ case ARM_CPU_PART_CORTEX_A12:
+ case ARM_CPU_PART_CORTEX_A17:
+ {
+ extern char __kvm_hyp_vector_bp_inv[];
+ return kvm_ksym_ref(__kvm_hyp_vector_bp_inv);
+ }
+
+ case ARM_CPU_PART_BRAHMA_B15:
+ case ARM_CPU_PART_CORTEX_A15:
+ {
+ extern char __kvm_hyp_vector_ic_inv[];
+ return kvm_ksym_ref(__kvm_hyp_vector_ic_inv);
+ }
+#endif
+ default:
+ {
+ extern char __kvm_hyp_vector[];
+ return kvm_ksym_ref(__kvm_hyp_vector);
+ }
+ }
}
static inline int kvm_map_vectors(void)
/* ID_MMFR0 data relevant to MPU */
#define MMFR0_PMSA (0xF << 4)
#define MMFR0_PMSAv7 (3 << 4)
+#define MMFR0_PMSAv8 (4 << 4)
/* MPU D/I Size Register fields */
-#define MPU_RSR_SZ 1
-#define MPU_RSR_EN 0
-#define MPU_RSR_SD 8
+#define PMSAv7_RSR_SZ 1
+#define PMSAv7_RSR_EN 0
+#define PMSAv7_RSR_SD 8
/* Number of subregions (SD) */
-#define MPU_NR_SUBREGS 8
-#define MPU_MIN_SUBREG_SIZE 256
+#define PMSAv7_NR_SUBREGS 8
+#define PMSAv7_MIN_SUBREG_SIZE 256
/* The D/I RSR value for an enabled region spanning the whole of memory */
-#define MPU_RSR_ALL_MEM 63
+#define PMSAv7_RSR_ALL_MEM 63
/* Individual bits in the DR/IR ACR */
-#define MPU_ACR_XN (1 << 12)
-#define MPU_ACR_SHARED (1 << 2)
+#define PMSAv7_ACR_XN (1 << 12)
+#define PMSAv7_ACR_SHARED (1 << 2)
/* C, B and TEX[2:0] bits only have semantic meanings when grouped */
-#define MPU_RGN_CACHEABLE 0xB
-#define MPU_RGN_SHARED_CACHEABLE (MPU_RGN_CACHEABLE | MPU_ACR_SHARED)
-#define MPU_RGN_STRONGLY_ORDERED 0
+#define PMSAv7_RGN_CACHEABLE 0xB
+#define PMSAv7_RGN_SHARED_CACHEABLE (PMSAv7_RGN_CACHEABLE | PMSAv7_ACR_SHARED)
+#define PMSAv7_RGN_STRONGLY_ORDERED 0
/* Main region should only be shared for SMP */
#ifdef CONFIG_SMP
-#define MPU_RGN_NORMAL (MPU_RGN_CACHEABLE | MPU_ACR_SHARED)
+#define PMSAv7_RGN_NORMAL (PMSAv7_RGN_CACHEABLE | PMSAv7_ACR_SHARED)
#else
-#define MPU_RGN_NORMAL MPU_RGN_CACHEABLE
+#define PMSAv7_RGN_NORMAL PMSAv7_RGN_CACHEABLE
#endif
/* Access permission bits of ACR (only define those that we use)*/
-#define MPU_AP_PL1RO_PL0NA (0x5 << 8)
-#define MPU_AP_PL1RW_PL0RW (0x3 << 8)
-#define MPU_AP_PL1RW_PL0R0 (0x2 << 8)
-#define MPU_AP_PL1RW_PL0NA (0x1 << 8)
+#define PMSAv7_AP_PL1RO_PL0NA (0x5 << 8)
+#define PMSAv7_AP_PL1RW_PL0RW (0x3 << 8)
+#define PMSAv7_AP_PL1RW_PL0R0 (0x2 << 8)
+#define PMSAv7_AP_PL1RW_PL0NA (0x1 << 8)
+
+#define PMSAv8_BAR_XN 1
+
+#define PMSAv8_LAR_EN 1
+#define PMSAv8_LAR_IDX(n) (((n) & 0x7) << 1)
+
+
+#define PMSAv8_AP_PL1RW_PL0NA (0 << 1)
+#define PMSAv8_AP_PL1RW_PL0RW (1 << 1)
+#define PMSAv8_AP_PL1RO_PL0RO (3 << 1)
+
+#ifdef CONFIG_SMP
+#define PMSAv8_RGN_SHARED (3 << 3) // inner sharable
+#else
+#define PMSAv8_RGN_SHARED (0 << 3)
+#endif
+
+#define PMSAv8_RGN_DEVICE_nGnRnE 0
+#define PMSAv8_RGN_NORMAL 1
+
+#define PMSAv8_MAIR(attr, mt) ((attr) << ((mt) * 8))
+
+#ifdef CONFIG_CPU_V7M
+#define PMSAv8_MINALIGN 32
+#else
+#define PMSAv8_MINALIGN 64
+#endif
/* For minimal static MPU region configurations */
-#define MPU_PROBE_REGION 0
-#define MPU_BG_REGION 1
-#define MPU_RAM_REGION 2
-#define MPU_ROM_REGION 3
+#define PMSAv7_PROBE_REGION 0
+#define PMSAv7_BG_REGION 1
+#define PMSAv7_RAM_REGION 2
+#define PMSAv7_ROM_REGION 3
+
+/* Fixed for PMSAv8 only */
+#define PMSAv8_XIP_REGION 0
+#define PMSAv8_KERNEL_REGION 1
/* Maximum number of regions Linux is interested in */
-#define MPU_MAX_REGIONS 16
+#define MPU_MAX_REGIONS 16
-#define MPU_DATA_SIDE 0
-#define MPU_INSTR_SIDE 1
+#define PMSAv7_DATA_SIDE 0
+#define PMSAv7_INSTR_SIDE 1
#ifndef __ASSEMBLY__
struct mpu_rgn {
/* Assume same attributes for d/i-side */
- u32 drbar;
- u32 drsr;
- u32 dracr;
+ union {
+ u32 drbar; /* PMSAv7 */
+ u32 prbar; /* PMSAv8 */
+ };
+ union {
+ u32 drsr; /* PMSAv7 */
+ u32 prlar; /* PMSAv8 */
+ };
+ union {
+ u32 dracr; /* PMSAv7 */
+ u32 unused; /* not used in PMSAv8 */
+ };
};
struct mpu_rgn_info {
extern struct mpu_rgn_info mpu_rgn_info;
#ifdef CONFIG_ARM_MPU
+extern void __init pmsav7_adjust_lowmem_bounds(void);
+extern void __init pmsav8_adjust_lowmem_bounds(void);
-extern void __init adjust_lowmem_bounds_mpu(void);
-extern void __init mpu_setup(void);
-
+extern void __init pmsav7_setup(void);
+extern void __init pmsav8_setup(void);
#else
-
-static inline void adjust_lowmem_bounds_mpu(void) {}
-static inline void mpu_setup(void) {}
-
-#endif /* !CONFIG_ARM_MPU */
+static inline void pmsav7_adjust_lowmem_bounds(void) {};
+static inline void pmsav8_adjust_lowmem_bounds(void) {};
+static inline void pmsav7_setup(void) {};
+static inline void pmsav8_setup(void) {};
+#endif
#endif /* __ASSEMBLY__ */
* Set up any processor specifics
*/
void (*_proc_init)(void);
+ /*
+ * Check for processor bugs
+ */
+ void (*check_bugs)(void);
/*
* Disable any processor specifics
*/
#include <linux/linkage.h>
#include <linux/irqflags.h>
#include <linux/reboot.h>
+#include <linux/percpu.h>
extern void cpu_init(void);
extern void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd);
extern void (*arm_pm_idle)(void);
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+typedef void (*harden_branch_predictor_fn_t)(void);
+DECLARE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
+static inline void harden_branch_predictor(void)
+{
+ harden_branch_predictor_fn_t fn = per_cpu(harden_branch_predictor_fn,
+ smp_processor_id());
+ if (fn)
+ fn();
+}
+#else
+#define harden_branch_predictor() do { } while (0)
+#endif
+
#define UDBG_UNDEFINED (1 << 0)
#define UDBG_SYSCALL (1 << 1)
#define UDBG_BADABORT (1 << 2)
#define __get_user_check(x, p) \
({ \
unsigned long __limit = current_thread_info()->addr_limit - 1; \
- register const typeof(*(p)) __user *__p asm("r0") = (p);\
+ register typeof(*(p)) __user *__p asm("r0") = (p); \
register typeof(x) __r2 asm("r2"); \
register unsigned long __l asm("r1") = __limit; \
register int __e asm("r0"); \
#define MPU_CTRL_ENABLE 1
#define MPU_CTRL_PRIVDEFENA (1 << 2)
-#define MPU_RNR 0x98
-#define MPU_RBAR 0x9c
-#define MPU_RASR 0xa0
+#define PMSAv7_RNR 0x98
+#define PMSAv7_RBAR 0x9c
+#define PMSAv7_RASR 0xa0
+
+#define PMSAv8_RNR 0x98
+#define PMSAv8_RBAR 0x9c
+#define PMSAv8_RLAR 0xa0
+#define PMSAv8_RBAR_A(n) (PMSAv8_RBAR + 8*(n))
+#define PMSAv8_RLAR_A(n) (PMSAv8_RLAR + 8*(n))
+#define PMSAv8_MAIR0 0xc0
+#define PMSAv8_MAIR1 0xc4
/* Cache opeartions */
#define V7M_SCB_ICIALLU 0x250 /* I-cache invalidate all to PoU */
obj-y += entry-armv.o
endif
+obj-$(CONFIG_MMU) += bugs.o
obj-$(CONFIG_CPU_IDLE) += cpuidle.o
obj-$(CONFIG_ISA_DMA_API) += dma.o
obj-$(CONFIG_FIQ) += fiq.o fiqasm.o
DEFINE(MPU_RNG_INFO_USED, offsetof(struct mpu_rgn_info, used));
DEFINE(MPU_RNG_SIZE, sizeof(struct mpu_rgn));
- DEFINE(MPU_RGN_DRBAR, offsetof(struct mpu_rgn, drbar));
- DEFINE(MPU_RGN_DRSR, offsetof(struct mpu_rgn, drsr));
- DEFINE(MPU_RGN_DRACR, offsetof(struct mpu_rgn, dracr));
+ DEFINE(MPU_RGN_DRBAR, offsetof(struct mpu_rgn, drbar));
+ DEFINE(MPU_RGN_DRSR, offsetof(struct mpu_rgn, drsr));
+ DEFINE(MPU_RGN_DRACR, offsetof(struct mpu_rgn, dracr));
+ DEFINE(MPU_RGN_PRBAR, offsetof(struct mpu_rgn, prbar));
+ DEFINE(MPU_RGN_PRLAR, offsetof(struct mpu_rgn, prlar));
#endif
return 0;
}
--- /dev/null
+// SPDX-Identifier: GPL-2.0
+#include <linux/init.h>
+#include <asm/bugs.h>
+#include <asm/proc-fns.h>
+
+void check_other_bugs(void)
+{
+#ifdef MULTI_CPU
+ if (processor.check_bugs)
+ processor.check_bugs();
+#endif
+}
+
+void __init check_bugs(void)
+{
+ check_writebuffer_bugs();
+ check_other_bugs();
+}
tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls?
bne __sys_trace
- cmp scno, #NR_syscalls @ check upper syscall limit
- badr lr, ret_fast_syscall @ return address
- ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine
+ invoke_syscall tbl, scno, r10, ret_fast_syscall
add r1, sp, #S_OFF
2: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
mov r1, scno
add r0, sp, #S_OFF
bl syscall_trace_enter
-
- badr lr, __sys_trace_return @ return address
- mov scno, r0 @ syscall number (possibly new)
- add r1, sp, #S_R0 + S_OFF @ pointer to regs
- cmp scno, #NR_syscalls @ check upper syscall limit
- ldmccia r1, {r0 - r6} @ have to reload r0 - r6
- stmccia sp, {r4, r5} @ and update the stack args
- ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine
+ mov scno, r0
+ invoke_syscall tbl, scno, r10, __sys_trace_return, reload=1
cmp scno, #-1 @ skip the syscall?
bne 2b
add sp, sp, #S_OFF @ restore stack
bic scno, r0, #__NR_OABI_SYSCALL_BASE
cmp scno, #__NR_syscall - __NR_SYSCALL_BASE
cmpne scno, #NR_syscalls @ check range
+#ifdef CONFIG_CPU_SPECTRE
+ movhs scno, #0
+ csdb
+#endif
stmloia sp, {r5, r6} @ shuffle args
movlo r0, r1
movlo r1, r2
#endif
.endm
+ .macro invoke_syscall, table, nr, tmp, ret, reload=0
+#ifdef CONFIG_CPU_SPECTRE
+ mov \tmp, \nr
+ cmp \tmp, #NR_syscalls @ check upper syscall limit
+ movcs \tmp, #0
+ csdb
+ badr lr, \ret @ return address
+ .if \reload
+ add r1, sp, #S_R0 + S_OFF @ pointer to regs
+ ldmccia r1, {r0 - r6} @ reload r0-r6
+ stmccia sp, {r4, r5} @ update stack arguments
+ .endif
+ ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine
+#else
+ cmp \nr, #NR_syscalls @ check upper syscall limit
+ badr lr, \ret @ return address
+ .if \reload
+ add r1, sp, #S_R0 + S_OFF @ pointer to regs
+ ldmccia r1, {r0 - r6} @ reload r0-r6
+ stmccia sp, {r4, r5} @ update stack arguments
+ .endif
+ ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine
+#endif
+ .endm
+
/*
* These are the registers used in the syscall handler, and allow us to
* have in theory up to 7 arguments to a function - r0 to r6.
beq __error_p @ yes, error 'p'
#ifdef CONFIG_ARM_MPU
- /* Calculate the size of a region covering just the kernel */
- ldr r5, =PLAT_PHYS_OFFSET @ Region start: PHYS_OFFSET
- ldr r6, =(_end) @ Cover whole kernel
- sub r6, r6, r5 @ Minimum size of region to map
- clz r6, r6 @ Region size must be 2^N...
- rsb r6, r6, #31 @ ...so round up region size
- lsl r6, r6, #MPU_RSR_SZ @ Put size in right field
- orr r6, r6, #(1 << MPU_RSR_EN) @ Set region enabled bit
bl __setup_mpu
#endif
ldr r12, [r10, #PROCINFO_INITFUNC]
add r12, r12, r10
ret r12
-1: bl __after_proc_init
- b __mmap_switched
+1: ldr lr, =__mmap_switched
+ b __after_proc_init
ENDPROC(stext)
#ifdef CONFIG_SMP
ldr r7, __secondary_data
#ifdef CONFIG_ARM_MPU
- /* Use MPU region info supplied by __cpu_up */
- ldr r6, [r7] @ get secondary_data.mpu_rgn_info
bl __secondary_setup_mpu @ Initialize the MPU
#endif
/*
* Set the Control Register and Read the process ID.
*/
+ .text
__after_proc_init:
+#ifdef CONFIG_ARM_MPU
+M_CLASS(movw r12, #:lower16:BASEADDR_V7M_SCB)
+M_CLASS(movt r12, #:upper16:BASEADDR_V7M_SCB)
+M_CLASS(ldr r3, [r12, 0x50])
+AR_CLASS(mrc p15, 0, r3, c0, c1, 4) @ Read ID_MMFR0
+ and r3, r3, #(MMFR0_PMSA) @ PMSA field
+ teq r3, #(MMFR0_PMSAv7) @ PMSA v7
+ beq 1f
+ teq r3, #(MMFR0_PMSAv8) @ PMSA v8
+ /*
+ * Memory region attributes for PMSAv8:
+ *
+ * n = AttrIndx[2:0]
+ * n MAIR
+ * DEVICE_nGnRnE 000 00000000
+ * NORMAL 001 11111111
+ */
+ ldreq r3, =PMSAv8_MAIR(0x00, PMSAv8_RGN_DEVICE_nGnRnE) | \
+ PMSAv8_MAIR(0xff, PMSAv8_RGN_NORMAL)
+AR_CLASS(mcreq p15, 0, r3, c10, c2, 0) @ MAIR 0
+M_CLASS(streq r3, [r12, #PMSAv8_MAIR0])
+ moveq r3, #0
+AR_CLASS(mcreq p15, 0, r3, c10, c2, 1) @ MAIR 1
+M_CLASS(streq r3, [r12, #PMSAv8_MAIR1])
+
+1:
+#endif
#ifdef CONFIG_CPU_CP15
/*
* CP15 system control register value returned in r0 from
* the CPU init function.
*/
+
+#ifdef CONFIG_ARM_MPU
+ biceq r0, r0, #CR_BR @ Disable the 'default mem-map'
+ orreq r0, r0, #CR_M @ Set SCTRL.M (MPU on)
+#endif
#if defined(CONFIG_ALIGNMENT_TRAP) && __LINUX_ARM_ARCH__ < 6
orr r0, r0, #CR_A
#else
bic r0, r0, #CR_I
#endif
mcr p15, 0, r0, c1, c0, 0 @ write control reg
+ isb
#elif defined (CONFIG_CPU_V7M)
+#ifdef CONFIG_ARM_MPU
+ ldreq r3, [r12, MPU_CTRL]
+ biceq r3, #MPU_CTRL_PRIVDEFENA
+ orreq r3, #MPU_CTRL_ENABLE
+ streq r3, [r12, MPU_CTRL]
+ isb
+#endif
/* For V7M systems we want to modify the CCR similarly to the SCTLR */
#ifdef CONFIG_CPU_DCACHE_DISABLE
bic r0, r0, #V7M_SCB_CCR_DC
#ifdef CONFIG_CPU_ICACHE_DISABLE
bic r0, r0, #V7M_SCB_CCR_IC
#endif
- movw r3, #:lower16:(BASEADDR_V7M_SCB + V7M_SCB_CCR)
- movt r3, #:upper16:(BASEADDR_V7M_SCB + V7M_SCB_CCR)
- str r0, [r3]
+ str r0, [r12, V7M_SCB_CCR]
#endif /* CONFIG_CPU_CP15 elif CONFIG_CPU_V7M */
ret lr
ENDPROC(__after_proc_init)
.endm
/* Setup a single MPU region, either D or I side (D-side for unified) */
-.macro setup_region bar, acr, sr, side = MPU_DATA_SIDE, unused
+.macro setup_region bar, acr, sr, side = PMSAv7_DATA_SIDE, unused
mcr p15, 0, \bar, c6, c1, (0 + \side) @ I/DRBAR
mcr p15, 0, \acr, c6, c1, (4 + \side) @ I/DRACR
mcr p15, 0, \sr, c6, c1, (2 + \side) @ I/DRSR
#else
.macro set_region_nr tmp, rgnr, base
mov \tmp, \rgnr
- str \tmp, [\base, #MPU_RNR]
+ str \tmp, [\base, #PMSAv7_RNR]
.endm
.macro setup_region bar, acr, sr, unused, base
lsl \acr, \acr, #16
orr \acr, \acr, \sr
- str \bar, [\base, #MPU_RBAR]
- str \acr, [\base, #MPU_RASR]
+ str \bar, [\base, #PMSAv7_RBAR]
+ str \acr, [\base, #PMSAv7_RASR]
.endm
#endif
* Region 2: Normal, Shared, cacheable for RAM. From PHYS_OFFSET, size from r6
* Region 3: Normal, shared, inaccessible from PL0 to protect the vectors page
*
- * r6: Value to be written to DRSR (and IRSR if required) for MPU_RAM_REGION
+ * r6: Value to be written to DRSR (and IRSR if required) for PMSAv7_RAM_REGION
*/
+ __HEAD
ENTRY(__setup_mpu)
M_CLASS(ldr r0, [r12, 0x50])
and r0, r0, #(MMFR0_PMSA) @ PMSA field
teq r0, #(MMFR0_PMSAv7) @ PMSA v7
- bxne lr
+ beq __setup_pmsa_v7
+ teq r0, #(MMFR0_PMSAv8) @ PMSA v8
+ beq __setup_pmsa_v8
+
+ ret lr
+ENDPROC(__setup_mpu)
+
+ENTRY(__setup_pmsa_v7)
+ /* Calculate the size of a region covering just the kernel */
+ ldr r5, =PLAT_PHYS_OFFSET @ Region start: PHYS_OFFSET
+ ldr r6, =(_end) @ Cover whole kernel
+ sub r6, r6, r5 @ Minimum size of region to map
+ clz r6, r6 @ Region size must be 2^N...
+ rsb r6, r6, #31 @ ...so round up region size
+ lsl r6, r6, #PMSAv7_RSR_SZ @ Put size in right field
+ orr r6, r6, #(1 << PMSAv7_RSR_EN) @ Set region enabled bit
/* Determine whether the D/I-side memory map is unified. We set the
* flags here and continue to use them for the rest of this function */
tst r0, #MPUIR_nU @ MPUIR_nU = 0 for unified
/* Setup second region first to free up r6 */
- set_region_nr r0, #MPU_RAM_REGION, r12
+ set_region_nr r0, #PMSAv7_RAM_REGION, r12
isb
/* Full access from PL0, PL1, shared for CONFIG_SMP, cacheable */
ldr r0, =PLAT_PHYS_OFFSET @ RAM starts at PHYS_OFFSET
- ldr r5,=(MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL)
+ ldr r5,=(PMSAv7_AP_PL1RW_PL0RW | PMSAv7_RGN_NORMAL)
- setup_region r0, r5, r6, MPU_DATA_SIDE, r12 @ PHYS_OFFSET, shared, enabled
+ setup_region r0, r5, r6, PMSAv7_DATA_SIDE, r12 @ PHYS_OFFSET, shared, enabled
beq 1f @ Memory-map not unified
- setup_region r0, r5, r6, MPU_INSTR_SIDE, r12 @ PHYS_OFFSET, shared, enabled
+ setup_region r0, r5, r6, PMSAv7_INSTR_SIDE, r12 @ PHYS_OFFSET, shared, enabled
1: isb
/* First/background region */
- set_region_nr r0, #MPU_BG_REGION, r12
+ set_region_nr r0, #PMSAv7_BG_REGION, r12
isb
/* Execute Never, strongly ordered, inaccessible to PL0, rw PL1 */
mov r0, #0 @ BG region starts at 0x0
- ldr r5,=(MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0NA)
- mov r6, #MPU_RSR_ALL_MEM @ 4GB region, enabled
+ ldr r5,=(PMSAv7_ACR_XN | PMSAv7_RGN_STRONGLY_ORDERED | PMSAv7_AP_PL1RW_PL0NA)
+ mov r6, #PMSAv7_RSR_ALL_MEM @ 4GB region, enabled
- setup_region r0, r5, r6, MPU_DATA_SIDE, r12 @ 0x0, BG region, enabled
+ setup_region r0, r5, r6, PMSAv7_DATA_SIDE, r12 @ 0x0, BG region, enabled
beq 2f @ Memory-map not unified
- setup_region r0, r5, r6, MPU_INSTR_SIDE r12 @ 0x0, BG region, enabled
+ setup_region r0, r5, r6, PMSAv7_INSTR_SIDE r12 @ 0x0, BG region, enabled
2: isb
#ifdef CONFIG_XIP_KERNEL
- set_region_nr r0, #MPU_ROM_REGION, r12
+ set_region_nr r0, #PMSAv7_ROM_REGION, r12
isb
- ldr r5,=(MPU_AP_PL1RO_PL0NA | MPU_RGN_NORMAL)
+ ldr r5,=(PMSAv7_AP_PL1RO_PL0NA | PMSAv7_RGN_NORMAL)
ldr r0, =CONFIG_XIP_PHYS_ADDR @ ROM start
ldr r6, =(_exiprom) @ ROM end
sub r6, r6, r0 @ Minimum size of region to map
clz r6, r6 @ Region size must be 2^N...
rsb r6, r6, #31 @ ...so round up region size
- lsl r6, r6, #MPU_RSR_SZ @ Put size in right field
- orr r6, r6, #(1 << MPU_RSR_EN) @ Set region enabled bit
+ lsl r6, r6, #PMSAv7_RSR_SZ @ Put size in right field
+ orr r6, r6, #(1 << PMSAv7_RSR_EN) @ Set region enabled bit
- setup_region r0, r5, r6, MPU_DATA_SIDE, r12 @ XIP_PHYS_ADDR, shared, enabled
+ setup_region r0, r5, r6, PMSAv7_DATA_SIDE, r12 @ XIP_PHYS_ADDR, shared, enabled
beq 3f @ Memory-map not unified
- setup_region r0, r5, r6, MPU_INSTR_SIDE, r12 @ XIP_PHYS_ADDR, shared, enabled
+ setup_region r0, r5, r6, PMSAv7_INSTR_SIDE, r12 @ XIP_PHYS_ADDR, shared, enabled
3: isb
#endif
+ ret lr
+ENDPROC(__setup_pmsa_v7)
+
+ENTRY(__setup_pmsa_v8)
+ mov r0, #0
+AR_CLASS(mcr p15, 0, r0, c6, c2, 1) @ PRSEL
+M_CLASS(str r0, [r12, #PMSAv8_RNR])
+ isb
+
+#ifdef CONFIG_XIP_KERNEL
+ ldr r5, =CONFIG_XIP_PHYS_ADDR @ ROM start
+ ldr r6, =(_exiprom) @ ROM end
+ sub r6, r6, #1
+ bic r6, r6, #(PMSAv8_MINALIGN - 1)
+
+ orr r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED)
+ orr r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN)
+
+AR_CLASS(mcr p15, 0, r5, c6, c8, 0) @ PRBAR0
+AR_CLASS(mcr p15, 0, r6, c6, c8, 1) @ PRLAR0
+M_CLASS(str r5, [r12, #PMSAv8_RBAR_A(0)])
+M_CLASS(str r6, [r12, #PMSAv8_RLAR_A(0)])
+#endif
+
+ ldr r5, =KERNEL_START
+ ldr r6, =KERNEL_END
+ sub r6, r6, #1
+ bic r6, r6, #(PMSAv8_MINALIGN - 1)
+
+ orr r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED)
+ orr r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN)
+
+AR_CLASS(mcr p15, 0, r5, c6, c8, 4) @ PRBAR1
+AR_CLASS(mcr p15, 0, r6, c6, c8, 5) @ PRLAR1
+M_CLASS(str r5, [r12, #PMSAv8_RBAR_A(1)])
+M_CLASS(str r6, [r12, #PMSAv8_RLAR_A(1)])
+
+ /* Setup Background: 0x0 - min(KERNEL_START, XIP_PHYS_ADDR) */
+#ifdef CONFIG_XIP_KERNEL
+ ldr r6, =KERNEL_START
+ ldr r5, =CONFIG_XIP_PHYS_ADDR
+ cmp r6, r5
+ movcs r6, r5
+#else
+ ldr r6, =KERNEL_START
+#endif
+ cmp r6, #0
+ beq 1f
+
+ mov r5, #0
+ sub r6, r6, #1
+ bic r6, r6, #(PMSAv8_MINALIGN - 1)
+
+ orr r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN)
+ orr r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN)
+
+AR_CLASS(mcr p15, 0, r5, c6, c9, 0) @ PRBAR2
+AR_CLASS(mcr p15, 0, r6, c6, c9, 1) @ PRLAR2
+M_CLASS(str r5, [r12, #PMSAv8_RBAR_A(2)])
+M_CLASS(str r6, [r12, #PMSAv8_RLAR_A(2)])
+
+1:
+ /* Setup Background: max(KERNEL_END, _exiprom) - 0xffffffff */
+#ifdef CONFIG_XIP_KERNEL
+ ldr r5, =KERNEL_END
+ ldr r6, =(_exiprom)
+ cmp r5, r6
+ movcc r5, r6
+#else
+ ldr r5, =KERNEL_END
+#endif
+ mov r6, #0xffffffff
+ bic r6, r6, #(PMSAv8_MINALIGN - 1)
+
+ orr r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN)
+ orr r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN)
- /* Enable the MPU */
-AR_CLASS(mrc p15, 0, r0, c1, c0, 0) @ Read SCTLR
-AR_CLASS(bic r0, r0, #CR_BR) @ Disable the 'default mem-map'
-AR_CLASS(orr r0, r0, #CR_M) @ Set SCTRL.M (MPU on)
-AR_CLASS(mcr p15, 0, r0, c1, c0, 0) @ Enable MPU
+AR_CLASS(mcr p15, 0, r5, c6, c9, 4) @ PRBAR3
+AR_CLASS(mcr p15, 0, r6, c6, c9, 5) @ PRLAR3
+M_CLASS(str r5, [r12, #PMSAv8_RBAR_A(3)])
+M_CLASS(str r6, [r12, #PMSAv8_RLAR_A(3)])
-M_CLASS(ldr r0, [r12, #MPU_CTRL])
-M_CLASS(bic r0, #MPU_CTRL_PRIVDEFENA)
-M_CLASS(orr r0, #MPU_CTRL_ENABLE)
-M_CLASS(str r0, [r12, #MPU_CTRL])
+#ifdef CONFIG_XIP_KERNEL
+ /* Setup Background: min(_exiprom, KERNEL_END) - max(KERNEL_START, XIP_PHYS_ADDR) */
+ ldr r5, =(_exiprom)
+ ldr r6, =KERNEL_END
+ cmp r5, r6
+ movcs r5, r6
+
+ ldr r6, =KERNEL_START
+ ldr r0, =CONFIG_XIP_PHYS_ADDR
+ cmp r6, r0
+ movcc r6, r0
+
+ sub r6, r6, #1
+ bic r6, r6, #(PMSAv8_MINALIGN - 1)
+
+ orr r5, r5, #(PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN)
+ orr r6, r6, #(PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN)
+
+#ifdef CONFIG_CPU_V7M
+ /* There is no alias for n == 4 */
+ mov r0, #4
+ str r0, [r12, #PMSAv8_RNR] @ PRSEL
isb
+ str r5, [r12, #PMSAv8_RBAR_A(0)]
+ str r6, [r12, #PMSAv8_RLAR_A(0)]
+#else
+ mcr p15, 0, r5, c6, c10, 1 @ PRBAR4
+ mcr p15, 0, r6, c6, c10, 2 @ PRLAR4
+#endif
+#endif
ret lr
-ENDPROC(__setup_mpu)
+ENDPROC(__setup_pmsa_v8)
#ifdef CONFIG_SMP
/*
* r6: pointer at mpu_rgn_info
*/
+ .text
ENTRY(__secondary_setup_mpu)
+ /* Use MPU region info supplied by __cpu_up */
+ ldr r6, [r7] @ get secondary_data.mpu_rgn_info
+
/* Probe for v7 PMSA compliance */
mrc p15, 0, r0, c0, c1, 4 @ Read ID_MMFR0
and r0, r0, #(MMFR0_PMSA) @ PMSA field
teq r0, #(MMFR0_PMSAv7) @ PMSA v7
- bne __error_p
+ beq __secondary_setup_pmsa_v7
+ teq r0, #(MMFR0_PMSAv8) @ PMSA v8
+ beq __secondary_setup_pmsa_v8
+ b __error_p
+ENDPROC(__secondary_setup_mpu)
+/*
+ * r6: pointer at mpu_rgn_info
+ */
+ENTRY(__secondary_setup_pmsa_v7)
/* Determine whether the D/I-side memory map is unified. We set the
* flags here and continue to use them for the rest of this function */
mrc p15, 0, r0, c0, c0, 4 @ MPUIR
ldr r6, [r3, #MPU_RGN_DRSR]
ldr r5, [r3, #MPU_RGN_DRACR]
- setup_region r0, r5, r6, MPU_DATA_SIDE
+ setup_region r0, r5, r6, PMSAv7_DATA_SIDE
beq 2f
- setup_region r0, r5, r6, MPU_INSTR_SIDE
+ setup_region r0, r5, r6, PMSAv7_INSTR_SIDE
2: isb
mrc p15, 0, r0, c0, c0, 4 @ Reevaluate the MPUIR
cmp r4, #0
bgt 1b
- /* Enable the MPU */
- mrc p15, 0, r0, c1, c0, 0 @ Read SCTLR
- bic r0, r0, #CR_BR @ Disable the 'default mem-map'
- orr r0, r0, #CR_M @ Set SCTRL.M (MPU on)
- mcr p15, 0, r0, c1, c0, 0 @ Enable MPU
+ ret lr
+ENDPROC(__secondary_setup_pmsa_v7)
+
+ENTRY(__secondary_setup_pmsa_v8)
+ ldr r4, [r6, #MPU_RNG_INFO_USED]
+#ifndef CONFIG_XIP_KERNEL
+ add r4, r4, #1
+#endif
+ mov r5, #MPU_RNG_SIZE
+ add r3, r6, #MPU_RNG_INFO_RNGS
+ mla r3, r4, r5, r3
+
+1:
+ sub r3, r3, #MPU_RNG_SIZE
+ sub r4, r4, #1
+
+ mcr p15, 0, r4, c6, c2, 1 @ PRSEL
isb
- ret lr
-ENDPROC(__secondary_setup_mpu)
+ ldr r5, [r3, #MPU_RGN_PRBAR]
+ ldr r6, [r3, #MPU_RGN_PRLAR]
+ mcr p15, 0, r5, c6, c3, 0 @ PRBAR
+ mcr p15, 0, r6, c6, c3, 1 @ PRLAR
+
+ cmp r4, #0
+ bgt 1b
+
+ ret lr
+ENDPROC(__secondary_setup_pmsa_v8)
#endif /* CONFIG_SMP */
#endif /* CONFIG_ARM_MPU */
#include "head-common.S"
#include <linux/irq_work.h>
#include <linux/atomic.h>
+#include <asm/bugs.h>
#include <asm/smp.h>
#include <asm/cacheflush.h>
#include <asm/cpu.h>
flush_cache_louis();
local_flush_tlb_all();
- clear_tasks_mm_cpumask(cpu);
-
return 0;
}
}
pr_debug("CPU%u: shutdown\n", cpu);
+ clear_tasks_mm_cpumask(cpu);
/*
* platform_cpu_kill() is generally expected to do the powering off
* and/or cutting of clocks to the dying CPU. Optionally, this may
* before we continue - which happens after __cpu_up returns.
*/
set_cpu_online(cpu, true);
+
+ check_other_bugs();
+
complete(&cpu_running);
local_irq_enable();
#include <linux/slab.h>
#include <linux/mm_types.h>
+#include <asm/bugs.h>
#include <asm/cacheflush.h>
#include <asm/idmap.h>
#include <asm/pgalloc.h>
cpu_switch_mm(mm->pgd, mm);
local_flush_bp_all();
local_flush_tlb_all();
+ check_other_bugs();
}
return ret;
#include <asm/cache.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
+#include <asm/mpu.h>
#include <asm/page.h>
#include "vmlinux.lds.h"
__init_end = .;
BSS_SECTION(0, 0, 8)
+#ifdef CONFIG_ARM_MPU
+ . = ALIGN(PMSAv8_MINALIGN);
+#endif
_end = .;
STABS_DEBUG
#include <asm/cache.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
+#include <asm/mpu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
. = ALIGN(1<<SECTION_SHIFT);
#endif
+#ifdef CONFIG_ARM_MPU
+ . = ALIGN(PMSAv8_MINALIGN);
+#endif
.text : { /* Real text segment */
_stext = .; /* Text and read-only data */
ARM_TEXT
_edata = .;
BSS_SECTION(0, 0, 0)
+#ifdef CONFIG_ARM_MPU
+ . = ALIGN(PMSAv8_MINALIGN);
+#endif
_end = .;
STABS_DEBUG
#define PROC_INFO \
. = ALIGN(4); \
- VMLINUX_SYMBOL(__proc_info_begin) = .; \
+ __proc_info_begin = .; \
*(.proc.info.init) \
- VMLINUX_SYMBOL(__proc_info_end) = .;
+ __proc_info_end = .;
#define HYPERVISOR_TEXT \
- VMLINUX_SYMBOL(__hyp_text_start) = .; \
+ __hyp_text_start = .; \
*(.hyp.text) \
- VMLINUX_SYMBOL(__hyp_text_end) = .;
+ __hyp_text_end = .;
#define IDMAP_TEXT \
ALIGN_FUNCTION(); \
- VMLINUX_SYMBOL(__idmap_text_start) = .; \
+ __idmap_text_start = .; \
*(.idmap.text) \
- VMLINUX_SYMBOL(__idmap_text_end) = .; \
+ __idmap_text_end = .; \
. = ALIGN(PAGE_SIZE); \
- VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \
+ __hyp_idmap_text_start = .; \
*(.hyp.idmap.text) \
- VMLINUX_SYMBOL(__hyp_idmap_text_end) = .;
+ __hyp_idmap_text_end = .;
#define ARM_DISCARD \
*(.ARM.exidx.exit.text) \
* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
+#include <linux/arm-smccc.h>
#include <linux/linkage.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
W(b) hyp_irq
W(b) hyp_fiq
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+ .align 5
+__kvm_hyp_vector_ic_inv:
+ .global __kvm_hyp_vector_ic_inv
+
+ /*
+ * We encode the exception entry in the bottom 3 bits of
+ * SP, and we have to guarantee to be 8 bytes aligned.
+ */
+ W(add) sp, sp, #1 /* Reset 7 */
+ W(add) sp, sp, #1 /* Undef 6 */
+ W(add) sp, sp, #1 /* Syscall 5 */
+ W(add) sp, sp, #1 /* Prefetch abort 4 */
+ W(add) sp, sp, #1 /* Data abort 3 */
+ W(add) sp, sp, #1 /* HVC 2 */
+ W(add) sp, sp, #1 /* IRQ 1 */
+ W(nop) /* FIQ 0 */
+
+ mcr p15, 0, r0, c7, c5, 0 /* ICIALLU */
+ isb
+
+ b decode_vectors
+
+ .align 5
+__kvm_hyp_vector_bp_inv:
+ .global __kvm_hyp_vector_bp_inv
+
+ /*
+ * We encode the exception entry in the bottom 3 bits of
+ * SP, and we have to guarantee to be 8 bytes aligned.
+ */
+ W(add) sp, sp, #1 /* Reset 7 */
+ W(add) sp, sp, #1 /* Undef 6 */
+ W(add) sp, sp, #1 /* Syscall 5 */
+ W(add) sp, sp, #1 /* Prefetch abort 4 */
+ W(add) sp, sp, #1 /* Data abort 3 */
+ W(add) sp, sp, #1 /* HVC 2 */
+ W(add) sp, sp, #1 /* IRQ 1 */
+ W(nop) /* FIQ 0 */
+
+ mcr p15, 0, r0, c7, c5, 6 /* BPIALL */
+ isb
+
+decode_vectors:
+
+#ifdef CONFIG_THUMB2_KERNEL
+ /*
+ * Yet another silly hack: Use VPIDR as a temp register.
+ * Thumb2 is really a pain, as SP cannot be used with most
+ * of the bitwise instructions. The vect_br macro ensures
+ * things gets cleaned-up.
+ */
+ mcr p15, 4, r0, c0, c0, 0 /* VPIDR */
+ mov r0, sp
+ and r0, r0, #7
+ sub sp, sp, r0
+ push {r1, r2}
+ mov r1, r0
+ mrc p15, 4, r0, c0, c0, 0 /* VPIDR */
+ mrc p15, 0, r2, c0, c0, 0 /* MIDR */
+ mcr p15, 4, r2, c0, c0, 0 /* VPIDR */
+#endif
+
+.macro vect_br val, targ
+ARM( eor sp, sp, #\val )
+ARM( tst sp, #7 )
+ARM( eorne sp, sp, #\val )
+
+THUMB( cmp r1, #\val )
+THUMB( popeq {r1, r2} )
+
+ beq \targ
+.endm
+
+ vect_br 0, hyp_fiq
+ vect_br 1, hyp_irq
+ vect_br 2, hyp_hvc
+ vect_br 3, hyp_dabt
+ vect_br 4, hyp_pabt
+ vect_br 5, hyp_svc
+ vect_br 6, hyp_undef
+ vect_br 7, hyp_reset
+#endif
+
.macro invalid_vector label, cause
.align
\label: mov r0, #\cause
lsr r2, r2, #16
and r2, r2, #0xff
cmp r2, #0
- bne guest_trap @ Guest called HVC
+ bne guest_hvc_trap @ Guest called HVC
/*
* Getting here means host called HVC, we shift parameters and branch
bx ip
1:
- push {lr}
+ /*
+ * Pushing r2 here is just a way of keeping the stack aligned to
+ * 8 bytes on any path that can trigger a HYP exception. Here,
+ * we may well be about to jump into the guest, and the guest
+ * exit would otherwise be badly decoded by our fancy
+ * "decode-exception-without-a-branch" code...
+ */
+ push {r2, lr}
mov lr, r0
mov r0, r1
THUMB( orr lr, #1)
blx lr @ Call the HYP function
- pop {lr}
+ pop {r2, lr}
+ eret
+
+guest_hvc_trap:
+ movw r2, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
+ movt r2, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
+ ldr r0, [sp] @ Guest's r0
+ teq r0, r2
+ bne guest_trap
+ add sp, sp, #12
+ @ Returns:
+ @ r0 = 0
+ @ r1 = HSR value (perfectly predictable)
+ @ r2 = ARM_SMCCC_ARCH_WORKAROUND_1
+ mov r0, #0
eret
guest_trap:
select CPU_CP15_MPU if !MMU
select CPU_HAS_ASID if MMU
select CPU_PABRT_V7
+ select CPU_SPECTRE if MMU
select CPU_THUMB_CAPABLE
select CPU_TLB_V7 if MMU
help
Say Y here to disable branch prediction. If unsure, say N.
+config CPU_SPECTRE
+ bool
+
+config HARDEN_BRANCH_PREDICTOR
+ bool "Harden the branch predictor against aliasing attacks" if EXPERT
+ depends on CPU_SPECTRE
+ default y
+ help
+ Speculation attacks against some high-performance processors rely
+ on being able to manipulate the branch predictor for a victim
+ context by executing aliasing branches in the attacker context.
+ Such attacks can be partially mitigated against by clearing
+ internal branch predictor state and limiting the prediction
+ logic in some situations.
+
+ This config option will take CPU-specific actions to harden
+ the branch predictor against aliasing attacks and may rely on
+ specific instruction sequences or control bits being set by
+ the system firmware.
+
+ If unsure, say Y.
+
config TLS_REG_EMUL
bool
select NEED_KUSER_HELPERS
ifneq ($(CONFIG_MMU),y)
obj-y += nommu.o
-obj-$(CONFIG_ARM_MPU) += pmsa-v7.o
+obj-$(CONFIG_ARM_MPU) += pmsa-v7.o pmsa-v8.o
endif
obj-$(CONFIG_ARM_PTDUMP_CORE) += dump.o
obj-$(CONFIG_CPU_FEROCEON) += proc-feroceon.o
obj-$(CONFIG_CPU_V6) += proc-v6.o
obj-$(CONFIG_CPU_V6K) += proc-v6.o
-obj-$(CONFIG_CPU_V7) += proc-v7.o
+obj-$(CONFIG_CPU_V7) += proc-v7.o proc-v7-bugs.o
obj-$(CONFIG_CPU_V7M) += proc-v7m.o
AFLAGS_proc-v6.o :=-Wa,-march=armv6
unsigned long attrs)
{
int ret;
- unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+ unsigned long nr_vma_pages = vma_pages(vma);
unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned long pfn = dma_to_pfn(dev, dma_addr);
unsigned long off = vma->vm_pgoff;
{
struct siginfo si;
+ if (addr > TASK_SIZE)
+ harden_branch_predictor();
+
#ifdef CONFIG_DEBUG_USER
if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
((user_debug & UDBG_BUS) && (sig == SIGBUS))) {
memblock_reserve(0, 1);
}
+static void __init adjust_lowmem_bounds_mpu(void)
+{
+ unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA;
+
+ switch (pmsa) {
+ case MMFR0_PMSAv7:
+ pmsav7_adjust_lowmem_bounds();
+ break;
+ case MMFR0_PMSAv8:
+ pmsav8_adjust_lowmem_bounds();
+ break;
+ default:
+ break;
+ }
+}
+
+static void __init mpu_setup(void)
+{
+ unsigned long pmsa = read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA;
+
+ switch (pmsa) {
+ case MMFR0_PMSAv7:
+ pmsav7_setup();
+ break;
+ case MMFR0_PMSAv8:
+ pmsav8_setup();
+ break;
+ default:
+ break;
+ }
+}
+
void __init adjust_lowmem_bounds(void)
{
phys_addr_t end;
static inline void rgnr_write(u32 v)
{
- writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RNR);
+ writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv7_RNR);
}
/* Data-side / unified region attributes */
/* Region access control register */
static inline void dracr_write(u32 v)
{
- u32 rsr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(15, 0);
+ u32 rsr = readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RASR) & GENMASK(15, 0);
- writel_relaxed((v << 16) | rsr, BASEADDR_V7M_SCB + MPU_RASR);
+ writel_relaxed((v << 16) | rsr, BASEADDR_V7M_SCB + PMSAv7_RASR);
}
/* Region size register */
static inline void drsr_write(u32 v)
{
- u32 racr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(31, 16);
+ u32 racr = readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RASR) & GENMASK(31, 16);
- writel_relaxed(v | racr, BASEADDR_V7M_SCB + MPU_RASR);
+ writel_relaxed(v | racr, BASEADDR_V7M_SCB + PMSAv7_RASR);
}
/* Region base address register */
static inline void drbar_write(u32 v)
{
- writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RBAR);
+ writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv7_RBAR);
}
static inline u32 drbar_read(void)
{
- return readl_relaxed(BASEADDR_V7M_SCB + MPU_RBAR);
+ return readl_relaxed(BASEADDR_V7M_SCB + PMSAv7_RBAR);
}
/* ARMv7-M only supports a unified MPU, so I-side operations are nop */
#endif
-static int __init mpu_present(void)
-{
- return ((read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA) == MMFR0_PMSAv7);
-}
-
static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct region *region)
{
unsigned long subreg, bslots, sslots;
bdiff = base - abase;
sdiff = p2size - asize;
- subreg = p2size / MPU_NR_SUBREGS;
+ subreg = p2size / PMSAv7_NR_SUBREGS;
if ((bdiff % subreg) || (sdiff % subreg))
return false;
if (bslots || sslots) {
int i;
- if (subreg < MPU_MIN_SUBREG_SIZE)
+ if (subreg < PMSAv7_MIN_SUBREG_SIZE)
return false;
- if (bslots + sslots > MPU_NR_SUBREGS)
+ if (bslots + sslots > PMSAv7_NR_SUBREGS)
return false;
for (i = 0; i < bslots; i++)
_set_bit(i, ®ion->subreg);
for (i = 1; i <= sslots; i++)
- _set_bit(MPU_NR_SUBREGS - i, ®ion->subreg);
+ _set_bit(PMSAv7_NR_SUBREGS - i, ®ion->subreg);
}
region->base = abase;
}
/* MPU initialisation functions */
-void __init adjust_lowmem_bounds_mpu(void)
+void __init pmsav7_adjust_lowmem_bounds(void)
{
phys_addr_t specified_mem_size = 0, total_mem_size = 0;
struct memblock_region *reg;
unsigned int mem_max_regions;
int num, i;
- if (!mpu_present())
- return;
-
- /* Free-up MPU_PROBE_REGION */
+ /* Free-up PMSAv7_PROBE_REGION */
mpu_min_region_order = __mpu_min_region_order();
/* How many regions are supported */
num = allocate_region(mem_start, specified_mem_size, mem_max_regions, mem);
for (i = 0; i < num; i++) {
- unsigned long subreg = mem[i].size / MPU_NR_SUBREGS;
+ unsigned long subreg = mem[i].size / PMSAv7_NR_SUBREGS;
total_mem_size += mem[i].size - subreg * hweight_long(mem[i].subreg);
pr_debug("MPU: base %pa size %pa disable subregions: %*pbl\n",
- &mem[i].base, &mem[i].size, MPU_NR_SUBREGS, &mem[i].subreg);
+ &mem[i].base, &mem[i].size, PMSAv7_NR_SUBREGS, &mem[i].subreg);
}
if (total_mem_size != specified_mem_size) {
u32 drbar_result, irbar_result;
/* We've kept a region free for this probing */
- rgnr_write(MPU_PROBE_REGION);
+ rgnr_write(PMSAv7_PROBE_REGION);
isb();
/*
* As per ARM ARM, write 0xFFFFFFFC to DRBAR to find the minimum
return -ENOMEM;
/* Writing N to bits 5:1 (RSR_SZ) specifies region size 2^N+1 */
- size_data = ((size_order - 1) << MPU_RSR_SZ) | 1 << MPU_RSR_EN;
- size_data |= subregions << MPU_RSR_SD;
+ size_data = ((size_order - 1) << PMSAv7_RSR_SZ) | 1 << PMSAv7_RSR_EN;
+ size_data |= subregions << PMSAv7_RSR_SD;
if (need_flush)
flush_cache_all();
/*
* Set up default MPU regions, doing nothing if there is no MPU
*/
-void __init mpu_setup(void)
+void __init pmsav7_setup(void)
{
int i, region = 0, err = 0;
- if (!mpu_present())
- return;
-
/* Setup MPU (order is important) */
/* Background */
err |= mpu_setup_region(region++, 0, 32,
- MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0RW,
+ PMSAv7_ACR_XN | PMSAv7_RGN_STRONGLY_ORDERED | PMSAv7_AP_PL1RW_PL0RW,
0, false);
#ifdef CONFIG_XIP_KERNEL
* with BG region (which is uncachable), thus we need
* to clean and invalidate cache.
*/
- bool need_flush = region == MPU_RAM_REGION;
+ bool need_flush = region == PMSAv7_RAM_REGION;
if (!xip[i].size)
continue;
err |= mpu_setup_region(region++, xip[i].base, ilog2(xip[i].size),
- MPU_AP_PL1RO_PL0NA | MPU_RGN_NORMAL,
+ PMSAv7_AP_PL1RO_PL0NA | PMSAv7_RGN_NORMAL,
xip[i].subreg, need_flush);
}
#endif
continue;
err |= mpu_setup_region(region++, mem[i].base, ilog2(mem[i].size),
- MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL,
+ PMSAv7_AP_PL1RW_PL0RW | PMSAv7_RGN_NORMAL,
mem[i].subreg, false);
}
/* Vectors */
#ifndef CONFIG_CPU_V7M
err |= mpu_setup_region(region++, vectors_base, ilog2(2 * PAGE_SIZE),
- MPU_AP_PL1RW_PL0NA | MPU_RGN_NORMAL,
+ PMSAv7_AP_PL1RW_PL0NA | PMSAv7_RGN_NORMAL,
0, false);
#endif
if (err) {
--- /dev/null
+/*
+ * Based on linux/arch/arm/pmsa-v7.c
+ *
+ * ARM PMSAv8 supporting functions.
+ */
+
+#include <linux/memblock.h>
+#include <linux/range.h>
+
+#include <asm/cp15.h>
+#include <asm/cputype.h>
+#include <asm/mpu.h>
+
+#include <asm/memory.h>
+#include <asm/sections.h>
+
+#include "mm.h"
+
+#ifndef CONFIG_CPU_V7M
+
+#define PRSEL __ACCESS_CP15(c6, 0, c2, 1)
+#define PRBAR __ACCESS_CP15(c6, 0, c3, 0)
+#define PRLAR __ACCESS_CP15(c6, 0, c3, 1)
+
+static inline u32 prlar_read(void)
+{
+ return read_sysreg(PRLAR);
+}
+
+static inline u32 prbar_read(void)
+{
+ return read_sysreg(PRBAR);
+}
+
+static inline void prsel_write(u32 v)
+{
+ write_sysreg(v, PRSEL);
+}
+
+static inline void prbar_write(u32 v)
+{
+ write_sysreg(v, PRBAR);
+}
+
+static inline void prlar_write(u32 v)
+{
+ write_sysreg(v, PRLAR);
+}
+#else
+
+static inline u32 prlar_read(void)
+{
+ return readl_relaxed(BASEADDR_V7M_SCB + PMSAv8_RLAR);
+}
+
+static inline u32 prbar_read(void)
+{
+ return readl_relaxed(BASEADDR_V7M_SCB + PMSAv8_RBAR);
+}
+
+static inline void prsel_write(u32 v)
+{
+ writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RNR);
+}
+
+static inline void prbar_write(u32 v)
+{
+ writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RBAR);
+}
+
+static inline void prlar_write(u32 v)
+{
+ writel_relaxed(v, BASEADDR_V7M_SCB + PMSAv8_RLAR);
+}
+
+#endif
+
+static struct range __initdata io[MPU_MAX_REGIONS];
+static struct range __initdata mem[MPU_MAX_REGIONS];
+
+static unsigned int __initdata mpu_max_regions;
+
+static __init bool is_region_fixed(int number)
+{
+ switch (number) {
+ case PMSAv8_XIP_REGION:
+ case PMSAv8_KERNEL_REGION:
+ return true;
+ default:
+ return false;
+ }
+}
+
+void __init pmsav8_adjust_lowmem_bounds(void)
+{
+ phys_addr_t mem_end;
+ struct memblock_region *reg;
+ bool first = true;
+
+ for_each_memblock(memory, reg) {
+ if (first) {
+ phys_addr_t phys_offset = PHYS_OFFSET;
+
+ /*
+ * Initially only use memory continuous from
+ * PHYS_OFFSET */
+ if (reg->base != phys_offset)
+ panic("First memory bank must be contiguous from PHYS_OFFSET");
+ mem_end = reg->base + reg->size;
+ first = false;
+ } else {
+ /*
+ * memblock auto merges contiguous blocks, remove
+ * all blocks afterwards in one go (we can't remove
+ * blocks separately while iterating)
+ */
+ pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n",
+ &mem_end, ®->base);
+ memblock_remove(reg->base, 0 - reg->base);
+ break;
+ }
+ }
+}
+
+static int __init __mpu_max_regions(void)
+{
+ static int max_regions;
+ u32 mpuir;
+
+ if (max_regions)
+ return max_regions;
+
+ mpuir = read_cpuid_mputype();
+
+ max_regions = (mpuir & MPUIR_DREGION_SZMASK) >> MPUIR_DREGION;
+
+ return max_regions;
+}
+
+static int __init __pmsav8_setup_region(unsigned int number, u32 bar, u32 lar)
+{
+ if (number > mpu_max_regions
+ || number >= MPU_MAX_REGIONS)
+ return -ENOENT;
+
+ dsb();
+ prsel_write(number);
+ isb();
+ prbar_write(bar);
+ prlar_write(lar);
+
+ mpu_rgn_info.rgns[number].prbar = bar;
+ mpu_rgn_info.rgns[number].prlar = lar;
+
+ mpu_rgn_info.used++;
+
+ return 0;
+}
+
+static int __init pmsav8_setup_ram(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+ u32 bar, lar;
+
+ if (is_region_fixed(number))
+ return -EINVAL;
+
+ bar = start;
+ lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);;
+
+ bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED;
+ lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN;
+
+ return __pmsav8_setup_region(number, bar, lar);
+}
+
+static int __init pmsav8_setup_io(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+ u32 bar, lar;
+
+ if (is_region_fixed(number))
+ return -EINVAL;
+
+ bar = start;
+ lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);;
+
+ bar |= PMSAv8_AP_PL1RW_PL0RW | PMSAv8_RGN_SHARED | PMSAv8_BAR_XN;
+ lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_DEVICE_nGnRnE) | PMSAv8_LAR_EN;
+
+ return __pmsav8_setup_region(number, bar, lar);
+}
+
+static int __init pmsav8_setup_fixed(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+ u32 bar, lar;
+
+ if (!is_region_fixed(number))
+ return -EINVAL;
+
+ bar = start;
+ lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);
+
+ bar |= PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED;
+ lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN;
+
+ prsel_write(number);
+ isb();
+
+ if (prbar_read() != bar || prlar_read() != lar)
+ return -EINVAL;
+
+ /* Reserved region was set up early, we just need a record for secondaries */
+ mpu_rgn_info.rgns[number].prbar = bar;
+ mpu_rgn_info.rgns[number].prlar = lar;
+
+ mpu_rgn_info.used++;
+
+ return 0;
+}
+
+#ifndef CONFIG_CPU_V7M
+static int __init pmsav8_setup_vector(unsigned int number, phys_addr_t start,phys_addr_t end)
+{
+ u32 bar, lar;
+
+ if (number == PMSAv8_KERNEL_REGION)
+ return -EINVAL;
+
+ bar = start;
+ lar = (end - 1) & ~(PMSAv8_MINALIGN - 1);
+
+ bar |= PMSAv8_AP_PL1RW_PL0NA | PMSAv8_RGN_SHARED;
+ lar |= PMSAv8_LAR_IDX(PMSAv8_RGN_NORMAL) | PMSAv8_LAR_EN;
+
+ return __pmsav8_setup_region(number, bar, lar);
+}
+#endif
+
+void __init pmsav8_setup(void)
+{
+ int i, err = 0;
+ int region = PMSAv8_KERNEL_REGION;
+
+ /* How many regions are supported ? */
+ mpu_max_regions = __mpu_max_regions();
+
+ /* RAM: single chunk of memory */
+ add_range(mem, ARRAY_SIZE(mem), 0, memblock.memory.regions[0].base,
+ memblock.memory.regions[0].base + memblock.memory.regions[0].size);
+
+ /* IO: cover full 4G range */
+ add_range(io, ARRAY_SIZE(io), 0, 0, 0xffffffff);
+
+ /* RAM and IO: exclude kernel */
+ subtract_range(mem, ARRAY_SIZE(mem), __pa(KERNEL_START), __pa(KERNEL_END));
+ subtract_range(io, ARRAY_SIZE(io), __pa(KERNEL_START), __pa(KERNEL_END));
+
+#ifdef CONFIG_XIP_KERNEL
+ /* RAM and IO: exclude xip */
+ subtract_range(mem, ARRAY_SIZE(mem), CONFIG_XIP_PHYS_ADDR, __pa(_exiprom));
+ subtract_range(io, ARRAY_SIZE(io), CONFIG_XIP_PHYS_ADDR, __pa(_exiprom));
+#endif
+
+#ifndef CONFIG_CPU_V7M
+ /* RAM and IO: exclude vectors */
+ subtract_range(mem, ARRAY_SIZE(mem), vectors_base, vectors_base + 2 * PAGE_SIZE);
+ subtract_range(io, ARRAY_SIZE(io), vectors_base, vectors_base + 2 * PAGE_SIZE);
+#endif
+ /* IO: exclude RAM */
+ for (i = 0; i < ARRAY_SIZE(mem); i++)
+ subtract_range(io, ARRAY_SIZE(io), mem[i].start, mem[i].end);
+
+ /* Now program MPU */
+
+#ifdef CONFIG_XIP_KERNEL
+ /* ROM */
+ err |= pmsav8_setup_fixed(PMSAv8_XIP_REGION, CONFIG_XIP_PHYS_ADDR, __pa(_exiprom));
+#endif
+ /* Kernel */
+ err |= pmsav8_setup_fixed(region++, __pa(KERNEL_START), __pa(KERNEL_END));
+
+
+ /* IO */
+ for (i = 0; i < ARRAY_SIZE(io); i++) {
+ if (!io[i].end)
+ continue;
+
+ err |= pmsav8_setup_io(region++, io[i].start, io[i].end);
+ }
+
+ /* RAM */
+ for (i = 0; i < ARRAY_SIZE(mem); i++) {
+ if (!mem[i].end)
+ continue;
+
+ err |= pmsav8_setup_ram(region++, mem[i].start, mem[i].end);
+ }
+
+ /* Vectors */
+#ifndef CONFIG_CPU_V7M
+ err |= pmsav8_setup_vector(region++, vectors_base, vectors_base + 2 * PAGE_SIZE);
+#endif
+ if (err)
+ pr_warn("MPU region initialization failure! %d", err);
+ else
+ pr_info("Using ARM PMSAv8 Compliant MPU. Used %d of %d regions\n",
+ mpu_rgn_info.used, mpu_max_regions);
+}
mcr p15, 0, ip, c7, c10, 4 @ data write barrier
.endm
-.macro define_processor_functions name:req, dabort:req, pabort:req, nommu=0, suspend=0
+.macro define_processor_functions name:req, dabort:req, pabort:req, nommu=0, suspend=0, bugs=0
.type \name\()_processor_functions, #object
.align 2
ENTRY(\name\()_processor_functions)
.word \dabort
.word \pabort
.word cpu_\name\()_proc_init
+ .word \bugs
.word cpu_\name\()_proc_fin
.word cpu_\name\()_reset
.word cpu_\name\()_do_idle
* even on Cortex-A8 revisions not affected by 430973.
* If IBE is not set, the flush BTAC/BTB won't do anything.
*/
-ENTRY(cpu_ca8_switch_mm)
-#ifdef CONFIG_MMU
- mov r2, #0
- mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
-#endif
ENTRY(cpu_v7_switch_mm)
#ifdef CONFIG_MMU
mmid r1, r1 @ get mm->context.id
#endif
bx lr
ENDPROC(cpu_v7_switch_mm)
-ENDPROC(cpu_ca8_switch_mm)
/*
* cpu_v7_set_pte_ext(ptep, pte)
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/arm-smccc.h>
+#include <linux/kernel.h>
+#include <linux/psci.h>
+#include <linux/smp.h>
+
+#include <asm/cp15.h>
+#include <asm/cputype.h>
+#include <asm/proc-fns.h>
+#include <asm/system_misc.h>
+
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn);
+
+extern void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+extern void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+extern void cpu_v7_smc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+extern void cpu_v7_hvc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
+
+static void harden_branch_predictor_bpiall(void)
+{
+ write_sysreg(0, BPIALL);
+}
+
+static void harden_branch_predictor_iciallu(void)
+{
+ write_sysreg(0, ICIALLU);
+}
+
+static void __maybe_unused call_smc_arch_workaround_1(void)
+{
+ arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static void __maybe_unused call_hvc_arch_workaround_1(void)
+{
+ arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static void cpu_v7_spectre_init(void)
+{
+ const char *spectre_v2_method = NULL;
+ int cpu = smp_processor_id();
+
+ if (per_cpu(harden_branch_predictor_fn, cpu))
+ return;
+
+ switch (read_cpuid_part()) {
+ case ARM_CPU_PART_CORTEX_A8:
+ case ARM_CPU_PART_CORTEX_A9:
+ case ARM_CPU_PART_CORTEX_A12:
+ case ARM_CPU_PART_CORTEX_A17:
+ case ARM_CPU_PART_CORTEX_A73:
+ case ARM_CPU_PART_CORTEX_A75:
+ if (processor.switch_mm != cpu_v7_bpiall_switch_mm)
+ goto bl_error;
+ per_cpu(harden_branch_predictor_fn, cpu) =
+ harden_branch_predictor_bpiall;
+ spectre_v2_method = "BPIALL";
+ break;
+
+ case ARM_CPU_PART_CORTEX_A15:
+ case ARM_CPU_PART_BRAHMA_B15:
+ if (processor.switch_mm != cpu_v7_iciallu_switch_mm)
+ goto bl_error;
+ per_cpu(harden_branch_predictor_fn, cpu) =
+ harden_branch_predictor_iciallu;
+ spectre_v2_method = "ICIALLU";
+ break;
+
+#ifdef CONFIG_ARM_PSCI
+ default:
+ /* Other ARM CPUs require no workaround */
+ if (read_cpuid_implementor() == ARM_CPU_IMP_ARM)
+ break;
+ /* fallthrough */
+ /* Cortex A57/A72 require firmware workaround */
+ case ARM_CPU_PART_CORTEX_A57:
+ case ARM_CPU_PART_CORTEX_A72: {
+ struct arm_smccc_res res;
+
+ if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
+ break;
+
+ switch (psci_ops.conduit) {
+ case PSCI_CONDUIT_HVC:
+ arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+ ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+ if ((int)res.a0 != 0)
+ break;
+ if (processor.switch_mm != cpu_v7_hvc_switch_mm && cpu)
+ goto bl_error;
+ per_cpu(harden_branch_predictor_fn, cpu) =
+ call_hvc_arch_workaround_1;
+ processor.switch_mm = cpu_v7_hvc_switch_mm;
+ spectre_v2_method = "hypervisor";
+ break;
+
+ case PSCI_CONDUIT_SMC:
+ arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+ ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+ if ((int)res.a0 != 0)
+ break;
+ if (processor.switch_mm != cpu_v7_smc_switch_mm && cpu)
+ goto bl_error;
+ per_cpu(harden_branch_predictor_fn, cpu) =
+ call_smc_arch_workaround_1;
+ processor.switch_mm = cpu_v7_smc_switch_mm;
+ spectre_v2_method = "firmware";
+ break;
+
+ default:
+ break;
+ }
+ }
+#endif
+ }
+
+ if (spectre_v2_method)
+ pr_info("CPU%u: Spectre v2: using %s workaround\n",
+ smp_processor_id(), spectre_v2_method);
+ return;
+
+bl_error:
+ pr_err("CPU%u: Spectre v2: incorrect context switching function, system vulnerable\n",
+ cpu);
+}
+#else
+static void cpu_v7_spectre_init(void)
+{
+}
+#endif
+
+static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned,
+ u32 mask, const char *msg)
+{
+ u32 aux_cr;
+
+ asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr));
+
+ if ((aux_cr & mask) != mask) {
+ if (!*warned)
+ pr_err("CPU%u: %s", smp_processor_id(), msg);
+ *warned = true;
+ return false;
+ }
+ return true;
+}
+
+static DEFINE_PER_CPU(bool, spectre_warned);
+
+static bool check_spectre_auxcr(bool *warned, u32 bit)
+{
+ return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) &&
+ cpu_v7_check_auxcr_set(warned, bit,
+ "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n");
+}
+
+void cpu_v7_ca8_ibe(void)
+{
+ if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6)))
+ cpu_v7_spectre_init();
+}
+
+void cpu_v7_ca15_ibe(void)
+{
+ if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0)))
+ cpu_v7_spectre_init();
+}
+
+void cpu_v7_bugs_init(void)
+{
+ cpu_v7_spectre_init();
+}
*
* This is the "shell" of the ARMv7 processor support.
*/
+#include <linux/arm-smccc.h>
#include <linux/init.h>
#include <linux/linkage.h>
#include <asm/assembler.h>
ret lr
ENDPROC(cpu_v7_dcache_clean_area)
+#ifdef CONFIG_ARM_PSCI
+ .arch_extension sec
+ENTRY(cpu_v7_smc_switch_mm)
+ stmfd sp!, {r0 - r3}
+ movw r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
+ movt r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
+ smc #0
+ ldmfd sp!, {r0 - r3}
+ b cpu_v7_switch_mm
+ENDPROC(cpu_v7_smc_switch_mm)
+ .arch_extension virt
+ENTRY(cpu_v7_hvc_switch_mm)
+ stmfd sp!, {r0 - r3}
+ movw r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
+ movt r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
+ hvc #0
+ ldmfd sp!, {r0 - r3}
+ b cpu_v7_switch_mm
+ENDPROC(cpu_v7_smc_switch_mm)
+#endif
+ENTRY(cpu_v7_iciallu_switch_mm)
+ mov r3, #0
+ mcr p15, 0, r3, c7, c5, 0 @ ICIALLU
+ b cpu_v7_switch_mm
+ENDPROC(cpu_v7_iciallu_switch_mm)
+ENTRY(cpu_v7_bpiall_switch_mm)
+ mov r3, #0
+ mcr p15, 0, r3, c7, c5, 6 @ flush BTAC/BTB
+ b cpu_v7_switch_mm
+ENDPROC(cpu_v7_bpiall_switch_mm)
+
string cpu_v7_name, "ARMv7 Processor"
.align
ENDPROC(cpu_v7_do_resume)
#endif
-/*
- * Cortex-A8
- */
- globl_equ cpu_ca8_proc_init, cpu_v7_proc_init
- globl_equ cpu_ca8_proc_fin, cpu_v7_proc_fin
- globl_equ cpu_ca8_reset, cpu_v7_reset
- globl_equ cpu_ca8_do_idle, cpu_v7_do_idle
- globl_equ cpu_ca8_dcache_clean_area, cpu_v7_dcache_clean_area
- globl_equ cpu_ca8_set_pte_ext, cpu_v7_set_pte_ext
- globl_equ cpu_ca8_suspend_size, cpu_v7_suspend_size
-#ifdef CONFIG_ARM_CPU_SUSPEND
- globl_equ cpu_ca8_do_suspend, cpu_v7_do_suspend
- globl_equ cpu_ca8_do_resume, cpu_v7_do_resume
-#endif
-
-/*
- * Cortex-A9 processor functions
- */
- globl_equ cpu_ca9mp_proc_init, cpu_v7_proc_init
- globl_equ cpu_ca9mp_proc_fin, cpu_v7_proc_fin
- globl_equ cpu_ca9mp_reset, cpu_v7_reset
- globl_equ cpu_ca9mp_do_idle, cpu_v7_do_idle
- globl_equ cpu_ca9mp_dcache_clean_area, cpu_v7_dcache_clean_area
- globl_equ cpu_ca9mp_switch_mm, cpu_v7_switch_mm
- globl_equ cpu_ca9mp_set_pte_ext, cpu_v7_set_pte_ext
.globl cpu_ca9mp_suspend_size
.equ cpu_ca9mp_suspend_size, cpu_v7_suspend_size + 4 * 2
#ifdef CONFIG_ARM_CPU_SUSPEND
__INITDATA
+ .weak cpu_v7_bugs_init
+
@ define struct processor (see <asm/proc-fns.h> and proc-macros.S)
- define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
+ define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
+
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+ @ generic v7 bpiall on context switch
+ globl_equ cpu_v7_bpiall_proc_init, cpu_v7_proc_init
+ globl_equ cpu_v7_bpiall_proc_fin, cpu_v7_proc_fin
+ globl_equ cpu_v7_bpiall_reset, cpu_v7_reset
+ globl_equ cpu_v7_bpiall_do_idle, cpu_v7_do_idle
+ globl_equ cpu_v7_bpiall_dcache_clean_area, cpu_v7_dcache_clean_area
+ globl_equ cpu_v7_bpiall_set_pte_ext, cpu_v7_set_pte_ext
+ globl_equ cpu_v7_bpiall_suspend_size, cpu_v7_suspend_size
+#ifdef CONFIG_ARM_CPU_SUSPEND
+ globl_equ cpu_v7_bpiall_do_suspend, cpu_v7_do_suspend
+ globl_equ cpu_v7_bpiall_do_resume, cpu_v7_do_resume
+#endif
+ define_processor_functions v7_bpiall, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
+
+#define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_bpiall_processor_functions
+#else
+#define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_processor_functions
+#endif
+
#ifndef CONFIG_ARM_LPAE
- define_processor_functions ca8, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
- define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
+ @ Cortex-A8 - always needs bpiall switch_mm implementation
+ globl_equ cpu_ca8_proc_init, cpu_v7_proc_init
+ globl_equ cpu_ca8_proc_fin, cpu_v7_proc_fin
+ globl_equ cpu_ca8_reset, cpu_v7_reset
+ globl_equ cpu_ca8_do_idle, cpu_v7_do_idle
+ globl_equ cpu_ca8_dcache_clean_area, cpu_v7_dcache_clean_area
+ globl_equ cpu_ca8_set_pte_ext, cpu_v7_set_pte_ext
+ globl_equ cpu_ca8_switch_mm, cpu_v7_bpiall_switch_mm
+ globl_equ cpu_ca8_suspend_size, cpu_v7_suspend_size
+#ifdef CONFIG_ARM_CPU_SUSPEND
+ globl_equ cpu_ca8_do_suspend, cpu_v7_do_suspend
+ globl_equ cpu_ca8_do_resume, cpu_v7_do_resume
+#endif
+ define_processor_functions ca8, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_ca8_ibe
+
+ @ Cortex-A9 - needs more registers preserved across suspend/resume
+ @ and bpiall switch_mm for hardening
+ globl_equ cpu_ca9mp_proc_init, cpu_v7_proc_init
+ globl_equ cpu_ca9mp_proc_fin, cpu_v7_proc_fin
+ globl_equ cpu_ca9mp_reset, cpu_v7_reset
+ globl_equ cpu_ca9mp_do_idle, cpu_v7_do_idle
+ globl_equ cpu_ca9mp_dcache_clean_area, cpu_v7_dcache_clean_area
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+ globl_equ cpu_ca9mp_switch_mm, cpu_v7_bpiall_switch_mm
+#else
+ globl_equ cpu_ca9mp_switch_mm, cpu_v7_switch_mm
+#endif
+ globl_equ cpu_ca9mp_set_pte_ext, cpu_v7_set_pte_ext
+ define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init
#endif
+
+ @ Cortex-A15 - needs iciallu switch_mm for hardening
+ globl_equ cpu_ca15_proc_init, cpu_v7_proc_init
+ globl_equ cpu_ca15_proc_fin, cpu_v7_proc_fin
+ globl_equ cpu_ca15_reset, cpu_v7_reset
+ globl_equ cpu_ca15_do_idle, cpu_v7_do_idle
+ globl_equ cpu_ca15_dcache_clean_area, cpu_v7_dcache_clean_area
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+ globl_equ cpu_ca15_switch_mm, cpu_v7_iciallu_switch_mm
+#else
+ globl_equ cpu_ca15_switch_mm, cpu_v7_switch_mm
+#endif
+ globl_equ cpu_ca15_set_pte_ext, cpu_v7_set_pte_ext
+ globl_equ cpu_ca15_suspend_size, cpu_v7_suspend_size
+ globl_equ cpu_ca15_do_suspend, cpu_v7_do_suspend
+ globl_equ cpu_ca15_do_resume, cpu_v7_do_resume
+ define_processor_functions ca15, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_ca15_ibe
#ifdef CONFIG_CPU_PJ4B
define_processor_functions pj4b, dabort=v7_early_abort, pabort=v7_pabort, suspend=1
#endif
__v7_ca12mp_proc_info:
.long 0x410fc0d0
.long 0xff0ffff0
- __v7_proc __v7_ca12mp_proc_info, __v7_ca12mp_setup
+ __v7_proc __v7_ca12mp_proc_info, __v7_ca12mp_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
.size __v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info
/*
__v7_ca15mp_proc_info:
.long 0x410fc0f0
.long 0xff0ffff0
- __v7_proc __v7_ca15mp_proc_info, __v7_ca15mp_setup
+ __v7_proc __v7_ca15mp_proc_info, __v7_ca15mp_setup, proc_fns = ca15_processor_functions
.size __v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info
/*
__v7_b15mp_proc_info:
.long 0x420f00f0
.long 0xff0ffff0
- __v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup, cache_fns = b15_cache_fns
+ __v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup, proc_fns = ca15_processor_functions, cache_fns = b15_cache_fns
.size __v7_b15mp_proc_info, . - __v7_b15mp_proc_info
/*
__v7_ca17mp_proc_info:
.long 0x410fc0e0
.long 0xff0ffff0
- __v7_proc __v7_ca17mp_proc_info, __v7_ca17mp_setup
+ __v7_proc __v7_ca17mp_proc_info, __v7_ca17mp_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
.size __v7_ca17mp_proc_info, . - __v7_ca17mp_proc_info
+ /* ARM Ltd. Cortex A73 processor */
+ .type __v7_ca73_proc_info, #object
+__v7_ca73_proc_info:
+ .long 0x410fd090
+ .long 0xff0ffff0
+ __v7_proc __v7_ca73_proc_info, __v7_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
+ .size __v7_ca73_proc_info, . - __v7_ca73_proc_info
+
+ /* ARM Ltd. Cortex A75 processor */
+ .type __v7_ca75_proc_info, #object
+__v7_ca75_proc_info:
+ .long 0x410fd0a0
+ .long 0xff0ffff0
+ __v7_proc __v7_ca75_proc_info, __v7_setup, proc_fns = HARDENED_BPIALL_PROCESSOR_FUNCTIONS
+ .size __v7_ca75_proc_info, . - __v7_ca75_proc_info
+
/*
* Qualcomm Inc. Krait processors.
*/
.pm = &amba_pm,
.force_dma = true,
};
+EXPORT_SYMBOL_GPL(amba_bustype);
static int __init amba_init(void)
{
git.samba.org / sfrench / cifs-2.6.git / commitdiff