1 // SPDX-License-Identifier: GPL-2.0+
3 * PowerPC Memory Protection Keys management
5 * Copyright 2017, Ram Pai, IBM Corporation.
9 #include <asm/mmu_context.h>
11 #include <asm/setup.h>
12 #include <linux/pkeys.h>
13 #include <linux/of_device.h>
15 DEFINE_STATIC_KEY_TRUE(pkey_disabled);
16 int pkeys_total; /* Total pkeys as per device tree */
17 u32 initial_allocation_mask; /* Bits set for the initially allocated keys */
18 u32 reserved_allocation_mask; /* Bits set for reserved keys */
19 static bool pkey_execute_disable_supported;
20 static bool pkeys_devtree_defined; /* property exported by device tree */
21 static u64 pkey_amr_mask; /* Bits in AMR not to be touched */
22 static u64 pkey_iamr_mask; /* Bits in AMR not to be touched */
23 static u64 pkey_uamor_mask; /* Bits in UMOR not to be touched */
24 static int execute_only_key = 2;
26 #define AMR_BITS_PER_PKEY 2
27 #define AMR_RD_BIT 0x1UL
28 #define AMR_WR_BIT 0x2UL
29 #define IAMR_EX_BIT 0x1UL
30 #define PKEY_REG_BITS (sizeof(u64)*8)
31 #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
33 static void scan_pkey_feature(void)
36 struct device_node *cpu;
38 cpu = of_find_node_by_type(NULL, "cpu");
42 if (of_property_read_u32_array(cpu,
43 "ibm,processor-storage-keys", vals, 2))
47 * Since any pkey can be used for data or execute, we will just treat
48 * all keys as equal and track them as one entity.
50 pkeys_total = vals[0];
51 pkeys_devtree_defined = true;
54 static inline bool pkey_mmu_enabled(void)
56 if (firmware_has_feature(FW_FEATURE_LPAR))
59 return cpu_has_feature(CPU_FTR_PKEY);
62 static int pkey_initialize(void)
67 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
68 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
69 * Ensure that the bits a distinct.
71 BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
72 (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
75 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
76 * in the vmaflag. Make sure that is really the case.
78 BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
79 __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
80 != (sizeof(u64) * BITS_PER_BYTE));
82 /* scan the device tree for pkey feature */
86 * Let's assume 32 pkeys on P8 bare metal, if its not defined by device
87 * tree. We make this exception since skiboot forgot to expose this
90 if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) &&
91 cpu_has_feature(CPU_FTRS_POWER8))
95 * Adjust the upper limit, based on the number of bits supported by
98 pkeys_total = min_t(int, pkeys_total,
99 ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)+1));
101 if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total)
102 static_branch_enable(&pkey_disabled);
104 static_branch_disable(&pkey_disabled);
106 if (static_branch_likely(&pkey_disabled))
110 * The device tree cannot be relied to indicate support for
111 * execute_disable support. Instead we use a PVR check.
113 if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
114 pkey_execute_disable_supported = false;
116 pkey_execute_disable_supported = true;
118 #ifdef CONFIG_PPC_4K_PAGES
120 * The OS can manage only 8 pkeys due to its inability to represent them
121 * in the Linux 4K PTE.
123 os_reserved = pkeys_total - 8;
127 /* Bits are in LE format. */
128 reserved_allocation_mask = (0x1 << 1) | (0x1 << execute_only_key);
130 /* register mask is in BE format */
131 pkey_amr_mask = ~0x0ul;
132 pkey_amr_mask &= ~(0x3ul << pkeyshift(0));
134 pkey_iamr_mask = ~0x0ul;
135 pkey_iamr_mask &= ~(0x3ul << pkeyshift(0));
136 pkey_iamr_mask &= ~(0x3ul << pkeyshift(execute_only_key));
138 pkey_uamor_mask = ~0x0ul;
139 pkey_uamor_mask &= ~(0x3ul << pkeyshift(0));
140 pkey_uamor_mask &= ~(0x3ul << pkeyshift(execute_only_key));
142 /* mark the rest of the keys as reserved and hence unavailable */
143 for (i = (pkeys_total - os_reserved); i < pkeys_total; i++) {
144 reserved_allocation_mask |= (0x1 << i);
145 pkey_uamor_mask &= ~(0x3ul << pkeyshift(i));
147 initial_allocation_mask = reserved_allocation_mask | (0x1 << 0);
149 if (unlikely((pkeys_total - os_reserved) <= execute_only_key)) {
151 * Insufficient number of keys to support
152 * execute only key. Mark it unavailable.
153 * Any AMR, UAMOR, IAMR bit set for
154 * this key is irrelevant since this key
155 * can never be allocated.
157 execute_only_key = -1;
163 arch_initcall(pkey_initialize);
165 void pkey_mm_init(struct mm_struct *mm)
167 if (static_branch_likely(&pkey_disabled))
169 mm_pkey_allocation_map(mm) = initial_allocation_mask;
170 mm->context.execute_only_pkey = execute_only_key;
173 static inline u64 read_amr(void)
175 return mfspr(SPRN_AMR);
178 static inline void write_amr(u64 value)
180 mtspr(SPRN_AMR, value);
183 static inline u64 read_iamr(void)
185 if (!likely(pkey_execute_disable_supported))
188 return mfspr(SPRN_IAMR);
191 static inline void write_iamr(u64 value)
193 if (!likely(pkey_execute_disable_supported))
196 mtspr(SPRN_IAMR, value);
199 static inline u64 read_uamor(void)
201 return mfspr(SPRN_UAMOR);
204 static inline void write_uamor(u64 value)
206 mtspr(SPRN_UAMOR, value);
209 static bool is_pkey_enabled(int pkey)
211 u64 uamor = read_uamor();
212 u64 pkey_bits = 0x3ul << pkeyshift(pkey);
213 u64 uamor_pkey_bits = (uamor & pkey_bits);
216 * Both the bits in UAMOR corresponding to the key should be set or
219 WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits));
220 return !!(uamor_pkey_bits);
223 static inline void init_amr(int pkey, u8 init_bits)
225 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
226 u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
228 write_amr(old_amr | new_amr_bits);
231 static inline void init_iamr(int pkey, u8 init_bits)
233 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
234 u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
236 write_iamr(old_iamr | new_iamr_bits);
240 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
241 * specified in @init_val.
243 int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
244 unsigned long init_val)
246 u64 new_amr_bits = 0x0ul;
247 u64 new_iamr_bits = 0x0ul;
249 if (!is_pkey_enabled(pkey))
252 if (init_val & PKEY_DISABLE_EXECUTE) {
253 if (!pkey_execute_disable_supported)
255 new_iamr_bits |= IAMR_EX_BIT;
257 init_iamr(pkey, new_iamr_bits);
259 /* Set the bits we need in AMR: */
260 if (init_val & PKEY_DISABLE_ACCESS)
261 new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
262 else if (init_val & PKEY_DISABLE_WRITE)
263 new_amr_bits |= AMR_WR_BIT;
265 init_amr(pkey, new_amr_bits);
269 void thread_pkey_regs_save(struct thread_struct *thread)
271 if (static_branch_likely(&pkey_disabled))
275 * TODO: Skip saving registers if @thread hasn't used any keys yet.
277 thread->amr = read_amr();
278 thread->iamr = read_iamr();
279 thread->uamor = read_uamor();
282 void thread_pkey_regs_restore(struct thread_struct *new_thread,
283 struct thread_struct *old_thread)
285 if (static_branch_likely(&pkey_disabled))
288 if (old_thread->amr != new_thread->amr)
289 write_amr(new_thread->amr);
290 if (old_thread->iamr != new_thread->iamr)
291 write_iamr(new_thread->iamr);
292 if (old_thread->uamor != new_thread->uamor)
293 write_uamor(new_thread->uamor);
296 void thread_pkey_regs_init(struct thread_struct *thread)
298 if (static_branch_likely(&pkey_disabled))
301 thread->amr = pkey_amr_mask;
302 thread->iamr = pkey_iamr_mask;
303 thread->uamor = pkey_uamor_mask;
305 write_uamor(pkey_uamor_mask);
306 write_amr(pkey_amr_mask);
307 write_iamr(pkey_iamr_mask);
310 int __execute_only_pkey(struct mm_struct *mm)
312 return mm->context.execute_only_pkey;
315 static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma)
317 /* Do this check first since the vm_flags should be hot */
318 if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC)
321 return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey);
325 * This should only be called for *plain* mprotect calls.
327 int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot,
331 * If the currently associated pkey is execute-only, but the requested
332 * protection is not execute-only, move it back to the default pkey.
334 if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC))
338 * The requested protection is execute-only. Hence let's use an
341 if (prot == PROT_EXEC) {
342 pkey = execute_only_pkey(vma->vm_mm);
347 /* Nothing to override. */
348 return vma_pkey(vma);
351 static bool pkey_access_permitted(int pkey, bool write, bool execute)
356 if (!is_pkey_enabled(pkey))
359 pkey_shift = pkeyshift(pkey);
360 if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift)))
363 amr = read_amr(); /* Delay reading amr until absolutely needed */
364 return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) ||
365 (write && !(amr & (AMR_WR_BIT << pkey_shift))));
368 bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
370 if (static_branch_likely(&pkey_disabled))
373 return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
377 * We only want to enforce protection keys on the current thread because we
378 * effectively have no access to AMR/IAMR for other threads or any way to tell
379 * which AMR/IAMR in a threaded process we could use.
381 * So do not enforce things if the VMA is not from the current mm, or if we are
382 * in a kernel thread.
384 static inline bool vma_is_foreign(struct vm_area_struct *vma)
389 /* if it is not our ->mm, it has to be foreign */
390 if (current->mm != vma->vm_mm)
396 bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
397 bool execute, bool foreign)
399 if (static_branch_likely(&pkey_disabled))
402 * Do not enforce our key-permissions on a foreign vma.
404 if (foreign || vma_is_foreign(vma))
407 return pkey_access_permitted(vma_pkey(vma), write, execute);
410 void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm)
412 if (static_branch_likely(&pkey_disabled))
415 /* Duplicate the oldmm pkey state in mm: */
416 mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm);
417 mm->context.execute_only_pkey = oldmm->context.execute_only_pkey;