arch/powerpc/include/asm/book3s/64/tlbflush.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
   3 #define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
   4
   5 #define MMU_NO_CONTEXT  ~0UL
   6
   7 #include <linux/mm_types.h>
   8 #include <asm/book3s/64/tlbflush-hash.h>
   9 #include <asm/book3s/64/tlbflush-radix.h>
  10
  11 /* TLB flush actions. Used as argument to tlbiel_all() */
  12 enum {
  13         TLB_INVAL_SCOPE_GLOBAL = 0,     /* invalidate all TLBs */
  14         TLB_INVAL_SCOPE_LPID = 1,       /* invalidate TLBs for current LPID */
  15 };
  16
  17 static inline void tlbiel_all(void)
  18 {
  19         /*
  20          * This is used for host machine check and bootup.
  21          *
  22          * This uses early_radix_enabled and implementations use
  23          * early_cpu_has_feature etc because that works early in boot
  24          * and this is the machine check path which is not performance
  25          * critical.
  26          */
  27         if (early_radix_enabled())
  28                 radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
  29         else
  30                 hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
  31 }
  32
  33 static inline void tlbiel_all_lpid(bool radix)
  34 {
  35         /*
  36          * This is used for guest machine check.
  37          */
  38         if (radix)
  39                 radix__tlbiel_all(TLB_INVAL_SCOPE_LPID);
  40         else
  41                 hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
  42 }
  43
  44
  45 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
  46 static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
  47                                        unsigned long start, unsigned long end)
  48 {
  49         if (radix_enabled())
  50                 return radix__flush_pmd_tlb_range(vma, start, end);
  51         return hash__flush_tlb_range(vma, start, end);
  52 }
  53
  54 #define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
  55 static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
  56                                            unsigned long start,
  57                                            unsigned long end)
  58 {
  59         if (radix_enabled())
  60                 return radix__flush_hugetlb_tlb_range(vma, start, end);
  61         return hash__flush_tlb_range(vma, start, end);
  62 }
  63
  64 static inline void flush_tlb_range(struct vm_area_struct *vma,
  65                                    unsigned long start, unsigned long end)
  66 {
  67         if (radix_enabled())
  68                 return radix__flush_tlb_range(vma, start, end);
  69         return hash__flush_tlb_range(vma, start, end);
  70 }
  71
  72 static inline void flush_tlb_kernel_range(unsigned long start,
  73                                           unsigned long end)
  74 {
  75         if (radix_enabled())
  76                 return radix__flush_tlb_kernel_range(start, end);
  77         return hash__flush_tlb_kernel_range(start, end);
  78 }
  79
  80 static inline void local_flush_tlb_mm(struct mm_struct *mm)
  81 {
  82         if (radix_enabled())
  83                 return radix__local_flush_tlb_mm(mm);
  84         return hash__local_flush_tlb_mm(mm);
  85 }
  86
  87 static inline void local_flush_tlb_page(struct vm_area_struct *vma,
  88                                         unsigned long vmaddr)
  89 {
  90         if (radix_enabled())
  91                 return radix__local_flush_tlb_page(vma, vmaddr);
  92         return hash__local_flush_tlb_page(vma, vmaddr);
  93 }
  94
  95 static inline void local_flush_all_mm(struct mm_struct *mm)
  96 {
  97         if (radix_enabled())
  98                 return radix__local_flush_all_mm(mm);
  99         return hash__local_flush_all_mm(mm);
 100 }
 101
 102 static inline void tlb_flush(struct mmu_gather *tlb)
 103 {
 104         if (radix_enabled())
 105                 return radix__tlb_flush(tlb);
 106         return hash__tlb_flush(tlb);
 107 }
 108
 109 #ifdef CONFIG_SMP
 110 static inline void flush_tlb_mm(struct mm_struct *mm)
 111 {
 112         if (radix_enabled())
 113                 return radix__flush_tlb_mm(mm);
 114         return hash__flush_tlb_mm(mm);
 115 }
 116
 117 static inline void flush_tlb_page(struct vm_area_struct *vma,
 118                                   unsigned long vmaddr)
 119 {
 120         if (radix_enabled())
 121                 return radix__flush_tlb_page(vma, vmaddr);
 122         return hash__flush_tlb_page(vma, vmaddr);
 123 }
 124
 125 static inline void flush_all_mm(struct mm_struct *mm)
 126 {
 127         if (radix_enabled())
 128                 return radix__flush_all_mm(mm);
 129         return hash__flush_all_mm(mm);
 130 }
 131 #else
 132 #define flush_tlb_mm(mm)                local_flush_tlb_mm(mm)
 133 #define flush_tlb_page(vma, addr)       local_flush_tlb_page(vma, addr)
 134 #define flush_all_mm(mm)                local_flush_all_mm(mm)
 135 #endif /* CONFIG_SMP */
 136
 137 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
 138 static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
 139                                                 unsigned long address)
 140 {
 141         /*
 142          * Book3S 64 does not require spurious fault flushes because the PTE
 143          * must be re-fetched in case of an access permission problem. So the
 144          * only reason for a spurious fault should be concurrent modification
 145          * to the PTE, in which case the PTE will eventually be re-fetched by
 146          * the MMU when it attempts the access again.
 147          *
 148          * See: Power ISA Version 3.1B, 6.10.1.2 Modifying a Translation Table
 149          * Entry, Setting a Reference or Change Bit or Upgrading Access
 150          * Authority (PTE Subject to Atomic Hardware Updates):
 151          *
 152          * "If the only change being made to a valid PTE that is subject to
 153          *  atomic hardware updates is to set the Reference or Change bit to
 154          *  1 or to upgrade access authority, a simpler sequence suffices
 155          *  because the translation hardware will refetch the PTE if an
 156          *  access is attempted for which the only problems were reference
 157          *  and/or change bits needing to be set or insufficient access
 158          *  authority."
 159          *
 160          * The nest MMU in POWER9 does not perform this PTE re-fetch, but
 161          * it avoids the spurious fault problem by flushing the TLB before
 162          * upgrading PTE permissions, see radix__ptep_set_access_flags.
 163          */
 164 }
 165
 166 static inline bool __pte_flags_need_flush(unsigned long oldval,
 167                                           unsigned long newval)
 168 {
 169         unsigned long delta = oldval ^ newval;
 170
 171         /*
 172          * The return value of this function doesn't matter for hash,
 173          * ptep_modify_prot_start() does a pte_update() which does or schedules
 174          * any necessary hash table update and flush.
 175          */
 176         if (!radix_enabled())
 177                 return true;
 178
 179         /*
 180          * We do not expect kernel mappings or non-PTEs or not-present PTEs.
 181          */
 182         VM_WARN_ON_ONCE(oldval & _PAGE_PRIVILEGED);
 183         VM_WARN_ON_ONCE(newval & _PAGE_PRIVILEGED);
 184         VM_WARN_ON_ONCE(!(oldval & _PAGE_PTE));
 185         VM_WARN_ON_ONCE(!(newval & _PAGE_PTE));
 186         VM_WARN_ON_ONCE(!(oldval & _PAGE_PRESENT));
 187         VM_WARN_ON_ONCE(!(newval & _PAGE_PRESENT));
 188
 189         /*
 190         *  Must flush on any change except READ, WRITE, EXEC, DIRTY, ACCESSED.
 191         *
 192          * In theory, some changed software bits could be tolerated, in
 193          * practice those should rarely if ever matter.
 194          */
 195
 196         if (delta & ~(_PAGE_RWX | _PAGE_DIRTY | _PAGE_ACCESSED))
 197                 return true;
 198
 199         /*
 200          * If any of the above was present in old but cleared in new, flush.
 201          * With the exception of _PAGE_ACCESSED, don't worry about flushing
 202          * if that was cleared (see the comment in ptep_clear_flush_young()).
 203          */
 204         if ((delta & ~_PAGE_ACCESSED) & oldval)
 205                 return true;
 206
 207         return false;
 208 }
 209
 210 static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)
 211 {
 212         return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte));
 213 }
 214 #define pte_needs_flush pte_needs_flush
 215
 216 static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
 217 {
 218         return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd));
 219 }
 220 #define huge_pmd_needs_flush huge_pmd_needs_flush
 221
 222 extern bool tlbie_capable;
 223 extern bool tlbie_enabled;
 224
 225 static inline bool cputlb_use_tlbie(void)
 226 {
 227         return tlbie_enabled;
 228 }
 229
 230 #endif /*  _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */