riscv: tlb: fix __p*d_free_tlb()

If non-leaf PTEs I.E pmd, pud or p4d is modified, a sfence.vma is
a must for safe, imagine if an implementation caches the non-leaf
translation in TLB, although I didn't meet this HW so far, but it's
possible in theory.

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
Fixes: c5e9b2c2ae82 ("riscv: Improve tlb_flush()")
Link: https://lore.kernel.org/r/20231219175046.2496-2-jszhang@kernel.org
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h
index d169a4f41a2e728276a97898e1270c7b4763f9ed..c80bb9990d32ef706452d7d4fcc1c049cd7436d9 100644 (file)
--- a/arch/riscv/include/asm/pgalloc.h
+++ b/arch/riscv/include/asm/pgalloc.h
@@ -95,7 +95,13 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
                __pud_free(mm, pud);
 }
 
-#define __pud_free_tlb(tlb, pud, addr)  pud_free((tlb)->mm, pud)
+#define __pud_free_tlb(tlb, pud, addr)                                 \
+do {                                                                   \
+       if (pgtable_l4_enabled) {                                       \
+               pagetable_pud_dtor(virt_to_ptdesc(pud));                \
+               tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pud));     \
+       }                                                               \
+} while (0)
 
 #define p4d_alloc_one p4d_alloc_one
 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
@@ -124,7 +130,11 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
                __p4d_free(mm, p4d);
 }
 
-#define __p4d_free_tlb(tlb, p4d, addr)  p4d_free((tlb)->mm, p4d)
+#define __p4d_free_tlb(tlb, p4d, addr)                                 \
+do {                                                                   \
+       if (pgtable_l5_enabled)                                         \
+               tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(p4d));     \
+} while (0)
 #endif /* __PAGETABLE_PMD_FOLDED */
 
 static inline void sync_kernel_mappings(pgd_t *pgd)
@@ -149,7 +159,11 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 
 #ifndef __PAGETABLE_PMD_FOLDED
 
-#define __pmd_free_tlb(tlb, pmd, addr)  pmd_free((tlb)->mm, pmd)
+#define __pmd_free_tlb(tlb, pmd, addr)                         \
+do {                                                           \
+       pagetable_pmd_dtor(virt_to_ptdesc(pmd));                \
+       tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pmd));     \
+} while (0)
 
 #endif /* __PAGETABLE_PMD_FOLDED */