Merge tag 'x86_shstk_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git...

[sfrench/cifs-2.6.git] / mm / userfaultfd.c

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index b322ac54ea205e995b398d4b096c15a03511246d..96d9eae5c7cc8e21e2fd7d6c8dacb172ab6e7e90 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -45,6 +45,22 @@ struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,
        return dst_vma;
 }
 
+/* Check if dst_addr is outside of file's size. Must be called with ptl held. */
+static bool mfill_file_over_size(struct vm_area_struct *dst_vma,
+                                unsigned long dst_addr)
+{
+       struct inode *inode;
+       pgoff_t offset, max_off;
+
+       if (!dst_vma->vm_file)
+               return false;
+
+       inode = dst_vma->vm_file->f_inode;
+       offset = linear_page_index(dst_vma, dst_addr);
+       max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
+       return offset >= max_off;
+}
+
 /*
  * Install PTEs, to map dst_addr (within dst_vma) to page.
  *
@@ -64,8 +80,6 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd,
        bool page_in_cache = page_mapping(page);
        spinlock_t *ptl;
        struct folio *folio;
-       struct inode *inode;
-       pgoff_t offset, max_off;
 
        _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
        _dst_pte = pte_mkdirty(_dst_pte);
@@ -81,14 +95,9 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd,
        if (!dst_pte)
                goto out;
 
-       if (vma_is_shmem(dst_vma)) {
-               /* serialize against truncate with the page table lock */
-               inode = dst_vma->vm_file->f_inode;
-               offset = linear_page_index(dst_vma, dst_addr);
-               max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
+       if (mfill_file_over_size(dst_vma, dst_addr)) {
                ret = -EFAULT;
-               if (unlikely(offset >= max_off))
-                       goto out_unlock;
+               goto out_unlock;
        }
 
        ret = -EEXIST;
@@ -211,8 +220,6 @@ static int mfill_atomic_pte_zeropage(pmd_t *dst_pmd,
        pte_t _dst_pte, *dst_pte;
        spinlock_t *ptl;
        int ret;
-       pgoff_t offset, max_off;
-       struct inode *inode;
 
        _dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr),
                                         dst_vma->vm_page_prot));
@@ -220,14 +227,9 @@ static int mfill_atomic_pte_zeropage(pmd_t *dst_pmd,
        dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl);
        if (!dst_pte)
                goto out;
-       if (dst_vma->vm_file) {
-               /* the shmem MAP_PRIVATE case requires checking the i_size */
-               inode = dst_vma->vm_file->f_inode;
-               offset = linear_page_index(dst_vma, dst_addr);
-               max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
+       if (mfill_file_over_size(dst_vma, dst_addr)) {
                ret = -EFAULT;
-               if (unlikely(offset >= max_off))
-                       goto out_unlock;
+               goto out_unlock;
        }
        ret = -EEXIST;
        if (!pte_none(ptep_get(dst_pte)))
@@ -286,6 +288,44 @@ out_release:
        goto out;
 }
 
+/* Handles UFFDIO_POISON for all non-hugetlb VMAs. */
+static int mfill_atomic_pte_poison(pmd_t *dst_pmd,
+                                  struct vm_area_struct *dst_vma,
+                                  unsigned long dst_addr,
+                                  uffd_flags_t flags)
+{
+       int ret;
+       struct mm_struct *dst_mm = dst_vma->vm_mm;
+       pte_t _dst_pte, *dst_pte;
+       spinlock_t *ptl;
+
+       _dst_pte = make_pte_marker(PTE_MARKER_POISONED);
+       ret = -EAGAIN;
+       dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
+       if (!dst_pte)
+               goto out;
+
+       if (mfill_file_over_size(dst_vma, dst_addr)) {
+               ret = -EFAULT;
+               goto out_unlock;
+       }
+
+       ret = -EEXIST;
+       /* Refuse to overwrite any PTE, even a PTE marker (e.g. UFFD WP). */
+       if (!pte_none(*dst_pte))
+               goto out_unlock;
+
+       set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
+
+       /* No need to invalidate - it was non-present before */
+       update_mmu_cache(dst_vma, dst_addr, dst_pte);
+       ret = 0;
+out_unlock:
+       pte_unmap_unlock(dst_pte, ptl);
+out:
+       return ret;
+}
+
 static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
 {
        pgd_t *pgd;
@@ -481,6 +521,9 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd,
        if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE)) {
                return mfill_atomic_pte_continue(dst_pmd, dst_vma,
                                                 dst_addr, flags);
+       } else if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) {
+               return mfill_atomic_pte_poison(dst_pmd, dst_vma,
+                                              dst_addr, flags);
        }
 
        /*
@@ -702,6 +745,14 @@ ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long start,
                            uffd_flags_set_mode(flags, MFILL_ATOMIC_CONTINUE));
 }
 
+ssize_t mfill_atomic_poison(struct mm_struct *dst_mm, unsigned long start,
+                           unsigned long len, atomic_t *mmap_changing,
+                           uffd_flags_t flags)
+{
+       return mfill_atomic(dst_mm, start, 0, len, mmap_changing,
+                           uffd_flags_set_mode(flags, MFILL_ATOMIC_POISON));
+}
+
 long uffd_wp_range(struct vm_area_struct *dst_vma,
                   unsigned long start, unsigned long len, bool enable_wp)
 {