mm/hugetlb: allow uffd wr-protect none ptes

Teach hugetlbfs code to wr-protect none ptes just in case the page cache
existed for that pte.  Meanwhile we also need to be able to recognize a
uffd-wp marker pte and remove it for uffd_wp_resolve.

Since at it, introduce a variable "psize" to replace all references to the
huge page size fetcher.

Link: https://lkml.kernel.org/r/20220405014912.14815-1-peterx@redhat.com
Signed-off-by: Peter Xu <peterx@redhat.com>
Reviewed-by: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jerome Glisse <jglisse@redhat.com>
Cc: "Kirill A . Shutemov" <kirill@shutemov.name>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mike Rapoport <rppt@linux.vnet.ibm.com>
Cc: Nadav Amit <nadav.amit@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 25f4ac5ca6fe7462ca66de9ffa7f99ed138507e7..ec9774ed84c018ae67994d9e62e485682a476559 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6251,7 +6251,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
        pte_t *ptep;
        pte_t pte;
        struct hstate *h = hstate_vma(vma);
-       unsigned long pages = 0;
+       unsigned long pages = 0, psize = huge_page_size(h);
        bool shared_pmd = false;
        struct mmu_notifier_range range;
        bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
@@ -6271,13 +6271,19 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
 
        mmu_notifier_invalidate_range_start(&range);
        i_mmap_lock_write(vma->vm_file->f_mapping);
-       for (; address < end; address += huge_page_size(h)) {
+       for (; address < end; address += psize) {
                spinlock_t *ptl;
-               ptep = huge_pte_offset(mm, address, huge_page_size(h));
+               ptep = huge_pte_offset(mm, address, psize);
                if (!ptep)
                        continue;
                ptl = huge_pte_lock(h, mm, ptep);
                if (huge_pmd_unshare(mm, vma, &address, ptep)) {
+                       /*
+                        * When uffd-wp is enabled on the vma, unshare
+                        * shouldn't happen at all.  Warn about it if it
+                        * happened due to some reason.
+                        */
+                       WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
                        pages++;
                        spin_unlock(ptl);
                        shared_pmd = true;
@@ -6307,12 +6313,20 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
                                else if (uffd_wp_resolve)
                                        newpte = pte_swp_clear_uffd_wp(newpte);
                                set_huge_swap_pte_at(mm, address, ptep,
-                                                    newpte, huge_page_size(h));
+                                                    newpte, psize);
                                pages++;
                        }
                        spin_unlock(ptl);
                        continue;
                }
+               if (unlikely(pte_marker_uffd_wp(pte))) {
+                       /*
+                        * This is changing a non-present pte into a none pte,
+                        * no need for huge_ptep_modify_prot_start/commit().
+                        */
+                       if (uffd_wp_resolve)
+                               huge_pte_clear(mm, address, ptep, psize);
+               }
                if (!huge_pte_none(pte)) {
                        pte_t old_pte;
                        unsigned int shift = huge_page_shift(hstate_vma(vma));
@@ -6326,6 +6340,12 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
                                pte = huge_pte_clear_uffd_wp(pte);
                        huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
                        pages++;
+               } else {
+                       /* None pte */
+                       if (unlikely(uffd_wp))
+                               /* Safe to modify directly (none->non-present). */
+                               set_huge_pte_at(mm, address, ptep,
+                                               make_pte_marker(PTE_MARKER_UFFD_WP));
                }
                spin_unlock(ptl);
        }