mm: pagewalk: Fix race between unmap and page walker

commit 46642b56ef6991f008bb6e9589b2cadd79959ef9 upstream.

The mmap lock protects the page walker from changes to the page tables
during the walk.  However a read lock is insufficient to protect those
areas which don't have a VMA as munmap() detaches the VMAs before
downgrading to a read lock and actually tearing down PTEs/page tables.

For users of walk_page_range() the solution is to simply call pte_hole()
immediately without checking the actual page tables when a VMA is not
present. We now never call __walk_page_range() without a valid vma.

For walk_page_range_novma() the locking requirements are tightened to
require the mmap write lock to be taken, and then walking the pgd
directly with 'no_vma' set.

This in turn means that all page walkers either have a valid vma, or
it's that special 'novma' case for page table debugging.  As a result,
all the odd '(!walk->vma && !walk->no_vma)' tests can be removed.

Fixes: 5dcd5782f466 ("mm: mmap: zap pages with read mmap_sem in munmap")
Reported-by: Jann Horn <jannh@google.com>
Signed-off-by: Steven Price <steven.price@arm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
[manually backported. backport note: walk_page_range_novma() does not exist in
5.4, so I'm omitting it from the backport]
Signed-off-by: Jann Horn <jannh@google.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 4eb09e089881768c971d2b603a1302a56aeb59bb..ec41e7552f37ce8b3e5a8eab831fbbf28a2e6eab 100644 (file)
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -38,7 +38,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
        do {
 again:
                next = pmd_addr_end(addr, end);
-               if (pmd_none(*pmd) || !walk->vma) {
+               if (pmd_none(*pmd)) {
                        if (ops->pte_hole)
                                err = ops->pte_hole(addr, next, walk);
                        if (err)
@@ -84,7 +84,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
        do {
  again:
                next = pud_addr_end(addr, end);
-               if (pud_none(*pud) || !walk->vma) {
+               if (pud_none(*pud)) {
                        if (ops->pte_hole)
                                err = ops->pte_hole(addr, next, walk);
                        if (err)
@@ -254,7 +254,7 @@ static int __walk_page_range(unsigned long start, unsigned long end,
        int err = 0;
        struct vm_area_struct *vma = walk->vma;
 
-       if (vma && is_vm_hugetlb_page(vma)) {
+       if (is_vm_hugetlb_page(vma)) {
                if (walk->ops->hugetlb_entry)
                        err = walk_hugetlb_range(start, end, walk);
        } else
@@ -324,9 +324,13 @@ int walk_page_range(struct mm_struct *mm, unsigned long start,
                if (!vma) { /* after the last vma */
                        walk.vma = NULL;
                        next = end;
+                       if (ops->pte_hole)
+                               err = ops->pte_hole(start, next, &walk);
                } else if (start < vma->vm_start) { /* outside vma */
                        walk.vma = NULL;
                        next = min(end, vma->vm_start);
+                       if (ops->pte_hole)
+                               err = ops->pte_hole(start, next, &walk);
                } else { /* inside vma */
                        walk.vma = vma;
                        next = min(end, vma->vm_end);
@@ -344,9 +348,8 @@ int walk_page_range(struct mm_struct *mm, unsigned long start,
                        }
                        if (err < 0)
                                break;
-               }
-               if (walk.vma || walk.ops->pte_hole)
                        err = __walk_page_range(start, next, &walk);
+               }
                if (err)
                        break;
        } while (start = next, start < end);