kernel/fork: use maple tree for dup_mmap() during forking

The maple tree was already tracking VMAs in this function by an earlier
commit, but the rbtree iterator was being used to iterate the list.
Change the iterator to use a maple tree native iterator and switch to the
maple tree advanced API to avoid multiple walks of the tree during insert
operations.  Unexport the now-unused vma_store() function.

For performance reasons we bulk allocate the maple tree nodes.  The node
calculations are done internally to the tree and use the VMA count and
assume the worst-case node requirements.  The VM_DONT_COPY flag does not
allow for the most efficient copy method of the tree and so a bulk loading
algorithm is used.

Link: https://lkml.kernel.org/r/20220906194824.2110408-15-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Tested-by: Yu Zhao <yuzhao@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 3701da1fac5fdd73d940b0265f8ec92c784876cc..646ea4d3bd74eb3d00fc53316a4eb1aae23a7b64 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2599,8 +2599,6 @@ extern bool arch_has_descending_max_zone_pfns(void);
 /* nommu.c */
 extern atomic_long_t mmap_pages_allocated;
 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
-/* mmap.c */
-void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas);
 
 /* interval_tree.c */
 void vma_interval_tree_insert(struct vm_area_struct *node,

diff --git a/kernel/fork.c b/kernel/fork.c
index 273364207f17b56056639aa13b4b80b3aa1ec05e..16970c346b5b8bcd650a549c919e026a308b1433 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -583,8 +583,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
        struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
        struct rb_node **rb_link, *rb_parent;
        int retval;
-       unsigned long charge;
+       unsigned long charge = 0;
        LIST_HEAD(uf);
+       MA_STATE(old_mas, &oldmm->mm_mt, 0, 0);
        MA_STATE(mas, &mm->mm_mt, 0, 0);
 
        uprobe_start_dup_mmap();
@@ -620,7 +621,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                goto out;
 
        prev = NULL;
-       for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
+
+       retval = mas_expected_entries(&mas, oldmm->map_count);
+       if (retval)
+               goto out;
+
+       mas_for_each(&old_mas, mpnt, ULONG_MAX) {
                struct file *file;
 
                if (mpnt->vm_flags & VM_DONTCOPY) {
@@ -703,6 +709,8 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                mas.index = tmp->vm_start;
                mas.last = tmp->vm_end - 1;
                mas_store(&mas, tmp);
+               if (mas_is_err(&mas))
+                       goto fail_nomem_mas_store;
 
                mm->map_count++;
                if (!(tmp->vm_flags & VM_WIPEONFORK))
@@ -726,6 +734,9 @@ out:
 fail_uprobe_end:
        uprobe_end_dup_mmap();
        return retval;
+
+fail_nomem_mas_store:
+       unlink_anon_vmas(tmp);
 fail_nomem_anon_vma_fork:
        mpol_put(vma_policy(tmp));
 fail_nomem_policy: