thp: consolidate code between handle_mm_fault() and do_huge_pmd_anonymous_page()

do_huge_pmd_anonymous_page() has copy-pasted piece of handle_mm_fault()
to handle fallback path.

Let's consolidate code back by introducing VM_FAULT_FALLBACK return
code.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Hillf Danton <dhillf@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Hugh Dickins <hughd@google.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Matthew Wilcox <willy@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index b60de92e2edc4ecadc2681a77b8973c1087dd748..3935428c57cff80d01236428fdea979d7a9f911b 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -96,9 +96,6 @@ extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                          pmd_t *dst_pmd, pmd_t *src_pmd,
                          struct vm_area_struct *vma,
                          unsigned long addr, unsigned long end);
-extern int handle_pte_fault(struct mm_struct *mm,
-                           struct vm_area_struct *vma, unsigned long address,
-                           pte_t *pte, pmd_t *pmd, unsigned int flags);
 extern int split_huge_page_to_list(struct page *page, struct list_head *list);
 static inline int split_huge_page(struct page *page)
 {

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 928df792c0052346ba38a0befef0d22a66559200..8b6e55ee885576d3dc3613a23a990cd20da4bfed 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -877,11 +877,12 @@ static inline int page_mapped(struct page *page)
 #define VM_FAULT_NOPAGE        0x0100  /* ->fault installed the pte, not return page */
 #define VM_FAULT_LOCKED        0x0200  /* ->fault locked the returned page */
 #define VM_FAULT_RETRY 0x0400  /* ->fault blocked, must retry */
+#define VM_FAULT_FALLBACK 0x0800       /* huge page fault failed, fall back to small */
 
 #define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
 
 #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \
-                        VM_FAULT_HWPOISON_LARGE)
+                        VM_FAULT_FALLBACK | VM_FAULT_HWPOISON_LARGE)
 
 /* Encode hstate index for a hwpoisoned large page */
 #define VM_FAULT_SET_HINDEX(x) ((x) << 12)

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 6551dd06dd643cd5856b2c65f1e6186e1b34fc7e..243f4cc757775533f369a1d8acaee7d12c607e53 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -783,10 +783,9 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
 {
        struct page *page;
        unsigned long haddr = address & HPAGE_PMD_MASK;
-       pte_t *pte;
 
        if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
-               goto out;
+               return VM_FAULT_FALLBACK;
        if (unlikely(anon_vma_prepare(vma)))
                return VM_FAULT_OOM;
        if (unlikely(khugepaged_enter(vma)))
@@ -803,7 +802,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                if (unlikely(!zero_page)) {
                        pte_free(mm, pgtable);
                        count_vm_event(THP_FAULT_FALLBACK);
-                       goto out;
+                       return VM_FAULT_FALLBACK;
                }
                spin_lock(&mm->page_table_lock);
                set = set_huge_zero_page(pgtable, mm, vma, haddr, pmd,
@@ -819,40 +818,20 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                        vma, haddr, numa_node_id(), 0);
        if (unlikely(!page)) {
                count_vm_event(THP_FAULT_FALLBACK);
-               goto out;
+               return VM_FAULT_FALLBACK;
        }
        count_vm_event(THP_FAULT_ALLOC);
        if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
                put_page(page);
-               goto out;
+               return VM_FAULT_FALLBACK;
        }
        if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page))) {
                mem_cgroup_uncharge_page(page);
                put_page(page);
-               goto out;
+               return VM_FAULT_FALLBACK;
        }
 
        return 0;
-out:
-       /*
-        * Use __pte_alloc instead of pte_alloc_map, because we can't
-        * run pte_offset_map on the pmd, if an huge pmd could
-        * materialize from under us from a different thread.
-        */
-       if (unlikely(pmd_none(*pmd)) &&
-           unlikely(__pte_alloc(mm, vma, pmd, address)))
-               return VM_FAULT_OOM;
-       /* if an huge pmd materialized from under us just retry later */
-       if (unlikely(pmd_trans_huge(*pmd)))
-               return 0;
-       /*
-        * A regular pmd is established and it can't morph into a huge pmd
-        * from under us anymore at this point because we hold the mmap_sem
-        * read mode and khugepaged takes it in write mode. So now it's
-        * safe to run pte_offset_map().
-        */
-       pte = pte_offset_map(pmd, address);
-       return handle_pte_fault(mm, vma, address, pte, pmd, flags);
 }
 
 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,

diff --git a/mm/memory.c b/mm/memory.c
index 5ec6f199e68581bf62f2ba30efc2c207a991cd17..ca00039471152eae75bb8321a129a4edfc6f580e 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3695,7 +3695,7 @@ static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
  * but allow concurrent faults), and pte mapped but not yet locked.
  * We return with mmap_sem still held, but pte unmapped and unlocked.
  */
-int handle_pte_fault(struct mm_struct *mm,
+static int handle_pte_fault(struct mm_struct *mm,
                     struct vm_area_struct *vma, unsigned long address,
                     pte_t *pte, pmd_t *pmd, unsigned int flags)
 {
@@ -3774,9 +3774,12 @@ retry:
        if (!pmd)
                return VM_FAULT_OOM;
        if (pmd_none(*pmd) && transparent_hugepage_enabled(vma)) {
+               int ret = VM_FAULT_FALLBACK;
                if (!vma->vm_ops)
-                       return do_huge_pmd_anonymous_page(mm, vma, address,
-                                                         pmd, flags);
+                       ret = do_huge_pmd_anonymous_page(mm, vma, address,
+                                       pmd, flags);
+               if (!(ret & VM_FAULT_FALLBACK))
+                       return ret;
        } else {
                pmd_t orig_pmd = *pmd;
                int ret;