mm: make alloc_contig_range handle in-use hugetlb pages

alloc_contig_range() will fail if it finds a HugeTLB page within the
range, without a chance to handle them.  Since HugeTLB pages can be
migrated as any LRU or Movable page, it does not make sense to bail out
without trying.  Enable the interface to recognize in-use HugeTLB pages so
we can migrate them, and have much better chances to succeed the call.

Link: https://lkml.kernel.org/r/20210419075413.1064-7-osalvador@suse.de
Signed-off-by: Oscar Salvador <osalvador@suse.de>
Reviewed-by: Mike Kravetz <mike.kravetz@oracle.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: David Hildenbrand <david@redhat.com>
Cc: Muchun Song <songmuchun@bytedance.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index ec6a10b8860a3eba21b2e0e1b4ce76d97a527f44..d0f310ae3f82fc31a7b71ff48799bc418e0bb95a 100644 (file)
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -588,7 +588,7 @@ struct huge_bootmem_page {
        struct hstate *hstate;
 };
 
-int isolate_or_dissolve_huge_page(struct page *page);
+int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list);
 struct page *alloc_huge_page(struct vm_area_struct *vma,
                                unsigned long addr, int avoid_reserve);
 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
@@ -871,7 +871,8 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
 #else  /* CONFIG_HUGETLB_PAGE */
 struct hstate {};
 
-static inline int isolate_or_dissolve_huge_page(struct page *page)
+static inline int isolate_or_dissolve_huge_page(struct page *page,
+                                               struct list_head *list)
 {
        return -ENOMEM;
 }

diff --git a/mm/compaction.c b/mm/compaction.c
index b77e1382307f1da0511dd4ed16b2f981fd179b3d..335862f1661c17e0db135e672f56ec887deb8c00 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -907,7 +907,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                }
 
                if (PageHuge(page) && cc->alloc_contig) {
-                       ret = isolate_or_dissolve_huge_page(page);
+                       ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);
 
                        /*
                         * Fail isolation in case isolate_or_dissolve_huge_page()
@@ -921,6 +921,15 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                                goto isolate_fail;
                        }
 
+                       if (PageHuge(page)) {
+                               /*
+                                * Hugepage was successfully isolated and placed
+                                * on the cc->migratepages list.
+                                */
+                               low_pfn += compound_nr(page) - 1;
+                               goto isolate_success_no_list;
+                       }
+
                        /*
                         * Ok, the hugepage was dissolved. Now these pages are
                         * Buddy and cannot be re-allocated because they are
@@ -1062,6 +1071,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
 isolate_success:
                list_add(&page->lru, &cc->migratepages);
+isolate_success_no_list:
                cc->nr_migratepages += compound_nr(page);
                nr_isolated += compound_nr(page);
 

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 92f3cd08946f1e061274acf28ef8f0a521858333..b5977d9709ad1479f1b727b1ac2c3efa2f925cea 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -2271,9 +2271,11 @@ static void restore_reserve_on_error(struct hstate *h,
  * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
  * @h: struct hstate old page belongs to
  * @old_page: Old page to dissolve
+ * @list: List to isolate the page in case we need to
  * Returns 0 on success, otherwise negated error.
  */
-static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page)
+static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
+                                       struct list_head *list)
 {
        gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
        int nid = page_to_nid(old_page);
@@ -2300,9 +2302,13 @@ retry:
                goto free_new;
        } else if (page_count(old_page)) {
                /*
-                * Someone has grabbed the page, fail for now.
+                * Someone has grabbed the page, try to isolate it here.
+                * Fail with -EBUSY if not possible.
                 */
-               ret = -EBUSY;
+               spin_unlock_irq(&hugetlb_lock);
+               if (!isolate_huge_page(old_page, list))
+                       ret = -EBUSY;
+               spin_lock_irq(&hugetlb_lock);
                goto free_new;
        } else if (!HPageFreed(old_page)) {
                /*
@@ -2352,10 +2358,11 @@ free_new:
        return ret;
 }
 
-int isolate_or_dissolve_huge_page(struct page *page)
+int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
 {
        struct hstate *h;
        struct page *head;
+       int ret = -EBUSY;
 
        /*
         * The page might have been dissolved from under our feet, so make sure
@@ -2380,7 +2387,12 @@ int isolate_or_dissolve_huge_page(struct page *page)
        if (hstate_is_gigantic(h))
                return -ENOMEM;
 
-       return alloc_and_dissolve_huge_page(h, head);
+       if (page_count(head) && isolate_huge_page(head, list))
+               ret = 0;
+       else if (!page_count(head))
+               ret = alloc_and_dissolve_huge_page(h, head, list);
+
+       return ret;
 }
 
 struct page *alloc_huge_page(struct vm_area_struct *vma,

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 562e87cbd7a1ab94203489e9cd9b4e8bec278aea..42aaef30633e3ccefcf275d7b368aae56551b6f1 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1507,8 +1507,9 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
        LIST_HEAD(clean_pages);
 
        list_for_each_entry_safe(page, next, page_list, lru) {
-               if (page_is_file_lru(page) && !PageDirty(page) &&
-                   !__PageMovable(page) && !PageUnevictable(page)) {
+               if (!PageHuge(page) && page_is_file_lru(page) &&
+                   !PageDirty(page) && !__PageMovable(page) &&
+                   !PageUnevictable(page)) {
                        ClearPageActive(page);
                        list_move(&page->lru, &clean_pages);
                }