mm/migrate: fix race between lock page and clear PG_Isolated

When memory is tight, system may start to compact memory for large
continuous memory demands.  If one process tries to lock a memory page
that is being locked and isolated for compaction, it may wait a long time
or even forever.  This is because compaction will perform non-atomic
PG_Isolated clear while holding page lock, this may overwrite PG_waiters
set by the process that can't obtain the page lock and add itself to the
waiting queue to wait for the lock to be unlocked.

  CPU1                            CPU2
  lock_page(page); (successful)
                                  lock_page(); (failed)
  __ClearPageIsolated(page);      SetPageWaiters(page) (may be overwritten)
  unlock_page(page);

The solution is to not perform non-atomic operation on page flags while
holding page lock.

Link: https://lkml.kernel.org/r/20220315030515.20263-1-andrew.yang@mediatek.com
Signed-off-by: andrew.yang <andrew.yang@mediatek.com>
Cc: Matthias Brugger <matthias.bgg@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: "Vlastimil Babka" <vbabka@suse.cz>
Cc: David Howells <dhowells@redhat.com>
Cc: "William Kucharski" <william.kucharski@oracle.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Nicholas Tang <nicholas.tang@mediatek.com>
Cc: Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 340cb81565683f45134468efce45a71d6df6b9eb..88fe1d759cdd49eb5052f811d1cc8faa2702f66d 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -1000,7 +1000,7 @@ PAGE_TYPE_OPS(Guard, guard)
 
 extern bool is_free_buddy_page(struct page *page);
 
-__PAGEFLAG(Isolated, isolated, PF_ANY);
+PAGEFLAG(Isolated, isolated, PF_ANY);
 
 #ifdef CONFIG_MMU
 #define __PG_MLOCKED           (1UL << PG_mlocked)

diff --git a/mm/migrate.c b/mm/migrate.c
index 67616ee4aa26ad02331739dfecfee6f9a5c22355..c0d16f050fec2ba2a5f744ece3c3421ab71482ea 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -107,7 +107,7 @@ int isolate_movable_page(struct page *page, isolate_mode_t mode)
 
        /* Driver shouldn't use PG_isolated bit of page->flags */
        WARN_ON_ONCE(PageIsolated(page));
-       __SetPageIsolated(page);
+       SetPageIsolated(page);
        unlock_page(page);
 
        return 0;
@@ -126,7 +126,7 @@ static void putback_movable_page(struct page *page)
 
        mapping = page_mapping(page);
        mapping->a_ops->putback_page(page);
-       __ClearPageIsolated(page);
+       ClearPageIsolated(page);
 }
 
 /*
@@ -159,7 +159,7 @@ void putback_movable_pages(struct list_head *l)
                        if (PageMovable(page))
                                putback_movable_page(page);
                        else
-                               __ClearPageIsolated(page);
+                               ClearPageIsolated(page);
                        unlock_page(page);
                        put_page(page);
                } else {
@@ -883,7 +883,7 @@ static int move_to_new_page(struct page *newpage, struct page *page,
                VM_BUG_ON_PAGE(!PageIsolated(page), page);
                if (!PageMovable(page)) {
                        rc = MIGRATEPAGE_SUCCESS;
-                       __ClearPageIsolated(page);
+                       ClearPageIsolated(page);
                        goto out;
                }
 
@@ -905,7 +905,7 @@ static int move_to_new_page(struct page *newpage, struct page *page,
                         * We clear PG_movable under page_lock so any compactor
                         * cannot try to migrate this page.
                         */
-                       __ClearPageIsolated(page);
+                       ClearPageIsolated(page);
                }
 
                /*
@@ -1091,7 +1091,7 @@ static int unmap_and_move(new_page_t get_new_page,
                if (unlikely(__PageMovable(page))) {
                        lock_page(page);
                        if (!PageMovable(page))
-                               __ClearPageIsolated(page);
+                               ClearPageIsolated(page);
                        unlock_page(page);
                }
                goto out;