mm: introduce memalloc_retry_wait()

Various places in the kernel - largely in filesystems - respond to a
memory allocation failure by looping around and re-trying.  Some of
these cannot conveniently use __GFP_NOFAIL, for reasons such as:

 - a GFP_ATOMIC allocation, which __GFP_NOFAIL doesn't work on
 - a need to check for the process being signalled between failures
 - the possibility that other recovery actions could be performed
 - the allocation is quite deep in support code, and passing down an
   extra flag to say if __GFP_NOFAIL is wanted would be clumsy.

Many of these currently use congestion_wait() which (in almost all
cases) simply waits the given timeout - congestion isn't tracked for
most devices.

It isn't clear what the best delay is for loops, but it is clear that
the various filesystems shouldn't be responsible for choosing a timeout.

This patch introduces memalloc_retry_wait() with takes on that
responsibility.  Code that wants to retry a memory allocation can call
this function passing the GFP flags that were used.  It will wait
however is appropriate.

For now, it only considers __GFP_NORETRY and whatever
gfpflags_allow_blocking() tests.  If blocking is allowed without
__GFP_NORETRY, then alloc_page either made some reclaim progress, or
waited for a while, before failing.  So there is no need for much
further waiting.  memalloc_retry_wait() will wait until the current
jiffie ends.  If this condition is not met, then alloc_page() won't have
waited much if at all.  In that case memalloc_retry_wait() waits about
200ms.  This is the delay that most current loops uses.

linux/sched/mm.h needs to be included in some files now,
but linux/backing-dev.h does not.

Link: https://lkml.kernel.org/r/163754371968.13692.1277530886009912421@noble.neil.brown.name
Signed-off-by: NeilBrown <neilb@suse.de>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: "Theodore Ts'o" <tytso@mit.edu>
Cc: Jaegeuk Kim <jaegeuk@kernel.org>
Cc: Chao Yu <chao@kernel.org>
Cc: Darrick J. Wong <djwong@kernel.org>
Cc: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
index 0ecf819bf1891ab30ab7d53c6f0b19a74758bfd0..5582fba36b4461c40066ff446a8f61d97b0f28d7 100644 (file)
--- a/fs/ext4/extents.c
+++ b/fs/ext4/extents.c
@@ -27,8 +27,8 @@
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/fiemap.h>
-#include <linux/backing-dev.h>
 #include <linux/iomap.h>
+#include <linux/sched/mm.h>
 #include "ext4_jbd2.h"
 #include "ext4_extents.h"
 #include "xattr.h"
@@ -4407,8 +4407,7 @@ retry:
        err = ext4_es_remove_extent(inode, last_block,
                                    EXT_MAX_BLOCKS - last_block);
        if (err == -ENOMEM) {
-               cond_resched();
-               congestion_wait(BLK_RW_ASYNC, HZ/50);
+               memalloc_retry_wait(GFP_ATOMIC);
                goto retry;
        }
        if (err)
@@ -4416,8 +4415,7 @@ retry:
 retry_remove_space:
        err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1);
        if (err == -ENOMEM) {
-               cond_resched();
-               congestion_wait(BLK_RW_ASYNC, HZ/50);
+               memalloc_retry_wait(GFP_ATOMIC);
                goto retry_remove_space;
        }
        return err;

diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c
index 39a1ab129fdc94b2caa18a7febe8a5b0f8ffa7b4..635bcf68a67eca5a29f95e3d9ff6e8cc125e63a2 100644 (file)
--- a/fs/ext4/inline.c
+++ b/fs/ext4/inline.c
@@ -7,7 +7,7 @@
 #include <linux/iomap.h>
 #include <linux/fiemap.h>
 #include <linux/iversion.h>
-#include <linux/backing-dev.h>
+#include <linux/sched/mm.h>
 
 #include "ext4_jbd2.h"
 #include "ext4.h"
@@ -1929,8 +1929,7 @@ int ext4_inline_data_truncate(struct inode *inode, int *has_inline)
 retry:
                        err = ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS);
                        if (err == -ENOMEM) {
-                               cond_resched();
-                               congestion_wait(BLK_RW_ASYNC, HZ/50);
+                               memalloc_retry_wait(GFP_ATOMIC);
                                goto retry;
                        }
                        if (err)

diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
index 9cb2617149918faef7adea35906a4cbf3df6d276..1d370364230e8106fe3eae8d5172ccaf8fa10b3c 100644 (file)
--- a/fs/ext4/page-io.c
+++ b/fs/ext4/page-io.c
@@ -24,7 +24,7 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
-#include <linux/backing-dev.h>
+#include <linux/sched/mm.h>
 
 #include "ext4_jbd2.h"
 #include "xattr.h"
@@ -523,12 +523,13 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
                        ret = PTR_ERR(bounce_page);
                        if (ret == -ENOMEM &&
                            (io->io_bio || wbc->sync_mode == WB_SYNC_ALL)) {
-                               gfp_flags = GFP_NOFS;
+                               gfp_t new_gfp_flags = GFP_NOFS;
                                if (io->io_bio)
                                        ext4_io_submit(io);
                                else
-                                       gfp_flags |= __GFP_NOFAIL;
-                               congestion_wait(BLK_RW_ASYNC, HZ/50);
+                                       new_gfp_flags |= __GFP_NOFAIL;
+                               memalloc_retry_wait(gfp_flags);
+                               gfp_flags = new_gfp_flags;
                                goto retry_encrypt;
                        }
 

diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 9f754aaef558bef37d513f1f957fba471f0600ec..aacf5e4dcc57660c3c1031d2be4c14dc18ceb577 100644 (file)
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -8,9 +8,9 @@
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
 #include <linux/buffer_head.h>
+#include <linux/sched/mm.h>
 #include <linux/mpage.h>
 #include <linux/writeback.h>
-#include <linux/backing-dev.h>
 #include <linux/pagevec.h>
 #include <linux/blkdev.h>
 #include <linux/bio.h>
@@ -2542,7 +2542,7 @@ retry_encrypt:
                /* flush pending IOs and wait for a while in the ENOMEM case */
                if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
                        f2fs_flush_merged_writes(fio->sbi);
-                       congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
+                       memalloc_retry_wait(GFP_NOFS);
                        gfp_flags |= __GFP_NOFAIL;
                        goto retry_encrypt;
                }

diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index a946ce0ead34176b71b3a9afafbba2b458fc1229..374bbb5294d9d7587c5de6475bd130487671a413 100644 (file)
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -7,7 +7,6 @@
  */
 #include <linux/fs.h>
 #include <linux/module.h>
-#include <linux/backing-dev.h>
 #include <linux/init.h>
 #include <linux/f2fs_fs.h>
 #include <linux/kthread.h>
@@ -15,6 +14,7 @@
 #include <linux/freezer.h>
 #include <linux/sched/signal.h>
 #include <linux/random.h>
+#include <linux/sched/mm.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -1375,8 +1375,7 @@ retry:
                if (err) {
                        clear_page_private_gcing(page);
                        if (err == -ENOMEM) {
-                               congestion_wait(BLK_RW_ASYNC,
-                                               DEFAULT_IO_TIMEOUT);
+                               memalloc_retry_wait(GFP_NOFS);
                                goto retry;
                        }
                        if (is_dirty)

diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index 0f8b2df3e1e012ac49cedc3ce47bf3faeeb49ffe..4c11254a07d41ae7584ab7f6bb110544fa340e91 100644 (file)
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -8,8 +8,8 @@
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
 #include <linux/buffer_head.h>
-#include <linux/backing-dev.h>
 #include <linux/writeback.h>
+#include <linux/sched/mm.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -562,7 +562,7 @@ retry:
        inode = f2fs_iget(sb, ino);
        if (IS_ERR(inode)) {
                if (PTR_ERR(inode) == -ENOMEM) {
-                       congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
+                       memalloc_retry_wait(GFP_NOFS);
                        goto retry;
                }
        }

diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index 556fcd8457f3f26d3564842eeef0714106769661..219506ca9a97524e8f5ed8399f54fcba03b96a1d 100644 (file)
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -8,7 +8,7 @@
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
 #include <linux/mpage.h>
-#include <linux/backing-dev.h>
+#include <linux/sched/mm.h>
 #include <linux/blkdev.h>
 #include <linux/pagevec.h>
 #include <linux/swap.h>
@@ -2750,7 +2750,7 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct page *page)
 retry:
        ipage = f2fs_grab_cache_page(NODE_MAPPING(sbi), ino, false);
        if (!ipage) {
-               congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
+               memalloc_retry_wait(GFP_NOFS);
                goto retry;
        }
 

diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c
index 6a1b4668d933aad629a4c4426a12429da1d66b87..d1664a0567efe80bbef0b20bbcbb751af0491cbc 100644 (file)
--- a/fs/f2fs/recovery.c
+++ b/fs/f2fs/recovery.c
@@ -8,6 +8,7 @@
 #include <asm/unaligned.h>
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
+#include <linux/sched/mm.h>
 #include "f2fs.h"
 #include "node.h"
 #include "segment.h"
@@ -587,7 +588,7 @@ retry_dn:
        err = f2fs_get_dnode_of_data(&dn, start, ALLOC_NODE);
        if (err) {
                if (err == -ENOMEM) {
-                       congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
+                       memalloc_retry_wait(GFP_NOFS);
                        goto retry_dn;
                }
                goto out;
@@ -670,8 +671,7 @@ retry_prev:
                        err = check_index_in_prev_nodes(sbi, dest, &dn);
                        if (err) {
                                if (err == -ENOMEM) {
-                                       congestion_wait(BLK_RW_ASYNC,
-                                                       DEFAULT_IO_TIMEOUT);
+                                       memalloc_retry_wait(GFP_NOFS);
                                        goto retry_prev;
                                }
                                goto err;

diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index df9ed75f0b7a766a4466ee0bf2733e15c5e60d36..40fdb4a8daeb6224a9c01f91654016b45f3b962f 100644 (file)
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -9,6 +9,7 @@
 #include <linux/f2fs_fs.h>
 #include <linux/bio.h>
 #include <linux/blkdev.h>
+#include <linux/sched/mm.h>
 #include <linux/prefetch.h>
 #include <linux/kthread.h>
 #include <linux/swap.h>
@@ -245,9 +246,7 @@ retry:
                                                                LOOKUP_NODE);
                        if (err) {
                                if (err == -ENOMEM) {
-                                       congestion_wait(BLK_RW_ASYNC,
-                                                       DEFAULT_IO_TIMEOUT);
-                                       cond_resched();
+                                       memalloc_retry_wait(GFP_NOFS);
                                        goto retry;
                                }
                                err = -EAGAIN;
@@ -424,9 +423,7 @@ retry:
                        err = f2fs_do_write_data_page(&fio);
                        if (err) {
                                if (err == -ENOMEM) {
-                                       congestion_wait(BLK_RW_ASYNC,
-                                                       DEFAULT_IO_TIMEOUT);
-                                       cond_resched();
+                                       memalloc_retry_wait(GFP_NOFS);
                                        goto retry;
                                }
                                unlock_page(page);

diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 040b6d02e1d8a412edb8ddf47138f5abdc288009..3bace24f880052eb9980481fb4e9c46837e95756 100644 (file)
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -8,9 +8,9 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/fs.h>
+#include <linux/sched/mm.h>
 #include <linux/statfs.h>
 #include <linux/buffer_head.h>
-#include <linux/backing-dev.h>
 #include <linux/kthread.h>
 #include <linux/parser.h>
 #include <linux/mount.h>
@@ -2415,8 +2415,7 @@ repeat:
                page = read_cache_page_gfp(mapping, blkidx, GFP_NOFS);
                if (IS_ERR(page)) {
                        if (PTR_ERR(page) == -ENOMEM) {
-                               congestion_wait(BLK_RW_ASYNC,
-                                               DEFAULT_IO_TIMEOUT);
+                               memalloc_retry_wait(GFP_NOFS);
                                goto repeat;
                        }
                        set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR);

diff --git a/fs/xfs/kmem.c b/fs/xfs/kmem.c
index 6f49bf39183c78e7d3cc82f21408a3b06f405453..c557a030acfea42d7a48677cfecc227db3539dfa 100644 (file)
--- a/fs/xfs/kmem.c
+++ b/fs/xfs/kmem.c
@@ -4,7 +4,6 @@
  * All Rights Reserved.
  */
 #include "xfs.h"
-#include <linux/backing-dev.h>
 #include "xfs_message.h"
 #include "xfs_trace.h"
 
@@ -26,6 +25,6 @@ kmem_alloc(size_t size, xfs_km_flags_t flags)
        "%s(%u) possible memory allocation deadlock size %u in %s (mode:0x%x)",
                                current->comm, current->pid,
                                (unsigned int)size, __func__, lflags);
-               congestion_wait(BLK_RW_ASYNC, HZ/50);
+               memalloc_retry_wait(lflags);
        } while (1);
 }

diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index 631c5a61d89b717c9ae5f8e35ce9b000755f17c1..6c45e3fa56f4cd84c7090d38943dd14683e13f85 100644 (file)
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -394,7 +394,7 @@ xfs_buf_alloc_pages(
                }
 
                XFS_STATS_INC(bp->b_mount, xb_page_retries);
-               congestion_wait(BLK_RW_ASYNC, HZ / 50);
+               memalloc_retry_wait(gfp_mask);
        }
        return 0;
 }

diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index aca874d33fe6e541eee8d369fdd23431d9446e46..aa5f09ca5bcf437853dd27d8044a8ced187cb120 100644 (file)
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -214,6 +214,32 @@ static inline void fs_reclaim_acquire(gfp_t gfp_mask) { }
 static inline void fs_reclaim_release(gfp_t gfp_mask) { }
 #endif
 
+/* Any memory-allocation retry loop should use
+ * memalloc_retry_wait(), and pass the flags for the most
+ * constrained allocation attempt that might have failed.
+ * This provides useful documentation of where loops are,
+ * and a central place to fine tune the waiting as the MM
+ * implementation changes.
+ */
+static inline void memalloc_retry_wait(gfp_t gfp_flags)
+{
+       /* We use io_schedule_timeout because waiting for memory
+        * typically included waiting for dirty pages to be
+        * written out, which requires IO.
+        */
+       __set_current_state(TASK_UNINTERRUPTIBLE);
+       gfp_flags = current_gfp_context(gfp_flags);
+       if (gfpflags_allow_blocking(gfp_flags) &&
+           !(gfp_flags & __GFP_NORETRY))
+               /* Probably waited already, no need for much more */
+               io_schedule_timeout(1);
+       else
+               /* Probably didn't wait, and has now released a lock,
+                * so now is a good time to wait
+                */
+               io_schedule_timeout(HZ/50);
+}
+
 /**
  * might_alloc - Mark possible allocation sites
  * @gfp_mask: gfp_t flags that would be used to allocate

diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index 1e99ba1b9d723d007ec3a00044b5ff922b7d8e56..9cb18b822ab2569fcb5d0c58c9c966317382063b 100644 (file)
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/sched.h>
+#include <linux/sched/mm.h>
 #include <linux/errno.h>
 #include <linux/freezer.h>
 #include <linux/kthread.h>
@@ -688,7 +689,7 @@ static int svc_alloc_arg(struct svc_rqst *rqstp)
                        return -EINTR;
                }
                trace_svc_alloc_arg_err(pages);
-               schedule_timeout(msecs_to_jiffies(500));
+               memalloc_retry_wait(GFP_KERNEL);
        }
        rqstp->rq_page_end = &rqstp->rq_pages[pages];
        rqstp->rq_pages[pages] = NULL; /* this might be seen in nfsd_splice_actor() */