Add a configure option to group pages by mobility

The grouping mechanism has some memory overhead and a more complex allocation
path.  This patch allows the strategy to be disabled for small memory systems
or if it is known the workload is suffering because of the strategy.  It also
acts to show where the page groupings strategy interacts with the standard
buddy allocator.

Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Joel Schopp <jschopp@austin.ibm.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 57700038e669403db2c783791d6d5dcc2452411f..7d7e4fe0fda8325381a8aae3ffa2a50800b75915 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -33,9 +33,15 @@
  */
 #define PAGE_ALLOC_COSTLY_ORDER 3
 
+#ifdef CONFIG_PAGE_GROUP_BY_MOBILITY
 #define MIGRATE_UNMOVABLE     0
 #define MIGRATE_MOVABLE       1
 #define MIGRATE_TYPES         2
+#else
+#define MIGRATE_UNMOVABLE     0
+#define MIGRATE_MOVABLE       0
+#define MIGRATE_TYPES         1
+#endif
 
 #define for_each_migratetype_order(order, type) \
        for (order = 0; order < MAX_ORDER; order++) \

diff --git a/init/Kconfig b/init/Kconfig
index 54f31a191b8813a1bac92190cd237023c30a684d..bab643f7717f2a0be9b2b63d7c4d14acdb5a99e5 100644 (file)
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -607,6 +607,19 @@ config BASE_SMALL
        default 0 if BASE_FULL
        default 1 if !BASE_FULL
 
+config PAGE_GROUP_BY_MOBILITY
+       bool "Group pages based on their mobility in the page allocator"
+       def_bool y
+       help
+         The standard allocator will fragment memory over time which means
+         that high order allocations will fail even if kswapd is running. If
+         this option is set, the allocator will try and group page types
+         based on their ability to migrate or reclaim. This is a best effort
+         attempt at lowering fragmentation which a few workloads care about.
+         The loss is a more complex allocator that may perform slower. If
+         you are interested in working with large pages, say Y and set
+         /proc/sys/vm/min_free_bytes to 16374. Otherwise say N
+
 menuconfig MODULES
        bool "Enable loadable module support"
        help

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e3e726bd2858f6b97df2e92e4cf7f3ea9073fbd2..fea1e3b56c3d6626c5cf59bccd14f84ea37fbaf4 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -158,6 +158,7 @@ int nr_node_ids __read_mostly = MAX_NUMNODES;
 EXPORT_SYMBOL(nr_node_ids);
 #endif
 
+#ifdef CONFIG_PAGE_GROUP_BY_MOBILITY
 static inline int get_pageblock_migratetype(struct page *page)
 {
        return get_pageblock_flags_group(page, PB_migrate, PB_migrate_end);
@@ -174,6 +175,22 @@ static inline int gfpflags_to_migratetype(gfp_t gfp_flags)
        return ((gfp_flags & __GFP_MOVABLE) != 0);
 }
 
+#else
+static inline int get_pageblock_migratetype(struct page *page)
+{
+       return MIGRATE_UNMOVABLE;
+}
+
+static void set_pageblock_migratetype(struct page *page, int migratetype)
+{
+}
+
+static inline int gfpflags_to_migratetype(gfp_t gfp_flags)
+{
+       return MIGRATE_UNMOVABLE;
+}
+#endif /* CONFIG_PAGE_GROUP_BY_MOBILITY */
+
 #ifdef CONFIG_DEBUG_VM
 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 {
@@ -653,6 +670,7 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
        return 0;
 }
 
+#ifdef CONFIG_PAGE_GROUP_BY_MOBILITY
 /*
  * This array describes the order lists are fallen back to when
  * the free lists for the desirable migrate type are depleted
@@ -709,6 +727,13 @@ static struct page *__rmqueue_fallback(struct zone *zone, int order,
 
        return NULL;
 }
+#else
+static struct page *__rmqueue_fallback(struct zone *zone, int order,
+                                               int start_migratetype)
+{
+       return NULL;
+}
+#endif /* CONFIG_PAGE_GROUP_BY_MOBILITY */
 
 /* 
  * Do the hard work of removing an element from the buddy allocator.
@@ -953,27 +978,25 @@ again:
                        if (unlikely(!pcp->count))
                                goto failed;
                }
+
+#ifdef CONFIG_PAGE_GROUP_BY_MOBILITY
                /* Find a page of the appropriate migrate type */
-               list_for_each_entry(page, &pcp->list, lru) {
-                       if (page_private(page) == migratetype) {
-                               list_del(&page->lru);
-                               pcp->count--;
+               list_for_each_entry(page, &pcp->list, lru)
+                       if (page_private(page) == migratetype)
                                break;
-                       }
-               }
 
-               /*
-                * Check if a page of the appropriate migrate type
-                * was found. If not, allocate more to the pcp list
-                */
-               if (&page->lru == &pcp->list) {
+               /* Allocate more to the pcp list if necessary */
+               if (unlikely(&page->lru == &pcp->list)) {
                        pcp->count += rmqueue_bulk(zone, 0,
                                        pcp->batch, &pcp->list, migratetype);
                        page = list_entry(pcp->list.next, struct page, lru);
-                       VM_BUG_ON(page_private(page) != migratetype);
-                       list_del(&page->lru);
-                       pcp->count--;
                }
+#else
+               page = list_entry(pcp->list.next, struct page, lru);
+#endif /* CONFIG_PAGE_GROUP_BY_MOBILITY */
+
+               list_del(&page->lru);
+               pcp->count--;
        } else {
                spin_lock_irqsave(&zone->lock, flags);
                page = __rmqueue(zone, order, migratetype);