mm: fix GFP_THISNODE callers and clarify

GFP_THISNODE is for callers that implement their own clever fallback to
remote nodes.  It restricts the allocation to the specified node and
does not invoke reclaim, assuming that the caller will take care of it
when the fallback fails, e.g.  through a subsequent allocation request
without GFP_THISNODE set.

However, many current GFP_THISNODE users only want the node exclusive
aspect of the flag, without actually implementing their own fallback or
triggering reclaim if necessary.  This results in things like page
migration failing prematurely even when there is easily reclaimable
memory available, unless kswapd happens to be running already or a
concurrent allocation attempt triggers the necessary reclaim.

Convert all callsites that don't implement their own fallback strategy
to __GFP_THISNODE.  This restricts the allocation a single node too, but
at the same time allows the allocator to enter the slowpath, wake
kswapd, and invoke direct reclaim if necessary, to make the allocation
happen when memory is full.

Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Jan Stancek <jstancek@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c
index a96bcf83a735dae2f812f81e0fc54635bb3de072..20e8a9b21d7519ebf7d506e4825d83c05629ba37 100644 (file)
--- a/arch/ia64/kernel/uncached.c
+++ b/arch/ia64/kernel/uncached.c
@@ -98,7 +98,7 @@ static int uncached_add_chunk(struct uncached_pool *uc_pool, int nid)
        /* attempt to allocate a granule's worth of cached memory pages */
 
        page = alloc_pages_exact_node(nid,
-                               GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
+                               GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE,
                                IA64_GRANULE_SHIFT-PAGE_SHIFT);
        if (!page) {
                mutex_unlock(&uc_pool->add_chunk_mutex);

diff --git a/arch/powerpc/platforms/cell/ras.c b/arch/powerpc/platforms/cell/ras.c
index 5ec1e47a0d771eba56e8b8c42111a19aed7df25a..e865d748179b2ac1b0b7550c1d3bebbcae71e049 100644 (file)
--- a/arch/powerpc/platforms/cell/ras.c
+++ b/arch/powerpc/platforms/cell/ras.c
@@ -123,7 +123,8 @@ static int __init cbe_ptcal_enable_on_node(int nid, int order)
 
        area->nid = nid;
        area->order = order;
-       area->pages = alloc_pages_exact_node(area->nid, GFP_KERNEL|GFP_THISNODE,
+       area->pages = alloc_pages_exact_node(area->nid,
+                                               GFP_KERNEL|__GFP_THISNODE,
                                                area->order);
 
        if (!area->pages) {

diff --git a/drivers/misc/sgi-xp/xpc_uv.c b/drivers/misc/sgi-xp/xpc_uv.c
index b9e2000969f025894be4bcae2c6950d699ded3a9..95c894482fddf443d4516ffad39c54adda5be754 100644 (file)
--- a/drivers/misc/sgi-xp/xpc_uv.c
+++ b/drivers/misc/sgi-xp/xpc_uv.c
@@ -240,7 +240,7 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
 
        nid = cpu_to_node(cpu);
        page = alloc_pages_exact_node(nid,
-                                     GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
+                                     GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE,
                                      pg_order);
        if (page == NULL) {
                dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d "

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 0437439bc047bd4a99d7465132b1d9417667934e..39b81dc7d01acdb28e6e888fea3399f7866057f8 100644 (file)
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -123,6 +123,10 @@ struct vm_area_struct;
                         __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | \
                         __GFP_NO_KSWAPD)
 
+/*
+ * GFP_THISNODE does not perform any reclaim, you most likely want to
+ * use __GFP_THISNODE to allocate from a given node without fallback!
+ */
 #ifdef CONFIG_NUMA
 #define GFP_THISNODE   (__GFP_THISNODE | __GFP_NOWARN | __GFP_NORETRY)
 #else

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 5f2052c831547a33b66606d78ac5713086d285e3..9b61b9bf81ac86a90c1ce70d12400b745b7582eb 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -590,10 +590,10 @@ static inline bool zone_is_empty(struct zone *zone)
 
 /*
  * The NUMA zonelists are doubled because we need zonelists that restrict the
- * allocations to a single node for GFP_THISNODE.
+ * allocations to a single node for __GFP_THISNODE.
  *
  * [0] : Zonelist with fallback
- * [1] : No fallback (GFP_THISNODE)
+ * [1] : No fallback (__GFP_THISNODE)
  */
 #define MAX_ZONELISTS 2
 

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 9260abdd67df801e002f18faf3f17bd00d494867..b5b2df60299e25c5c33b3be39ff90090c974689f 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -410,7 +410,7 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
  *
  * %GFP_NOWAIT - Allocation will not sleep.
  *
- * %GFP_THISNODE - Allocate node-local memory only.
+ * %__GFP_THISNODE - Allocate node-local memory only.
  *
  * %GFP_DMA - Allocation suitable for DMA.
  *   Should only be used for kmalloc() caches. Otherwise, use a

diff --git a/kernel/profile.c b/kernel/profile.c
index 6631e1ef55ab0970f095b42ec350982abd8d9678..ebdd9c1a86b4dfd7840df57c64f13f72a1401156 100644 (file)
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -549,14 +549,14 @@ static int create_hash_tables(void)
                struct page *page;
 
                page = alloc_pages_exact_node(node,
-                               GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
+                               GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE,
                                0);
                if (!page)
                        goto out_cleanup;
                per_cpu(cpu_profile_hits, cpu)[1]
                                = (struct profile_hit *)page_address(page);
                page = alloc_pages_exact_node(node,
-                               GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
+                               GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE,
                                0);
                if (!page)
                        goto out_cleanup;

diff --git a/mm/migrate.c b/mm/migrate.c
index 482a33d89134fd83e15d5f6863dae57f5cb05aac..b494fdb9a6363e058f802069ba5de8e677005b45 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1158,7 +1158,7 @@ static struct page *new_page_node(struct page *p, unsigned long private,
                                        pm->node);
        else
                return alloc_pages_exact_node(pm->node,
-                               GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
+                               GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 0);
 }
 
 /*
@@ -1544,9 +1544,9 @@ static struct page *alloc_misplaced_dst_page(struct page *page,
        struct page *newpage;
 
        newpage = alloc_pages_exact_node(nid,
-                                        (GFP_HIGHUSER_MOVABLE | GFP_THISNODE |
-                                         __GFP_NOMEMALLOC | __GFP_NORETRY |
-                                         __GFP_NOWARN) &
+                                        (GFP_HIGHUSER_MOVABLE |
+                                         __GFP_THISNODE | __GFP_NOMEMALLOC |
+                                         __GFP_NORETRY | __GFP_NOWARN) &
                                         ~GFP_IOFS, 0);
 
        return newpage;
@@ -1747,7 +1747,8 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
                goto out_dropref;
 
        new_page = alloc_pages_node(node,
-               (GFP_TRANSHUGE | GFP_THISNODE) & ~__GFP_WAIT, HPAGE_PMD_ORDER);
+               (GFP_TRANSHUGE | __GFP_THISNODE) & ~__GFP_WAIT,
+               HPAGE_PMD_ORDER);
        if (!new_page)
                goto out_fail;