mm/memory_hotplug: mark pageblocks MIGRATE_ISOLATE while onlining memory

Currently, it can happen that pages are allocated (and freed) via the
buddy before we finished basic memory onlining.

For example, pages are exposed to the buddy and can be allocated before we
actually mark the sections online.  Allocated pages could suddenly fail
pfn_to_online_page() checks.  We had similar issues with pcp handling,
when pages are allocated+freed before we reach zone_pcp_update() in
online_pages() [1].

Instead, mark all pageblocks MIGRATE_ISOLATE, such that allocations are
impossible.  Once done with the heavy lifting, use
undo_isolate_page_range() to move the pages to the MIGRATE_MOVABLE
freelist, marking them ready for allocation.  Similar to offline_pages(),
we have to manually adjust zone->nr_isolate_pageblock.

[1] https://lkml.kernel.org/r/1597150703-19003-1-git-send-email-charante@codeaurora.org

Signed-off-by: David Hildenbrand <david@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: Oscar Salvador <osalvador@suse.de>
Acked-by: Michal Hocko <mhocko@suse.com>
Cc: Wei Yang <richard.weiyang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Pankaj Gupta <pankaj.gupta.linux@gmail.com>
Cc: Charan Teja Reddy <charante@codeaurora.org>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michel Lespinasse <walken@google.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Link: https://lkml.kernel.org/r/20200819175957.28465-11-david@redhat.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/Kconfig b/mm/Kconfig
index e72e61c1d62e1a358881811fe976594d6ab48191..c7f30f8b282be591a2519443a763f4e492df3dac 100644 (file)
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -152,6 +152,7 @@ config HAVE_BOOTMEM_INFO_NODE
 # eventually, we can have this option just 'select SPARSEMEM'
 config MEMORY_HOTPLUG
        bool "Allow for memory hot-add"
+       select MEMORY_ISOLATION
        depends on SPARSEMEM || X86_64_ACPI_NUMA
        depends on ARCH_ENABLE_MEMORY_HOTPLUG
        depends on 64BIT || BROKEN
@@ -178,7 +179,6 @@ config MEMORY_HOTPLUG_DEFAULT_ONLINE
 
 config MEMORY_HOTREMOVE
        bool "Allow for memory hot remove"
-       select MEMORY_ISOLATION
        select HAVE_BOOTMEM_INFO_NODE if (X86_64 || PPC64)
        depends on MEMORY_HOTPLUG && ARCH_ENABLE_MEMORY_HOTREMOVE
        depends on MIGRATION

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index bb30e99b73832d4d7869851338cac3577f733545..1b40eebae3e4104629ea6c38e2f869be27f6680a 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -813,7 +813,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
 
        /* associate pfn range with the zone */
        zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages);
-       move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_MOVABLE);
+       move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);
 
        arg.start_pfn = pfn;
        arg.nr_pages = nr_pages;
@@ -824,6 +824,14 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
        if (ret)
                goto failed_addition;
 
+       /*
+        * Fixup the number of isolated pageblocks before marking the sections
+        * onlining, such that undo_isolate_page_range() works correctly.
+        */
+       spin_lock_irqsave(&zone->lock, flags);
+       zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages;
+       spin_unlock_irqrestore(&zone->lock, flags);
+
        /*
         * If this zone is not populated, then it is not in zonelist.
         * This means the page allocator ignores this zone.
@@ -841,21 +849,25 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
        zone->zone_pgdat->node_present_pages += nr_pages;
        pgdat_resize_unlock(zone->zone_pgdat, &flags);
 
+       node_states_set_node(nid, &arg);
+       if (need_zonelists_rebuild)
+               build_all_zonelists(NULL);
+       zone_pcp_update(zone);
+
+       /* Basic onlining is complete, allow allocation of onlined pages. */
+       undo_isolate_page_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE);
+
        /*
         * When exposing larger, physically contiguous memory areas to the
         * buddy, shuffling in the buddy (when freeing onlined pages, putting
         * them either to the head or the tail of the freelist) is only helpful
         * for maintaining the shuffle, but not for creating the initial
         * shuffle. Shuffle the whole zone to make sure the just onlined pages
-        * are properly distributed across the whole freelist.
+        * are properly distributed across the whole freelist. Make sure to
+        * shuffle once pageblocks are no longer isolated.
         */
        shuffle_zone(zone);
 
-       node_states_set_node(nid, &arg);
-       if (need_zonelists_rebuild)
-               build_all_zonelists(NULL);
-       zone_pcp_update(zone);
-
        init_per_zone_wmark_min();
 
        kswapd_run(nid);
@@ -1577,9 +1589,9 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages)
        pr_info("Offlined Pages %ld\n", nr_pages);
 
        /*
-        * Onlining will reset pagetype flags and makes migrate type
-        * MOVABLE, so just need to decrease the number of isolated
-        * pageblocks zone counter here.
+        * The memory sections are marked offline, and the pageblock flags
+        * effectively stale; nobody should be touching them. Fixup the number
+        * of isolated pageblocks, memory onlining will properly revert this.
         */
        spin_lock_irqsave(&zone->lock, flags);
        zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages;