page_alloc.c (4ed7e02222aba062bd0ed3ab12dfc8e9fc0467b5) page_alloc.c (ee6f509c3274014d1f52e7a7a10aee9f85393c5e)
1/*
2 * linux/mm/page_alloc.c
3 *
4 * Manages the free list, the system allocates free pages here.
5 * Note that kmalloc() lives in slab.c
6 *
7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Swap reorganised 29.12.95, Stephen Tweedie

--- 37 unchanged lines hidden (view full) ---

46#include <linux/sort.h>
47#include <linux/pfn.h>
48#include <linux/backing-dev.h>
49#include <linux/fault-inject.h>
50#include <linux/page-isolation.h>
51#include <linux/page_cgroup.h>
52#include <linux/debugobjects.h>
53#include <linux/kmemleak.h>
1/*
2 * linux/mm/page_alloc.c
3 *
4 * Manages the free list, the system allocates free pages here.
5 * Note that kmalloc() lives in slab.c
6 *
7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Swap reorganised 29.12.95, Stephen Tweedie

--- 37 unchanged lines hidden (view full) ---

46#include <linux/sort.h>
47#include <linux/pfn.h>
48#include <linux/backing-dev.h>
49#include <linux/fault-inject.h>
50#include <linux/page-isolation.h>
51#include <linux/page_cgroup.h>
52#include <linux/debugobjects.h>
53#include <linux/kmemleak.h>
54#include <linux/memory.h>
55#include <linux/compaction.h>
56#include <trace/events/kmem.h>
57#include <linux/ftrace_event.h>
58#include <linux/memcontrol.h>
59#include <linux/prefetch.h>
60#include <linux/migrate.h>
61#include <linux/page-debug-flags.h>
62

--- 151 unchanged lines hidden (view full) ---

214int nr_node_ids __read_mostly = MAX_NUMNODES;
215int nr_online_nodes __read_mostly = 1;
216EXPORT_SYMBOL(nr_node_ids);
217EXPORT_SYMBOL(nr_online_nodes);
218#endif
219
220int page_group_by_mobility_disabled __read_mostly;
221
54#include <linux/compaction.h>
55#include <trace/events/kmem.h>
56#include <linux/ftrace_event.h>
57#include <linux/memcontrol.h>
58#include <linux/prefetch.h>
59#include <linux/migrate.h>
60#include <linux/page-debug-flags.h>
61

--- 151 unchanged lines hidden (view full) ---

213int nr_node_ids __read_mostly = MAX_NUMNODES;
214int nr_online_nodes __read_mostly = 1;
215EXPORT_SYMBOL(nr_node_ids);
216EXPORT_SYMBOL(nr_online_nodes);
217#endif
218
219int page_group_by_mobility_disabled __read_mostly;
220
222static void set_pageblock_migratetype(struct page *page, int migratetype)
221void set_pageblock_migratetype(struct page *page, int migratetype)
223{
224
225 if (unlikely(page_group_by_mobility_disabled))
226 migratetype = MIGRATE_UNMOVABLE;
227
228 set_pageblock_flags_group(page, (unsigned long)migratetype,
229 PB_migrate, PB_migrate_end);
230}

--- 718 unchanged lines hidden (view full) ---

949 &zone->free_area[order].free_list[migratetype]);
950 page += 1 << order;
951 pages_moved += 1 << order;
952 }
953
954 return pages_moved;
955}
956
222{
223
224 if (unlikely(page_group_by_mobility_disabled))
225 migratetype = MIGRATE_UNMOVABLE;
226
227 set_pageblock_flags_group(page, (unsigned long)migratetype,
228 PB_migrate, PB_migrate_end);
229}

--- 718 unchanged lines hidden (view full) ---

948 &zone->free_area[order].free_list[migratetype]);
949 page += 1 << order;
950 pages_moved += 1 << order;
951 }
952
953 return pages_moved;
954}
955
957static int move_freepages_block(struct zone *zone, struct page *page,
956int move_freepages_block(struct zone *zone, struct page *page,
958 int migratetype)
959{
960 unsigned long start_pfn, end_pfn;
961 struct page *start_page, *end_page;
962
963 start_pfn = page_to_pfn(page);
964 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
965 start_page = pfn_to_page(start_pfn);

--- 4492 unchanged lines hidden (view full) ---

5458/*
5459 * This function checks whether pageblock includes unmovable pages or not.
5460 * If @count is not zero, it is okay to include less @count unmovable pages
5461 *
5462 * PageLRU check wihtout isolation or lru_lock could race so that
5463 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
5464 * expect this function should be exact.
5465 */
957 int migratetype)
958{
959 unsigned long start_pfn, end_pfn;
960 struct page *start_page, *end_page;
961
962 start_pfn = page_to_pfn(page);
963 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
964 start_page = pfn_to_page(start_pfn);

--- 4492 unchanged lines hidden (view full) ---

5457/*
5458 * This function checks whether pageblock includes unmovable pages or not.
5459 * If @count is not zero, it is okay to include less @count unmovable pages
5460 *
5461 * PageLRU check wihtout isolation or lru_lock could race so that
5462 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
5463 * expect this function should be exact.
5464 */
5466static bool
5467__has_unmovable_pages(struct zone *zone, struct page *page, int count)
5465bool has_unmovable_pages(struct zone *zone, struct page *page, int count)
5468{
5469 unsigned long pfn, iter, found;
5470 int mt;
5471
5472 /*
5473 * For avoiding noise data, lru_add_drain_all() should be called
5474 * If ZONE_MOVABLE, the zone never contains unmovable pages
5475 */

--- 60 unchanged lines hidden (view full) ---

5536 return false;
5537
5538 zone = page_zone(page);
5539 pfn = page_to_pfn(page);
5540 if (zone->zone_start_pfn > pfn ||
5541 zone->zone_start_pfn + zone->spanned_pages <= pfn)
5542 return false;
5543
5466{
5467 unsigned long pfn, iter, found;
5468 int mt;
5469
5470 /*
5471 * For avoiding noise data, lru_add_drain_all() should be called
5472 * If ZONE_MOVABLE, the zone never contains unmovable pages
5473 */

--- 60 unchanged lines hidden (view full) ---

5534 return false;
5535
5536 zone = page_zone(page);
5537 pfn = page_to_pfn(page);
5538 if (zone->zone_start_pfn > pfn ||
5539 zone->zone_start_pfn + zone->spanned_pages <= pfn)
5540 return false;
5541
5544 return !__has_unmovable_pages(zone, page, 0);
5542 return !has_unmovable_pages(zone, page, 0);
5545}
5546
5543}
5544
5547int set_migratetype_isolate(struct page *page)
5548{
5549 struct zone *zone;
5550 unsigned long flags, pfn;
5551 struct memory_isolate_notify arg;
5552 int notifier_ret;
5553 int ret = -EBUSY;
5554
5555 zone = page_zone(page);
5556
5557 spin_lock_irqsave(&zone->lock, flags);
5558
5559 pfn = page_to_pfn(page);
5560 arg.start_pfn = pfn;
5561 arg.nr_pages = pageblock_nr_pages;
5562 arg.pages_found = 0;
5563
5564 /*
5565 * It may be possible to isolate a pageblock even if the
5566 * migratetype is not MIGRATE_MOVABLE. The memory isolation
5567 * notifier chain is used by balloon drivers to return the
5568 * number of pages in a range that are held by the balloon
5569 * driver to shrink memory. If all the pages are accounted for
5570 * by balloons, are free, or on the LRU, isolation can continue.
5571 * Later, for example, when memory hotplug notifier runs, these
5572 * pages reported as "can be isolated" should be isolated(freed)
5573 * by the balloon driver through the memory notifier chain.
5574 */
5575 notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
5576 notifier_ret = notifier_to_errno(notifier_ret);
5577 if (notifier_ret)
5578 goto out;
5579 /*
5580 * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
5581 * We just check MOVABLE pages.
5582 */
5583 if (!__has_unmovable_pages(zone, page, arg.pages_found))
5584 ret = 0;
5585 /*
5586 * Unmovable means "not-on-lru" pages. If Unmovable pages are
5587 * larger than removable-by-driver pages reported by notifier,
5588 * we'll fail.
5589 */
5590
5591out:
5592 if (!ret) {
5593 set_pageblock_migratetype(page, MIGRATE_ISOLATE);
5594 move_freepages_block(zone, page, MIGRATE_ISOLATE);
5595 }
5596
5597 spin_unlock_irqrestore(&zone->lock, flags);
5598 if (!ret)
5599 drain_all_pages();
5600 return ret;
5601}
5602
5603void unset_migratetype_isolate(struct page *page, unsigned migratetype)
5604{
5605 struct zone *zone;
5606 unsigned long flags;
5607 zone = page_zone(page);
5608 spin_lock_irqsave(&zone->lock, flags);
5609 if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
5610 goto out;
5611 set_pageblock_migratetype(page, migratetype);
5612 move_freepages_block(zone, page, migratetype);
5613out:
5614 spin_unlock_irqrestore(&zone->lock, flags);
5615}
5616
5617#ifdef CONFIG_CMA
5618
5619static unsigned long pfn_max_align_down(unsigned long pfn)
5620{
5621 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
5622 pageblock_nr_pages) - 1);
5623}
5624

--- 434 unchanged lines hidden --- 		5545#ifdef CONFIG_CMA
5546
5547static unsigned long pfn_max_align_down(unsigned long pfn)
5548{
5549 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
5550 pageblock_nr_pages) - 1);
5551}
5552

--- 434 unchanged lines hidden ---