diff options
Diffstat (limited to 'mm/page_alloc.c')
| -rw-r--r-- | mm/page_alloc.c | 101 |
1 files changed, 81 insertions, 20 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b5368a3e6120..c7126875d8dc 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -724,7 +724,7 @@ static inline void __free_one_page(struct page *page, struct page *buddy; unsigned int max_order; - max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1); + max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order); VM_BUG_ON(!zone_is_initialized(zone)); VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); @@ -739,7 +739,7 @@ static inline void __free_one_page(struct page *page, VM_BUG_ON_PAGE(bad_range(zone, page), page); continue_merging: - while (order < max_order - 1) { + while (order < max_order) { buddy_idx = __find_buddy_index(page_idx, order); buddy = page + (buddy_idx - page_idx); if (!page_is_buddy(page, buddy, order)) @@ -760,7 +760,7 @@ continue_merging: page_idx = combined_idx; order++; } - if (max_order < MAX_ORDER) { + if (order < MAX_ORDER - 1) { /* If we are here, it means order is >= pageblock_order. * We want to prevent merge between freepages on isolate * pageblock and normal pageblock. Without this, pageblock @@ -781,7 +781,7 @@ continue_merging: is_migrate_isolate(buddy_mt))) goto done_merging; } - max_order++; + max_order = order + 1; goto continue_merging; } @@ -859,7 +859,6 @@ static void free_pcppages_bulk(struct zone *zone, int count, { int migratetype = 0; int batch_free = 0; - int to_free = count; unsigned long nr_scanned; spin_lock(&zone->lock); @@ -867,7 +866,12 @@ static void free_pcppages_bulk(struct zone *zone, int count, if (nr_scanned) __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); - while (to_free) { + /* + * Ensure proper count is passed which otherwise would stuck in the + * below while (list_empty(list)) loop. + */ + count = min(pcp->count, count); + while (count) { struct page *page; struct list_head *list; @@ -887,7 +891,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, /* This is the only non-empty list. Free them all. */ if (batch_free == MIGRATE_PCPTYPES) - batch_free = to_free; + batch_free = count; do { int mt; /* migratetype of the to-be-freed page */ @@ -905,7 +909,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, __free_one_page(page, page_to_pfn(page), zone, 0, mt); trace_mm_page_pcpu_drain(page, 0, mt); - } while (--to_free && --batch_free && !list_empty(list)); + } while (--count && --batch_free && !list_empty(list)); } spin_unlock(&zone->lock); } @@ -1655,7 +1659,8 @@ static void change_pageblock_range(struct page *pageblock_page, * is worse than movable allocations stealing from unmovable and reclaimable * pageblocks. */ -static bool can_steal_fallback(unsigned int order, int start_mt) +static bool can_steal_fallback(unsigned int current_order, unsigned int start_order, + int start_mt, int fallback_mt) { /* * Leaving this order check is intended, although there is @@ -1664,12 +1669,17 @@ static bool can_steal_fallback(unsigned int order, int start_mt) * but, below check doesn't guarantee it and that is just heuristic * so could be changed anytime. */ - if (order >= pageblock_order) + if (current_order >= pageblock_order) return true; - if (order >= pageblock_order / 2 || + /* don't let unmovable allocations cause migrations simply because of free pages */ + if ((start_mt != MIGRATE_UNMOVABLE && current_order >= pageblock_order / 2) || + /* only steal reclaimable page blocks for unmovable allocations */ + (start_mt == MIGRATE_UNMOVABLE && fallback_mt != MIGRATE_MOVABLE && current_order >= pageblock_order / 2) || + /* reclaimable can steal aggressively */ start_mt == MIGRATE_RECLAIMABLE || - start_mt == MIGRATE_UNMOVABLE || + /* allow unmovable allocs up to 64K without migrating blocks */ + (start_mt == MIGRATE_UNMOVABLE && start_order >= 5) || page_group_by_mobility_disabled) return true; @@ -1709,8 +1719,9 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page, * we can steal other freepages all together. This would help to reduce * fragmentation due to mixed migratetype pages in one pageblock. */ -int find_suitable_fallback(struct free_area *area, unsigned int order, - int migratetype, bool only_stealable, bool *can_steal) +int find_suitable_fallback(struct free_area *area, unsigned int current_order, + int migratetype, bool only_stealable, + int start_order, bool *can_steal) { int i; int fallback_mt; @@ -1727,7 +1738,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order, if (list_empty(&area->free_list[fallback_mt])) continue; - if (can_steal_fallback(order, migratetype)) + if (can_steal_fallback(current_order, start_order, migratetype, fallback_mt)) *can_steal = true; if (!only_stealable) @@ -1863,7 +1874,7 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) --current_order) { area = &(zone->free_area[current_order]); fallback_mt = find_suitable_fallback(area, current_order, - start_migratetype, false, &can_steal); + start_migratetype, false, order, &can_steal); if (fallback_mt == -1) continue; @@ -2076,8 +2087,9 @@ static void drain_pages(unsigned int cpu) * The CPU has to be pinned. When zone parameter is non-NULL, spill just * the single zone's pages. */ -void drain_local_pages(struct zone *zone) +void drain_local_pages(void *z) { + struct zone *zone = (struct zone *)z; int cpu = smp_processor_id(); if (zone) @@ -2137,8 +2149,7 @@ void drain_all_pages(struct zone *zone) else cpumask_clear_cpu(cpu, &cpus_with_pcps); } - on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, - zone, 1); + on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, zone, 1); } #ifdef CONFIG_HIBERNATION @@ -3735,6 +3746,56 @@ static inline void show_node(struct zone *zone) printk("Node %d ", zone_to_nid(zone)); } +long si_mem_available(void) +{ + long available; + unsigned long pagecache; + unsigned long wmark_low = 0; + unsigned long pages[NR_LRU_LISTS]; + struct zone *zone; + int lru; + + for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) + pages[lru] = global_page_state(NR_LRU_BASE + lru); + + for_each_zone(zone) + wmark_low += zone->watermark[WMARK_LOW]; + + /* + * Estimate the amount of memory available for userspace allocations, + * without causing swapping. + */ + available = global_page_state(NR_FREE_PAGES) - totalreserve_pages; + + /* + * Not all the page cache can be freed, otherwise the system will + * start swapping. Assume at least half of the page cache, or the + * low watermark worth of cache, needs to stay. + */ + pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; + pagecache -= min(pagecache / 2, wmark_low); + available += pagecache; + + /* + * Part of the reclaimable slab consists of items that are in use, + * and cannot be freed. Cap this estimate at the low watermark. + */ + available += global_page_state(NR_SLAB_RECLAIMABLE) - + min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low); + + /* + * Part of the kernel memory, which can be released under memory + * pressure. + */ + available += global_page_state(NR_INDIRECTLY_RECLAIMABLE_BYTES) >> + PAGE_SHIFT; + + if (available < 0) + available = 0; + return available; +} +EXPORT_SYMBOL_GPL(si_mem_available); + void si_meminfo(struct sysinfo *val) { val->totalram = totalram_pages; @@ -6325,7 +6386,7 @@ int __meminit init_per_zone_wmark_min(void) setup_per_zone_inactive_ratio(); return 0; } -core_initcall(init_per_zone_wmark_min) +postcore_initcall(init_per_zone_wmark_min) /* * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so |