diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/Kconfig.debug | 53 | ||||
| -rw-r--r-- | mm/Makefile | 2 | ||||
| -rw-r--r-- | mm/page_alloc.c | 10 | ||||
| -rw-r--r-- | mm/page_ext.c | 13 | ||||
| -rw-r--r-- | mm/page_poison.c (renamed from mm/debug-pagealloc.c) | 93 | ||||
| -rw-r--r-- | mm/vmscan.c | 54 |
6 files changed, 158 insertions, 67 deletions
diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 9e55d6b141aa..7470fd60fc59 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -40,9 +40,6 @@ config DEBUG_PAGEALLOC_ENABLE_DEFAULT Enable debug page memory allocations by default? This value can be overridden by debug_pagealloc=off|on. -config PAGE_POISONING - bool - config SLUB_DEBUG_PANIC_ON bool "Enable to Panic on SLUB corruption detection" depends on SLUB_DEBUG @@ -52,3 +49,53 @@ config SLUB_DEBUG_PANIC_ON debug options this may not be desirable as it prevents from investigating the root cause which may be rooted within cache or memory. + +config PAGE_POISONING + bool "Poison pages after freeing" + select PAGE_EXTENSION + select PAGE_POISONING_NO_SANITY if HIBERNATION + ---help--- + Fill the pages with poison patterns after free_pages() and verify + the patterns before alloc_pages. The filling of the memory helps + reduce the risk of information leaks from freed data. This does + have a potential performance impact. + + Note that "poison" here is not the same thing as the "HWPoison" + for CONFIG_MEMORY_FAILURE. This is software poisoning only. + + If unsure, say N + +config PAGE_POISONING_ENABLE_DEFAULT + bool "Enable page poisoning by default?" + default n + depends on PAGE_POISONING + ---help--- + Enable page poisoning of free pages by default? This value + can be overridden by page_poison=off|on. This can be used + to avoid passing the kernel parameter and let page poisoning + feature enabled by default. + +config PAGE_POISONING_NO_SANITY + depends on PAGE_POISONING + bool "Only poison, don't sanity check" + ---help--- + Skip the sanity checking on alloc, only fill the pages with + poison on free. This reduces some of the overhead of the + poisoning feature. + + If you are only interested in sanitization, say Y. Otherwise + say N. + +config PAGE_POISONING_ZERO + bool "Use zero for poisoning instead of random data" + depends on PAGE_POISONING + ---help--- + Instead of using the existing poison value, fill the pages with + zeros. This makes it harder to detect when errors are occurring + due to sanitization but the zeroing at free means that it is + no longer necessary to write zeros when GFP_ZERO is used on + allocation. + + Enabling page poisoning with this option will disable hibernation + + If unsure, say N diff --git a/mm/Makefile b/mm/Makefile index 4b1a69abce7a..130d06ac56e0 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -52,7 +52,7 @@ obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o obj-$(CONFIG_SLOB) += slob.o obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o obj-$(CONFIG_KSM) += ksm.o -obj-$(CONFIG_PAGE_POISONING) += debug-pagealloc.o +obj-$(CONFIG_PAGE_POISONING) += page_poison.o obj-$(CONFIG_SLAB) += slab.o obj-$(CONFIG_SLUB) += slub.o obj-$(CONFIG_KMEMCHECK) += kmemcheck.o diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 28f60d9ea074..170c1486e5c9 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1012,6 +1012,7 @@ static bool free_pages_prepare(struct page *page, unsigned int order) PAGE_SIZE << order); } arch_free_page(page, order); + kernel_poison_pages(page, 1 << order, 0); kernel_map_pages(page, 1 << order, 0); kasan_free_pages(page, order); @@ -1396,6 +1397,12 @@ static inline int check_new_page(struct page *page) return 0; } +static inline bool free_pages_prezeroed(void) +{ + return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) && + page_poisoning_enabled(); +} + static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, int alloc_flags) { @@ -1413,8 +1420,9 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, kasan_alloc_pages(page, order); arch_alloc_page(page, order); kernel_map_pages(page, 1 << order, 1); + kernel_poison_pages(page, 1 << order, 1); - if (gfp_flags & __GFP_ZERO) + if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO)) for (i = 0; i < (1 << order); i++) clear_highpage(page + i); diff --git a/mm/page_ext.c b/mm/page_ext.c index 292ca7b8debd..916accfec86a 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -54,9 +54,6 @@ static struct page_ext_operations *page_ext_ops[] = { &debug_guardpage_ops, -#ifdef CONFIG_PAGE_POISONING - &page_poisoning_ops, -#endif #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif @@ -106,12 +103,15 @@ struct page_ext *lookup_page_ext(struct page *page) struct page_ext *base; base = NODE_DATA(page_to_nid(page))->node_page_ext; -#ifdef CONFIG_DEBUG_VM +#if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PAGE_POISONING) /* * The sanity checks the page allocator does upon freeing a * page can reach here before the page_ext arrays are * allocated when feeding a range of pages to the allocator * for the first time during bootup or memory hotplug. + * + * This check is also necessary for ensuring page poisoning + * works as expected when enabled */ if (unlikely(!base)) return NULL; @@ -180,12 +180,15 @@ struct page_ext *lookup_page_ext(struct page *page) { unsigned long pfn = page_to_pfn(page); struct mem_section *section = __pfn_to_section(pfn); -#ifdef CONFIG_DEBUG_VM +#if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PAGE_POISONING) /* * The sanity checks the page allocator does upon freeing a * page can reach here before the page_ext arrays are * allocated when feeding a range of pages to the allocator * for the first time during bootup or memory hotplug. + * + * This check is also necessary for ensuring page poisoning + * works as expected when enabled */ if (!section->page_ext) return NULL; diff --git a/mm/debug-pagealloc.c b/mm/page_poison.c index 100963091cc6..c8cf230dbfcb 100644 --- a/mm/debug-pagealloc.c +++ b/mm/page_poison.c @@ -6,68 +6,41 @@ #include <linux/poison.h> #include <linux/ratelimit.h> -#ifndef mark_addr_rdonly -#define mark_addr_rdonly(a) -#endif - -#ifndef mark_addr_rdwrite -#define mark_addr_rdwrite(a) -#endif - -static bool page_poisoning_enabled __read_mostly; - -static bool need_page_poisoning(void) -{ - if (!debug_pagealloc_enabled()) - return false; - - return true; -} - -static void init_page_poisoning(void) -{ - if (!debug_pagealloc_enabled()) - return; - - page_poisoning_enabled = true; -} - -struct page_ext_operations page_poisoning_ops = { - .need = need_page_poisoning, - .init = init_page_poisoning, -}; +static bool want_page_poisoning __read_mostly + = IS_ENABLED(CONFIG_PAGE_POISONING_ENABLE_DEFAULT); -static inline void set_page_poison(struct page *page) +static int early_page_poison_param(char *buf) { - struct page_ext *page_ext; + if (!buf) + return -EINVAL; - page_ext = lookup_page_ext(page); - __set_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags); -} - -static inline void clear_page_poison(struct page *page) -{ - struct page_ext *page_ext; + if (strcmp(buf, "on") == 0) + want_page_poisoning = true; + else if (strcmp(buf, "off") == 0) + want_page_poisoning = false; - page_ext = lookup_page_ext(page); - __clear_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags); + return 0; } +early_param("page_poison", early_page_poison_param); -static inline bool page_poison(struct page *page) +bool page_poisoning_enabled(void) { - struct page_ext *page_ext; - - page_ext = lookup_page_ext(page); - return test_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags); + /* + * Assumes that debug_pagealloc_enabled is set before + * free_all_bootmem. + * Page poisoning is debug page alloc for some arches. If + * either of those options are enabled, enable poisoning. + */ + return (want_page_poisoning || + (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && + debug_pagealloc_enabled())); } static void poison_page(struct page *page) { void *addr = kmap_atomic(page); - set_page_poison(page); memset(addr, PAGE_POISON, PAGE_SIZE); - mark_addr_rdonly(addr); kunmap_atomic(addr); } @@ -93,6 +66,9 @@ static void check_poison_mem(struct page *page, unsigned char *start; unsigned char *end; + if (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY)) + return; + start = memchr_inv(mem, PAGE_POISON, bytes); if (!start) return; @@ -121,13 +97,13 @@ static void unpoison_page(struct page *page) { void *addr; - if (!page_poison(page)) - return; - addr = kmap_atomic(page); + /* + * Page poisoning when enabled poisons each and every page + * that is freed to buddy. Thus no extra check is done to + * see if a page was posioned. + */ check_poison_mem(page, addr, PAGE_SIZE); - mark_addr_rdwrite(addr); - clear_page_poison(page); kunmap_atomic(addr); } @@ -139,9 +115,9 @@ static void unpoison_pages(struct page *page, int n) unpoison_page(page + i); } -void __kernel_map_pages(struct page *page, int numpages, int enable) +void kernel_poison_pages(struct page *page, int numpages, int enable) { - if (!page_poisoning_enabled) + if (!page_poisoning_enabled()) return; if (enable) @@ -149,3 +125,10 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) else poison_pages(page, numpages); } + +#ifndef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC +void __kernel_map_pages(struct page *page, int numpages, int enable) +{ + /* This function does nothing, all work is done via poison pages */ +} +#endif diff --git a/mm/vmscan.c b/mm/vmscan.c index 5e9e74955bd1..94fecacf0ddc 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -399,6 +399,35 @@ static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, return freed; } +static void shrink_slab_lmk(gfp_t gfp_mask, int nid, + struct mem_cgroup *memcg, + unsigned long nr_scanned, + unsigned long nr_eligible) +{ + struct shrinker *shrinker; + + if (nr_scanned == 0) + nr_scanned = SWAP_CLUSTER_MAX; + + if (!down_read_trylock(&shrinker_rwsem)) + goto out; + + list_for_each_entry(shrinker, &shrinker_list, list) { + struct shrink_control sc = { + .gfp_mask = gfp_mask, + }; + + if (!(shrinker->flags & SHRINKER_LMK)) + continue; + + do_shrink_slab(&sc, shrinker, nr_scanned, nr_eligible); + } + + up_read(&shrinker_rwsem); +out: + cond_resched(); +} + /** * shrink_slab - shrink slab caches * @gfp_mask: allocation context @@ -460,6 +489,9 @@ static unsigned long shrink_slab(gfp_t gfp_mask, int nid, .memcg = memcg, }; + if (shrinker->flags & SHRINKER_LMK) + continue; + if (memcg && !(shrinker->flags & SHRINKER_MEMCG_AWARE)) continue; @@ -2626,6 +2658,7 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) gfp_t orig_mask; enum zone_type requested_highidx = gfp_zone(sc->gfp_mask); bool reclaimable = false; + unsigned long lru_pages = 0; /* * If the number of buffer_heads in the machine exceeds the maximum @@ -2653,6 +2686,7 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) * to global LRU. */ if (global_reclaim(sc)) { + lru_pages += zone_reclaimable_pages(zone); if (!cpuset_zone_allowed(zone, GFP_KERNEL | __GFP_HARDWALL)) continue; @@ -2703,6 +2737,9 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) reclaimable = true; } + if (global_reclaim(sc)) + shrink_slab_lmk(sc->gfp_mask, 0, NULL, + sc->nr_scanned, lru_pages); /* * Restore to original mask to avoid the impact on the caller if we * promoted it to __GFP_HIGHMEM. @@ -3181,7 +3218,8 @@ static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, long remaining, */ static bool kswapd_shrink_zone(struct zone *zone, int classzone_idx, - struct scan_control *sc) + struct scan_control *sc, + unsigned long lru_pages) { unsigned long balance_gap; bool lowmem_pressure; @@ -3208,6 +3246,8 @@ static bool kswapd_shrink_zone(struct zone *zone, return true; shrink_zone(zone, sc, zone_idx(zone) == classzone_idx); + shrink_slab_lmk(sc->gfp_mask, zone_to_nid(zone), NULL, + sc->nr_scanned, lru_pages); clear_bit(ZONE_WRITEBACK, &zone->flags); @@ -3265,6 +3305,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx) do { bool raise_priority = true; + unsigned long lru_pages = 0; sc.nr_reclaimed = 0; @@ -3322,6 +3363,15 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx) if (sc.priority < DEF_PRIORITY - 2) sc.may_writepage = 1; + for (i = 0; i <= end_zone; i++) { + struct zone *zone = pgdat->node_zones + i; + + if (!populated_zone(zone)) + continue; + + lru_pages += zone_reclaimable_pages(zone); + } + /* * Now scan the zone in the dma->highmem direction, stopping * at the last zone which needs scanning. @@ -3358,7 +3408,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx) * that that high watermark would be met at 100% * efficiency. */ - if (kswapd_shrink_zone(zone, end_zone, &sc)) + if (kswapd_shrink_zone(zone, end_zone, &sc, lru_pages)) raise_priority = false; } |