summaryrefslogtreecommitdiff
path: root/mm/slub.c
diff options
context:
space:
mode:
authorSrinivasarao P <spathi@codeaurora.org>2018-01-09 15:59:02 +0530
committerSrinivasarao P <spathi@codeaurora.org>2018-01-18 12:45:07 +0530
commitdd4f1e35fa5f5a452d34e1fcc681472f2011f93d (patch)
tree8f7c973e9920c4df577c5b7093da541ce12da7d1 /mm/slub.c
parent33260fbfb3632f876669fa815db1edb6d688ace3 (diff)
parent2fea0397a8e708952a22918bd89cce28b3087c11 (diff)
Merge android-4.4.106 (2fea039) into msm-4.4
* refs/heads/tmp-2fea039 Linux 4.4.106 usb: gadget: ffs: Forbid usb_ep_alloc_request from sleeping arm: KVM: Fix VTTBR_BADDR_MASK BUG_ON off-by-one Revert "x86/mm/pat: Ensure cpa->pfn only contains page frame numbers" Revert "x86/efi: Hoist page table switching code into efi_call_virt()" Revert "x86/efi: Build our own page table structures" net/packet: fix a race in packet_bind() and packet_notifier() packet: fix crash in fanout_demux_rollover() sit: update frag_off info rds: Fix NULL pointer dereference in __rds_rdma_map tipc: fix memory leak in tipc_accept_from_sock() more bio_map_user_iov() leak fixes s390: always save and restore all registers on context switch ipmi: Stop timers before cleaning up the module audit: ensure that 'audit=1' actually enables audit for PID 1 ipvlan: fix ipv6 outbound device afs: Connect up the CB.ProbeUuid IB/mlx5: Assign send CQ and recv CQ of UMR QP IB/mlx4: Increase maximal message size under UD QP xfrm: Copy policy family in clone_policy jump_label: Invoke jump_label_test() via early_initcall() atm: horizon: Fix irq release error sctp: use the right sk after waking up from wait_buf sleep sctp: do not free asoc when it is already dead in sctp_sendmsg sparc64/mm: set fields in deferred pages block: wake up all tasks blocked in get_request() sunrpc: Fix rpc_task_begin trace point NFS: Fix a typo in nfs_rename() dynamic-debug-howto: fix optional/omitted ending line number to be LARGE instead of 0 lib/genalloc.c: make the avail variable an atomic_long_t route: update fnhe_expires for redirect when the fnhe exists route: also update fnhe_genid when updating a route cache mac80211_hwsim: Fix memory leak in hwsim_new_radio_nl() kbuild: pkg: use --transform option to prefix paths in tar EDAC, i5000, i5400: Fix definition of NRECMEMB register EDAC, i5000, i5400: Fix use of MTR_DRAM_WIDTH macro powerpc/powernv/ioda2: Gracefully fail if too many TCE levels requested drm/amd/amdgpu: fix console deadlock if late init failed axonram: Fix gendisk handling netfilter: don't track fragmented packets zram: set physical queue limits to avoid array out of bounds accesses i2c: riic: fix restart condition crypto: s5p-sss - Fix completing crypto request in IRQ handler ipv6: reorder icmpv6_init() and ip6_mr_init() bnx2x: do not rollback VF MAC/VLAN filters we did not configure bnx2x: fix possible overrun of VFPF multicast addresses array bnx2x: prevent crash when accessing PTP with interface down spi_ks8995: fix "BUG: key accdaa28 not in .data!" arm64: KVM: Survive unknown traps from guests arm: KVM: Survive unknown traps from guests KVM: nVMX: reset nested_run_pending if the vCPU is going to be reset irqchip/crossbar: Fix incorrect type of register size scsi: lpfc: Fix crash during Hardware error recovery on SLI3 adapters workqueue: trigger WARN if queue_delayed_work() is called with NULL @wq libata: drop WARN from protocol error in ata_sff_qc_issue() kvm: nVMX: VMCLEAR should not cause the vCPU to shut down USB: gadgetfs: Fix a potential memory leak in 'dev_config()' usb: gadget: configs: plug memory leak HID: chicony: Add support for another ASUS Zen AiO keyboard gpio: altera: Use handle_level_irq when configured as a level_high ARM: OMAP2+: Release device node after it is no longer needed. ARM: OMAP2+: Fix device node reference counts module: set __jump_table alignment to 8 selftest/powerpc: Fix false failures for skipped tests x86/hpet: Prevent might sleep splat on resume ARM: OMAP2+: gpmc-onenand: propagate error on initialization failure vti6: Don't report path MTU below IPV6_MIN_MTU. Revert "s390/kbuild: enable modversions for symbols exported from asm" Revert "spi: SPI_FSL_DSPI should depend on HAS_DMA" Revert "drm/armada: Fix compile fail" mm: drop unused pmdp_huge_get_and_clear_notify() thp: fix MADV_DONTNEED vs. numa balancing race thp: reduce indentation level in change_huge_pmd() scsi: storvsc: Workaround for virtual DVD SCSI version ARM: avoid faulting on qemu ARM: BUG if jumping to usermode address in kernel mode arm64: fpsimd: Prevent registers leaking from dead tasks KVM: VMX: remove I/O port 0x80 bypass on Intel hosts arm64: KVM: fix VTTBR_BADDR_MASK BUG_ON off-by-one media: dvb: i2c transfers over usb cannot be done from stack drm/exynos: gem: Drop NONCONTIG flag for buffers allocated without IOMMU drm: extra printk() wrapper macros kdb: Fix handling of kallsyms_symbol_next() return value s390: fix compat system call table iommu/vt-d: Fix scatterlist offset handling ALSA: usb-audio: Add check return value for usb_string() ALSA: usb-audio: Fix out-of-bound error ALSA: seq: Remove spurious WARN_ON() at timer check ALSA: pcm: prevent UAF in snd_pcm_info x86/PCI: Make broadcom_postcore_init() check acpi_disabled X.509: reject invalid BIT STRING for subjectPublicKey ASN.1: check for error from ASN1_OP_END__ACT actions ASN.1: fix out-of-bounds read when parsing indefinite length item efi: Move some sysfs files to be read-only by root scsi: libsas: align sata_device's rps_resp on a cacheline isa: Prevent NULL dereference in isa_bus driver callbacks hv: kvp: Avoid reading past allocated blocks from KVP file virtio: release virtio index when fail to device_register can: usb_8dev: cancel urb on -EPIPE and -EPROTO can: esd_usb2: cancel urb on -EPIPE and -EPROTO can: ems_usb: cancel urb on -EPIPE and -EPROTO can: kvaser_usb: cancel urb on -EPIPE and -EPROTO can: kvaser_usb: ratelimit errors if incomplete messages are received can: kvaser_usb: Fix comparison bug in kvaser_usb_read_bulk_callback() can: kvaser_usb: free buf in error paths can: ti_hecc: Fix napi poll return value for repoll BACKPORT: irq: Make the irqentry text section unconditional UPSTREAM: arch, ftrace: for KASAN put hard/soft IRQ entries into separate sections UPSTREAM: x86, kasan, ftrace: Put APIC interrupt handlers into .irqentry.text UPSTREAM: kasan: make get_wild_bug_type() static UPSTREAM: kasan: separate report parts by empty lines UPSTREAM: kasan: improve double-free report format UPSTREAM: kasan: print page description after stacks UPSTREAM: kasan: improve slab object description UPSTREAM: kasan: change report header UPSTREAM: kasan: simplify address description logic UPSTREAM: kasan: change allocation and freeing stack traces headers UPSTREAM: kasan: unify report headers UPSTREAM: kasan: introduce helper functions for determining bug type BACKPORT: kasan: report only the first error by default UPSTREAM: kasan: fix races in quarantine_remove_cache() UPSTREAM: kasan: resched in quarantine_remove_cache() BACKPORT: kasan, sched/headers: Uninline kasan_enable/disable_current() BACKPORT: kasan: drain quarantine of memcg slab objects UPSTREAM: kasan: eliminate long stalls during quarantine reduction UPSTREAM: kasan: support panic_on_warn UPSTREAM: x86/suspend: fix false positive KASAN warning on suspend/resume UPSTREAM: kasan: support use-after-scope detection UPSTREAM: kasan/tests: add tests for user memory access functions UPSTREAM: mm, kasan: add a ksize() test UPSTREAM: kasan: test fix: warn if the UAF could not be detected in kmalloc_uaf2 UPSTREAM: kasan: modify kmalloc_large_oob_right(), add kmalloc_pagealloc_oob_right() UPSTREAM: lib/stackdepot: export save/fetch stack for drivers UPSTREAM: lib/stackdepot.c: bump stackdepot capacity from 16MB to 128MB BACKPORT: kprobes: Unpoison stack in jprobe_return() for KASAN UPSTREAM: kasan: remove the unnecessary WARN_ONCE from quarantine.c UPSTREAM: kasan: avoid overflowing quarantine size on low memory systems UPSTREAM: kasan: improve double-free reports BACKPORT: mm: coalesce split strings BACKPORT: mm/kasan: get rid of ->state in struct kasan_alloc_meta UPSTREAM: mm/kasan: get rid of ->alloc_size in struct kasan_alloc_meta UPSTREAM: mm: kasan: remove unused 'reserved' field from struct kasan_alloc_meta UPSTREAM: mm/kasan, slub: don't disable interrupts when object leaves quarantine UPSTREAM: mm/kasan: don't reduce quarantine in atomic contexts UPSTREAM: mm/kasan: fix corruptions and false positive reports UPSTREAM: lib/stackdepot.c: use __GFP_NOWARN for stack allocations BACKPORT: mm, kasan: switch SLUB to stackdepot, enable memory quarantine for SLUB UPSTREAM: kasan/quarantine: fix bugs on qlist_move_cache() UPSTREAM: mm: mempool: kasan: don't poot mempool objects in quarantine UPSTREAM: kasan: change memory hot-add error messages to info messages BACKPORT: mm/kasan: add API to check memory regions UPSTREAM: mm/kasan: print name of mem[set,cpy,move]() caller in report UPSTREAM: mm: kasan: initial memory quarantine implementation UPSTREAM: lib/stackdepot: avoid to return 0 handle UPSTREAM: lib/stackdepot.c: allow the stack trace hash to be zero UPSTREAM: mm, kasan: fix compilation for CONFIG_SLAB BACKPORT: mm, kasan: stackdepot implementation. Enable stackdepot for SLAB BACKPORT: mm, kasan: add GFP flags to KASAN API UPSTREAM: mm, kasan: SLAB support UPSTREAM: mm/slab: align cache size first before determination of OFF_SLAB candidate UPSTREAM: mm/slab: use more appropriate condition check for debug_pagealloc UPSTREAM: mm/slab: factor out debugging initialization in cache_init_objs() UPSTREAM: mm/slab: remove object status buffer for DEBUG_SLAB_LEAK UPSTREAM: mm/slab: alternative implementation for DEBUG_SLAB_LEAK UPSTREAM: mm/slab: clean up DEBUG_PAGEALLOC processing code UPSTREAM: mm/slab: activate debug_pagealloc in SLAB when it is actually enabled sched: EAS/WALT: Don't take into account of running task's util BACKPORT: schedutil: Reset cached freq if it is not in sync with next_freq UPSTREAM: kasan: add functions to clear stack poison Conflicts: arch/arm/include/asm/kvm_arm.h arch/arm64/kernel/vmlinux.lds.S include/linux/kasan.h kernel/softirq.c lib/Kconfig lib/Kconfig.kasan lib/Makefile lib/stackdepot.c mm/kasan/kasan.c sound/usb/mixer.c Change-Id: If70ced6da5f19be3dd92d10a8d8cd4d5841e5870 Signed-off-by: Srinivasarao P <spathi@codeaurora.org>
Diffstat (limited to 'mm/slub.c')
-rw-r--r--mm/slub.c97
1 files changed, 66 insertions, 31 deletions
diff --git a/mm/slub.c b/mm/slub.c
index a5f6c6d107e9..fd3a044aaa4a 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -287,6 +287,9 @@ static inline size_t slab_ksize(const struct kmem_cache *s)
return s->object_size;
#endif
+ if (s->flags & SLAB_KASAN)
+ return s->object_size;
+
/*
* If we have the need to store the freelist pointer
* back there or track user information then we can
@@ -469,8 +472,6 @@ static inline void *restore_red_left(struct kmem_cache *s, void *p)
*/
#if defined(CONFIG_SLUB_DEBUG_ON)
static int slub_debug = DEBUG_DEFAULT_FLAGS;
-#elif defined(CONFIG_KASAN)
-static int slub_debug = SLAB_STORE_USER;
#else
static int slub_debug;
#endif
@@ -675,6 +676,8 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
if (s->flags & SLAB_STORE_USER)
off += 2 * sizeof(struct track);
+ off += kasan_metadata_size(s);
+
if (off != size_from_object(s))
/* Beginning of the filler is the free pointer */
print_section("Padding ", p + off, size_from_object(s) - off);
@@ -814,6 +817,8 @@ static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
/* We also have user information there */
off += 2 * sizeof(struct track);
+ off += kasan_metadata_size(s);
+
if (size_from_object(s) == off)
return 1;
@@ -977,14 +982,14 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
max_objects = MAX_OBJS_PER_PAGE;
if (page->objects != max_objects) {
- slab_err(s, page, "Wrong number of objects. Found %d but "
- "should be %d", page->objects, max_objects);
+ slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
+ page->objects, max_objects);
page->objects = max_objects;
slab_fix(s, "Number of objects adjusted.");
}
if (page->inuse != page->objects - nr) {
- slab_err(s, page, "Wrong object count. Counter is %d but "
- "counted were %d", page->inuse, page->objects - nr);
+ slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
+ page->inuse, page->objects - nr);
page->inuse = page->objects - nr;
slab_fix(s, "Object count adjusted.");
}
@@ -1148,8 +1153,8 @@ next_object:
if (unlikely(s != page->slab_cache)) {
if (!PageSlab(page)) {
- slab_err(s, page, "Attempt to free object(0x%p) "
- "outside of slab", object);
+ slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
+ object);
} else if (!page->slab_cache) {
pr_err("SLUB <none>: no slab for object 0x%p.\n",
object);
@@ -1319,7 +1324,7 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node,
static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
{
kmemleak_alloc(ptr, size, 1, flags);
- kasan_kmalloc_large(ptr, size);
+ kasan_kmalloc_large(ptr, size, flags);
}
static inline void kfree_hook(const void *x)
@@ -1353,13 +1358,15 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
kmemleak_alloc_recursive(object, s->object_size, 1,
s->flags, flags);
- kasan_slab_alloc(s, object);
+ kasan_slab_alloc(s, object, flags);
}
memcg_kmem_put_cache(s);
}
-static inline void slab_free_hook(struct kmem_cache *s, void *x)
+static inline void *slab_free_hook(struct kmem_cache *s, void *x)
{
+ void *freeptr;
+
kmemleak_free_recursive(x, s->flags);
/*
@@ -1380,7 +1387,13 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
if (!(s->flags & SLAB_DEBUG_OBJECTS))
debug_check_no_obj_freed(x, s->object_size);
+ freeptr = get_freepointer(s, x);
+ /*
+ * kasan_slab_free() may put x into memory quarantine, delaying its
+ * reuse. In this case the object's freelist pointer is changed.
+ */
kasan_slab_free(s, x);
+ return freeptr;
}
static inline void slab_free_freelist_hook(struct kmem_cache *s,
@@ -1398,11 +1411,11 @@ static inline void slab_free_freelist_hook(struct kmem_cache *s,
void *object = head;
void *tail_obj = tail ? : head;
+ void *freeptr;
do {
- slab_free_hook(s, object);
- } while ((object != tail_obj) &&
- (object = get_freepointer(s, object)));
+ freeptr = slab_free_hook(s, object);
+ } while ((object != tail_obj) && (object = freeptr));
#endif
}
@@ -1410,6 +1423,7 @@ static void setup_object(struct kmem_cache *s, struct page *page,
void *object)
{
setup_object_debug(s, page, object);
+ kasan_init_slab_obj(s, object);
if (unlikely(s->ctor)) {
kasan_unpoison_object_data(s, object);
s->ctor(object);
@@ -2638,7 +2652,7 @@ void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
void *ret = slab_alloc(s, gfpflags, _RET_IP_);
trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
- kasan_kmalloc(s, ret, size);
+ kasan_kmalloc(s, ret, size, gfpflags);
return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_trace);
@@ -2666,7 +2680,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
trace_kmalloc_node(_RET_IP_, ret,
size, s->size, gfpflags, node);
- kasan_kmalloc(s, ret, size);
+ kasan_kmalloc(s, ret, size, gfpflags);
return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
@@ -2811,16 +2825,13 @@ slab_empty:
* same page) possible by specifying head and tail ptr, plus objects
* count (cnt). Bulk free indicated by tail pointer being set.
*/
-static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
- void *head, void *tail, int cnt,
- unsigned long addr)
+static __always_inline void do_slab_free(struct kmem_cache *s,
+ struct page *page, void *head, void *tail,
+ int cnt, unsigned long addr)
{
void *tail_obj = tail ? : head;
struct kmem_cache_cpu *c;
unsigned long tid;
-
- slab_free_freelist_hook(s, head, tail);
-
redo:
/*
* Determine the currently cpus per cpu slab.
@@ -2854,6 +2865,27 @@ redo:
}
+static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
+ void *head, void *tail, int cnt,
+ unsigned long addr)
+{
+ slab_free_freelist_hook(s, head, tail);
+ /*
+ * slab_free_freelist_hook() could have put the items into quarantine.
+ * If so, no need to free them.
+ */
+ if (s->flags & SLAB_KASAN && !(s->flags & SLAB_DESTROY_BY_RCU))
+ return;
+ do_slab_free(s, page, head, tail, cnt, addr);
+}
+
+#ifdef CONFIG_KASAN
+void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
+{
+ do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
+}
+#endif
+
void kmem_cache_free(struct kmem_cache *s, void *x)
{
s = cache_from_obj(s, x);
@@ -3210,7 +3242,8 @@ static void early_kmem_cache_node_alloc(int node)
init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
init_tracking(kmem_cache_node, n);
#endif
- kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node));
+ kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
+ GFP_KERNEL);
init_kmem_cache_node(n);
inc_slabs_node(kmem_cache_node, node, page->objects);
@@ -3273,7 +3306,7 @@ static void set_min_partial(struct kmem_cache *s, unsigned long min)
static int calculate_sizes(struct kmem_cache *s, int forced_order)
{
unsigned long flags = s->flags;
- unsigned long size = s->object_size;
+ size_t size = s->object_size;
int order;
/*
@@ -3332,7 +3365,10 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
* the object.
*/
size += 2 * sizeof(struct track);
+#endif
+ kasan_cache_create(s, &size, &s->flags);
+#ifdef CONFIG_SLUB_DEBUG
if (flags & SLAB_RED_ZONE) {
/*
* Add some empty padding so that we can catch
@@ -3461,10 +3497,9 @@ static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
free_kmem_cache_nodes(s);
error:
if (flags & SLAB_PANIC)
- panic("Cannot create slab %s size=%lu realsize=%u "
- "order=%u offset=%u flags=%lx\n",
- s->name, (unsigned long)s->size, s->size,
- oo_order(s->oo), s->offset, flags);
+ panic("Cannot create slab %s size=%lu realsize=%u order=%u offset=%u flags=%lx\n",
+ s->name, (unsigned long)s->size, s->size,
+ oo_order(s->oo), s->offset, flags);
return -EINVAL;
}
@@ -3588,7 +3623,7 @@ void *__kmalloc(size_t size, gfp_t flags)
trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
- kasan_kmalloc(s, ret, size);
+ kasan_kmalloc(s, ret, size, flags);
return ret;
}
@@ -3633,7 +3668,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
- kasan_kmalloc(s, ret, size);
+ kasan_kmalloc(s, ret, size, flags);
return ret;
}
@@ -3702,7 +3737,7 @@ size_t ksize(const void *object)
size_t size = __ksize(object);
/* We assume that ksize callers could use whole allocated area,
so we need unpoison this area. */
- kasan_krealloc(object, size);
+ kasan_krealloc(object, size, GFP_NOWAIT);
return size;
}
EXPORT_SYMBOL(ksize);