From 48aab2f79dfc1357c48ce22ff5c989b52a590069 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 22 Mar 2012 17:01:41 -0700 Subject: security: optimize avc_audit() common path avc_audit() did a lot of jumping around and had a big stack frame, all for the uncommon case. Split up the uncommon case (which we really can't make go fast anyway) into its own slow function, and mark the conditional branches appropriately for the common likely case. This causes avc_audit() to no longer show up as one of the hottest functions on the branch profiles (the new "perf -b" thing), and makes the cycle profiles look really nice and dense too. The whole audit path is still annoyingly very much one of the biggest costs of name lookup, so these things are worth optimizing for. I wish we could just tell people to turn it off, but realistically we do need it: we just need to make sure that the overhead of the necessary evil is as low as possible. Signed-off-by: Linus Torvalds --- security/selinux/avc.c | 70 +++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 29 deletions(-) (limited to 'security/selinux/avc.c') diff --git a/security/selinux/avc.c b/security/selinux/avc.c index dca1c22d9276..6989472d0957 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -457,6 +457,42 @@ static void avc_audit_post_callback(struct audit_buffer *ab, void *a) ad->selinux_audit_data.tclass); } +/* This is the slow part of avc audit with big stack footprint */ +static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, + u32 requested, u32 audited, u32 denied, + struct av_decision *avd, struct common_audit_data *a, + unsigned flags) +{ + struct common_audit_data stack_data; + + if (!a) { + a = &stack_data; + COMMON_AUDIT_DATA_INIT(a, NONE); + } + + /* + * When in a RCU walk do the audit on the RCU retry. This is because + * the collection of the dname in an inode audit message is not RCU + * safe. Note this may drop some audits when the situation changes + * during retry. However this is logically just as if the operation + * happened a little later. + */ + if ((a->type == LSM_AUDIT_DATA_INODE) && + (flags & MAY_NOT_BLOCK)) + return -ECHILD; + + a->selinux_audit_data.tclass = tclass; + a->selinux_audit_data.requested = requested; + a->selinux_audit_data.ssid = ssid; + a->selinux_audit_data.tsid = tsid; + a->selinux_audit_data.audited = audited; + a->selinux_audit_data.denied = denied; + a->lsm_pre_audit = avc_audit_pre_callback; + a->lsm_post_audit = avc_audit_post_callback; + common_lsm_audit(a); + return 0; +} + /** * avc_audit - Audit the granting or denial of permissions. * @ssid: source security identifier @@ -482,10 +518,9 @@ int avc_audit(u32 ssid, u32 tsid, struct av_decision *avd, int result, struct common_audit_data *a, unsigned flags) { - struct common_audit_data stack_data; u32 denied, audited; denied = requested & ~avd->allowed; - if (denied) { + if (unlikely(denied)) { audited = denied & avd->auditdeny; /* * a->selinux_audit_data.auditdeny is TRICKY! Setting a bit in @@ -511,35 +546,12 @@ int avc_audit(u32 ssid, u32 tsid, audited = denied = requested; else audited = requested & avd->auditallow; - if (!audited) + if (likely(!audited)) return 0; - if (!a) { - a = &stack_data; - COMMON_AUDIT_DATA_INIT(a, NONE); - } - - /* - * When in a RCU walk do the audit on the RCU retry. This is because - * the collection of the dname in an inode audit message is not RCU - * safe. Note this may drop some audits when the situation changes - * during retry. However this is logically just as if the operation - * happened a little later. - */ - if ((a->type == LSM_AUDIT_DATA_INODE) && - (flags & MAY_NOT_BLOCK)) - return -ECHILD; - - a->selinux_audit_data.tclass = tclass; - a->selinux_audit_data.requested = requested; - a->selinux_audit_data.ssid = ssid; - a->selinux_audit_data.tsid = tsid; - a->selinux_audit_data.audited = audited; - a->selinux_audit_data.denied = denied; - a->lsm_pre_audit = avc_audit_pre_callback; - a->lsm_post_audit = avc_audit_post_callback; - common_lsm_audit(a); - return 0; + return slow_avc_audit(ssid, tsid, tclass, + requested, audited, denied, + avd, a, flags); } /** -- cgit v1.2.3 From a554bea89948dfb6d2f9c4c62ce2b12b2dac18ad Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sat, 31 Mar 2012 10:58:08 -0700 Subject: selinux: don't inline slow-path code into avc_has_perm_noaudit() The selinux AVC paths remain some of the hottest (and deepest) codepaths at filename lookup time, and we make it worse by having the slow path cases take up I$ and stack space even when they don't trigger. Gcc tends to always want to inline functions that are just called once - never mind that this might make for slower and worse code in the caller. So this tries to improve on it a bit by making the slow-path cases explicitly separate functions that are marked noinline, causing gcc to at least no longer allocate stack space for them unless they are actually called. It also seems to help register allocation a tiny bit, since gcc now doesn't take the slow case code into account. Uninlining the slow path may also allow us to inline the remaining hot path into the one caller that actually matters: avc_has_perm_flags(). I'll have to look at that separately, but both avc_audit() and avc_has_perm_noaudit() are now small and lean enough that inlining them may make sense. Signed-off-by: Linus Torvalds --- security/selinux/avc.c | 52 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 14 deletions(-) (limited to 'security/selinux/avc.c') diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 6989472d0957..07620bc94e2a 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -741,6 +741,41 @@ int avc_ss_reset(u32 seqno) return rc; } +/* + * Slow-path helper function for avc_has_perm_noaudit, + * when the avc_node lookup fails. We get called with + * the RCU read lock held, and need to return with it + * still held, but drop if for the security compute. + * + * Don't inline this, since it's the slow-path and just + * results in a bigger stack frame. + */ +static noinline struct avc_node *avc_compute_av(u32 ssid, u32 tsid, + u16 tclass, struct av_decision *avd) +{ + rcu_read_unlock(); + security_compute_av(ssid, tsid, tclass, avd); + rcu_read_lock(); + return avc_insert(ssid, tsid, tclass, avd); +} + +static noinline int avc_denied(u32 ssid, u32 tsid, + u16 tclass, u32 requested, + unsigned flags, + struct av_decision *avd) +{ + if (flags & AVC_STRICT) + return -EACCES; + + if (selinux_enforcing && !(avd->flags & AVD_FLAGS_PERMISSIVE)) + return -EACCES; + + avc_update_node(AVC_CALLBACK_GRANT, requested, ssid, + tsid, tclass, avd->seqno); + return 0; +} + + /** * avc_has_perm_noaudit - Check permissions but perform no auditing. * @ssid: source security identifier @@ -776,26 +811,15 @@ int avc_has_perm_noaudit(u32 ssid, u32 tsid, node = avc_lookup(ssid, tsid, tclass); if (unlikely(!node)) { - rcu_read_unlock(); - security_compute_av(ssid, tsid, tclass, avd); - rcu_read_lock(); - node = avc_insert(ssid, tsid, tclass, avd); + node = avc_compute_av(ssid, tsid, tclass, avd); } else { memcpy(avd, &node->ae.avd, sizeof(*avd)); avd = &node->ae.avd; } denied = requested & ~(avd->allowed); - - if (denied) { - if (flags & AVC_STRICT) - rc = -EACCES; - else if (!selinux_enforcing || (avd->flags & AVD_FLAGS_PERMISSIVE)) - avc_update_node(AVC_CALLBACK_GRANT, requested, ssid, - tsid, tclass, avd->seqno); - else - rc = -EACCES; - } + if (unlikely(denied)) + rc = avc_denied(ssid, tsid, tclass, requested, flags, avd); rcu_read_unlock(); return rc; -- cgit v1.2.3 From cdb0f9a1ad2ee3c11e21bc99f0c2021a02844666 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sat, 31 Mar 2012 11:12:57 -0700 Subject: selinux: inline avc_audit() and avc_has_perm_noaudit() into caller Now that all the slow-path code is gone from these functions, we can inline them into the main caller - avc_has_perm_flags(). Now the compiler can see that 'avc' is allocated on the stack for this case, which helps register pressure a bit. It also actually shrinks the total stack frame, because the stack frame that avc_has_perm_flags() always needed (for that 'avc' allocation) is now sufficient for the inlined functions too. Inlining isn't bad - but mindless inlining of cold code (see the previous commit) is. Signed-off-by: Linus Torvalds --- security/selinux/avc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'security/selinux/avc.c') diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 07620bc94e2a..1a70fa26da72 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -513,7 +513,7 @@ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, * be performed under a lock, to allow the lock to be released * before calling the auditing code. */ -int avc_audit(u32 ssid, u32 tsid, +inline int avc_audit(u32 ssid, u32 tsid, u16 tclass, u32 requested, struct av_decision *avd, int result, struct common_audit_data *a, unsigned flags) @@ -796,7 +796,7 @@ static noinline int avc_denied(u32 ssid, u32 tsid, * auditing, e.g. in cases where a lock must be held for the check but * should be released for the auditing. */ -int avc_has_perm_noaudit(u32 ssid, u32 tsid, +inline int avc_has_perm_noaudit(u32 ssid, u32 tsid, u16 tclass, u32 requested, unsigned flags, struct av_decision *avd) -- cgit v1.2.3 From 3b3b0e4fc15efa507b902d90cea39e496a523c3b Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Tue, 3 Apr 2012 09:37:02 -0700 Subject: LSM: shrink sizeof LSM specific portion of common_audit_data Linus found that the gigantic size of the common audit data caused a big perf hit on something as simple as running stat() in a loop. This patch requires LSMs to declare the LSM specific portion separately rather than doing it in a union. Thus each LSM can be responsible for shrinking their portion and don't have to pay a penalty just because other LSMs have a bigger space requirement. Signed-off-by: Eric Paris Signed-off-by: Linus Torvalds --- security/selinux/avc.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'security/selinux/avc.c') diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 1a70fa26da72..00f3860c2370 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -436,9 +436,9 @@ static void avc_audit_pre_callback(struct audit_buffer *ab, void *a) { struct common_audit_data *ad = a; audit_log_format(ab, "avc: %s ", - ad->selinux_audit_data.denied ? "denied" : "granted"); - avc_dump_av(ab, ad->selinux_audit_data.tclass, - ad->selinux_audit_data.audited); + ad->selinux_audit_data->denied ? "denied" : "granted"); + avc_dump_av(ab, ad->selinux_audit_data->tclass, + ad->selinux_audit_data->audited); audit_log_format(ab, " for "); } @@ -452,9 +452,9 @@ static void avc_audit_post_callback(struct audit_buffer *ab, void *a) { struct common_audit_data *ad = a; audit_log_format(ab, " "); - avc_dump_query(ab, ad->selinux_audit_data.ssid, - ad->selinux_audit_data.tsid, - ad->selinux_audit_data.tclass); + avc_dump_query(ab, ad->selinux_audit_data->ssid, + ad->selinux_audit_data->tsid, + ad->selinux_audit_data->tclass); } /* This is the slow part of avc audit with big stack footprint */ @@ -464,10 +464,12 @@ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, unsigned flags) { struct common_audit_data stack_data; + struct selinux_audit_data sad = {0,}; if (!a) { a = &stack_data; COMMON_AUDIT_DATA_INIT(a, NONE); + a->selinux_audit_data = &sad; } /* @@ -481,12 +483,12 @@ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, (flags & MAY_NOT_BLOCK)) return -ECHILD; - a->selinux_audit_data.tclass = tclass; - a->selinux_audit_data.requested = requested; - a->selinux_audit_data.ssid = ssid; - a->selinux_audit_data.tsid = tsid; - a->selinux_audit_data.audited = audited; - a->selinux_audit_data.denied = denied; + a->selinux_audit_data->tclass = tclass; + a->selinux_audit_data->requested = requested; + a->selinux_audit_data->ssid = ssid; + a->selinux_audit_data->tsid = tsid; + a->selinux_audit_data->audited = audited; + a->selinux_audit_data->denied = denied; a->lsm_pre_audit = avc_audit_pre_callback; a->lsm_post_audit = avc_audit_post_callback; common_lsm_audit(a); @@ -523,7 +525,7 @@ inline int avc_audit(u32 ssid, u32 tsid, if (unlikely(denied)) { audited = denied & avd->auditdeny; /* - * a->selinux_audit_data.auditdeny is TRICKY! Setting a bit in + * a->selinux_audit_data->auditdeny is TRICKY! Setting a bit in * this field means that ANY denials should NOT be audited if * the policy contains an explicit dontaudit rule for that * permission. Take notice that this is unrelated to the @@ -532,15 +534,15 @@ inline int avc_audit(u32 ssid, u32 tsid, * * denied == READ * avd.auditdeny & ACCESS == 0 (not set means explicit rule) - * selinux_audit_data.auditdeny & ACCESS == 1 + * selinux_audit_data->auditdeny & ACCESS == 1 * * We will NOT audit the denial even though the denied * permission was READ and the auditdeny checks were for * ACCESS */ if (a && - a->selinux_audit_data.auditdeny && - !(a->selinux_audit_data.auditdeny & avd->auditdeny)) + a->selinux_audit_data->auditdeny && + !(a->selinux_audit_data->auditdeny & avd->auditdeny)) audited = 0; } else if (result) audited = denied = requested; -- cgit v1.2.3 From f8294f1144ad0630075918df4bf94075f5384604 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 2 Apr 2012 13:15:55 -0400 Subject: SELinux: remove avd from slow_avc_audit() We don't use the argument, so remove it. Signed-off-by: Eric Paris Signed-off-by: Linus Torvalds --- security/selinux/avc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'security/selinux/avc.c') diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 00f3860c2370..b5545a84448a 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -460,7 +460,7 @@ static void avc_audit_post_callback(struct audit_buffer *ab, void *a) /* This is the slow part of avc audit with big stack footprint */ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, u32 requested, u32 audited, u32 denied, - struct av_decision *avd, struct common_audit_data *a, + struct common_audit_data *a, unsigned flags) { struct common_audit_data stack_data; @@ -553,7 +553,7 @@ inline int avc_audit(u32 ssid, u32 tsid, return slow_avc_audit(ssid, tsid, tclass, requested, audited, denied, - avd, a, flags); + a, flags); } /** -- cgit v1.2.3 From 3f0882c48286e7bdb0bbdec9c4bfa934e0db8e09 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Tue, 3 Apr 2012 09:38:00 -0700 Subject: SELinux: do not allocate stack space for AVC data unless needed Instead of declaring the entire selinux_audit_data on the stack when we start an operation on declare it on the stack if we are going to use it. We know it's usefulness at the end of the security decision and can declare it there. Signed-off-by: Eric Paris Signed-off-by: Linus Torvalds --- security/selinux/avc.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'security/selinux/avc.c') diff --git a/security/selinux/avc.c b/security/selinux/avc.c index b5545a84448a..36c42bb52d81 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -436,9 +436,9 @@ static void avc_audit_pre_callback(struct audit_buffer *ab, void *a) { struct common_audit_data *ad = a; audit_log_format(ab, "avc: %s ", - ad->selinux_audit_data->denied ? "denied" : "granted"); - avc_dump_av(ab, ad->selinux_audit_data->tclass, - ad->selinux_audit_data->audited); + ad->selinux_audit_data->slad->denied ? "denied" : "granted"); + avc_dump_av(ab, ad->selinux_audit_data->slad->tclass, + ad->selinux_audit_data->slad->audited); audit_log_format(ab, " for "); } @@ -452,9 +452,9 @@ static void avc_audit_post_callback(struct audit_buffer *ab, void *a) { struct common_audit_data *ad = a; audit_log_format(ab, " "); - avc_dump_query(ab, ad->selinux_audit_data->ssid, - ad->selinux_audit_data->tsid, - ad->selinux_audit_data->tclass); + avc_dump_query(ab, ad->selinux_audit_data->slad->ssid, + ad->selinux_audit_data->slad->tsid, + ad->selinux_audit_data->slad->tclass); } /* This is the slow part of avc audit with big stack footprint */ @@ -465,6 +465,7 @@ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, { struct common_audit_data stack_data; struct selinux_audit_data sad = {0,}; + struct selinux_late_audit_data slad; if (!a) { a = &stack_data; @@ -483,12 +484,14 @@ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, (flags & MAY_NOT_BLOCK)) return -ECHILD; - a->selinux_audit_data->tclass = tclass; - a->selinux_audit_data->requested = requested; - a->selinux_audit_data->ssid = ssid; - a->selinux_audit_data->tsid = tsid; - a->selinux_audit_data->audited = audited; - a->selinux_audit_data->denied = denied; + slad.tclass = tclass; + slad.requested = requested; + slad.ssid = ssid; + slad.tsid = tsid; + slad.audited = audited; + slad.denied = denied; + + a->selinux_audit_data->slad = &slad; a->lsm_pre_audit = avc_audit_pre_callback; a->lsm_post_audit = avc_audit_post_callback; common_lsm_audit(a); -- cgit v1.2.3 From b61c37f57988567c84359645f8202a7c84bc798a Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Mon, 2 Apr 2012 15:48:12 -0700 Subject: lsm_audit: don't specify the audit pre/post callbacks in 'struct common_audit_data' It just bloats the audit data structure for no good reason, since the only time those fields are filled are just before calling the common_lsm_audit() function, which is also the only user of those fields. So just make them be the arguments to common_lsm_audit(), rather than bloating that structure that is passed around everywhere, and is initialized in hot paths. Signed-off-by: Linus Torvalds --- security/selinux/avc.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'security/selinux/avc.c') diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 36c42bb52d81..8ee42b2a5f19 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -492,9 +492,7 @@ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, slad.denied = denied; a->selinux_audit_data->slad = &slad; - a->lsm_pre_audit = avc_audit_pre_callback; - a->lsm_post_audit = avc_audit_post_callback; - common_lsm_audit(a); + common_lsm_audit(a, avc_audit_pre_callback, avc_audit_post_callback); return 0; } -- cgit v1.2.3