From 0a14842f5a3c0e88a1e59fac5c3025db39721f74 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 20 Apr 2011 09:27:32 +0000 Subject: net: filter: Just In Time compiler for x86-64 In order to speedup packet filtering, here is an implementation of a JIT compiler for x86_64 It is disabled by default, and must be enabled by the admin. echo 1 >/proc/sys/net/core/bpf_jit_enable It uses module_alloc() and module_free() to get memory in the 2GB text kernel range since we call helpers functions from the generated code. EAX : BPF A accumulator EBX : BPF X accumulator RDI : pointer to skb (first argument given to JIT function) RBP : frame pointer (even if CONFIG_FRAME_POINTER=n) r9d : skb->len - skb->data_len (headlen) r8 : skb->data To get a trace of generated code, use : echo 2 >/proc/sys/net/core/bpf_jit_enable Example of generated code : # tcpdump -p -n -s 0 -i eth1 host 192.168.20.0/24 flen=18 proglen=147 pass=3 image=ffffffffa00b5000 JIT code: ffffffffa00b5000: 55 48 89 e5 48 83 ec 60 48 89 5d f8 44 8b 4f 60 JIT code: ffffffffa00b5010: 44 2b 4f 64 4c 8b 87 b8 00 00 00 be 0c 00 00 00 JIT code: ffffffffa00b5020: e8 24 7b f7 e0 3d 00 08 00 00 75 28 be 1a 00 00 JIT code: ffffffffa00b5030: 00 e8 fe 7a f7 e0 24 00 3d 00 14 a8 c0 74 49 be JIT code: ffffffffa00b5040: 1e 00 00 00 e8 eb 7a f7 e0 24 00 3d 00 14 a8 c0 JIT code: ffffffffa00b5050: 74 36 eb 3b 3d 06 08 00 00 74 07 3d 35 80 00 00 JIT code: ffffffffa00b5060: 75 2d be 1c 00 00 00 e8 c8 7a f7 e0 24 00 3d 00 JIT code: ffffffffa00b5070: 14 a8 c0 74 13 be 26 00 00 00 e8 b5 7a f7 e0 24 JIT code: ffffffffa00b5080: 00 3d 00 14 a8 c0 75 07 b8 ff ff 00 00 eb 02 31 JIT code: ffffffffa00b5090: c0 c9 c3 BPF program is 144 bytes long, so native program is almost same size ;) (000) ldh [12] (001) jeq #0x800 jt 2 jf 8 (002) ld [26] (003) and #0xffffff00 (004) jeq #0xc0a81400 jt 16 jf 5 (005) ld [30] (006) and #0xffffff00 (007) jeq #0xc0a81400 jt 16 jf 17 (008) jeq #0x806 jt 10 jf 9 (009) jeq #0x8035 jt 10 jf 17 (010) ld [28] (011) and #0xffffff00 (012) jeq #0xc0a81400 jt 16 jf 13 (013) ld [38] (014) and #0xffffff00 (015) jeq #0xc0a81400 jt 16 jf 17 (016) ret #65535 (017) ret #0 Signed-off-by: Eric Dumazet Cc: Arnaldo Carvalho de Melo Cc: Ben Hutchings Cc: Hagen Paul Pfeifer Signed-off-by: David S. Miller --- include/linux/skbuff.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d0ae90af0b40..79aafbbf430a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -391,8 +391,8 @@ struct sk_buff { __u32 rxhash; + __u16 queue_mapping; kmemcheck_bitfield_begin(flags2); - __u16 queue_mapping:16; #ifdef CONFIG_IPV6_NDISC_NODETYPE __u8 ndisc_nodetype:2; #endif -- cgit v1.2.3 From 34ea646c9f8c18fd2e4332ff3b2b509f878c56f1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 22 May 2011 18:55:10 +0200 Subject: net: add missing prefetch.h include Fixes build errors on s390 and probably other archs as well: In file included from net/ipv4/ip_forward.c:32:0: include/net/udp.h: In function 'udp_csum_outgoing': include/net/udp.h:141:2: error: implicit declaration of function 'prefetch' Signed-off-by: Heiko Carstens Signed-off-by: Linus Torvalds --- include/linux/skbuff.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 79aafbbf430a..827681540d6f 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -28,6 +28,7 @@ #include #include #include +#include #include /* Don't change this without changing skb_csum_unnecessary! */ -- cgit v1.2.3 From 0fcbe742eaac14bd5032b369c09e9d94be9058ad Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sun, 22 May 2011 20:35:29 -0400 Subject: net: Remove prefetches from SKB list handlers. Noticed by Linus. Signed-off-by: David S. Miller --- include/linux/skbuff.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 827681540d6f..09901fdd73ae 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1783,7 +1783,7 @@ static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) #define skb_queue_walk(queue, skb) \ for (skb = (queue)->next; \ - prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \ + (skb != (struct sk_buff *)(queue)); \ skb = skb->next) #define skb_queue_walk_safe(queue, skb, tmp) \ @@ -1792,7 +1792,7 @@ static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) skb = tmp, tmp = skb->next) #define skb_queue_walk_from(queue, skb) \ - for (; prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \ + for (; (skb != (struct sk_buff *)(queue)); \ skb = skb->next) #define skb_queue_walk_from_safe(queue, skb, tmp) \ @@ -1802,7 +1802,7 @@ static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) #define skb_queue_reverse_walk(queue, skb) \ for (skb = (queue)->prev; \ - prefetch(skb->prev), (skb != (struct sk_buff *)(queue)); \ + (skb != (struct sk_buff *)(queue)); \ skb = skb->prev) #define skb_queue_reverse_walk_safe(queue, skb, tmp) \ -- cgit v1.2.3 From 67f11f4deda0818640decb19a28c537dbe5d429e Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sun, 22 May 2011 20:54:11 -0400 Subject: net: Remove linux/prefetch.h include from linux/skbuff.h No longer needed. Signed-off-by: David S. Miller --- include/linux/skbuff.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 09901fdd73ae..8cac356b77b2 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -28,7 +28,6 @@ #include #include #include -#include #include /* Don't change this without changing skb_csum_unnecessary! */ -- cgit v1.2.3 From c4264f27e83968ddfe3f0cfe7a33adfb320e1e42 Mon Sep 17 00:00:00 2001 From: Emmanuel Grumbach Date: Sat, 21 May 2011 19:46:09 +0000 Subject: net: skb_trim explicitely check the linearity instead of data_len The purpose of the check on data_len is to check linearity, so use the inline helper for this. No overhead and more explicit. Signed-off-by: Emmanuel Grumbach Signed-off-by: David S. Miller --- include/linux/skbuff.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 8cac356b77b2..aeaad97e6815 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1442,7 +1442,7 @@ extern int ___pskb_trim(struct sk_buff *skb, unsigned int len); static inline void __skb_trim(struct sk_buff *skb, unsigned int len) { - if (unlikely(skb->data_len)) { + if (unlikely(skb_is_nonlinear(skb))) { WARN_ON(1); return; } -- cgit v1.2.3 From a1e4891fd48d298870b704c6eb48cba0da5ed6b1 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 22 May 2011 16:51:43 -0700 Subject: Remove prefetch() from and "netlabel_addrlist.h" Commit e66eed651fd1 ("list: remove prefetching from regular list iterators") removed the include of prefetch.h from list.h. The skbuff list traversal still had them. Quoth David Miller: "Please just remove the prefetches. Those are modelled after list.h as I intend to eventually convert SKB list handling to "struct list_head" but we're not there yet. Therefore if we kill prefetches from list.h we should kill it from these things in skbuff.h too." Requested-by: David Miller Signed-off-by: Linus Torvalds --- include/linux/skbuff.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 827681540d6f..16c9c091555d 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -28,7 +28,6 @@ #include #include #include -#include #include /* Don't change this without changing skb_csum_unnecessary! */ @@ -1783,7 +1782,7 @@ static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) #define skb_queue_walk(queue, skb) \ for (skb = (queue)->next; \ - prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \ + skb != (struct sk_buff *)(queue); \ skb = skb->next) #define skb_queue_walk_safe(queue, skb, tmp) \ @@ -1792,7 +1791,7 @@ static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) skb = tmp, tmp = skb->next) #define skb_queue_walk_from(queue, skb) \ - for (; prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \ + for (; skb != (struct sk_buff *)(queue); \ skb = skb->next) #define skb_queue_walk_from_safe(queue, skb, tmp) \ @@ -1802,7 +1801,7 @@ static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) #define skb_queue_reverse_walk(queue, skb) \ for (skb = (queue)->prev; \ - prefetch(skb->prev), (skb != (struct sk_buff *)(queue)); \ + skb != (struct sk_buff *)(queue); \ skb = skb->prev) #define skb_queue_reverse_walk_safe(queue, skb, tmp) \ -- cgit v1.2.3 From 0b5c9db1b11d3175bb42b80663a9f072f801edf5 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 10 Jun 2011 06:56:58 +0000 Subject: vlan: Fix the ingress VLAN_FLAG_REORDER_HDR check Testing of VLAN_FLAG_REORDER_HDR does not belong in vlan_untag but rather in vlan_do_receive. Otherwise the vlan header will not be properly put on the packet in the case of vlan header accelleration. As we remove the check from vlan_check_reorder_header rename it vlan_reorder_header to keep the naming clean. Fix up the skb->pkt_type early so we don't look at the packet after adding the vlan tag, which guarantees we don't goof and look at the wrong field. Use a simple if statement instead of a complicated switch statement to decided that we need to increment rx_stats for a multicast packet. Hopefully at somepoint we will just declare the case where VLAN_FLAG_REORDER_HDR is cleared as unsupported and remove the code. Until then this keeps it working correctly. Signed-off-by: Eric W. Biederman Signed-off-by: Jiri Pirko Acked-by: Changli Gao Signed-off-by: David S. Miller --- include/linux/skbuff.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index e8b78ce14474..c0a4f3ab0cc0 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1256,6 +1256,11 @@ static inline void skb_reserve(struct sk_buff *skb, int len) skb->tail += len; } +static inline void skb_reset_mac_len(struct sk_buff *skb) +{ + skb->mac_len = skb->network_header - skb->mac_header; +} + #ifdef NET_SKBUFF_DATA_USES_OFFSET static inline unsigned char *skb_transport_header(const struct sk_buff *skb) { -- cgit v1.2.3