ipv6: make lookups simpler and faster
TCP listener refactoring, part 4 :
To speed up inet lookups, we moved IPv4 addresses from inet to struct
sock_common
Now is time to do the same for IPv6, because it permits us to have fast
lookups for all kind of sockets, including upcoming SYN_RECV.
Getting IPv6 addresses in TCP lookups currently requires two extra cache
lines, plus a dereference (and memory stall).
inet6_sk(sk) does the dereference of inet_sk(__sk)->pinet6
This patch is way bigger than its IPv4 counter part, because for IPv4,
we could add aliases (inet_daddr, inet_rcv_saddr), while on IPv6,
it's not doable easily.
inet6_sk(sk)->daddr becomes sk->sk_v6_daddr
inet6_sk(sk)->rcv_saddr becomes sk->sk_v6_rcv_saddr
And timewait socket also have tw->tw_v6_daddr & tw->tw_v6_rcv_saddr
at the same offset.
We get rid of INET6_TW_MATCH() as INET6_MATCH() is now the generic
macro.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h
index b7f1f3b..35f6c1b 100644
--- a/include/linux/ipv6.h
+++ b/include/linux/ipv6.h
@@ -141,8 +141,6 @@
*/
struct ipv6_pinfo {
struct in6_addr saddr;
- struct in6_addr rcv_saddr;
- struct in6_addr daddr;
struct in6_pktinfo sticky_pktinfo;
const struct in6_addr *daddr_cache;
#ifdef CONFIG_IPV6_SUBTREES
@@ -256,22 +254,10 @@
extern int inet6_sk_rebuild_header(struct sock *sk);
-struct inet6_timewait_sock {
- struct in6_addr tw_v6_daddr;
- struct in6_addr tw_v6_rcv_saddr;
-};
-
struct tcp6_timewait_sock {
struct tcp_timewait_sock tcp6tw_tcp;
- struct inet6_timewait_sock tcp6tw_inet6;
};
-static inline struct inet6_timewait_sock *inet6_twsk(const struct sock *sk)
-{
- return (struct inet6_timewait_sock *)(((u8 *)sk) +
- inet_twsk(sk)->tw_ipv6_offset);
-}
-
#if IS_ENABLED(CONFIG_IPV6)
static inline struct ipv6_pinfo * inet6_sk(const struct sock *__sk)
{
@@ -321,21 +307,11 @@
#define __ipv6_only_sock(sk) (inet6_sk(sk)->ipv6only)
#define ipv6_only_sock(sk) ((sk)->sk_family == PF_INET6 && __ipv6_only_sock(sk))
-static inline u16 inet6_tw_offset(const struct proto *prot)
+static inline const struct in6_addr *inet6_rcv_saddr(const struct sock *sk)
{
- return prot->twsk_prot->twsk_obj_size -
- sizeof(struct inet6_timewait_sock);
-}
-
-static inline struct in6_addr *__inet6_rcv_saddr(const struct sock *sk)
-{
- return likely(sk->sk_state != TCP_TIME_WAIT) ?
- &inet6_sk(sk)->rcv_saddr : &inet6_twsk(sk)->tw_v6_rcv_saddr;
-}
-
-static inline struct in6_addr *inet6_rcv_saddr(const struct sock *sk)
-{
- return sk->sk_family == AF_INET6 ? __inet6_rcv_saddr(sk) : NULL;
+ if (sk->sk_family == AF_INET6)
+ return &sk->sk_v6_rcv_saddr;
+ return NULL;
}
static inline int inet_v6_ipv6only(const struct sock *sk)
@@ -363,7 +339,6 @@
return NULL;
}
-#define __inet6_rcv_saddr(__sk) NULL
#define inet6_rcv_saddr(__sk) NULL
#define tcp_twsk_ipv6only(__sk) 0
#define inet_v6_ipv6only(__sk) 0
@@ -372,19 +347,10 @@
#define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \
(((__sk)->sk_portpair == (__ports)) && \
((__sk)->sk_family == AF_INET6) && \
- ipv6_addr_equal(&inet6_sk(__sk)->daddr, (__saddr)) && \
- ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) && \
+ ipv6_addr_equal(&(__sk)->sk_v6_daddr, (__saddr)) && \
+ ipv6_addr_equal(&(__sk)->sk_v6_rcv_saddr, (__daddr)) && \
(!(__sk)->sk_bound_dev_if || \
((__sk)->sk_bound_dev_if == (__dif))) && \
net_eq(sock_net(__sk), (__net)))
-#define INET6_TW_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \
- (((__sk)->sk_portpair == (__ports)) && \
- ((__sk)->sk_family == AF_INET6) && \
- ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_daddr, (__saddr)) && \
- ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_rcv_saddr, (__daddr)) && \
- (!(__sk)->sk_bound_dev_if || \
- ((__sk)->sk_bound_dev_if == (__dif))) && \
- net_eq(sock_net(__sk), (__net)))
-
#endif /* _IPV6_H */