tcp: replace ehash_size by ehash_mask
Storing the mask (size - 1) instead of the size allows fast path to be
a bit faster.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index d522dcf..5f11c4a 100644
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -125,7 +125,7 @@
*/
struct inet_ehash_bucket *ehash;
spinlock_t *ehash_locks;
- unsigned int ehash_size;
+ unsigned int ehash_mask;
unsigned int ehash_locks_mask;
/* Ok, let's try this, I give up, we do need a local binding
@@ -158,7 +158,7 @@
struct inet_hashinfo *hashinfo,
unsigned int hash)
{
- return &hashinfo->ehash[hash & (hashinfo->ehash_size - 1)];
+ return &hashinfo->ehash[hash & hashinfo->ehash_mask];
}
static inline spinlock_t *inet_ehash_lockp(
diff --git a/net/dccp/proto.c b/net/dccp/proto.c
index a156319..ecb203f 100644
--- a/net/dccp/proto.c
+++ b/net/dccp/proto.c
@@ -1060,11 +1060,12 @@
for (ehash_order = 0; (1UL << ehash_order) < goal; ehash_order++)
;
do {
- dccp_hashinfo.ehash_size = (1UL << ehash_order) * PAGE_SIZE /
+ unsigned long hash_size = (1UL << ehash_order) * PAGE_SIZE /
sizeof(struct inet_ehash_bucket);
- while (dccp_hashinfo.ehash_size &
- (dccp_hashinfo.ehash_size - 1))
- dccp_hashinfo.ehash_size--;
+
+ while (hash_size & (hash_size - 1))
+ hash_size--;
+ dccp_hashinfo.ehash_mask = hash_size - 1;
dccp_hashinfo.ehash = (struct inet_ehash_bucket *)
__get_free_pages(GFP_ATOMIC|__GFP_NOWARN, ehash_order);
} while (!dccp_hashinfo.ehash && --ehash_order > 0);
@@ -1074,7 +1075,7 @@
goto out_free_bind_bucket_cachep;
}
- for (i = 0; i < dccp_hashinfo.ehash_size; i++) {
+ for (i = 0; i <= dccp_hashinfo.ehash_mask; i++) {
INIT_HLIST_NULLS_HEAD(&dccp_hashinfo.ehash[i].chain, i);
INIT_HLIST_NULLS_HEAD(&dccp_hashinfo.ehash[i].twchain, i);
}
@@ -1153,7 +1154,7 @@
get_order(dccp_hashinfo.bhash_size *
sizeof(struct inet_bind_hashbucket)));
free_pages((unsigned long)dccp_hashinfo.ehash,
- get_order(dccp_hashinfo.ehash_size *
+ get_order((dccp_hashinfo.ehash_mask + 1) *
sizeof(struct inet_ehash_bucket)));
inet_ehash_locks_free(&dccp_hashinfo);
kmem_cache_destroy(dccp_hashinfo.bind_bucket_cachep);
diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c
index a706a47..cb73fde 100644
--- a/net/ipv4/inet_diag.c
+++ b/net/ipv4/inet_diag.c
@@ -774,7 +774,7 @@
if (!(r->idiag_states & ~(TCPF_LISTEN | TCPF_SYN_RECV)))
goto unlock;
- for (i = s_i; i < hashinfo->ehash_size; i++) {
+ for (i = s_i; i <= hashinfo->ehash_mask; i++) {
struct inet_ehash_bucket *head = &hashinfo->ehash[i];
spinlock_t *lock = inet_ehash_lockp(hashinfo, i);
struct sock *sk;
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index 625cc5f..a45aaf3 100644
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -209,7 +209,7 @@
* have wildcards anyways.
*/
unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
- unsigned int slot = hash & (hashinfo->ehash_size - 1);
+ unsigned int slot = hash & hashinfo->ehash_mask;
struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
rcu_read_lock();
diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c
index 13f0781..2fe5711 100644
--- a/net/ipv4/inet_timewait_sock.c
+++ b/net/ipv4/inet_timewait_sock.c
@@ -430,7 +430,7 @@
int h;
local_bh_disable();
- for (h = 0; h < (hashinfo->ehash_size); h++) {
+ for (h = 0; h <= hashinfo->ehash_mask; h++) {
struct inet_ehash_bucket *head =
inet_ehash_bucket(hashinfo, h);
spinlock_t *lock = inet_ehash_lockp(hashinfo, h);
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 64d0af6..cf13726 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2865,11 +2865,10 @@
(totalram_pages >= 128 * 1024) ?
13 : 15,
0,
- &tcp_hashinfo.ehash_size,
NULL,
+ &tcp_hashinfo.ehash_mask,
thash_entries ? 0 : 512 * 1024);
- tcp_hashinfo.ehash_size = 1 << tcp_hashinfo.ehash_size;
- for (i = 0; i < tcp_hashinfo.ehash_size; i++) {
+ for (i = 0; i <= tcp_hashinfo.ehash_mask; i++) {
INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].twchain, i);
}
@@ -2878,7 +2877,7 @@
tcp_hashinfo.bhash =
alloc_large_system_hash("TCP bind",
sizeof(struct inet_bind_hashbucket),
- tcp_hashinfo.ehash_size,
+ tcp_hashinfo.ehash_mask + 1,
(totalram_pages >= 128 * 1024) ?
13 : 15,
0,
@@ -2933,8 +2932,8 @@
sysctl_tcp_rmem[2] = max(87380, max_share);
printk(KERN_INFO "TCP: Hash tables configured "
- "(established %d bind %d)\n",
- tcp_hashinfo.ehash_size, tcp_hashinfo.bhash_size);
+ "(established %u bind %u)\n",
+ tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
tcp_register_congestion_control(&tcp_reno);
}
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 7cda24b..9971870 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -2000,7 +2000,7 @@
struct net *net = seq_file_net(seq);
void *rc = NULL;
- for (st->bucket = 0; st->bucket < tcp_hashinfo.ehash_size; ++st->bucket) {
+ for (st->bucket = 0; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
struct sock *sk;
struct hlist_nulls_node *node;
struct inet_timewait_sock *tw;
@@ -2061,10 +2061,10 @@
st->state = TCP_SEQ_STATE_ESTABLISHED;
/* Look for next non empty bucket */
- while (++st->bucket < tcp_hashinfo.ehash_size &&
+ while (++st->bucket <= tcp_hashinfo.ehash_mask &&
empty_bucket(st))
;
- if (st->bucket >= tcp_hashinfo.ehash_size)
+ if (st->bucket > tcp_hashinfo.ehash_mask)
return NULL;
spin_lock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
index 1bcc343..874aed8 100644
--- a/net/ipv6/inet6_hashtables.c
+++ b/net/ipv6/inet6_hashtables.c
@@ -73,7 +73,7 @@
* have wildcards anyways.
*/
unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
- unsigned int slot = hash & (hashinfo->ehash_size - 1);
+ unsigned int slot = hash & hashinfo->ehash_mask;
struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
