1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/module.h>
17#include <linux/random.h>
18#include <linux/sched.h>
19#include <linux/slab.h>
20#include <linux/wait.h>
21
22#include <net/inet_connection_sock.h>
23#include <net/inet_hashtables.h>
24#include <net/ip.h>
25
26
27
28
29
30struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
31 struct net *net,
32 struct inet_bind_hashbucket *head,
33 const unsigned short snum)
34{
35 struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
36
37 if (tb != NULL) {
38 write_pnet(&tb->ib_net, hold_net(net));
39 tb->port = snum;
40 tb->fastreuse = 0;
41 tb->num_owners = 0;
42 INIT_HLIST_HEAD(&tb->owners);
43 hlist_add_head(&tb->node, &head->chain);
44 }
45 return tb;
46}
47
48
49
50
51void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
52{
53 if (hlist_empty(&tb->owners)) {
54 __hlist_del(&tb->node);
55 release_net(ib_net(tb));
56 kmem_cache_free(cachep, tb);
57 }
58}
59
60void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
61 const unsigned short snum)
62{
63 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
64
65 atomic_inc(&hashinfo->bsockets);
66
67 inet_sk(sk)->num = snum;
68 sk_add_bind_node(sk, &tb->owners);
69 tb->num_owners++;
70 inet_csk(sk)->icsk_bind_hash = tb;
71}
72
73
74
75
76static void __inet_put_port(struct sock *sk)
77{
78 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
79 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->num,
80 hashinfo->bhash_size);
81 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
82 struct inet_bind_bucket *tb;
83
84 atomic_dec(&hashinfo->bsockets);
85
86 spin_lock(&head->lock);
87 tb = inet_csk(sk)->icsk_bind_hash;
88 __sk_del_bind_node(sk);
89 tb->num_owners--;
90 inet_csk(sk)->icsk_bind_hash = NULL;
91 inet_sk(sk)->num = 0;
92 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
93 spin_unlock(&head->lock);
94}
95
96void inet_put_port(struct sock *sk)
97{
98 local_bh_disable();
99 __inet_put_port(sk);
100 local_bh_enable();
101}
102
103EXPORT_SYMBOL(inet_put_port);
104
105void __inet_inherit_port(struct sock *sk, struct sock *child)
106{
107 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
108 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(child)->num,
109 table->bhash_size);
110 struct inet_bind_hashbucket *head = &table->bhash[bhash];
111 struct inet_bind_bucket *tb;
112
113 spin_lock(&head->lock);
114 tb = inet_csk(sk)->icsk_bind_hash;
115 sk_add_bind_node(child, &tb->owners);
116 inet_csk(child)->icsk_bind_hash = tb;
117 spin_unlock(&head->lock);
118}
119
120EXPORT_SYMBOL_GPL(__inet_inherit_port);
121
122static inline int compute_score(struct sock *sk, struct net *net,
123 const unsigned short hnum, const __be32 daddr,
124 const int dif)
125{
126 int score = -1;
127 struct inet_sock *inet = inet_sk(sk);
128
129 if (net_eq(sock_net(sk), net) && inet->num == hnum &&
130 !ipv6_only_sock(sk)) {
131 __be32 rcv_saddr = inet->rcv_saddr;
132 score = sk->sk_family == PF_INET ? 1 : 0;
133 if (rcv_saddr) {
134 if (rcv_saddr != daddr)
135 return -1;
136 score += 2;
137 }
138 if (sk->sk_bound_dev_if) {
139 if (sk->sk_bound_dev_if != dif)
140 return -1;
141 score += 2;
142 }
143 }
144 return score;
145}
146
147
148
149
150
151
152
153
154
155struct sock *__inet_lookup_listener(struct net *net,
156 struct inet_hashinfo *hashinfo,
157 const __be32 daddr, const unsigned short hnum,
158 const int dif)
159{
160 struct sock *sk, *result;
161 struct hlist_nulls_node *node;
162 unsigned int hash = inet_lhashfn(net, hnum);
163 struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
164 int score, hiscore;
165
166 rcu_read_lock();
167begin:
168 result = NULL;
169 hiscore = -1;
170 sk_nulls_for_each_rcu(sk, node, &ilb->head) {
171 score = compute_score(sk, net, hnum, daddr, dif);
172 if (score > hiscore) {
173 result = sk;
174 hiscore = score;
175 }
176 }
177
178
179
180
181
182 if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
183 goto begin;
184 if (result) {
185 if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
186 result = NULL;
187 else if (unlikely(compute_score(result, net, hnum, daddr,
188 dif) < hiscore)) {
189 sock_put(result);
190 goto begin;
191 }
192 }
193 rcu_read_unlock();
194 return result;
195}
196EXPORT_SYMBOL_GPL(__inet_lookup_listener);
197
198struct sock * __inet_lookup_established(struct net *net,
199 struct inet_hashinfo *hashinfo,
200 const __be32 saddr, const __be16 sport,
201 const __be32 daddr, const u16 hnum,
202 const int dif)
203{
204 INET_ADDR_COOKIE(acookie, saddr, daddr)
205 const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
206 struct sock *sk;
207 const struct hlist_nulls_node *node;
208
209
210
211 unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
212 unsigned int slot = hash & (hashinfo->ehash_size - 1);
213 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
214
215 rcu_read_lock();
216begin:
217 sk_nulls_for_each_rcu(sk, node, &head->chain) {
218 if (INET_MATCH(sk, net, hash, acookie,
219 saddr, daddr, ports, dif)) {
220 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
221 goto begintw;
222 if (unlikely(!INET_MATCH(sk, net, hash, acookie,
223 saddr, daddr, ports, dif))) {
224 sock_put(sk);
225 goto begin;
226 }
227 goto out;
228 }
229 }
230
231
232
233
234
235 if (get_nulls_value(node) != slot)
236 goto begin;
237
238begintw:
239
240 sk_nulls_for_each_rcu(sk, node, &head->twchain) {
241 if (INET_TW_MATCH(sk, net, hash, acookie,
242 saddr, daddr, ports, dif)) {
243 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) {
244 sk = NULL;
245 goto out;
246 }
247 if (unlikely(!INET_TW_MATCH(sk, net, hash, acookie,
248 saddr, daddr, ports, dif))) {
249 sock_put(sk);
250 goto begintw;
251 }
252 goto out;
253 }
254 }
255
256
257
258
259
260 if (get_nulls_value(node) != slot)
261 goto begintw;
262 sk = NULL;
263out:
264 rcu_read_unlock();
265 return sk;
266}
267EXPORT_SYMBOL_GPL(__inet_lookup_established);
268
269
270static int __inet_check_established(struct inet_timewait_death_row *death_row,
271 struct sock *sk, __u16 lport,
272 struct inet_timewait_sock **twp)
273{
274 struct inet_hashinfo *hinfo = death_row->hashinfo;
275 struct inet_sock *inet = inet_sk(sk);
276 __be32 daddr = inet->rcv_saddr;
277 __be32 saddr = inet->daddr;
278 int dif = sk->sk_bound_dev_if;
279 INET_ADDR_COOKIE(acookie, saddr, daddr)
280 const __portpair ports = INET_COMBINED_PORTS(inet->dport, lport);
281 struct net *net = sock_net(sk);
282 unsigned int hash = inet_ehashfn(net, daddr, lport, saddr, inet->dport);
283 struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
284 spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
285 struct sock *sk2;
286 const struct hlist_nulls_node *node;
287 struct inet_timewait_sock *tw;
288
289 spin_lock(lock);
290
291
292 sk_nulls_for_each(sk2, node, &head->twchain) {
293 tw = inet_twsk(sk2);
294
295 if (INET_TW_MATCH(sk2, net, hash, acookie,
296 saddr, daddr, ports, dif)) {
297 if (twsk_unique(sk, sk2, twp))
298 goto unique;
299 else
300 goto not_unique;
301 }
302 }
303 tw = NULL;
304
305
306 sk_nulls_for_each(sk2, node, &head->chain) {
307 if (INET_MATCH(sk2, net, hash, acookie,
308 saddr, daddr, ports, dif))
309 goto not_unique;
310 }
311
312unique:
313
314
315 inet->num = lport;
316 inet->sport = htons(lport);
317 sk->sk_hash = hash;
318 WARN_ON(!sk_unhashed(sk));
319 __sk_nulls_add_node_rcu(sk, &head->chain);
320 spin_unlock(lock);
321 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
322
323 if (twp) {
324 *twp = tw;
325 NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
326 } else if (tw) {
327
328 inet_twsk_deschedule(tw, death_row);
329 NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
330
331 inet_twsk_put(tw);
332 }
333
334 return 0;
335
336not_unique:
337 spin_unlock(lock);
338 return -EADDRNOTAVAIL;
339}
340
341static inline u32 inet_sk_port_offset(const struct sock *sk)
342{
343 const struct inet_sock *inet = inet_sk(sk);
344 return secure_ipv4_port_ephemeral(inet->rcv_saddr, inet->daddr,
345 inet->dport);
346}
347
348void __inet_hash_nolisten(struct sock *sk)
349{
350 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
351 struct hlist_nulls_head *list;
352 spinlock_t *lock;
353 struct inet_ehash_bucket *head;
354
355 WARN_ON(!sk_unhashed(sk));
356
357 sk->sk_hash = inet_sk_ehashfn(sk);
358 head = inet_ehash_bucket(hashinfo, sk->sk_hash);
359 list = &head->chain;
360 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
361
362 spin_lock(lock);
363 __sk_nulls_add_node_rcu(sk, list);
364 spin_unlock(lock);
365 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
366}
367EXPORT_SYMBOL_GPL(__inet_hash_nolisten);
368
369static void __inet_hash(struct sock *sk)
370{
371 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
372 struct inet_listen_hashbucket *ilb;
373
374 if (sk->sk_state != TCP_LISTEN) {
375 __inet_hash_nolisten(sk);
376 return;
377 }
378
379 WARN_ON(!sk_unhashed(sk));
380 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
381
382 spin_lock(&ilb->lock);
383 __sk_nulls_add_node_rcu(sk, &ilb->head);
384 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
385 spin_unlock(&ilb->lock);
386}
387
388void inet_hash(struct sock *sk)
389{
390 if (sk->sk_state != TCP_CLOSE) {
391 local_bh_disable();
392 __inet_hash(sk);
393 local_bh_enable();
394 }
395}
396EXPORT_SYMBOL_GPL(inet_hash);
397
398void inet_unhash(struct sock *sk)
399{
400 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
401 spinlock_t *lock;
402 int done;
403
404 if (sk_unhashed(sk))
405 return;
406
407 if (sk->sk_state == TCP_LISTEN)
408 lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;
409 else
410 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
411
412 spin_lock_bh(lock);
413 done =__sk_nulls_del_node_init_rcu(sk);
414 if (done)
415 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
416 spin_unlock_bh(lock);
417}
418EXPORT_SYMBOL_GPL(inet_unhash);
419
420int __inet_hash_connect(struct inet_timewait_death_row *death_row,
421 struct sock *sk, u32 port_offset,
422 int (*check_established)(struct inet_timewait_death_row *,
423 struct sock *, __u16, struct inet_timewait_sock **),
424 void (*hash)(struct sock *sk))
425{
426 struct inet_hashinfo *hinfo = death_row->hashinfo;
427 const unsigned short snum = inet_sk(sk)->num;
428 struct inet_bind_hashbucket *head;
429 struct inet_bind_bucket *tb;
430 int ret;
431 struct net *net = sock_net(sk);
432
433 if (!snum) {
434 int i, remaining, low, high, port;
435 static u32 hint;
436 u32 offset = hint + port_offset;
437 struct hlist_node *node;
438 struct inet_timewait_sock *tw = NULL;
439
440 inet_get_local_port_range(&low, &high);
441 remaining = (high - low) + 1;
442
443 local_bh_disable();
444 for (i = 1; i <= remaining; i++) {
445 port = low + (i + offset) % remaining;
446 head = &hinfo->bhash[inet_bhashfn(net, port,
447 hinfo->bhash_size)];
448 spin_lock(&head->lock);
449
450
451
452
453
454 inet_bind_bucket_for_each(tb, node, &head->chain) {
455 if (ib_net(tb) == net && tb->port == port) {
456 if (tb->fastreuse >= 0)
457 goto next_port;
458 WARN_ON(hlist_empty(&tb->owners));
459 if (!check_established(death_row, sk,
460 port, &tw))
461 goto ok;
462 goto next_port;
463 }
464 }
465
466 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
467 net, head, port);
468 if (!tb) {
469 spin_unlock(&head->lock);
470 break;
471 }
472 tb->fastreuse = -1;
473 goto ok;
474
475 next_port:
476 spin_unlock(&head->lock);
477 }
478 local_bh_enable();
479
480 return -EADDRNOTAVAIL;
481
482ok:
483 hint += i;
484
485
486 inet_bind_hash(sk, tb, port);
487 if (sk_unhashed(sk)) {
488 inet_sk(sk)->sport = htons(port);
489 hash(sk);
490 }
491 spin_unlock(&head->lock);
492
493 if (tw) {
494 inet_twsk_deschedule(tw, death_row);
495 inet_twsk_put(tw);
496 }
497
498 ret = 0;
499 goto out;
500 }
501
502 head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)];
503 tb = inet_csk(sk)->icsk_bind_hash;
504 spin_lock_bh(&head->lock);
505 if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
506 hash(sk);
507 spin_unlock_bh(&head->lock);
508 return 0;
509 } else {
510 spin_unlock(&head->lock);
511
512 ret = check_established(death_row, sk, snum, NULL);
513out:
514 local_bh_enable();
515 return ret;
516 }
517}
518
519
520
521
522int inet_hash_connect(struct inet_timewait_death_row *death_row,
523 struct sock *sk)
524{
525 return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk),
526 __inet_check_established, __inet_hash_nolisten);
527}
528
529EXPORT_SYMBOL_GPL(inet_hash_connect);
530
531void inet_hashinfo_init(struct inet_hashinfo *h)
532{
533 int i;
534
535 atomic_set(&h->bsockets, 0);
536 for (i = 0; i < INET_LHTABLE_SIZE; i++) {
537 spin_lock_init(&h->listening_hash[i].lock);
538 INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head,
539 i + LISTENING_NULLS_BASE);
540 }
541}
542
543EXPORT_SYMBOL_GPL(inet_hashinfo_init);
544