1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80#define pr_fmt(fmt) "UDP: " fmt
81
82#include <asm/uaccess.h>
83#include <asm/ioctls.h>
84#include <linux/bootmem.h>
85#include <linux/highmem.h>
86#include <linux/swap.h>
87#include <linux/types.h>
88#include <linux/fcntl.h>
89#include <linux/module.h>
90#include <linux/socket.h>
91#include <linux/sockios.h>
92#include <linux/igmp.h>
93#include <linux/inetdevice.h>
94#include <linux/in.h>
95#include <linux/errno.h>
96#include <linux/timer.h>
97#include <linux/mm.h>
98#include <linux/inet.h>
99#include <linux/netdevice.h>
100#include <linux/slab.h>
101#include <net/tcp_states.h>
102#include <linux/skbuff.h>
103#include <linux/proc_fs.h>
104#include <linux/seq_file.h>
105#include <net/net_namespace.h>
106#include <net/icmp.h>
107#include <net/inet_hashtables.h>
108#include <net/route.h>
109#include <net/checksum.h>
110#include <net/xfrm.h>
111#include <trace/events/udp.h>
112#include <linux/static_key.h>
113#include <trace/events/skb.h>
114#include <net/busy_poll.h>
115#include "udp_impl.h"
116#include <net/sock_reuseport.h>
117
118struct udp_table udp_table __read_mostly;
119EXPORT_SYMBOL(udp_table);
120
121long sysctl_udp_mem[3] __read_mostly;
122EXPORT_SYMBOL(sysctl_udp_mem);
123
124int sysctl_udp_rmem_min __read_mostly;
125EXPORT_SYMBOL(sysctl_udp_rmem_min);
126
127int sysctl_udp_wmem_min __read_mostly;
128EXPORT_SYMBOL(sysctl_udp_wmem_min);
129
130atomic_long_t udp_memory_allocated;
131EXPORT_SYMBOL(udp_memory_allocated);
132
133#define MAX_UDP_PORTS 65536
134#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
135
136static int udp_lib_lport_inuse(struct net *net, __u16 num,
137 const struct udp_hslot *hslot,
138 unsigned long *bitmap,
139 struct sock *sk,
140 int (*saddr_comp)(const struct sock *sk1,
141 const struct sock *sk2,
142 bool match_wildcard),
143 unsigned int log)
144{
145 struct sock *sk2;
146 struct hlist_nulls_node *node;
147 kuid_t uid = sock_i_uid(sk);
148
149 sk_nulls_for_each(sk2, node, &hslot->head) {
150 if (net_eq(sock_net(sk2), net) &&
151 sk2 != sk &&
152 (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
153 (!sk2->sk_reuse || !sk->sk_reuse) &&
154 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
155 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
156 (!sk2->sk_reuseport || !sk->sk_reuseport ||
157 rcu_access_pointer(sk->sk_reuseport_cb) ||
158 !uid_eq(uid, sock_i_uid(sk2))) &&
159 saddr_comp(sk, sk2, true)) {
160 if (!bitmap)
161 return 1;
162 __set_bit(udp_sk(sk2)->udp_port_hash >> log, bitmap);
163 }
164 }
165 return 0;
166}
167
168
169
170
171
172static int udp_lib_lport_inuse2(struct net *net, __u16 num,
173 struct udp_hslot *hslot2,
174 struct sock *sk,
175 int (*saddr_comp)(const struct sock *sk1,
176 const struct sock *sk2,
177 bool match_wildcard))
178{
179 struct sock *sk2;
180 struct hlist_nulls_node *node;
181 kuid_t uid = sock_i_uid(sk);
182 int res = 0;
183
184 spin_lock(&hslot2->lock);
185 udp_portaddr_for_each_entry(sk2, node, &hslot2->head) {
186 if (net_eq(sock_net(sk2), net) &&
187 sk2 != sk &&
188 (udp_sk(sk2)->udp_port_hash == num) &&
189 (!sk2->sk_reuse || !sk->sk_reuse) &&
190 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
191 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
192 (!sk2->sk_reuseport || !sk->sk_reuseport ||
193 rcu_access_pointer(sk->sk_reuseport_cb) ||
194 !uid_eq(uid, sock_i_uid(sk2))) &&
195 saddr_comp(sk, sk2, true)) {
196 res = 1;
197 break;
198 }
199 }
200 spin_unlock(&hslot2->lock);
201 return res;
202}
203
204static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot,
205 int (*saddr_same)(const struct sock *sk1,
206 const struct sock *sk2,
207 bool match_wildcard))
208{
209 struct net *net = sock_net(sk);
210 struct hlist_nulls_node *node;
211 kuid_t uid = sock_i_uid(sk);
212 struct sock *sk2;
213
214 sk_nulls_for_each(sk2, node, &hslot->head) {
215 if (net_eq(sock_net(sk2), net) &&
216 sk2 != sk &&
217 sk2->sk_family == sk->sk_family &&
218 ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
219 (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
220 (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
221 sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
222 (*saddr_same)(sk, sk2, false)) {
223 return reuseport_add_sock(sk, sk2);
224 }
225 }
226
227
228 if (!rcu_access_pointer(sk->sk_reuseport_cb))
229 return reuseport_alloc(sk);
230 return 0;
231}
232
233
234
235
236
237
238
239
240
241
242int udp_lib_get_port(struct sock *sk, unsigned short snum,
243 int (*saddr_comp)(const struct sock *sk1,
244 const struct sock *sk2,
245 bool match_wildcard),
246 unsigned int hash2_nulladdr)
247{
248 struct udp_hslot *hslot, *hslot2;
249 struct udp_table *udptable = sk->sk_prot->h.udp_table;
250 int error = 1;
251 struct net *net = sock_net(sk);
252
253 if (!snum) {
254 int low, high, remaining;
255 unsigned int rand;
256 unsigned short first, last;
257 DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
258
259 inet_get_local_port_range(net, &low, &high);
260 remaining = (high - low) + 1;
261
262 rand = prandom_u32();
263 first = reciprocal_scale(rand, remaining) + low;
264
265
266
267 rand = (rand | 1) * (udptable->mask + 1);
268 last = first + udptable->mask + 1;
269 do {
270 hslot = udp_hashslot(udptable, net, first);
271 bitmap_zero(bitmap, PORTS_PER_CHAIN);
272 spin_lock_bh(&hslot->lock);
273 udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
274 saddr_comp, udptable->log);
275
276 snum = first;
277
278
279
280
281
282 do {
283 if (low <= snum && snum <= high &&
284 !test_bit(snum >> udptable->log, bitmap) &&
285 !inet_is_local_reserved_port(net, snum))
286 goto found;
287 snum += rand;
288 } while (snum != first);
289 spin_unlock_bh(&hslot->lock);
290 } while (++first != last);
291 goto fail;
292 } else {
293 hslot = udp_hashslot(udptable, net, snum);
294 spin_lock_bh(&hslot->lock);
295 if (hslot->count > 10) {
296 int exist;
297 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
298
299 slot2 &= udptable->mask;
300 hash2_nulladdr &= udptable->mask;
301
302 hslot2 = udp_hashslot2(udptable, slot2);
303 if (hslot->count < hslot2->count)
304 goto scan_primary_hash;
305
306 exist = udp_lib_lport_inuse2(net, snum, hslot2,
307 sk, saddr_comp);
308 if (!exist && (hash2_nulladdr != slot2)) {
309 hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
310 exist = udp_lib_lport_inuse2(net, snum, hslot2,
311 sk, saddr_comp);
312 }
313 if (exist)
314 goto fail_unlock;
315 else
316 goto found;
317 }
318scan_primary_hash:
319 if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk,
320 saddr_comp, 0))
321 goto fail_unlock;
322 }
323found:
324 inet_sk(sk)->inet_num = snum;
325 udp_sk(sk)->udp_port_hash = snum;
326 udp_sk(sk)->udp_portaddr_hash ^= snum;
327 if (sk_unhashed(sk)) {
328 if (sk->sk_reuseport &&
329 udp_reuseport_add_sock(sk, hslot, saddr_comp)) {
330 inet_sk(sk)->inet_num = 0;
331 udp_sk(sk)->udp_port_hash = 0;
332 udp_sk(sk)->udp_portaddr_hash ^= snum;
333 goto fail_unlock;
334 }
335
336 sk_nulls_add_node_rcu(sk, &hslot->head);
337 hslot->count++;
338 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
339
340 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
341 spin_lock(&hslot2->lock);
342 hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
343 &hslot2->head);
344 hslot2->count++;
345 spin_unlock(&hslot2->lock);
346 }
347 error = 0;
348fail_unlock:
349 spin_unlock_bh(&hslot->lock);
350fail:
351 return error;
352}
353EXPORT_SYMBOL(udp_lib_get_port);
354
355
356
357
358
359static int ipv4_rcv_saddr_equal(const struct sock *sk1, const struct sock *sk2,
360 bool match_wildcard)
361{
362 struct inet_sock *inet1 = inet_sk(sk1), *inet2 = inet_sk(sk2);
363
364 if (!ipv6_only_sock(sk2)) {
365 if (inet1->inet_rcv_saddr == inet2->inet_rcv_saddr)
366 return 1;
367 if (!inet1->inet_rcv_saddr || !inet2->inet_rcv_saddr)
368 return match_wildcard;
369 }
370 return 0;
371}
372
373static u32 udp4_portaddr_hash(const struct net *net, __be32 saddr,
374 unsigned int port)
375{
376 return jhash_1word((__force u32)saddr, net_hash_mix(net)) ^ port;
377}
378
379int udp_v4_get_port(struct sock *sk, unsigned short snum)
380{
381 unsigned int hash2_nulladdr =
382 udp4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
383 unsigned int hash2_partial =
384 udp4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
385
386
387 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
388 return udp_lib_get_port(sk, snum, ipv4_rcv_saddr_equal, hash2_nulladdr);
389}
390
391static inline int compute_score(struct sock *sk, struct net *net,
392 __be32 saddr, unsigned short hnum, __be16 sport,
393 __be32 daddr, __be16 dport, int dif)
394{
395 int score;
396 struct inet_sock *inet;
397
398 if (!net_eq(sock_net(sk), net) ||
399 udp_sk(sk)->udp_port_hash != hnum ||
400 ipv6_only_sock(sk))
401 return -1;
402
403 score = (sk->sk_family == PF_INET) ? 2 : 1;
404 inet = inet_sk(sk);
405
406 if (inet->inet_rcv_saddr) {
407 if (inet->inet_rcv_saddr != daddr)
408 return -1;
409 score += 4;
410 }
411
412 if (inet->inet_daddr) {
413 if (inet->inet_daddr != saddr)
414 return -1;
415 score += 4;
416 }
417
418 if (inet->inet_dport) {
419 if (inet->inet_dport != sport)
420 return -1;
421 score += 4;
422 }
423
424 if (sk->sk_bound_dev_if) {
425 if (sk->sk_bound_dev_if != dif)
426 return -1;
427 score += 4;
428 }
429 if (sk->sk_incoming_cpu == raw_smp_processor_id())
430 score++;
431 return score;
432}
433
434
435
436
437static inline int compute_score2(struct sock *sk, struct net *net,
438 __be32 saddr, __be16 sport,
439 __be32 daddr, unsigned int hnum, int dif)
440{
441 int score;
442 struct inet_sock *inet;
443
444 if (!net_eq(sock_net(sk), net) ||
445 ipv6_only_sock(sk))
446 return -1;
447
448 inet = inet_sk(sk);
449
450 if (inet->inet_rcv_saddr != daddr ||
451 inet->inet_num != hnum)
452 return -1;
453
454 score = (sk->sk_family == PF_INET) ? 2 : 1;
455
456 if (inet->inet_daddr) {
457 if (inet->inet_daddr != saddr)
458 return -1;
459 score += 4;
460 }
461
462 if (inet->inet_dport) {
463 if (inet->inet_dport != sport)
464 return -1;
465 score += 4;
466 }
467
468 if (sk->sk_bound_dev_if) {
469 if (sk->sk_bound_dev_if != dif)
470 return -1;
471 score += 4;
472 }
473
474 if (sk->sk_incoming_cpu == raw_smp_processor_id())
475 score++;
476
477 return score;
478}
479
480static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
481 const __u16 lport, const __be32 faddr,
482 const __be16 fport)
483{
484 static u32 udp_ehash_secret __read_mostly;
485
486 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
487
488 return __inet_ehashfn(laddr, lport, faddr, fport,
489 udp_ehash_secret + net_hash_mix(net));
490}
491
492
493static struct sock *udp4_lib_lookup2(struct net *net,
494 __be32 saddr, __be16 sport,
495 __be32 daddr, unsigned int hnum, int dif,
496 struct udp_hslot *hslot2, unsigned int slot2,
497 struct sk_buff *skb)
498{
499 struct sock *sk, *result;
500 struct hlist_nulls_node *node;
501 int score, badness, matches = 0, reuseport = 0;
502 bool select_ok = true;
503 u32 hash = 0;
504
505begin:
506 result = NULL;
507 badness = 0;
508 udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
509 score = compute_score2(sk, net, saddr, sport,
510 daddr, hnum, dif);
511 if (score > badness) {
512 result = sk;
513 badness = score;
514 reuseport = sk->sk_reuseport;
515 if (reuseport) {
516 hash = udp_ehashfn(net, daddr, hnum,
517 saddr, sport);
518 if (select_ok) {
519 struct sock *sk2;
520
521 sk2 = reuseport_select_sock(sk, hash, skb,
522 sizeof(struct udphdr));
523 if (sk2) {
524 result = sk2;
525 select_ok = false;
526 goto found;
527 }
528 }
529 matches = 1;
530 }
531 } else if (score == badness && reuseport) {
532 matches++;
533 if (reciprocal_scale(hash, matches) == 0)
534 result = sk;
535 hash = next_pseudo_random32(hash);
536 }
537 }
538
539
540
541
542
543 if (get_nulls_value(node) != slot2)
544 goto begin;
545 if (result) {
546found:
547 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
548 result = NULL;
549 else if (unlikely(compute_score2(result, net, saddr, sport,
550 daddr, hnum, dif) < badness)) {
551 sock_put(result);
552 goto begin;
553 }
554 }
555 return result;
556}
557
558
559
560
561struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
562 __be16 sport, __be32 daddr, __be16 dport,
563 int dif, struct udp_table *udptable, struct sk_buff *skb)
564{
565 struct sock *sk, *result;
566 struct hlist_nulls_node *node;
567 unsigned short hnum = ntohs(dport);
568 unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
569 struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
570 int score, badness, matches = 0, reuseport = 0;
571 bool select_ok = true;
572 u32 hash = 0;
573
574 rcu_read_lock();
575 if (hslot->count > 10) {
576 hash2 = udp4_portaddr_hash(net, daddr, hnum);
577 slot2 = hash2 & udptable->mask;
578 hslot2 = &udptable->hash2[slot2];
579 if (hslot->count < hslot2->count)
580 goto begin;
581
582 result = udp4_lib_lookup2(net, saddr, sport,
583 daddr, hnum, dif,
584 hslot2, slot2, skb);
585 if (!result) {
586 hash2 = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
587 slot2 = hash2 & udptable->mask;
588 hslot2 = &udptable->hash2[slot2];
589 if (hslot->count < hslot2->count)
590 goto begin;
591
592 result = udp4_lib_lookup2(net, saddr, sport,
593 htonl(INADDR_ANY), hnum, dif,
594 hslot2, slot2, skb);
595 }
596 rcu_read_unlock();
597 return result;
598 }
599begin:
600 result = NULL;
601 badness = 0;
602 sk_nulls_for_each_rcu(sk, node, &hslot->head) {
603 score = compute_score(sk, net, saddr, hnum, sport,
604 daddr, dport, dif);
605 if (score > badness) {
606 result = sk;
607 badness = score;
608 reuseport = sk->sk_reuseport;
609 if (reuseport) {
610 hash = udp_ehashfn(net, daddr, hnum,
611 saddr, sport);
612 if (select_ok) {
613 struct sock *sk2;
614
615 sk2 = reuseport_select_sock(sk, hash, skb,
616 sizeof(struct udphdr));
617 if (sk2) {
618 result = sk2;
619 select_ok = false;
620 goto found;
621 }
622 }
623 matches = 1;
624 }
625 } else if (score == badness && reuseport) {
626 matches++;
627 if (reciprocal_scale(hash, matches) == 0)
628 result = sk;
629 hash = next_pseudo_random32(hash);
630 }
631 }
632
633
634
635
636
637 if (get_nulls_value(node) != slot)
638 goto begin;
639
640 if (result) {
641found:
642 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
643 result = NULL;
644 else if (unlikely(compute_score(result, net, saddr, hnum, sport,
645 daddr, dport, dif) < badness)) {
646 sock_put(result);
647 goto begin;
648 }
649 }
650 rcu_read_unlock();
651 return result;
652}
653EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
654
655static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
656 __be16 sport, __be16 dport,
657 struct udp_table *udptable)
658{
659 const struct iphdr *iph = ip_hdr(skb);
660
661 return __udp4_lib_lookup(dev_net(skb_dst(skb)->dev), iph->saddr, sport,
662 iph->daddr, dport, inet_iif(skb),
663 udptable, skb);
664}
665
666struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
667 __be32 daddr, __be16 dport, int dif)
668{
669 return __udp4_lib_lookup(net, saddr, sport, daddr, dport, dif,
670 &udp_table, NULL);
671}
672EXPORT_SYMBOL_GPL(udp4_lib_lookup);
673
674static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
675 __be16 loc_port, __be32 loc_addr,
676 __be16 rmt_port, __be32 rmt_addr,
677 int dif, unsigned short hnum)
678{
679 struct inet_sock *inet = inet_sk(sk);
680
681 if (!net_eq(sock_net(sk), net) ||
682 udp_sk(sk)->udp_port_hash != hnum ||
683 (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
684 (inet->inet_dport != rmt_port && inet->inet_dport) ||
685 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
686 ipv6_only_sock(sk) ||
687 (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif))
688 return false;
689 if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif))
690 return false;
691 return true;
692}
693
694
695
696
697
698
699
700
701
702
703
704
705void __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
706{
707 struct inet_sock *inet;
708 const struct iphdr *iph = (const struct iphdr *)skb->data;
709 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
710 const int type = icmp_hdr(skb)->type;
711 const int code = icmp_hdr(skb)->code;
712 struct sock *sk;
713 int harderr;
714 int err;
715 struct net *net = dev_net(skb->dev);
716
717 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
718 iph->saddr, uh->source, skb->dev->ifindex, udptable,
719 NULL);
720 if (!sk) {
721 ICMP_INC_STATS_BH(net, ICMP_MIB_INERRORS);
722 return;
723 }
724
725 err = 0;
726 harderr = 0;
727 inet = inet_sk(sk);
728
729 switch (type) {
730 default:
731 case ICMP_TIME_EXCEEDED:
732 err = EHOSTUNREACH;
733 break;
734 case ICMP_SOURCE_QUENCH:
735 goto out;
736 case ICMP_PARAMETERPROB:
737 err = EPROTO;
738 harderr = 1;
739 break;
740 case ICMP_DEST_UNREACH:
741 if (code == ICMP_FRAG_NEEDED) {
742 ipv4_sk_update_pmtu(skb, sk, info);
743 if (inet->pmtudisc != IP_PMTUDISC_DONT) {
744 err = EMSGSIZE;
745 harderr = 1;
746 break;
747 }
748 goto out;
749 }
750 err = EHOSTUNREACH;
751 if (code <= NR_ICMP_UNREACH) {
752 harderr = icmp_err_convert[code].fatal;
753 err = icmp_err_convert[code].errno;
754 }
755 break;
756 case ICMP_REDIRECT:
757 ipv4_sk_redirect(skb, sk);
758 goto out;
759 }
760
761
762
763
764
765 if (!inet->recverr) {
766 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
767 goto out;
768 } else
769 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
770
771 sk->sk_err = err;
772 sk->sk_error_report(sk);
773out:
774 sock_put(sk);
775}
776
777void udp_err(struct sk_buff *skb, u32 info)
778{
779 __udp4_lib_err(skb, info, &udp_table);
780}
781
782
783
784
785void udp_flush_pending_frames(struct sock *sk)
786{
787 struct udp_sock *up = udp_sk(sk);
788
789 if (up->pending) {
790 up->len = 0;
791 up->pending = 0;
792 ip_flush_pending_frames(sk);
793 }
794}
795EXPORT_SYMBOL(udp_flush_pending_frames);
796
797
798
799
800
801
802
803
804void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
805{
806 struct udphdr *uh = udp_hdr(skb);
807 int offset = skb_transport_offset(skb);
808 int len = skb->len - offset;
809 int hlen = len;
810 __wsum csum = 0;
811
812 if (!skb_has_frag_list(skb)) {
813
814
815
816 skb->csum_start = skb_transport_header(skb) - skb->head;
817 skb->csum_offset = offsetof(struct udphdr, check);
818 uh->check = ~csum_tcpudp_magic(src, dst, len,
819 IPPROTO_UDP, 0);
820 } else {
821 struct sk_buff *frags;
822
823
824
825
826
827
828 skb_walk_frags(skb, frags) {
829 csum = csum_add(csum, frags->csum);
830 hlen -= frags->len;
831 }
832
833 csum = skb_checksum(skb, offset, hlen, csum);
834 skb->ip_summed = CHECKSUM_NONE;
835
836 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
837 if (uh->check == 0)
838 uh->check = CSUM_MANGLED_0;
839 }
840}
841EXPORT_SYMBOL_GPL(udp4_hwcsum);
842
843
844
845
846void udp_set_csum(bool nocheck, struct sk_buff *skb,
847 __be32 saddr, __be32 daddr, int len)
848{
849 struct udphdr *uh = udp_hdr(skb);
850
851 if (nocheck)
852 uh->check = 0;
853 else if (skb_is_gso(skb))
854 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
855 else if (skb_dst(skb) && skb_dst(skb)->dev &&
856 (skb_dst(skb)->dev->features &
857 (NETIF_F_IP_CSUM | NETIF_F_HW_CSUM))) {
858
859 BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
860
861 skb->ip_summed = CHECKSUM_PARTIAL;
862 skb->csum_start = skb_transport_header(skb) - skb->head;
863 skb->csum_offset = offsetof(struct udphdr, check);
864 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
865 } else {
866 __wsum csum;
867
868 BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
869
870 uh->check = 0;
871 csum = skb_checksum(skb, 0, len, 0);
872 uh->check = udp_v4_check(len, saddr, daddr, csum);
873 if (uh->check == 0)
874 uh->check = CSUM_MANGLED_0;
875
876 skb->ip_summed = CHECKSUM_UNNECESSARY;
877 }
878}
879EXPORT_SYMBOL(udp_set_csum);
880
881static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4)
882{
883 struct sock *sk = skb->sk;
884 struct inet_sock *inet = inet_sk(sk);
885 struct udphdr *uh;
886 int err = 0;
887 int is_udplite = IS_UDPLITE(sk);
888 int offset = skb_transport_offset(skb);
889 int len = skb->len - offset;
890 __wsum csum = 0;
891
892
893
894
895 uh = udp_hdr(skb);
896 uh->source = inet->inet_sport;
897 uh->dest = fl4->fl4_dport;
898 uh->len = htons(len);
899 uh->check = 0;
900
901 if (is_udplite)
902 csum = udplite_csum(skb);
903
904 else if (sk->sk_no_check_tx) {
905
906 skb->ip_summed = CHECKSUM_NONE;
907 goto send;
908
909 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
910
911 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
912 goto send;
913
914 } else
915 csum = udp_csum(skb);
916
917
918 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
919 sk->sk_protocol, csum);
920 if (uh->check == 0)
921 uh->check = CSUM_MANGLED_0;
922
923send:
924 err = ip_send_skb(sock_net(sk), skb);
925 if (err) {
926 if (err == -ENOBUFS && !inet->recverr) {
927 UDP_INC_STATS_USER(sock_net(sk),
928 UDP_MIB_SNDBUFERRORS, is_udplite);
929 err = 0;
930 }
931 } else
932 UDP_INC_STATS_USER(sock_net(sk),
933 UDP_MIB_OUTDATAGRAMS, is_udplite);
934 return err;
935}
936
937
938
939
940int udp_push_pending_frames(struct sock *sk)
941{
942 struct udp_sock *up = udp_sk(sk);
943 struct inet_sock *inet = inet_sk(sk);
944 struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
945 struct sk_buff *skb;
946 int err = 0;
947
948 skb = ip_finish_skb(sk, fl4);
949 if (!skb)
950 goto out;
951
952 err = udp_send_skb(skb, fl4);
953
954out:
955 up->len = 0;
956 up->pending = 0;
957 return err;
958}
959EXPORT_SYMBOL(udp_push_pending_frames);
960
961int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
962{
963 struct inet_sock *inet = inet_sk(sk);
964 struct udp_sock *up = udp_sk(sk);
965 struct flowi4 fl4_stack;
966 struct flowi4 *fl4;
967 int ulen = len;
968 struct ipcm_cookie ipc;
969 struct rtable *rt = NULL;
970 int free = 0;
971 int connected = 0;
972 __be32 daddr, faddr, saddr;
973 __be16 dport;
974 u8 tos;
975 int err, is_udplite = IS_UDPLITE(sk);
976 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
977 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
978 struct sk_buff *skb;
979 struct ip_options_data opt_copy;
980
981 if (len > 0xFFFF)
982 return -EMSGSIZE;
983
984
985
986
987
988 if (msg->msg_flags & MSG_OOB)
989 return -EOPNOTSUPP;
990
991 ipc.opt = NULL;
992 ipc.tx_flags = 0;
993 ipc.ttl = 0;
994 ipc.tos = -1;
995
996 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
997
998 fl4 = &inet->cork.fl.u.ip4;
999 if (up->pending) {
1000
1001
1002
1003
1004 lock_sock(sk);
1005 if (likely(up->pending)) {
1006 if (unlikely(up->pending != AF_INET)) {
1007 release_sock(sk);
1008 return -EINVAL;
1009 }
1010 goto do_append_data;
1011 }
1012 release_sock(sk);
1013 }
1014 ulen += sizeof(struct udphdr);
1015
1016
1017
1018
1019 if (msg->msg_name) {
1020 DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
1021 if (msg->msg_namelen < sizeof(*usin))
1022 return -EINVAL;
1023 if (usin->sin_family != AF_INET) {
1024 if (usin->sin_family != AF_UNSPEC)
1025 return -EAFNOSUPPORT;
1026 }
1027
1028 daddr = usin->sin_addr.s_addr;
1029 dport = usin->sin_port;
1030 if (dport == 0)
1031 return -EINVAL;
1032 } else {
1033 if (sk->sk_state != TCP_ESTABLISHED)
1034 return -EDESTADDRREQ;
1035 daddr = inet->inet_daddr;
1036 dport = inet->inet_dport;
1037
1038
1039
1040 connected = 1;
1041 }
1042 ipc.addr = inet->inet_saddr;
1043
1044 ipc.oif = sk->sk_bound_dev_if;
1045
1046 sock_tx_timestamp(sk, &ipc.tx_flags);
1047
1048 if (msg->msg_controllen) {
1049 err = ip_cmsg_send(sock_net(sk), msg, &ipc,
1050 sk->sk_family == AF_INET6);
1051 if (unlikely(err)) {
1052 kfree(ipc.opt);
1053 return err;
1054 }
1055 if (ipc.opt)
1056 free = 1;
1057 connected = 0;
1058 }
1059 if (!ipc.opt) {
1060 struct ip_options_rcu *inet_opt;
1061
1062 rcu_read_lock();
1063 inet_opt = rcu_dereference(inet->inet_opt);
1064 if (inet_opt) {
1065 memcpy(&opt_copy, inet_opt,
1066 sizeof(*inet_opt) + inet_opt->opt.optlen);
1067 ipc.opt = &opt_copy.opt;
1068 }
1069 rcu_read_unlock();
1070 }
1071
1072 saddr = ipc.addr;
1073 ipc.addr = faddr = daddr;
1074
1075 if (ipc.opt && ipc.opt->opt.srr) {
1076 if (!daddr)
1077 return -EINVAL;
1078 faddr = ipc.opt->opt.faddr;
1079 connected = 0;
1080 }
1081 tos = get_rttos(&ipc, inet);
1082 if (sock_flag(sk, SOCK_LOCALROUTE) ||
1083 (msg->msg_flags & MSG_DONTROUTE) ||
1084 (ipc.opt && ipc.opt->opt.is_strictroute)) {
1085 tos |= RTO_ONLINK;
1086 connected = 0;
1087 }
1088
1089 if (ipv4_is_multicast(daddr)) {
1090 if (!ipc.oif)
1091 ipc.oif = inet->mc_index;
1092 if (!saddr)
1093 saddr = inet->mc_addr;
1094 connected = 0;
1095 } else if (!ipc.oif)
1096 ipc.oif = inet->uc_index;
1097
1098 if (connected)
1099 rt = (struct rtable *)sk_dst_check(sk, 0);
1100
1101 if (!rt) {
1102 struct net *net = sock_net(sk);
1103 __u8 flow_flags = inet_sk_flowi_flags(sk);
1104
1105 fl4 = &fl4_stack;
1106
1107 flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
1108 RT_SCOPE_UNIVERSE, sk->sk_protocol,
1109 flow_flags,
1110 faddr, saddr, dport, inet->inet_sport);
1111
1112 if (!saddr && ipc.oif) {
1113 err = l3mdev_get_saddr(net, ipc.oif, fl4);
1114 if (err < 0)
1115 goto out;
1116 }
1117
1118 security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
1119 rt = ip_route_output_flow(net, fl4, sk);
1120 if (IS_ERR(rt)) {
1121 err = PTR_ERR(rt);
1122 rt = NULL;
1123 if (err == -ENETUNREACH)
1124 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
1125 goto out;
1126 }
1127
1128 err = -EACCES;
1129 if ((rt->rt_flags & RTCF_BROADCAST) &&
1130 !sock_flag(sk, SOCK_BROADCAST))
1131 goto out;
1132 if (connected)
1133 sk_dst_set(sk, dst_clone(&rt->dst));
1134 }
1135
1136 if (msg->msg_flags&MSG_CONFIRM)
1137 goto do_confirm;
1138back_from_confirm:
1139
1140 saddr = fl4->saddr;
1141 if (!ipc.addr)
1142 daddr = ipc.addr = fl4->daddr;
1143
1144
1145 if (!corkreq) {
1146 skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
1147 sizeof(struct udphdr), &ipc, &rt,
1148 msg->msg_flags);
1149 err = PTR_ERR(skb);
1150 if (!IS_ERR_OR_NULL(skb))
1151 err = udp_send_skb(skb, fl4);
1152 goto out;
1153 }
1154
1155 lock_sock(sk);
1156 if (unlikely(up->pending)) {
1157
1158
1159 release_sock(sk);
1160
1161 net_dbg_ratelimited("cork app bug 2\n");
1162 err = -EINVAL;
1163 goto out;
1164 }
1165
1166
1167
1168 fl4 = &inet->cork.fl.u.ip4;
1169 fl4->daddr = daddr;
1170 fl4->saddr = saddr;
1171 fl4->fl4_dport = dport;
1172 fl4->fl4_sport = inet->inet_sport;
1173 up->pending = AF_INET;
1174
1175do_append_data:
1176 up->len += ulen;
1177 err = ip_append_data(sk, fl4, getfrag, msg, ulen,
1178 sizeof(struct udphdr), &ipc, &rt,
1179 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
1180 if (err)
1181 udp_flush_pending_frames(sk);
1182 else if (!corkreq)
1183 err = udp_push_pending_frames(sk);
1184 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1185 up->pending = 0;
1186 release_sock(sk);
1187
1188out:
1189 ip_rt_put(rt);
1190 if (free)
1191 kfree(ipc.opt);
1192 if (!err)
1193 return len;
1194
1195
1196
1197
1198
1199
1200
1201 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1202 UDP_INC_STATS_USER(sock_net(sk),
1203 UDP_MIB_SNDBUFERRORS, is_udplite);
1204 }
1205 return err;
1206
1207do_confirm:
1208 dst_confirm(&rt->dst);
1209 if (!(msg->msg_flags&MSG_PROBE) || len)
1210 goto back_from_confirm;
1211 err = 0;
1212 goto out;
1213}
1214EXPORT_SYMBOL(udp_sendmsg);
1215
1216int udp_sendpage(struct sock *sk, struct page *page, int offset,
1217 size_t size, int flags)
1218{
1219 struct inet_sock *inet = inet_sk(sk);
1220 struct udp_sock *up = udp_sk(sk);
1221 int ret;
1222
1223 if (flags & MSG_SENDPAGE_NOTLAST)
1224 flags |= MSG_MORE;
1225
1226 if (!up->pending) {
1227 struct msghdr msg = { .msg_flags = flags|MSG_MORE };
1228
1229
1230
1231
1232
1233 ret = udp_sendmsg(sk, &msg, 0);
1234 if (ret < 0)
1235 return ret;
1236 }
1237
1238 lock_sock(sk);
1239
1240 if (unlikely(!up->pending)) {
1241 release_sock(sk);
1242
1243 net_dbg_ratelimited("udp cork app bug 3\n");
1244 return -EINVAL;
1245 }
1246
1247 ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
1248 page, offset, size, flags);
1249 if (ret == -EOPNOTSUPP) {
1250 release_sock(sk);
1251 return sock_no_sendpage(sk->sk_socket, page, offset,
1252 size, flags);
1253 }
1254 if (ret < 0) {
1255 udp_flush_pending_frames(sk);
1256 goto out;
1257 }
1258
1259 up->len += size;
1260 if (!(up->corkflag || (flags&MSG_MORE)))
1261 ret = udp_push_pending_frames(sk);
1262 if (!ret)
1263 ret = size;
1264out:
1265 release_sock(sk);
1266 return ret;
1267}
1268
1269
1270
1271
1272
1273
1274
1275
1276static unsigned int first_packet_length(struct sock *sk)
1277{
1278 struct sk_buff_head list_kill, *rcvq = &sk->sk_receive_queue;
1279 struct sk_buff *skb;
1280 unsigned int res;
1281
1282 __skb_queue_head_init(&list_kill);
1283
1284 spin_lock_bh(&rcvq->lock);
1285 while ((skb = skb_peek(rcvq)) != NULL &&
1286 udp_lib_checksum_complete(skb)) {
1287 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS,
1288 IS_UDPLITE(sk));
1289 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
1290 IS_UDPLITE(sk));
1291 atomic_inc(&sk->sk_drops);
1292 __skb_unlink(skb, rcvq);
1293 __skb_queue_tail(&list_kill, skb);
1294 }
1295 res = skb ? skb->len : 0;
1296 spin_unlock_bh(&rcvq->lock);
1297
1298 if (!skb_queue_empty(&list_kill)) {
1299 bool slow = lock_sock_fast(sk);
1300
1301 __skb_queue_purge(&list_kill);
1302 sk_mem_reclaim_partial(sk);
1303 unlock_sock_fast(sk, slow);
1304 }
1305 return res;
1306}
1307
1308
1309
1310
1311
1312int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
1313{
1314 switch (cmd) {
1315 case SIOCOUTQ:
1316 {
1317 int amount = sk_wmem_alloc_get(sk);
1318
1319 return put_user(amount, (int __user *)arg);
1320 }
1321
1322 case SIOCINQ:
1323 {
1324 unsigned int amount = first_packet_length(sk);
1325
1326 if (amount)
1327
1328
1329
1330
1331
1332 amount -= sizeof(struct udphdr);
1333
1334 return put_user(amount, (int __user *)arg);
1335 }
1336
1337 default:
1338 return -ENOIOCTLCMD;
1339 }
1340
1341 return 0;
1342}
1343EXPORT_SYMBOL(udp_ioctl);
1344
1345
1346
1347
1348
1349
1350int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,
1351 int flags, int *addr_len)
1352{
1353 struct inet_sock *inet = inet_sk(sk);
1354 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
1355 struct sk_buff *skb;
1356 unsigned int ulen, copied;
1357 int peeked, off = 0;
1358 int err;
1359 int is_udplite = IS_UDPLITE(sk);
1360 bool checksum_valid = false;
1361 bool slow;
1362
1363 if (flags & MSG_ERRQUEUE)
1364 return ip_recv_error(sk, msg, len, addr_len);
1365
1366try_again:
1367 skb = __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),
1368 &peeked, &off, &err);
1369 if (!skb)
1370 goto out;
1371
1372 ulen = skb->len - sizeof(struct udphdr);
1373 copied = len;
1374 if (copied > ulen)
1375 copied = ulen;
1376 else if (copied < ulen)
1377 msg->msg_flags |= MSG_TRUNC;
1378
1379
1380
1381
1382
1383
1384
1385 if (copied < ulen || UDP_SKB_CB(skb)->partial_cov) {
1386 checksum_valid = !udp_lib_checksum_complete(skb);
1387 if (!checksum_valid)
1388 goto csum_copy_err;
1389 }
1390
1391 if (checksum_valid || skb_csum_unnecessary(skb))
1392 err = skb_copy_datagram_msg(skb, sizeof(struct udphdr),
1393 msg, copied);
1394 else {
1395 err = skb_copy_and_csum_datagram_msg(skb, sizeof(struct udphdr),
1396 msg);
1397
1398 if (err == -EINVAL)
1399 goto csum_copy_err;
1400 }
1401
1402 if (unlikely(err)) {
1403 trace_kfree_skb(skb, udp_recvmsg);
1404 if (!peeked) {
1405 atomic_inc(&sk->sk_drops);
1406 UDP_INC_STATS_USER(sock_net(sk),
1407 UDP_MIB_INERRORS, is_udplite);
1408 }
1409 goto out_free;
1410 }
1411
1412 if (!peeked)
1413 UDP_INC_STATS_USER(sock_net(sk),
1414 UDP_MIB_INDATAGRAMS, is_udplite);
1415
1416 sock_recv_ts_and_drops(msg, sk, skb);
1417
1418
1419 if (sin) {
1420 sin->sin_family = AF_INET;
1421 sin->sin_port = udp_hdr(skb)->source;
1422 sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1423 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
1424 *addr_len = sizeof(*sin);
1425 }
1426 if (inet->cmsg_flags)
1427 ip_cmsg_recv_offset(msg, skb, sizeof(struct udphdr));
1428
1429 err = copied;
1430 if (flags & MSG_TRUNC)
1431 err = ulen;
1432
1433out_free:
1434 skb_free_datagram_locked(sk, skb);
1435out:
1436 return err;
1437
1438csum_copy_err:
1439 slow = lock_sock_fast(sk);
1440 if (!skb_kill_datagram(sk, skb, flags)) {
1441 UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1442 UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1443 }
1444 unlock_sock_fast(sk, slow);
1445
1446
1447 cond_resched();
1448 msg->msg_flags &= ~MSG_TRUNC;
1449 goto try_again;
1450}
1451
1452int udp_disconnect(struct sock *sk, int flags)
1453{
1454 struct inet_sock *inet = inet_sk(sk);
1455
1456
1457
1458
1459 sk->sk_state = TCP_CLOSE;
1460 inet->inet_daddr = 0;
1461 inet->inet_dport = 0;
1462 sock_rps_reset_rxhash(sk);
1463 sk->sk_bound_dev_if = 0;
1464 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1465 inet_reset_saddr(sk);
1466
1467 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1468 sk->sk_prot->unhash(sk);
1469 inet->inet_sport = 0;
1470 }
1471 sk_dst_reset(sk);
1472 return 0;
1473}
1474EXPORT_SYMBOL(udp_disconnect);
1475
1476void udp_lib_unhash(struct sock *sk)
1477{
1478 if (sk_hashed(sk)) {
1479 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1480 struct udp_hslot *hslot, *hslot2;
1481
1482 hslot = udp_hashslot(udptable, sock_net(sk),
1483 udp_sk(sk)->udp_port_hash);
1484 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1485
1486 spin_lock_bh(&hslot->lock);
1487 if (rcu_access_pointer(sk->sk_reuseport_cb))
1488 reuseport_detach_sock(sk);
1489 if (sk_nulls_del_node_init_rcu(sk)) {
1490 hslot->count--;
1491 inet_sk(sk)->inet_num = 0;
1492 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
1493
1494 spin_lock(&hslot2->lock);
1495 hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1496 hslot2->count--;
1497 spin_unlock(&hslot2->lock);
1498 }
1499 spin_unlock_bh(&hslot->lock);
1500 }
1501}
1502EXPORT_SYMBOL(udp_lib_unhash);
1503
1504
1505
1506
1507void udp_lib_rehash(struct sock *sk, u16 newhash)
1508{
1509 if (sk_hashed(sk)) {
1510 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1511 struct udp_hslot *hslot, *hslot2, *nhslot2;
1512
1513 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1514 nhslot2 = udp_hashslot2(udptable, newhash);
1515 udp_sk(sk)->udp_portaddr_hash = newhash;
1516
1517 if (hslot2 != nhslot2 ||
1518 rcu_access_pointer(sk->sk_reuseport_cb)) {
1519 hslot = udp_hashslot(udptable, sock_net(sk),
1520 udp_sk(sk)->udp_port_hash);
1521
1522 spin_lock_bh(&hslot->lock);
1523 if (rcu_access_pointer(sk->sk_reuseport_cb))
1524 reuseport_detach_sock(sk);
1525
1526 if (hslot2 != nhslot2) {
1527 spin_lock(&hslot2->lock);
1528 hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1529 hslot2->count--;
1530 spin_unlock(&hslot2->lock);
1531
1532 spin_lock(&nhslot2->lock);
1533 hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
1534 &nhslot2->head);
1535 nhslot2->count++;
1536 spin_unlock(&nhslot2->lock);
1537 }
1538
1539 spin_unlock_bh(&hslot->lock);
1540 }
1541 }
1542}
1543EXPORT_SYMBOL(udp_lib_rehash);
1544
1545static void udp_v4_rehash(struct sock *sk)
1546{
1547 u16 new_hash = udp4_portaddr_hash(sock_net(sk),
1548 inet_sk(sk)->inet_rcv_saddr,
1549 inet_sk(sk)->inet_num);
1550 udp_lib_rehash(sk, new_hash);
1551}
1552
1553static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1554{
1555 int rc;
1556
1557 if (inet_sk(sk)->inet_daddr) {
1558 sock_rps_save_rxhash(sk, skb);
1559 sk_mark_napi_id(sk, skb);
1560 sk_incoming_cpu_update(sk);
1561 }
1562
1563 rc = sock_queue_rcv_skb(sk, skb);
1564 if (rc < 0) {
1565 int is_udplite = IS_UDPLITE(sk);
1566
1567
1568 if (rc == -ENOMEM)
1569 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1570 is_udplite);
1571 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1572 kfree_skb(skb);
1573 trace_udp_fail_queue_rcv_skb(rc, sk);
1574 return -1;
1575 }
1576
1577 return 0;
1578
1579}
1580
1581static struct static_key udp_encap_needed __read_mostly;
1582void udp_encap_enable(void)
1583{
1584 if (!static_key_enabled(&udp_encap_needed))
1585 static_key_slow_inc(&udp_encap_needed);
1586}
1587EXPORT_SYMBOL(udp_encap_enable);
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1598{
1599 struct udp_sock *up = udp_sk(sk);
1600 int rc;
1601 int is_udplite = IS_UDPLITE(sk);
1602
1603
1604
1605
1606 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1607 goto drop;
1608 nf_reset(skb);
1609
1610 if (static_key_false(&udp_encap_needed) && up->encap_type) {
1611 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625 encap_rcv = ACCESS_ONCE(up->encap_rcv);
1626 if (skb->len > sizeof(struct udphdr) && encap_rcv) {
1627 int ret;
1628
1629
1630 if (udp_lib_checksum_complete(skb))
1631 goto csum_error;
1632
1633 ret = encap_rcv(sk, skb);
1634 if (ret <= 0) {
1635 UDP_INC_STATS_BH(sock_net(sk),
1636 UDP_MIB_INDATAGRAMS,
1637 is_udplite);
1638 return -ret;
1639 }
1640 }
1641
1642
1643 }
1644
1645
1646
1647
1648 if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661 if (up->pcrlen == 0) {
1662 net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
1663 UDP_SKB_CB(skb)->cscov, skb->len);
1664 goto drop;
1665 }
1666
1667
1668
1669
1670
1671
1672 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
1673 net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
1674 UDP_SKB_CB(skb)->cscov, up->pcrlen);
1675 goto drop;
1676 }
1677 }
1678
1679 if (rcu_access_pointer(sk->sk_filter) &&
1680 udp_lib_checksum_complete(skb))
1681 goto csum_error;
1682
1683 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
1684 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1685 is_udplite);
1686 goto drop;
1687 }
1688
1689 rc = 0;
1690
1691 ipv4_pktinfo_prepare(sk, skb);
1692 bh_lock_sock(sk);
1693 if (!sock_owned_by_user(sk))
1694 rc = __udp_queue_rcv_skb(sk, skb);
1695 else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
1696 bh_unlock_sock(sk);
1697 goto drop;
1698 }
1699 bh_unlock_sock(sk);
1700
1701 return rc;
1702
1703csum_error:
1704 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1705drop:
1706 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1707 atomic_inc(&sk->sk_drops);
1708 kfree_skb(skb);
1709 return -1;
1710}
1711
1712static void flush_stack(struct sock **stack, unsigned int count,
1713 struct sk_buff *skb, unsigned int final)
1714{
1715 unsigned int i;
1716 struct sk_buff *skb1 = NULL;
1717 struct sock *sk;
1718
1719 for (i = 0; i < count; i++) {
1720 sk = stack[i];
1721 if (likely(!skb1))
1722 skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
1723
1724 if (!skb1) {
1725 atomic_inc(&sk->sk_drops);
1726 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1727 IS_UDPLITE(sk));
1728 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
1729 IS_UDPLITE(sk));
1730 }
1731
1732 if (skb1 && udp_queue_rcv_skb(sk, skb1) <= 0)
1733 skb1 = NULL;
1734
1735 sock_put(sk);
1736 }
1737 if (unlikely(skb1))
1738 kfree_skb(skb1);
1739}
1740
1741
1742
1743
1744static void udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
1745{
1746 struct dst_entry *old;
1747
1748 dst_hold(dst);
1749 old = xchg(&sk->sk_rx_dst, dst);
1750 dst_release(old);
1751}
1752
1753
1754
1755
1756
1757
1758static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
1759 struct udphdr *uh,
1760 __be32 saddr, __be32 daddr,
1761 struct udp_table *udptable,
1762 int proto)
1763{
1764 struct sock *sk, *stack[256 / sizeof(struct sock *)];
1765 struct hlist_nulls_node *node;
1766 unsigned short hnum = ntohs(uh->dest);
1767 struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
1768 int dif = skb->dev->ifindex;
1769 unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
1770 unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
1771 bool inner_flushed = false;
1772
1773 if (use_hash2) {
1774 hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
1775 udp_table.mask;
1776 hash2 = udp4_portaddr_hash(net, daddr, hnum) & udp_table.mask;
1777start_lookup:
1778 hslot = &udp_table.hash2[hash2];
1779 offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
1780 }
1781
1782 spin_lock(&hslot->lock);
1783 sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {
1784 if (__udp_is_mcast_sock(net, sk,
1785 uh->dest, daddr,
1786 uh->source, saddr,
1787 dif, hnum)) {
1788 if (unlikely(count == ARRAY_SIZE(stack))) {
1789 flush_stack(stack, count, skb, ~0);
1790 inner_flushed = true;
1791 count = 0;
1792 }
1793 stack[count++] = sk;
1794 sock_hold(sk);
1795 }
1796 }
1797
1798 spin_unlock(&hslot->lock);
1799
1800
1801 if (use_hash2 && hash2 != hash2_any) {
1802 hash2 = hash2_any;
1803 goto start_lookup;
1804 }
1805
1806
1807
1808
1809 if (count) {
1810 flush_stack(stack, count, skb, count - 1);
1811 } else {
1812 if (!inner_flushed)
1813 UDP_INC_STATS_BH(net, UDP_MIB_IGNOREDMULTI,
1814 proto == IPPROTO_UDPLITE);
1815 consume_skb(skb);
1816 }
1817 return 0;
1818}
1819
1820
1821
1822
1823
1824
1825static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
1826 int proto)
1827{
1828 int err;
1829
1830 UDP_SKB_CB(skb)->partial_cov = 0;
1831 UDP_SKB_CB(skb)->cscov = skb->len;
1832
1833 if (proto == IPPROTO_UDPLITE) {
1834 err = udplite_checksum_init(skb, uh);
1835 if (err)
1836 return err;
1837 }
1838
1839 return skb_checksum_init_zero_check(skb, proto, uh->check,
1840 inet_compute_pseudo);
1841}
1842
1843
1844
1845
1846
1847int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
1848 int proto)
1849{
1850 struct sock *sk;
1851 struct udphdr *uh;
1852 unsigned short ulen;
1853 struct rtable *rt = skb_rtable(skb);
1854 __be32 saddr, daddr;
1855 struct net *net = dev_net(skb->dev);
1856
1857
1858
1859
1860 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
1861 goto drop;
1862
1863 uh = udp_hdr(skb);
1864 ulen = ntohs(uh->len);
1865 saddr = ip_hdr(skb)->saddr;
1866 daddr = ip_hdr(skb)->daddr;
1867
1868 if (ulen > skb->len)
1869 goto short_packet;
1870
1871 if (proto == IPPROTO_UDP) {
1872
1873 if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
1874 goto short_packet;
1875 uh = udp_hdr(skb);
1876 }
1877
1878 if (udp4_csum_init(skb, uh, proto))
1879 goto csum_error;
1880
1881 sk = skb_steal_sock(skb);
1882 if (sk) {
1883 struct dst_entry *dst = skb_dst(skb);
1884 int ret;
1885
1886 if (unlikely(sk->sk_rx_dst != dst))
1887 udp_sk_rx_dst_set(sk, dst);
1888
1889 ret = udp_queue_rcv_skb(sk, skb);
1890 sock_put(sk);
1891
1892
1893
1894 if (ret > 0)
1895 return -ret;
1896 return 0;
1897 }
1898
1899 if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
1900 return __udp4_lib_mcast_deliver(net, skb, uh,
1901 saddr, daddr, udptable, proto);
1902
1903 sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
1904 if (sk) {
1905 int ret;
1906
1907 if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
1908 skb_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
1909 inet_compute_pseudo);
1910
1911 ret = udp_queue_rcv_skb(sk, skb);
1912 sock_put(sk);
1913
1914
1915
1916
1917 if (ret > 0)
1918 return -ret;
1919 return 0;
1920 }
1921
1922 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
1923 goto drop;
1924 nf_reset(skb);
1925
1926
1927 if (udp_lib_checksum_complete(skb))
1928 goto csum_error;
1929
1930 UDP_INC_STATS_BH(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
1931 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
1932
1933
1934
1935
1936
1937 kfree_skb(skb);
1938 return 0;
1939
1940short_packet:
1941 net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
1942 proto == IPPROTO_UDPLITE ? "Lite" : "",
1943 &saddr, ntohs(uh->source),
1944 ulen, skb->len,
1945 &daddr, ntohs(uh->dest));
1946 goto drop;
1947
1948csum_error:
1949
1950
1951
1952
1953 net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
1954 proto == IPPROTO_UDPLITE ? "Lite" : "",
1955 &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
1956 ulen);
1957 UDP_INC_STATS_BH(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
1958drop:
1959 UDP_INC_STATS_BH(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
1960 kfree_skb(skb);
1961 return 0;
1962}
1963
1964
1965
1966
1967static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
1968 __be16 loc_port, __be32 loc_addr,
1969 __be16 rmt_port, __be32 rmt_addr,
1970 int dif)
1971{
1972 struct sock *sk, *result;
1973 struct hlist_nulls_node *node;
1974 unsigned short hnum = ntohs(loc_port);
1975 unsigned int count, slot = udp_hashfn(net, hnum, udp_table.mask);
1976 struct udp_hslot *hslot = &udp_table.hash[slot];
1977
1978
1979 if (hslot->count > 10)
1980 return NULL;
1981
1982 rcu_read_lock();
1983begin:
1984 count = 0;
1985 result = NULL;
1986 sk_nulls_for_each_rcu(sk, node, &hslot->head) {
1987 if (__udp_is_mcast_sock(net, sk,
1988 loc_port, loc_addr,
1989 rmt_port, rmt_addr,
1990 dif, hnum)) {
1991 result = sk;
1992 ++count;
1993 }
1994 }
1995
1996
1997
1998
1999
2000 if (get_nulls_value(node) != slot)
2001 goto begin;
2002
2003 if (result) {
2004 if (count != 1 ||
2005 unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
2006 result = NULL;
2007 else if (unlikely(!__udp_is_mcast_sock(net, result,
2008 loc_port, loc_addr,
2009 rmt_port, rmt_addr,
2010 dif, hnum))) {
2011 sock_put(result);
2012 result = NULL;
2013 }
2014 }
2015 rcu_read_unlock();
2016 return result;
2017}
2018
2019
2020
2021
2022
2023static struct sock *__udp4_lib_demux_lookup(struct net *net,
2024 __be16 loc_port, __be32 loc_addr,
2025 __be16 rmt_port, __be32 rmt_addr,
2026 int dif)
2027{
2028 struct sock *sk, *result;
2029 struct hlist_nulls_node *node;
2030 unsigned short hnum = ntohs(loc_port);
2031 unsigned int hash2 = udp4_portaddr_hash(net, loc_addr, hnum);
2032 unsigned int slot2 = hash2 & udp_table.mask;
2033 struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
2034 INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
2035 const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
2036
2037 rcu_read_lock();
2038 result = NULL;
2039 udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
2040 if (INET_MATCH(sk, net, acookie,
2041 rmt_addr, loc_addr, ports, dif))
2042 result = sk;
2043
2044 break;
2045 }
2046
2047 if (result) {
2048 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
2049 result = NULL;
2050 else if (unlikely(!INET_MATCH(sk, net, acookie,
2051 rmt_addr, loc_addr,
2052 ports, dif))) {
2053 sock_put(result);
2054 result = NULL;
2055 }
2056 }
2057 rcu_read_unlock();
2058 return result;
2059}
2060
2061void udp_v4_early_demux(struct sk_buff *skb)
2062{
2063 struct net *net = dev_net(skb->dev);
2064 const struct iphdr *iph;
2065 const struct udphdr *uh;
2066 struct sock *sk;
2067 struct dst_entry *dst;
2068 int dif = skb->dev->ifindex;
2069 int ours;
2070
2071
2072 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
2073 return;
2074
2075 iph = ip_hdr(skb);
2076 uh = udp_hdr(skb);
2077
2078 if (skb->pkt_type == PACKET_BROADCAST ||
2079 skb->pkt_type == PACKET_MULTICAST) {
2080 struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
2081
2082 if (!in_dev)
2083 return;
2084
2085 ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
2086 iph->protocol);
2087 if (!ours)
2088 return;
2089 sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
2090 uh->source, iph->saddr, dif);
2091 } else if (skb->pkt_type == PACKET_HOST) {
2092 sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
2093 uh->source, iph->saddr, dif);
2094 } else {
2095 return;
2096 }
2097
2098 if (!sk)
2099 return;
2100
2101 skb->sk = sk;
2102 skb->destructor = sock_efree;
2103 dst = READ_ONCE(sk->sk_rx_dst);
2104
2105 if (dst)
2106 dst = dst_check(dst, 0);
2107 if (dst) {
2108
2109 if (dst->flags & DST_NOCACHE) {
2110 if (likely(atomic_inc_not_zero(&dst->__refcnt)))
2111 skb_dst_set(skb, dst);
2112 } else {
2113 skb_dst_set_noref(skb, dst);
2114 }
2115 }
2116}
2117
2118int udp_rcv(struct sk_buff *skb)
2119{
2120 return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP);
2121}
2122
2123void udp_destroy_sock(struct sock *sk)
2124{
2125 struct udp_sock *up = udp_sk(sk);
2126 bool slow = lock_sock_fast(sk);
2127 udp_flush_pending_frames(sk);
2128 unlock_sock_fast(sk, slow);
2129 if (static_key_false(&udp_encap_needed) && up->encap_type) {
2130 void (*encap_destroy)(struct sock *sk);
2131 encap_destroy = ACCESS_ONCE(up->encap_destroy);
2132 if (encap_destroy)
2133 encap_destroy(sk);
2134 }
2135}
2136
2137
2138
2139
2140int udp_lib_setsockopt(struct sock *sk, int level, int optname,
2141 char __user *optval, unsigned int optlen,
2142 int (*push_pending_frames)(struct sock *))
2143{
2144 struct udp_sock *up = udp_sk(sk);
2145 int val, valbool;
2146 int err = 0;
2147 int is_udplite = IS_UDPLITE(sk);
2148
2149 if (optlen < sizeof(int))
2150 return -EINVAL;
2151
2152 if (get_user(val, (int __user *)optval))
2153 return -EFAULT;
2154
2155 valbool = val ? 1 : 0;
2156
2157 switch (optname) {
2158 case UDP_CORK:
2159 if (val != 0) {
2160 up->corkflag = 1;
2161 } else {
2162 up->corkflag = 0;
2163 lock_sock(sk);
2164 push_pending_frames(sk);
2165 release_sock(sk);
2166 }
2167 break;
2168
2169 case UDP_ENCAP:
2170 switch (val) {
2171 case 0:
2172 case UDP_ENCAP_ESPINUDP:
2173 case UDP_ENCAP_ESPINUDP_NON_IKE:
2174 up->encap_rcv = xfrm4_udp_encap_rcv;
2175
2176 case UDP_ENCAP_L2TPINUDP:
2177 up->encap_type = val;
2178 udp_encap_enable();
2179 break;
2180 default:
2181 err = -ENOPROTOOPT;
2182 break;
2183 }
2184 break;
2185
2186 case UDP_NO_CHECK6_TX:
2187 up->no_check6_tx = valbool;
2188 break;
2189
2190 case UDP_NO_CHECK6_RX:
2191 up->no_check6_rx = valbool;
2192 break;
2193
2194
2195
2196
2197
2198
2199 case UDPLITE_SEND_CSCOV:
2200 if (!is_udplite)
2201 return -ENOPROTOOPT;
2202 if (val != 0 && val < 8)
2203 val = 8;
2204 else if (val > USHRT_MAX)
2205 val = USHRT_MAX;
2206 up->pcslen = val;
2207 up->pcflag |= UDPLITE_SEND_CC;
2208 break;
2209
2210
2211
2212
2213 case UDPLITE_RECV_CSCOV:
2214 if (!is_udplite)
2215 return -ENOPROTOOPT;
2216 if (val != 0 && val < 8)
2217 val = 8;
2218 else if (val > USHRT_MAX)
2219 val = USHRT_MAX;
2220 up->pcrlen = val;
2221 up->pcflag |= UDPLITE_RECV_CC;
2222 break;
2223
2224 default:
2225 err = -ENOPROTOOPT;
2226 break;
2227 }
2228
2229 return err;
2230}
2231EXPORT_SYMBOL(udp_lib_setsockopt);
2232
2233int udp_setsockopt(struct sock *sk, int level, int optname,
2234 char __user *optval, unsigned int optlen)
2235{
2236 if (level == SOL_UDP || level == SOL_UDPLITE)
2237 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2238 udp_push_pending_frames);
2239 return ip_setsockopt(sk, level, optname, optval, optlen);
2240}
2241
2242#ifdef CONFIG_COMPAT
2243int compat_udp_setsockopt(struct sock *sk, int level, int optname,
2244 char __user *optval, unsigned int optlen)
2245{
2246 if (level == SOL_UDP || level == SOL_UDPLITE)
2247 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2248 udp_push_pending_frames);
2249 return compat_ip_setsockopt(sk, level, optname, optval, optlen);
2250}
2251#endif
2252
2253int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2254 char __user *optval, int __user *optlen)
2255{
2256 struct udp_sock *up = udp_sk(sk);
2257 int val, len;
2258
2259 if (get_user(len, optlen))
2260 return -EFAULT;
2261
2262 len = min_t(unsigned int, len, sizeof(int));
2263
2264 if (len < 0)
2265 return -EINVAL;
2266
2267 switch (optname) {
2268 case UDP_CORK:
2269 val = up->corkflag;
2270 break;
2271
2272 case UDP_ENCAP:
2273 val = up->encap_type;
2274 break;
2275
2276 case UDP_NO_CHECK6_TX:
2277 val = up->no_check6_tx;
2278 break;
2279
2280 case UDP_NO_CHECK6_RX:
2281 val = up->no_check6_rx;
2282 break;
2283
2284
2285
2286 case UDPLITE_SEND_CSCOV:
2287 val = up->pcslen;
2288 break;
2289
2290 case UDPLITE_RECV_CSCOV:
2291 val = up->pcrlen;
2292 break;
2293
2294 default:
2295 return -ENOPROTOOPT;
2296 }
2297
2298 if (put_user(len, optlen))
2299 return -EFAULT;
2300 if (copy_to_user(optval, &val, len))
2301 return -EFAULT;
2302 return 0;
2303}
2304EXPORT_SYMBOL(udp_lib_getsockopt);
2305
2306int udp_getsockopt(struct sock *sk, int level, int optname,
2307 char __user *optval, int __user *optlen)
2308{
2309 if (level == SOL_UDP || level == SOL_UDPLITE)
2310 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2311 return ip_getsockopt(sk, level, optname, optval, optlen);
2312}
2313
2314#ifdef CONFIG_COMPAT
2315int compat_udp_getsockopt(struct sock *sk, int level, int optname,
2316 char __user *optval, int __user *optlen)
2317{
2318 if (level == SOL_UDP || level == SOL_UDPLITE)
2319 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2320 return compat_ip_getsockopt(sk, level, optname, optval, optlen);
2321}
2322#endif
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336unsigned int udp_poll(struct file *file, struct socket *sock, poll_table *wait)
2337{
2338 unsigned int mask = datagram_poll(file, sock, wait);
2339 struct sock *sk = sock->sk;
2340
2341 sock_rps_record_flow(sk);
2342
2343
2344 if ((mask & POLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
2345 !(sk->sk_shutdown & RCV_SHUTDOWN) && !first_packet_length(sk))
2346 mask &= ~(POLLIN | POLLRDNORM);
2347
2348 return mask;
2349
2350}
2351EXPORT_SYMBOL(udp_poll);
2352
2353struct proto udp_prot = {
2354 .name = "UDP",
2355 .owner = THIS_MODULE,
2356 .close = udp_lib_close,
2357 .connect = ip4_datagram_connect,
2358 .disconnect = udp_disconnect,
2359 .ioctl = udp_ioctl,
2360 .destroy = udp_destroy_sock,
2361 .setsockopt = udp_setsockopt,
2362 .getsockopt = udp_getsockopt,
2363 .sendmsg = udp_sendmsg,
2364 .recvmsg = udp_recvmsg,
2365 .sendpage = udp_sendpage,
2366 .backlog_rcv = __udp_queue_rcv_skb,
2367 .release_cb = ip4_datagram_release_cb,
2368 .hash = udp_lib_hash,
2369 .unhash = udp_lib_unhash,
2370 .rehash = udp_v4_rehash,
2371 .get_port = udp_v4_get_port,
2372 .memory_allocated = &udp_memory_allocated,
2373 .sysctl_mem = sysctl_udp_mem,
2374 .sysctl_wmem = &sysctl_udp_wmem_min,
2375 .sysctl_rmem = &sysctl_udp_rmem_min,
2376 .obj_size = sizeof(struct udp_sock),
2377 .slab_flags = SLAB_DESTROY_BY_RCU,
2378 .h.udp_table = &udp_table,
2379#ifdef CONFIG_COMPAT
2380 .compat_setsockopt = compat_udp_setsockopt,
2381 .compat_getsockopt = compat_udp_getsockopt,
2382#endif
2383 .clear_sk = sk_prot_clear_portaddr_nulls,
2384};
2385EXPORT_SYMBOL(udp_prot);
2386
2387
2388#ifdef CONFIG_PROC_FS
2389
2390static struct sock *udp_get_first(struct seq_file *seq, int start)
2391{
2392 struct sock *sk;
2393 struct udp_iter_state *state = seq->private;
2394 struct net *net = seq_file_net(seq);
2395
2396 for (state->bucket = start; state->bucket <= state->udp_table->mask;
2397 ++state->bucket) {
2398 struct hlist_nulls_node *node;
2399 struct udp_hslot *hslot = &state->udp_table->hash[state->bucket];
2400
2401 if (hlist_nulls_empty(&hslot->head))
2402 continue;
2403
2404 spin_lock_bh(&hslot->lock);
2405 sk_nulls_for_each(sk, node, &hslot->head) {
2406 if (!net_eq(sock_net(sk), net))
2407 continue;
2408 if (sk->sk_family == state->family)
2409 goto found;
2410 }
2411 spin_unlock_bh(&hslot->lock);
2412 }
2413 sk = NULL;
2414found:
2415 return sk;
2416}
2417
2418static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
2419{
2420 struct udp_iter_state *state = seq->private;
2421 struct net *net = seq_file_net(seq);
2422
2423 do {
2424 sk = sk_nulls_next(sk);
2425 } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family));
2426
2427 if (!sk) {
2428 if (state->bucket <= state->udp_table->mask)
2429 spin_unlock_bh(&state->udp_table->hash[state->bucket].lock);
2430 return udp_get_first(seq, state->bucket + 1);
2431 }
2432 return sk;
2433}
2434
2435static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
2436{
2437 struct sock *sk = udp_get_first(seq, 0);
2438
2439 if (sk)
2440 while (pos && (sk = udp_get_next(seq, sk)) != NULL)
2441 --pos;
2442 return pos ? NULL : sk;
2443}
2444
2445static void *udp_seq_start(struct seq_file *seq, loff_t *pos)
2446{
2447 struct udp_iter_state *state = seq->private;
2448 state->bucket = MAX_UDP_PORTS;
2449
2450 return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
2451}
2452
2453static void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2454{
2455 struct sock *sk;
2456
2457 if (v == SEQ_START_TOKEN)
2458 sk = udp_get_idx(seq, 0);
2459 else
2460 sk = udp_get_next(seq, v);
2461
2462 ++*pos;
2463 return sk;
2464}
2465
2466static void udp_seq_stop(struct seq_file *seq, void *v)
2467{
2468 struct udp_iter_state *state = seq->private;
2469
2470 if (state->bucket <= state->udp_table->mask)
2471 spin_unlock_bh(&state->udp_table->hash[state->bucket].lock);
2472}
2473
2474int udp_seq_open(struct inode *inode, struct file *file)
2475{
2476 struct udp_seq_afinfo *afinfo = PDE_DATA(inode);
2477 struct udp_iter_state *s;
2478 int err;
2479
2480 err = seq_open_net(inode, file, &afinfo->seq_ops,
2481 sizeof(struct udp_iter_state));
2482 if (err < 0)
2483 return err;
2484
2485 s = ((struct seq_file *)file->private_data)->private;
2486 s->family = afinfo->family;
2487 s->udp_table = afinfo->udp_table;
2488 return err;
2489}
2490EXPORT_SYMBOL(udp_seq_open);
2491
2492
2493int udp_proc_register(struct net *net, struct udp_seq_afinfo *afinfo)
2494{
2495 struct proc_dir_entry *p;
2496 int rc = 0;
2497
2498 afinfo->seq_ops.start = udp_seq_start;
2499 afinfo->seq_ops.next = udp_seq_next;
2500 afinfo->seq_ops.stop = udp_seq_stop;
2501
2502 p = proc_create_data(afinfo->name, S_IRUGO, net->proc_net,
2503 afinfo->seq_fops, afinfo);
2504 if (!p)
2505 rc = -ENOMEM;
2506 return rc;
2507}
2508EXPORT_SYMBOL(udp_proc_register);
2509
2510void udp_proc_unregister(struct net *net, struct udp_seq_afinfo *afinfo)
2511{
2512 remove_proc_entry(afinfo->name, net->proc_net);
2513}
2514EXPORT_SYMBOL(udp_proc_unregister);
2515
2516
2517static void udp4_format_sock(struct sock *sp, struct seq_file *f,
2518 int bucket)
2519{
2520 struct inet_sock *inet = inet_sk(sp);
2521 __be32 dest = inet->inet_daddr;
2522 __be32 src = inet->inet_rcv_saddr;
2523 __u16 destp = ntohs(inet->inet_dport);
2524 __u16 srcp = ntohs(inet->inet_sport);
2525
2526 seq_printf(f, "%5d: %08X:%04X %08X:%04X"
2527 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d",
2528 bucket, src, srcp, dest, destp, sp->sk_state,
2529 sk_wmem_alloc_get(sp),
2530 sk_rmem_alloc_get(sp),
2531 0, 0L, 0,
2532 from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
2533 0, sock_i_ino(sp),
2534 atomic_read(&sp->sk_refcnt), sp,
2535 atomic_read(&sp->sk_drops));
2536}
2537
2538int udp4_seq_show(struct seq_file *seq, void *v)
2539{
2540 seq_setwidth(seq, 127);
2541 if (v == SEQ_START_TOKEN)
2542 seq_puts(seq, " sl local_address rem_address st tx_queue "
2543 "rx_queue tr tm->when retrnsmt uid timeout "
2544 "inode ref pointer drops");
2545 else {
2546 struct udp_iter_state *state = seq->private;
2547
2548 udp4_format_sock(v, seq, state->bucket);
2549 }
2550 seq_pad(seq, '\n');
2551 return 0;
2552}
2553
2554static const struct file_operations udp_afinfo_seq_fops = {
2555 .owner = THIS_MODULE,
2556 .open = udp_seq_open,
2557 .read = seq_read,
2558 .llseek = seq_lseek,
2559 .release = seq_release_net
2560};
2561
2562
2563static struct udp_seq_afinfo udp4_seq_afinfo = {
2564 .name = "udp",
2565 .family = AF_INET,
2566 .udp_table = &udp_table,
2567 .seq_fops = &udp_afinfo_seq_fops,
2568 .seq_ops = {
2569 .show = udp4_seq_show,
2570 },
2571};
2572
2573static int __net_init udp4_proc_init_net(struct net *net)
2574{
2575 return udp_proc_register(net, &udp4_seq_afinfo);
2576}
2577
2578static void __net_exit udp4_proc_exit_net(struct net *net)
2579{
2580 udp_proc_unregister(net, &udp4_seq_afinfo);
2581}
2582
2583static struct pernet_operations udp4_net_ops = {
2584 .init = udp4_proc_init_net,
2585 .exit = udp4_proc_exit_net,
2586};
2587
2588int __init udp4_proc_init(void)
2589{
2590 return register_pernet_subsys(&udp4_net_ops);
2591}
2592
2593void udp4_proc_exit(void)
2594{
2595 unregister_pernet_subsys(&udp4_net_ops);
2596}
2597#endif
2598
2599static __initdata unsigned long uhash_entries;
2600static int __init set_uhash_entries(char *str)
2601{
2602 ssize_t ret;
2603
2604 if (!str)
2605 return 0;
2606
2607 ret = kstrtoul(str, 0, &uhash_entries);
2608 if (ret)
2609 return 0;
2610
2611 if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
2612 uhash_entries = UDP_HTABLE_SIZE_MIN;
2613 return 1;
2614}
2615__setup("uhash_entries=", set_uhash_entries);
2616
2617void __init udp_table_init(struct udp_table *table, const char *name)
2618{
2619 unsigned int i;
2620
2621 table->hash = alloc_large_system_hash(name,
2622 2 * sizeof(struct udp_hslot),
2623 uhash_entries,
2624 21,
2625 0,
2626 &table->log,
2627 &table->mask,
2628 UDP_HTABLE_SIZE_MIN,
2629 64 * 1024);
2630
2631 table->hash2 = table->hash + (table->mask + 1);
2632 for (i = 0; i <= table->mask; i++) {
2633 INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i);
2634 table->hash[i].count = 0;
2635 spin_lock_init(&table->hash[i].lock);
2636 }
2637 for (i = 0; i <= table->mask; i++) {
2638 INIT_HLIST_NULLS_HEAD(&table->hash2[i].head, i);
2639 table->hash2[i].count = 0;
2640 spin_lock_init(&table->hash2[i].lock);
2641 }
2642}
2643
2644u32 udp_flow_hashrnd(void)
2645{
2646 static u32 hashrnd __read_mostly;
2647
2648 net_get_random_once(&hashrnd, sizeof(hashrnd));
2649
2650 return hashrnd;
2651}
2652EXPORT_SYMBOL(udp_flow_hashrnd);
2653
2654void __init udp_init(void)
2655{
2656 unsigned long limit;
2657
2658 udp_table_init(&udp_table, "UDP");
2659 limit = nr_free_buffer_pages() / 8;
2660 limit = max(limit, 128UL);
2661 sysctl_udp_mem[0] = limit / 4 * 3;
2662 sysctl_udp_mem[1] = limit;
2663 sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
2664
2665 sysctl_udp_rmem_min = SK_MEM_QUANTUM;
2666 sysctl_udp_wmem_min = SK_MEM_QUANTUM;
2667}
2668