1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109#define pr_fmt(fmt) "IPv4: " fmt
110
111#include <linux/module.h>
112#include <linux/types.h>
113#include <linux/kernel.h>
114#include <linux/string.h>
115#include <linux/errno.h>
116#include <linux/slab.h>
117
118#include <linux/net.h>
119#include <linux/socket.h>
120#include <linux/sockios.h>
121#include <linux/in.h>
122#include <linux/inet.h>
123#include <linux/inetdevice.h>
124#include <linux/netdevice.h>
125#include <linux/etherdevice.h>
126#include <linux/indirect_call_wrapper.h>
127
128#include <net/snmp.h>
129#include <net/ip.h>
130#include <net/protocol.h>
131#include <net/route.h>
132#include <linux/skbuff.h>
133#include <net/sock.h>
134#include <net/arp.h>
135#include <net/icmp.h>
136#include <net/raw.h>
137#include <net/checksum.h>
138#include <net/inet_ecn.h>
139#include <linux/netfilter_ipv4.h>
140#include <net/xfrm.h>
141#include <linux/mroute.h>
142#include <linux/netlink.h>
143#include <net/dst_metadata.h>
144
145
146
147
148bool ip_call_ra_chain(struct sk_buff *skb)
149{
150 struct ip_ra_chain *ra;
151 u8 protocol = ip_hdr(skb)->protocol;
152 struct sock *last = NULL;
153 struct net_device *dev = skb->dev;
154 struct net *net = dev_net(dev);
155
156 for (ra = rcu_dereference(net->ipv4.ra_chain); ra; ra = rcu_dereference(ra->next)) {
157 struct sock *sk = ra->sk;
158
159
160
161
162 if (sk && inet_sk(sk)->inet_num == protocol &&
163 (!sk->sk_bound_dev_if ||
164 sk->sk_bound_dev_if == dev->ifindex)) {
165 if (ip_is_fragment(ip_hdr(skb))) {
166 if (ip_defrag(net, skb, IP_DEFRAG_CALL_RA_CHAIN))
167 return true;
168 }
169 if (last) {
170 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
171 if (skb2)
172 raw_rcv(last, skb2);
173 }
174 last = sk;
175 }
176 }
177
178 if (last) {
179 raw_rcv(last, skb);
180 return true;
181 }
182 return false;
183}
184
185INDIRECT_CALLABLE_DECLARE(int udp_rcv(struct sk_buff *));
186INDIRECT_CALLABLE_DECLARE(int tcp_v4_rcv(struct sk_buff *));
187void ip_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int protocol)
188{
189 const struct net_protocol *ipprot;
190 int raw, ret;
191
192resubmit:
193 raw = raw_local_deliver(skb, protocol);
194
195 ipprot = rcu_dereference(inet_protos[protocol]);
196 if (ipprot) {
197 if (!ipprot->no_policy) {
198 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
199 kfree_skb_reason(skb,
200 SKB_DROP_REASON_XFRM_POLICY);
201 return;
202 }
203 nf_reset_ct(skb);
204 }
205 ret = INDIRECT_CALL_2(ipprot->handler, tcp_v4_rcv, udp_rcv,
206 skb);
207 if (ret < 0) {
208 protocol = -ret;
209 goto resubmit;
210 }
211 __IP_INC_STATS(net, IPSTATS_MIB_INDELIVERS);
212 } else {
213 if (!raw) {
214 if (xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
215 __IP_INC_STATS(net, IPSTATS_MIB_INUNKNOWNPROTOS);
216 icmp_send(skb, ICMP_DEST_UNREACH,
217 ICMP_PROT_UNREACH, 0);
218 }
219 kfree_skb_reason(skb, SKB_DROP_REASON_IP_NOPROTO);
220 } else {
221 __IP_INC_STATS(net, IPSTATS_MIB_INDELIVERS);
222 consume_skb(skb);
223 }
224 }
225}
226
227static int ip_local_deliver_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
228{
229 skb_clear_delivery_time(skb);
230 __skb_pull(skb, skb_network_header_len(skb));
231
232 rcu_read_lock();
233 ip_protocol_deliver_rcu(net, skb, ip_hdr(skb)->protocol);
234 rcu_read_unlock();
235
236 return 0;
237}
238
239
240
241
242int ip_local_deliver(struct sk_buff *skb)
243{
244
245
246
247 struct net *net = dev_net(skb->dev);
248
249 if (ip_is_fragment(ip_hdr(skb))) {
250 if (ip_defrag(net, skb, IP_DEFRAG_LOCAL_DELIVER))
251 return 0;
252 }
253
254 return NF_HOOK(NFPROTO_IPV4, NF_INET_LOCAL_IN,
255 net, NULL, skb, skb->dev, NULL,
256 ip_local_deliver_finish);
257}
258EXPORT_SYMBOL(ip_local_deliver);
259
260static inline bool ip_rcv_options(struct sk_buff *skb, struct net_device *dev)
261{
262 struct ip_options *opt;
263 const struct iphdr *iph;
264
265
266
267
268
269
270
271
272 if (skb_cow(skb, skb_headroom(skb))) {
273 __IP_INC_STATS(dev_net(dev), IPSTATS_MIB_INDISCARDS);
274 goto drop;
275 }
276
277 iph = ip_hdr(skb);
278 opt = &(IPCB(skb)->opt);
279 opt->optlen = iph->ihl*4 - sizeof(struct iphdr);
280
281 if (ip_options_compile(dev_net(dev), opt, skb)) {
282 __IP_INC_STATS(dev_net(dev), IPSTATS_MIB_INHDRERRORS);
283 goto drop;
284 }
285
286 if (unlikely(opt->srr)) {
287 struct in_device *in_dev = __in_dev_get_rcu(dev);
288
289 if (in_dev) {
290 if (!IN_DEV_SOURCE_ROUTE(in_dev)) {
291 if (IN_DEV_LOG_MARTIANS(in_dev))
292 net_info_ratelimited("source route option %pI4 -> %pI4\n",
293 &iph->saddr,
294 &iph->daddr);
295 goto drop;
296 }
297 }
298
299 if (ip_options_rcv_srr(skb, dev))
300 goto drop;
301 }
302
303 return false;
304drop:
305 return true;
306}
307
308static bool ip_can_use_hint(const struct sk_buff *skb, const struct iphdr *iph,
309 const struct sk_buff *hint)
310{
311 return hint && !skb_dst(skb) && ip_hdr(hint)->daddr == iph->daddr &&
312 ip_hdr(hint)->tos == iph->tos;
313}
314
315INDIRECT_CALLABLE_DECLARE(int udp_v4_early_demux(struct sk_buff *));
316INDIRECT_CALLABLE_DECLARE(int tcp_v4_early_demux(struct sk_buff *));
317static int ip_rcv_finish_core(struct net *net, struct sock *sk,
318 struct sk_buff *skb, struct net_device *dev,
319 const struct sk_buff *hint)
320{
321 const struct iphdr *iph = ip_hdr(skb);
322 int (*edemux)(struct sk_buff *skb);
323 int err, drop_reason;
324 struct rtable *rt;
325
326 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
327
328 if (ip_can_use_hint(skb, iph, hint)) {
329 err = ip_route_use_hint(skb, iph->daddr, iph->saddr, iph->tos,
330 dev, hint);
331 if (unlikely(err))
332 goto drop_error;
333 }
334
335 if (net->ipv4.sysctl_ip_early_demux &&
336 !skb_dst(skb) &&
337 !skb->sk &&
338 !ip_is_fragment(iph)) {
339 const struct net_protocol *ipprot;
340 int protocol = iph->protocol;
341
342 ipprot = rcu_dereference(inet_protos[protocol]);
343 if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) {
344 err = INDIRECT_CALL_2(edemux, tcp_v4_early_demux,
345 udp_v4_early_demux, skb);
346 if (unlikely(err))
347 goto drop_error;
348
349 iph = ip_hdr(skb);
350 }
351 }
352
353
354
355
356
357 if (!skb_valid_dst(skb)) {
358 err = ip_route_input_noref(skb, iph->daddr, iph->saddr,
359 iph->tos, dev);
360 if (unlikely(err))
361 goto drop_error;
362 }
363
364#ifdef CONFIG_IP_ROUTE_CLASSID
365 if (unlikely(skb_dst(skb)->tclassid)) {
366 struct ip_rt_acct *st = this_cpu_ptr(ip_rt_acct);
367 u32 idx = skb_dst(skb)->tclassid;
368 st[idx&0xFF].o_packets++;
369 st[idx&0xFF].o_bytes += skb->len;
370 st[(idx>>16)&0xFF].i_packets++;
371 st[(idx>>16)&0xFF].i_bytes += skb->len;
372 }
373#endif
374
375 if (iph->ihl > 5 && ip_rcv_options(skb, dev))
376 goto drop;
377
378 rt = skb_rtable(skb);
379 if (rt->rt_type == RTN_MULTICAST) {
380 __IP_UPD_PO_STATS(net, IPSTATS_MIB_INMCAST, skb->len);
381 } else if (rt->rt_type == RTN_BROADCAST) {
382 __IP_UPD_PO_STATS(net, IPSTATS_MIB_INBCAST, skb->len);
383 } else if (skb->pkt_type == PACKET_BROADCAST ||
384 skb->pkt_type == PACKET_MULTICAST) {
385 struct in_device *in_dev = __in_dev_get_rcu(dev);
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402 if (in_dev &&
403 IN_DEV_ORCONF(in_dev, DROP_UNICAST_IN_L2_MULTICAST)) {
404 drop_reason = SKB_DROP_REASON_UNICAST_IN_L2_MULTICAST;
405 goto drop;
406 }
407 }
408
409 return NET_RX_SUCCESS;
410
411drop:
412 kfree_skb_reason(skb, drop_reason);
413 return NET_RX_DROP;
414
415drop_error:
416 if (err == -EXDEV) {
417 drop_reason = SKB_DROP_REASON_IP_RPFILTER;
418 __NET_INC_STATS(net, LINUX_MIB_IPRPFILTER);
419 }
420 goto drop;
421}
422
423static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
424{
425 struct net_device *dev = skb->dev;
426 int ret;
427
428
429
430
431 skb = l3mdev_ip_rcv(skb);
432 if (!skb)
433 return NET_RX_SUCCESS;
434
435 ret = ip_rcv_finish_core(net, sk, skb, dev, NULL);
436 if (ret != NET_RX_DROP)
437 ret = dst_input(skb);
438 return ret;
439}
440
441
442
443
444static struct sk_buff *ip_rcv_core(struct sk_buff *skb, struct net *net)
445{
446 const struct iphdr *iph;
447 int drop_reason;
448 u32 len;
449
450
451
452
453 if (skb->pkt_type == PACKET_OTHERHOST) {
454 drop_reason = SKB_DROP_REASON_OTHERHOST;
455 goto drop;
456 }
457
458 __IP_UPD_PO_STATS(net, IPSTATS_MIB_IN, skb->len);
459
460 skb = skb_share_check(skb, GFP_ATOMIC);
461 if (!skb) {
462 __IP_INC_STATS(net, IPSTATS_MIB_INDISCARDS);
463 goto out;
464 }
465
466 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
467 if (!pskb_may_pull(skb, sizeof(struct iphdr)))
468 goto inhdr_error;
469
470 iph = ip_hdr(skb);
471
472
473
474
475
476
477
478
479
480
481
482
483 if (iph->ihl < 5 || iph->version != 4)
484 goto inhdr_error;
485
486 BUILD_BUG_ON(IPSTATS_MIB_ECT1PKTS != IPSTATS_MIB_NOECTPKTS + INET_ECN_ECT_1);
487 BUILD_BUG_ON(IPSTATS_MIB_ECT0PKTS != IPSTATS_MIB_NOECTPKTS + INET_ECN_ECT_0);
488 BUILD_BUG_ON(IPSTATS_MIB_CEPKTS != IPSTATS_MIB_NOECTPKTS + INET_ECN_CE);
489 __IP_ADD_STATS(net,
490 IPSTATS_MIB_NOECTPKTS + (iph->tos & INET_ECN_MASK),
491 max_t(unsigned short, 1, skb_shinfo(skb)->gso_segs));
492
493 if (!pskb_may_pull(skb, iph->ihl*4))
494 goto inhdr_error;
495
496 iph = ip_hdr(skb);
497
498 if (unlikely(ip_fast_csum((u8 *)iph, iph->ihl)))
499 goto csum_error;
500
501 len = ntohs(iph->tot_len);
502 if (skb->len < len) {
503 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
504 __IP_INC_STATS(net, IPSTATS_MIB_INTRUNCATEDPKTS);
505 goto drop;
506 } else if (len < (iph->ihl*4))
507 goto inhdr_error;
508
509
510
511
512
513 if (pskb_trim_rcsum(skb, len)) {
514 __IP_INC_STATS(net, IPSTATS_MIB_INDISCARDS);
515 goto drop;
516 }
517
518 iph = ip_hdr(skb);
519 skb->transport_header = skb->network_header + iph->ihl*4;
520
521
522 memset(IPCB(skb), 0, sizeof(struct inet_skb_parm));
523 IPCB(skb)->iif = skb->skb_iif;
524
525
526 if (!skb_sk_is_prefetched(skb))
527 skb_orphan(skb);
528
529 return skb;
530
531csum_error:
532 drop_reason = SKB_DROP_REASON_IP_CSUM;
533 __IP_INC_STATS(net, IPSTATS_MIB_CSUMERRORS);
534inhdr_error:
535 if (drop_reason == SKB_DROP_REASON_NOT_SPECIFIED)
536 drop_reason = SKB_DROP_REASON_IP_INHDR;
537 __IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS);
538drop:
539 kfree_skb_reason(skb, drop_reason);
540out:
541 return NULL;
542}
543
544
545
546
547int ip_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt,
548 struct net_device *orig_dev)
549{
550 struct net *net = dev_net(dev);
551
552 skb = ip_rcv_core(skb, net);
553 if (skb == NULL)
554 return NET_RX_DROP;
555
556 return NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING,
557 net, NULL, skb, dev, NULL,
558 ip_rcv_finish);
559}
560
561static void ip_sublist_rcv_finish(struct list_head *head)
562{
563 struct sk_buff *skb, *next;
564
565 list_for_each_entry_safe(skb, next, head, list) {
566 skb_list_del_init(skb);
567 dst_input(skb);
568 }
569}
570
571static struct sk_buff *ip_extract_route_hint(const struct net *net,
572 struct sk_buff *skb, int rt_type)
573{
574 if (fib4_has_custom_rules(net) || rt_type == RTN_BROADCAST)
575 return NULL;
576
577 return skb;
578}
579
580static void ip_list_rcv_finish(struct net *net, struct sock *sk,
581 struct list_head *head)
582{
583 struct sk_buff *skb, *next, *hint = NULL;
584 struct dst_entry *curr_dst = NULL;
585 struct list_head sublist;
586
587 INIT_LIST_HEAD(&sublist);
588 list_for_each_entry_safe(skb, next, head, list) {
589 struct net_device *dev = skb->dev;
590 struct dst_entry *dst;
591
592 skb_list_del_init(skb);
593
594
595
596 skb = l3mdev_ip_rcv(skb);
597 if (!skb)
598 continue;
599 if (ip_rcv_finish_core(net, sk, skb, dev, hint) == NET_RX_DROP)
600 continue;
601
602 dst = skb_dst(skb);
603 if (curr_dst != dst) {
604 hint = ip_extract_route_hint(net, skb,
605 ((struct rtable *)dst)->rt_type);
606
607
608 if (!list_empty(&sublist))
609 ip_sublist_rcv_finish(&sublist);
610
611 INIT_LIST_HEAD(&sublist);
612 curr_dst = dst;
613 }
614 list_add_tail(&skb->list, &sublist);
615 }
616
617 ip_sublist_rcv_finish(&sublist);
618}
619
620static void ip_sublist_rcv(struct list_head *head, struct net_device *dev,
621 struct net *net)
622{
623 NF_HOOK_LIST(NFPROTO_IPV4, NF_INET_PRE_ROUTING, net, NULL,
624 head, dev, NULL, ip_rcv_finish);
625 ip_list_rcv_finish(net, NULL, head);
626}
627
628
629void ip_list_rcv(struct list_head *head, struct packet_type *pt,
630 struct net_device *orig_dev)
631{
632 struct net_device *curr_dev = NULL;
633 struct net *curr_net = NULL;
634 struct sk_buff *skb, *next;
635 struct list_head sublist;
636
637 INIT_LIST_HEAD(&sublist);
638 list_for_each_entry_safe(skb, next, head, list) {
639 struct net_device *dev = skb->dev;
640 struct net *net = dev_net(dev);
641
642 skb_list_del_init(skb);
643 skb = ip_rcv_core(skb, net);
644 if (skb == NULL)
645 continue;
646
647 if (curr_dev != dev || curr_net != net) {
648
649 if (!list_empty(&sublist))
650 ip_sublist_rcv(&sublist, curr_dev, curr_net);
651
652 INIT_LIST_HEAD(&sublist);
653 curr_dev = dev;
654 curr_net = net;
655 }
656 list_add_tail(&skb->list, &sublist);
657 }
658
659 if (!list_empty(&sublist))
660 ip_sublist_rcv(&sublist, curr_dev, curr_net);
661}
662