1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109#define pr_fmt(fmt) "IPv4: " fmt
110
111#include <linux/module.h>
112#include <linux/types.h>
113#include <linux/kernel.h>
114#include <linux/string.h>
115#include <linux/errno.h>
116#include <linux/slab.h>
117
118#include <linux/net.h>
119#include <linux/socket.h>
120#include <linux/sockios.h>
121#include <linux/in.h>
122#include <linux/inet.h>
123#include <linux/inetdevice.h>
124#include <linux/netdevice.h>
125#include <linux/etherdevice.h>
126#include <linux/indirect_call_wrapper.h>
127
128#include <net/snmp.h>
129#include <net/ip.h>
130#include <net/protocol.h>
131#include <net/route.h>
132#include <linux/skbuff.h>
133#include <net/sock.h>
134#include <net/arp.h>
135#include <net/icmp.h>
136#include <net/raw.h>
137#include <net/checksum.h>
138#include <net/inet_ecn.h>
139#include <linux/netfilter_ipv4.h>
140#include <net/xfrm.h>
141#include <linux/mroute.h>
142#include <linux/netlink.h>
143#include <net/dst_metadata.h>
144
145
146
147
148bool ip_call_ra_chain(struct sk_buff *skb)
149{
150 struct ip_ra_chain *ra;
151 u8 protocol = ip_hdr(skb)->protocol;
152 struct sock *last = NULL;
153 struct net_device *dev = skb->dev;
154 struct net *net = dev_net(dev);
155
156 for (ra = rcu_dereference(net->ipv4.ra_chain); ra; ra = rcu_dereference(ra->next)) {
157 struct sock *sk = ra->sk;
158
159
160
161
162 if (sk && inet_sk(sk)->inet_num == protocol &&
163 (!sk->sk_bound_dev_if ||
164 sk->sk_bound_dev_if == dev->ifindex)) {
165 if (ip_is_fragment(ip_hdr(skb))) {
166 if (ip_defrag(net, skb, IP_DEFRAG_CALL_RA_CHAIN))
167 return true;
168 }
169 if (last) {
170 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
171 if (skb2)
172 raw_rcv(last, skb2);
173 }
174 last = sk;
175 }
176 }
177
178 if (last) {
179 raw_rcv(last, skb);
180 return true;
181 }
182 return false;
183}
184
185INDIRECT_CALLABLE_DECLARE(int udp_rcv(struct sk_buff *));
186INDIRECT_CALLABLE_DECLARE(int tcp_v4_rcv(struct sk_buff *));
187void ip_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int protocol)
188{
189 const struct net_protocol *ipprot;
190 int raw, ret;
191
192resubmit:
193 raw = raw_local_deliver(skb, protocol);
194
195 ipprot = rcu_dereference(inet_protos[protocol]);
196 if (ipprot) {
197 if (!ipprot->no_policy) {
198 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
199 kfree_skb(skb);
200 return;
201 }
202 nf_reset_ct(skb);
203 }
204 ret = INDIRECT_CALL_2(ipprot->handler, tcp_v4_rcv, udp_rcv,
205 skb);
206 if (ret < 0) {
207 protocol = -ret;
208 goto resubmit;
209 }
210 __IP_INC_STATS(net, IPSTATS_MIB_INDELIVERS);
211 } else {
212 if (!raw) {
213 if (xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
214 __IP_INC_STATS(net, IPSTATS_MIB_INUNKNOWNPROTOS);
215 icmp_send(skb, ICMP_DEST_UNREACH,
216 ICMP_PROT_UNREACH, 0);
217 }
218 kfree_skb(skb);
219 } else {
220 __IP_INC_STATS(net, IPSTATS_MIB_INDELIVERS);
221 consume_skb(skb);
222 }
223 }
224}
225
226static int ip_local_deliver_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
227{
228 __skb_pull(skb, skb_network_header_len(skb));
229
230 rcu_read_lock();
231 ip_protocol_deliver_rcu(net, skb, ip_hdr(skb)->protocol);
232 rcu_read_unlock();
233
234 return 0;
235}
236
237
238
239
240int ip_local_deliver(struct sk_buff *skb)
241{
242
243
244
245 struct net *net = dev_net(skb->dev);
246
247 if (ip_is_fragment(ip_hdr(skb))) {
248 if (ip_defrag(net, skb, IP_DEFRAG_LOCAL_DELIVER))
249 return 0;
250 }
251
252 return NF_HOOK(NFPROTO_IPV4, NF_INET_LOCAL_IN,
253 net, NULL, skb, skb->dev, NULL,
254 ip_local_deliver_finish);
255}
256EXPORT_SYMBOL(ip_local_deliver);
257
258static inline bool ip_rcv_options(struct sk_buff *skb, struct net_device *dev)
259{
260 struct ip_options *opt;
261 const struct iphdr *iph;
262
263
264
265
266
267
268
269
270 if (skb_cow(skb, skb_headroom(skb))) {
271 __IP_INC_STATS(dev_net(dev), IPSTATS_MIB_INDISCARDS);
272 goto drop;
273 }
274
275 iph = ip_hdr(skb);
276 opt = &(IPCB(skb)->opt);
277 opt->optlen = iph->ihl*4 - sizeof(struct iphdr);
278
279 if (ip_options_compile(dev_net(dev), opt, skb)) {
280 __IP_INC_STATS(dev_net(dev), IPSTATS_MIB_INHDRERRORS);
281 goto drop;
282 }
283
284 if (unlikely(opt->srr)) {
285 struct in_device *in_dev = __in_dev_get_rcu(dev);
286
287 if (in_dev) {
288 if (!IN_DEV_SOURCE_ROUTE(in_dev)) {
289 if (IN_DEV_LOG_MARTIANS(in_dev))
290 net_info_ratelimited("source route option %pI4 -> %pI4\n",
291 &iph->saddr,
292 &iph->daddr);
293 goto drop;
294 }
295 }
296
297 if (ip_options_rcv_srr(skb, dev))
298 goto drop;
299 }
300
301 return false;
302drop:
303 return true;
304}
305
306static bool ip_can_use_hint(const struct sk_buff *skb, const struct iphdr *iph,
307 const struct sk_buff *hint)
308{
309 return hint && !skb_dst(skb) && ip_hdr(hint)->daddr == iph->daddr &&
310 ip_hdr(hint)->tos == iph->tos;
311}
312
313INDIRECT_CALLABLE_DECLARE(int udp_v4_early_demux(struct sk_buff *));
314INDIRECT_CALLABLE_DECLARE(int tcp_v4_early_demux(struct sk_buff *));
315static int ip_rcv_finish_core(struct net *net, struct sock *sk,
316 struct sk_buff *skb, struct net_device *dev,
317 const struct sk_buff *hint)
318{
319 const struct iphdr *iph = ip_hdr(skb);
320 int (*edemux)(struct sk_buff *skb);
321 struct rtable *rt;
322 int err;
323
324 if (ip_can_use_hint(skb, iph, hint)) {
325 err = ip_route_use_hint(skb, iph->daddr, iph->saddr, iph->tos,
326 dev, hint);
327 if (unlikely(err))
328 goto drop_error;
329 }
330
331 if (net->ipv4.sysctl_ip_early_demux &&
332 !skb_dst(skb) &&
333 !skb->sk &&
334 !ip_is_fragment(iph)) {
335 const struct net_protocol *ipprot;
336 int protocol = iph->protocol;
337
338 ipprot = rcu_dereference(inet_protos[protocol]);
339 if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) {
340 err = INDIRECT_CALL_2(edemux, tcp_v4_early_demux,
341 udp_v4_early_demux, skb);
342 if (unlikely(err))
343 goto drop_error;
344
345 iph = ip_hdr(skb);
346 }
347 }
348
349
350
351
352
353 if (!skb_valid_dst(skb)) {
354 err = ip_route_input_noref(skb, iph->daddr, iph->saddr,
355 iph->tos, dev);
356 if (unlikely(err))
357 goto drop_error;
358 }
359
360#ifdef CONFIG_IP_ROUTE_CLASSID
361 if (unlikely(skb_dst(skb)->tclassid)) {
362 struct ip_rt_acct *st = this_cpu_ptr(ip_rt_acct);
363 u32 idx = skb_dst(skb)->tclassid;
364 st[idx&0xFF].o_packets++;
365 st[idx&0xFF].o_bytes += skb->len;
366 st[(idx>>16)&0xFF].i_packets++;
367 st[(idx>>16)&0xFF].i_bytes += skb->len;
368 }
369#endif
370
371 if (iph->ihl > 5 && ip_rcv_options(skb, dev))
372 goto drop;
373
374 rt = skb_rtable(skb);
375 if (rt->rt_type == RTN_MULTICAST) {
376 __IP_UPD_PO_STATS(net, IPSTATS_MIB_INMCAST, skb->len);
377 } else if (rt->rt_type == RTN_BROADCAST) {
378 __IP_UPD_PO_STATS(net, IPSTATS_MIB_INBCAST, skb->len);
379 } else if (skb->pkt_type == PACKET_BROADCAST ||
380 skb->pkt_type == PACKET_MULTICAST) {
381 struct in_device *in_dev = __in_dev_get_rcu(dev);
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398 if (in_dev &&
399 IN_DEV_ORCONF(in_dev, DROP_UNICAST_IN_L2_MULTICAST))
400 goto drop;
401 }
402
403 return NET_RX_SUCCESS;
404
405drop:
406 kfree_skb(skb);
407 return NET_RX_DROP;
408
409drop_error:
410 if (err == -EXDEV)
411 __NET_INC_STATS(net, LINUX_MIB_IPRPFILTER);
412 goto drop;
413}
414
415static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
416{
417 struct net_device *dev = skb->dev;
418 int ret;
419
420
421
422
423 skb = l3mdev_ip_rcv(skb);
424 if (!skb)
425 return NET_RX_SUCCESS;
426
427 ret = ip_rcv_finish_core(net, sk, skb, dev, NULL);
428 if (ret != NET_RX_DROP)
429 ret = dst_input(skb);
430 return ret;
431}
432
433
434
435
436static struct sk_buff *ip_rcv_core(struct sk_buff *skb, struct net *net)
437{
438 const struct iphdr *iph;
439 u32 len;
440
441
442
443
444 if (skb->pkt_type == PACKET_OTHERHOST)
445 goto drop;
446
447 __IP_UPD_PO_STATS(net, IPSTATS_MIB_IN, skb->len);
448
449 skb = skb_share_check(skb, GFP_ATOMIC);
450 if (!skb) {
451 __IP_INC_STATS(net, IPSTATS_MIB_INDISCARDS);
452 goto out;
453 }
454
455 if (!pskb_may_pull(skb, sizeof(struct iphdr)))
456 goto inhdr_error;
457
458 iph = ip_hdr(skb);
459
460
461
462
463
464
465
466
467
468
469
470
471 if (iph->ihl < 5 || iph->version != 4)
472 goto inhdr_error;
473
474 BUILD_BUG_ON(IPSTATS_MIB_ECT1PKTS != IPSTATS_MIB_NOECTPKTS + INET_ECN_ECT_1);
475 BUILD_BUG_ON(IPSTATS_MIB_ECT0PKTS != IPSTATS_MIB_NOECTPKTS + INET_ECN_ECT_0);
476 BUILD_BUG_ON(IPSTATS_MIB_CEPKTS != IPSTATS_MIB_NOECTPKTS + INET_ECN_CE);
477 __IP_ADD_STATS(net,
478 IPSTATS_MIB_NOECTPKTS + (iph->tos & INET_ECN_MASK),
479 max_t(unsigned short, 1, skb_shinfo(skb)->gso_segs));
480
481 if (!pskb_may_pull(skb, iph->ihl*4))
482 goto inhdr_error;
483
484 iph = ip_hdr(skb);
485
486 if (unlikely(ip_fast_csum((u8 *)iph, iph->ihl)))
487 goto csum_error;
488
489 len = ntohs(iph->tot_len);
490 if (skb->len < len) {
491 __IP_INC_STATS(net, IPSTATS_MIB_INTRUNCATEDPKTS);
492 goto drop;
493 } else if (len < (iph->ihl*4))
494 goto inhdr_error;
495
496
497
498
499
500 if (pskb_trim_rcsum(skb, len)) {
501 __IP_INC_STATS(net, IPSTATS_MIB_INDISCARDS);
502 goto drop;
503 }
504
505 iph = ip_hdr(skb);
506 skb->transport_header = skb->network_header + iph->ihl*4;
507
508
509 memset(IPCB(skb), 0, sizeof(struct inet_skb_parm));
510 IPCB(skb)->iif = skb->skb_iif;
511
512
513 if (!skb_sk_is_prefetched(skb))
514 skb_orphan(skb);
515
516 return skb;
517
518csum_error:
519 __IP_INC_STATS(net, IPSTATS_MIB_CSUMERRORS);
520inhdr_error:
521 __IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS);
522drop:
523 kfree_skb(skb);
524out:
525 return NULL;
526}
527
528
529
530
531int ip_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt,
532 struct net_device *orig_dev)
533{
534 struct net *net = dev_net(dev);
535
536 skb = ip_rcv_core(skb, net);
537 if (skb == NULL)
538 return NET_RX_DROP;
539
540 return NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING,
541 net, NULL, skb, dev, NULL,
542 ip_rcv_finish);
543}
544
545static void ip_sublist_rcv_finish(struct list_head *head)
546{
547 struct sk_buff *skb, *next;
548
549 list_for_each_entry_safe(skb, next, head, list) {
550 skb_list_del_init(skb);
551 dst_input(skb);
552 }
553}
554
555static struct sk_buff *ip_extract_route_hint(const struct net *net,
556 struct sk_buff *skb, int rt_type)
557{
558 if (fib4_has_custom_rules(net) || rt_type == RTN_BROADCAST)
559 return NULL;
560
561 return skb;
562}
563
564static void ip_list_rcv_finish(struct net *net, struct sock *sk,
565 struct list_head *head)
566{
567 struct sk_buff *skb, *next, *hint = NULL;
568 struct dst_entry *curr_dst = NULL;
569 struct list_head sublist;
570
571 INIT_LIST_HEAD(&sublist);
572 list_for_each_entry_safe(skb, next, head, list) {
573 struct net_device *dev = skb->dev;
574 struct dst_entry *dst;
575
576 skb_list_del_init(skb);
577
578
579
580 skb = l3mdev_ip_rcv(skb);
581 if (!skb)
582 continue;
583 if (ip_rcv_finish_core(net, sk, skb, dev, hint) == NET_RX_DROP)
584 continue;
585
586 dst = skb_dst(skb);
587 if (curr_dst != dst) {
588 hint = ip_extract_route_hint(net, skb,
589 ((struct rtable *)dst)->rt_type);
590
591
592 if (!list_empty(&sublist))
593 ip_sublist_rcv_finish(&sublist);
594
595 INIT_LIST_HEAD(&sublist);
596 curr_dst = dst;
597 }
598 list_add_tail(&skb->list, &sublist);
599 }
600
601 ip_sublist_rcv_finish(&sublist);
602}
603
604static void ip_sublist_rcv(struct list_head *head, struct net_device *dev,
605 struct net *net)
606{
607 NF_HOOK_LIST(NFPROTO_IPV4, NF_INET_PRE_ROUTING, net, NULL,
608 head, dev, NULL, ip_rcv_finish);
609 ip_list_rcv_finish(net, NULL, head);
610}
611
612
613void ip_list_rcv(struct list_head *head, struct packet_type *pt,
614 struct net_device *orig_dev)
615{
616 struct net_device *curr_dev = NULL;
617 struct net *curr_net = NULL;
618 struct sk_buff *skb, *next;
619 struct list_head sublist;
620
621 INIT_LIST_HEAD(&sublist);
622 list_for_each_entry_safe(skb, next, head, list) {
623 struct net_device *dev = skb->dev;
624 struct net *net = dev_net(dev);
625
626 skb_list_del_init(skb);
627 skb = ip_rcv_core(skb, net);
628 if (skb == NULL)
629 continue;
630
631 if (curr_dev != dev || curr_net != net) {
632
633 if (!list_empty(&sublist))
634 ip_sublist_rcv(&sublist, curr_dev, curr_net);
635
636 INIT_LIST_HEAD(&sublist);
637 curr_dev = dev;
638 curr_net = net;
639 }
640 list_add_tail(&skb->list, &sublist);
641 }
642
643 if (!list_empty(&sublist))
644 ip_sublist_rcv(&sublist, curr_dev, curr_net);
645}
646