1
2#include <linux/kernel.h>
3#include <linux/skbuff.h>
4#include <linux/export.h>
5#include <linux/ip.h>
6#include <linux/ipv6.h>
7#include <linux/if_vlan.h>
8#include <net/dsa.h>
9#include <net/dst_metadata.h>
10#include <net/ip.h>
11#include <net/ipv6.h>
12#include <net/gre.h>
13#include <net/pptp.h>
14#include <net/tipc.h>
15#include <linux/igmp.h>
16#include <linux/icmp.h>
17#include <linux/sctp.h>
18#include <linux/dccp.h>
19#include <linux/if_tunnel.h>
20#include <linux/if_pppox.h>
21#include <linux/ppp_defs.h>
22#include <linux/stddef.h>
23#include <linux/if_ether.h>
24#include <linux/mpls.h>
25#include <linux/tcp.h>
26#include <linux/ptp_classify.h>
27#include <net/flow_dissector.h>
28#include <scsi/fc/fc_fcoe.h>
29#include <uapi/linux/batadv_packet.h>
30#include <linux/bpf.h>
31#if IS_ENABLED(CONFIG_NF_CONNTRACK)
32#include <net/netfilter/nf_conntrack_core.h>
33#include <net/netfilter/nf_conntrack_labels.h>
34#endif
35#include <linux/bpf-netns.h>
36
37static void dissector_set_key(struct flow_dissector *flow_dissector,
38 enum flow_dissector_key_id key_id)
39{
40 flow_dissector->used_keys |= (1 << key_id);
41}
42
43void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
44 const struct flow_dissector_key *key,
45 unsigned int key_count)
46{
47 unsigned int i;
48
49 memset(flow_dissector, 0, sizeof(*flow_dissector));
50
51 for (i = 0; i < key_count; i++, key++) {
52
53
54
55 BUG_ON(key->offset > USHRT_MAX);
56 BUG_ON(dissector_uses_key(flow_dissector,
57 key->key_id));
58
59 dissector_set_key(flow_dissector, key->key_id);
60 flow_dissector->offset[key->key_id] = key->offset;
61 }
62
63
64
65
66 BUG_ON(!dissector_uses_key(flow_dissector,
67 FLOW_DISSECTOR_KEY_CONTROL));
68 BUG_ON(!dissector_uses_key(flow_dissector,
69 FLOW_DISSECTOR_KEY_BASIC));
70}
71EXPORT_SYMBOL(skb_flow_dissector_init);
72
73#ifdef CONFIG_BPF_SYSCALL
74int flow_dissector_bpf_prog_attach_check(struct net *net,
75 struct bpf_prog *prog)
76{
77 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
78
79 if (net == &init_net) {
80
81
82
83
84
85 struct net *ns;
86
87 for_each_net(ns) {
88 if (ns == &init_net)
89 continue;
90 if (rcu_access_pointer(ns->bpf.run_array[type]))
91 return -EEXIST;
92 }
93 } else {
94
95
96
97 if (rcu_access_pointer(init_net.bpf.run_array[type]))
98 return -EEXIST;
99 }
100
101 return 0;
102}
103#endif
104
105
106
107
108
109
110
111
112
113
114
115
116__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
117 const void *data, int hlen)
118{
119 int poff = proto_ports_offset(ip_proto);
120
121 if (!data) {
122 data = skb->data;
123 hlen = skb_headlen(skb);
124 }
125
126 if (poff >= 0) {
127 __be32 *ports, _ports;
128
129 ports = __skb_header_pointer(skb, thoff + poff,
130 sizeof(_ports), data, hlen, &_ports);
131 if (ports)
132 return *ports;
133 }
134
135 return 0;
136}
137EXPORT_SYMBOL(__skb_flow_get_ports);
138
139static bool icmp_has_id(u8 type)
140{
141 switch (type) {
142 case ICMP_ECHO:
143 case ICMP_ECHOREPLY:
144 case ICMP_TIMESTAMP:
145 case ICMP_TIMESTAMPREPLY:
146 case ICMPV6_ECHO_REQUEST:
147 case ICMPV6_ECHO_REPLY:
148 return true;
149 }
150
151 return false;
152}
153
154
155
156
157
158
159
160
161
162void skb_flow_get_icmp_tci(const struct sk_buff *skb,
163 struct flow_dissector_key_icmp *key_icmp,
164 const void *data, int thoff, int hlen)
165{
166 struct icmphdr *ih, _ih;
167
168 ih = __skb_header_pointer(skb, thoff, sizeof(_ih), data, hlen, &_ih);
169 if (!ih)
170 return;
171
172 key_icmp->type = ih->type;
173 key_icmp->code = ih->code;
174
175
176
177
178 if (icmp_has_id(ih->type))
179 key_icmp->id = ih->un.echo.id ? ntohs(ih->un.echo.id) : 1;
180 else
181 key_icmp->id = 0;
182}
183EXPORT_SYMBOL(skb_flow_get_icmp_tci);
184
185
186
187
188static void __skb_flow_dissect_icmp(const struct sk_buff *skb,
189 struct flow_dissector *flow_dissector,
190 void *target_container, const void *data,
191 int thoff, int hlen)
192{
193 struct flow_dissector_key_icmp *key_icmp;
194
195 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ICMP))
196 return;
197
198 key_icmp = skb_flow_dissector_target(flow_dissector,
199 FLOW_DISSECTOR_KEY_ICMP,
200 target_container);
201
202 skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen);
203}
204
205void skb_flow_dissect_meta(const struct sk_buff *skb,
206 struct flow_dissector *flow_dissector,
207 void *target_container)
208{
209 struct flow_dissector_key_meta *meta;
210
211 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_META))
212 return;
213
214 meta = skb_flow_dissector_target(flow_dissector,
215 FLOW_DISSECTOR_KEY_META,
216 target_container);
217 meta->ingress_ifindex = skb->skb_iif;
218}
219EXPORT_SYMBOL(skb_flow_dissect_meta);
220
221static void
222skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type,
223 struct flow_dissector *flow_dissector,
224 void *target_container)
225{
226 struct flow_dissector_key_control *ctrl;
227
228 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL))
229 return;
230
231 ctrl = skb_flow_dissector_target(flow_dissector,
232 FLOW_DISSECTOR_KEY_ENC_CONTROL,
233 target_container);
234 ctrl->addr_type = type;
235}
236
237void
238skb_flow_dissect_ct(const struct sk_buff *skb,
239 struct flow_dissector *flow_dissector,
240 void *target_container, u16 *ctinfo_map,
241 size_t mapsize, bool post_ct)
242{
243#if IS_ENABLED(CONFIG_NF_CONNTRACK)
244 struct flow_dissector_key_ct *key;
245 enum ip_conntrack_info ctinfo;
246 struct nf_conn_labels *cl;
247 struct nf_conn *ct;
248
249 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CT))
250 return;
251
252 ct = nf_ct_get(skb, &ctinfo);
253 if (!ct && !post_ct)
254 return;
255
256 key = skb_flow_dissector_target(flow_dissector,
257 FLOW_DISSECTOR_KEY_CT,
258 target_container);
259
260 if (!ct) {
261 key->ct_state = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
262 TCA_FLOWER_KEY_CT_FLAGS_INVALID;
263 return;
264 }
265
266 if (ctinfo < mapsize)
267 key->ct_state = ctinfo_map[ctinfo];
268#if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)
269 key->ct_zone = ct->zone.id;
270#endif
271#if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
272 key->ct_mark = ct->mark;
273#endif
274
275 cl = nf_ct_labels_find(ct);
276 if (cl)
277 memcpy(key->ct_labels, cl->bits, sizeof(key->ct_labels));
278#endif
279}
280EXPORT_SYMBOL(skb_flow_dissect_ct);
281
282void
283skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
284 struct flow_dissector *flow_dissector,
285 void *target_container)
286{
287 struct ip_tunnel_info *info;
288 struct ip_tunnel_key *key;
289
290
291 if (!dissector_uses_key(flow_dissector,
292 FLOW_DISSECTOR_KEY_ENC_KEYID) &&
293 !dissector_uses_key(flow_dissector,
294 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) &&
295 !dissector_uses_key(flow_dissector,
296 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) &&
297 !dissector_uses_key(flow_dissector,
298 FLOW_DISSECTOR_KEY_ENC_CONTROL) &&
299 !dissector_uses_key(flow_dissector,
300 FLOW_DISSECTOR_KEY_ENC_PORTS) &&
301 !dissector_uses_key(flow_dissector,
302 FLOW_DISSECTOR_KEY_ENC_IP) &&
303 !dissector_uses_key(flow_dissector,
304 FLOW_DISSECTOR_KEY_ENC_OPTS))
305 return;
306
307 info = skb_tunnel_info(skb);
308 if (!info)
309 return;
310
311 key = &info->key;
312
313 switch (ip_tunnel_info_af(info)) {
314 case AF_INET:
315 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS,
316 flow_dissector,
317 target_container);
318 if (dissector_uses_key(flow_dissector,
319 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
320 struct flow_dissector_key_ipv4_addrs *ipv4;
321
322 ipv4 = skb_flow_dissector_target(flow_dissector,
323 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
324 target_container);
325 ipv4->src = key->u.ipv4.src;
326 ipv4->dst = key->u.ipv4.dst;
327 }
328 break;
329 case AF_INET6:
330 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS,
331 flow_dissector,
332 target_container);
333 if (dissector_uses_key(flow_dissector,
334 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
335 struct flow_dissector_key_ipv6_addrs *ipv6;
336
337 ipv6 = skb_flow_dissector_target(flow_dissector,
338 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
339 target_container);
340 ipv6->src = key->u.ipv6.src;
341 ipv6->dst = key->u.ipv6.dst;
342 }
343 break;
344 }
345
346 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
347 struct flow_dissector_key_keyid *keyid;
348
349 keyid = skb_flow_dissector_target(flow_dissector,
350 FLOW_DISSECTOR_KEY_ENC_KEYID,
351 target_container);
352 keyid->keyid = tunnel_id_to_key32(key->tun_id);
353 }
354
355 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
356 struct flow_dissector_key_ports *tp;
357
358 tp = skb_flow_dissector_target(flow_dissector,
359 FLOW_DISSECTOR_KEY_ENC_PORTS,
360 target_container);
361 tp->src = key->tp_src;
362 tp->dst = key->tp_dst;
363 }
364
365 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) {
366 struct flow_dissector_key_ip *ip;
367
368 ip = skb_flow_dissector_target(flow_dissector,
369 FLOW_DISSECTOR_KEY_ENC_IP,
370 target_container);
371 ip->tos = key->tos;
372 ip->ttl = key->ttl;
373 }
374
375 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) {
376 struct flow_dissector_key_enc_opts *enc_opt;
377
378 enc_opt = skb_flow_dissector_target(flow_dissector,
379 FLOW_DISSECTOR_KEY_ENC_OPTS,
380 target_container);
381
382 if (info->options_len) {
383 enc_opt->len = info->options_len;
384 ip_tunnel_info_opts_get(enc_opt->data, info);
385 enc_opt->dst_opt_type = info->key.tun_flags &
386 TUNNEL_OPTIONS_PRESENT;
387 }
388 }
389}
390EXPORT_SYMBOL(skb_flow_dissect_tunnel_info);
391
392void skb_flow_dissect_hash(const struct sk_buff *skb,
393 struct flow_dissector *flow_dissector,
394 void *target_container)
395{
396 struct flow_dissector_key_hash *key;
397
398 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_HASH))
399 return;
400
401 key = skb_flow_dissector_target(flow_dissector,
402 FLOW_DISSECTOR_KEY_HASH,
403 target_container);
404
405 key->hash = skb_get_hash_raw(skb);
406}
407EXPORT_SYMBOL(skb_flow_dissect_hash);
408
409static enum flow_dissect_ret
410__skb_flow_dissect_mpls(const struct sk_buff *skb,
411 struct flow_dissector *flow_dissector,
412 void *target_container, const void *data, int nhoff,
413 int hlen, int lse_index, bool *entropy_label)
414{
415 struct mpls_label *hdr, _hdr;
416 u32 entry, label, bos;
417
418 if (!dissector_uses_key(flow_dissector,
419 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) &&
420 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS))
421 return FLOW_DISSECT_RET_OUT_GOOD;
422
423 if (lse_index >= FLOW_DIS_MPLS_MAX)
424 return FLOW_DISSECT_RET_OUT_GOOD;
425
426 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
427 hlen, &_hdr);
428 if (!hdr)
429 return FLOW_DISSECT_RET_OUT_BAD;
430
431 entry = ntohl(hdr->entry);
432 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT;
433 bos = (entry & MPLS_LS_S_MASK) >> MPLS_LS_S_SHIFT;
434
435 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) {
436 struct flow_dissector_key_mpls *key_mpls;
437 struct flow_dissector_mpls_lse *lse;
438
439 key_mpls = skb_flow_dissector_target(flow_dissector,
440 FLOW_DISSECTOR_KEY_MPLS,
441 target_container);
442 lse = &key_mpls->ls[lse_index];
443
444 lse->mpls_ttl = (entry & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
445 lse->mpls_bos = bos;
446 lse->mpls_tc = (entry & MPLS_LS_TC_MASK) >> MPLS_LS_TC_SHIFT;
447 lse->mpls_label = label;
448 dissector_set_mpls_lse(key_mpls, lse_index);
449 }
450
451 if (*entropy_label &&
452 dissector_uses_key(flow_dissector,
453 FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) {
454 struct flow_dissector_key_keyid *key_keyid;
455
456 key_keyid = skb_flow_dissector_target(flow_dissector,
457 FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
458 target_container);
459 key_keyid->keyid = cpu_to_be32(label);
460 }
461
462 *entropy_label = label == MPLS_LABEL_ENTROPY;
463
464 return bos ? FLOW_DISSECT_RET_OUT_GOOD : FLOW_DISSECT_RET_PROTO_AGAIN;
465}
466
467static enum flow_dissect_ret
468__skb_flow_dissect_arp(const struct sk_buff *skb,
469 struct flow_dissector *flow_dissector,
470 void *target_container, const void *data,
471 int nhoff, int hlen)
472{
473 struct flow_dissector_key_arp *key_arp;
474 struct {
475 unsigned char ar_sha[ETH_ALEN];
476 unsigned char ar_sip[4];
477 unsigned char ar_tha[ETH_ALEN];
478 unsigned char ar_tip[4];
479 } *arp_eth, _arp_eth;
480 const struct arphdr *arp;
481 struct arphdr _arp;
482
483 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP))
484 return FLOW_DISSECT_RET_OUT_GOOD;
485
486 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
487 hlen, &_arp);
488 if (!arp)
489 return FLOW_DISSECT_RET_OUT_BAD;
490
491 if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
492 arp->ar_pro != htons(ETH_P_IP) ||
493 arp->ar_hln != ETH_ALEN ||
494 arp->ar_pln != 4 ||
495 (arp->ar_op != htons(ARPOP_REPLY) &&
496 arp->ar_op != htons(ARPOP_REQUEST)))
497 return FLOW_DISSECT_RET_OUT_BAD;
498
499 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
500 sizeof(_arp_eth), data,
501 hlen, &_arp_eth);
502 if (!arp_eth)
503 return FLOW_DISSECT_RET_OUT_BAD;
504
505 key_arp = skb_flow_dissector_target(flow_dissector,
506 FLOW_DISSECTOR_KEY_ARP,
507 target_container);
508
509 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip));
510 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip));
511
512
513
514
515 key_arp->op = ntohs(arp->ar_op) & 0xff;
516
517 ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
518 ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
519
520 return FLOW_DISSECT_RET_OUT_GOOD;
521}
522
523static enum flow_dissect_ret
524__skb_flow_dissect_gre(const struct sk_buff *skb,
525 struct flow_dissector_key_control *key_control,
526 struct flow_dissector *flow_dissector,
527 void *target_container, const void *data,
528 __be16 *p_proto, int *p_nhoff, int *p_hlen,
529 unsigned int flags)
530{
531 struct flow_dissector_key_keyid *key_keyid;
532 struct gre_base_hdr *hdr, _hdr;
533 int offset = 0;
534 u16 gre_ver;
535
536 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr),
537 data, *p_hlen, &_hdr);
538 if (!hdr)
539 return FLOW_DISSECT_RET_OUT_BAD;
540
541
542 if (hdr->flags & GRE_ROUTING)
543 return FLOW_DISSECT_RET_OUT_GOOD;
544
545
546 gre_ver = ntohs(hdr->flags & GRE_VERSION);
547 if (gre_ver > 1)
548 return FLOW_DISSECT_RET_OUT_GOOD;
549
550 *p_proto = hdr->protocol;
551 if (gre_ver) {
552
553 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
554 return FLOW_DISSECT_RET_OUT_GOOD;
555 }
556
557 offset += sizeof(struct gre_base_hdr);
558
559 if (hdr->flags & GRE_CSUM)
560 offset += sizeof_field(struct gre_full_hdr, csum) +
561 sizeof_field(struct gre_full_hdr, reserved1);
562
563 if (hdr->flags & GRE_KEY) {
564 const __be32 *keyid;
565 __be32 _keyid;
566
567 keyid = __skb_header_pointer(skb, *p_nhoff + offset,
568 sizeof(_keyid),
569 data, *p_hlen, &_keyid);
570 if (!keyid)
571 return FLOW_DISSECT_RET_OUT_BAD;
572
573 if (dissector_uses_key(flow_dissector,
574 FLOW_DISSECTOR_KEY_GRE_KEYID)) {
575 key_keyid = skb_flow_dissector_target(flow_dissector,
576 FLOW_DISSECTOR_KEY_GRE_KEYID,
577 target_container);
578 if (gre_ver == 0)
579 key_keyid->keyid = *keyid;
580 else
581 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
582 }
583 offset += sizeof_field(struct gre_full_hdr, key);
584 }
585
586 if (hdr->flags & GRE_SEQ)
587 offset += sizeof_field(struct pptp_gre_header, seq);
588
589 if (gre_ver == 0) {
590 if (*p_proto == htons(ETH_P_TEB)) {
591 const struct ethhdr *eth;
592 struct ethhdr _eth;
593
594 eth = __skb_header_pointer(skb, *p_nhoff + offset,
595 sizeof(_eth),
596 data, *p_hlen, &_eth);
597 if (!eth)
598 return FLOW_DISSECT_RET_OUT_BAD;
599 *p_proto = eth->h_proto;
600 offset += sizeof(*eth);
601
602
603
604
605
606 if (NET_IP_ALIGN)
607 *p_hlen = *p_nhoff + offset;
608 }
609 } else {
610 u8 _ppp_hdr[PPP_HDRLEN];
611 u8 *ppp_hdr;
612
613 if (hdr->flags & GRE_ACK)
614 offset += sizeof_field(struct pptp_gre_header, ack);
615
616 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset,
617 sizeof(_ppp_hdr),
618 data, *p_hlen, _ppp_hdr);
619 if (!ppp_hdr)
620 return FLOW_DISSECT_RET_OUT_BAD;
621
622 switch (PPP_PROTOCOL(ppp_hdr)) {
623 case PPP_IP:
624 *p_proto = htons(ETH_P_IP);
625 break;
626 case PPP_IPV6:
627 *p_proto = htons(ETH_P_IPV6);
628 break;
629 default:
630
631 break;
632 }
633
634 offset += PPP_HDRLEN;
635 }
636
637 *p_nhoff += offset;
638 key_control->flags |= FLOW_DIS_ENCAPSULATION;
639 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
640 return FLOW_DISSECT_RET_OUT_GOOD;
641
642 return FLOW_DISSECT_RET_PROTO_AGAIN;
643}
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664static enum flow_dissect_ret
665__skb_flow_dissect_batadv(const struct sk_buff *skb,
666 struct flow_dissector_key_control *key_control,
667 const void *data, __be16 *p_proto, int *p_nhoff,
668 int hlen, unsigned int flags)
669{
670 struct {
671 struct batadv_unicast_packet batadv_unicast;
672 struct ethhdr eth;
673 } *hdr, _hdr;
674
675 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen,
676 &_hdr);
677 if (!hdr)
678 return FLOW_DISSECT_RET_OUT_BAD;
679
680 if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION)
681 return FLOW_DISSECT_RET_OUT_BAD;
682
683 if (hdr->batadv_unicast.packet_type != BATADV_UNICAST)
684 return FLOW_DISSECT_RET_OUT_BAD;
685
686 *p_proto = hdr->eth.h_proto;
687 *p_nhoff += sizeof(*hdr);
688
689 key_control->flags |= FLOW_DIS_ENCAPSULATION;
690 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
691 return FLOW_DISSECT_RET_OUT_GOOD;
692
693 return FLOW_DISSECT_RET_PROTO_AGAIN;
694}
695
696static void
697__skb_flow_dissect_tcp(const struct sk_buff *skb,
698 struct flow_dissector *flow_dissector,
699 void *target_container, const void *data,
700 int thoff, int hlen)
701{
702 struct flow_dissector_key_tcp *key_tcp;
703 struct tcphdr *th, _th;
704
705 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP))
706 return;
707
708 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th);
709 if (!th)
710 return;
711
712 if (unlikely(__tcp_hdrlen(th) < sizeof(_th)))
713 return;
714
715 key_tcp = skb_flow_dissector_target(flow_dissector,
716 FLOW_DISSECTOR_KEY_TCP,
717 target_container);
718 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF));
719}
720
721static void
722__skb_flow_dissect_ports(const struct sk_buff *skb,
723 struct flow_dissector *flow_dissector,
724 void *target_container, const void *data,
725 int nhoff, u8 ip_proto, int hlen)
726{
727 enum flow_dissector_key_id dissector_ports = FLOW_DISSECTOR_KEY_MAX;
728 struct flow_dissector_key_ports *key_ports;
729
730 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
731 dissector_ports = FLOW_DISSECTOR_KEY_PORTS;
732 else if (dissector_uses_key(flow_dissector,
733 FLOW_DISSECTOR_KEY_PORTS_RANGE))
734 dissector_ports = FLOW_DISSECTOR_KEY_PORTS_RANGE;
735
736 if (dissector_ports == FLOW_DISSECTOR_KEY_MAX)
737 return;
738
739 key_ports = skb_flow_dissector_target(flow_dissector,
740 dissector_ports,
741 target_container);
742 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
743 data, hlen);
744}
745
746static void
747__skb_flow_dissect_ipv4(const struct sk_buff *skb,
748 struct flow_dissector *flow_dissector,
749 void *target_container, const void *data,
750 const struct iphdr *iph)
751{
752 struct flow_dissector_key_ip *key_ip;
753
754 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
755 return;
756
757 key_ip = skb_flow_dissector_target(flow_dissector,
758 FLOW_DISSECTOR_KEY_IP,
759 target_container);
760 key_ip->tos = iph->tos;
761 key_ip->ttl = iph->ttl;
762}
763
764static void
765__skb_flow_dissect_ipv6(const struct sk_buff *skb,
766 struct flow_dissector *flow_dissector,
767 void *target_container, const void *data,
768 const struct ipv6hdr *iph)
769{
770 struct flow_dissector_key_ip *key_ip;
771
772 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
773 return;
774
775 key_ip = skb_flow_dissector_target(flow_dissector,
776 FLOW_DISSECTOR_KEY_IP,
777 target_container);
778 key_ip->tos = ipv6_get_dsfield(iph);
779 key_ip->ttl = iph->hop_limit;
780}
781
782
783
784
785#define MAX_FLOW_DISSECT_HDRS 15
786
787static bool skb_flow_dissect_allowed(int *num_hdrs)
788{
789 ++*num_hdrs;
790
791 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS);
792}
793
794static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys,
795 struct flow_dissector *flow_dissector,
796 void *target_container)
797{
798 struct flow_dissector_key_ports *key_ports = NULL;
799 struct flow_dissector_key_control *key_control;
800 struct flow_dissector_key_basic *key_basic;
801 struct flow_dissector_key_addrs *key_addrs;
802 struct flow_dissector_key_tags *key_tags;
803
804 key_control = skb_flow_dissector_target(flow_dissector,
805 FLOW_DISSECTOR_KEY_CONTROL,
806 target_container);
807 key_control->thoff = flow_keys->thoff;
808 if (flow_keys->is_frag)
809 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
810 if (flow_keys->is_first_frag)
811 key_control->flags |= FLOW_DIS_FIRST_FRAG;
812 if (flow_keys->is_encap)
813 key_control->flags |= FLOW_DIS_ENCAPSULATION;
814
815 key_basic = skb_flow_dissector_target(flow_dissector,
816 FLOW_DISSECTOR_KEY_BASIC,
817 target_container);
818 key_basic->n_proto = flow_keys->n_proto;
819 key_basic->ip_proto = flow_keys->ip_proto;
820
821 if (flow_keys->addr_proto == ETH_P_IP &&
822 dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
823 key_addrs = skb_flow_dissector_target(flow_dissector,
824 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
825 target_container);
826 key_addrs->v4addrs.src = flow_keys->ipv4_src;
827 key_addrs->v4addrs.dst = flow_keys->ipv4_dst;
828 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
829 } else if (flow_keys->addr_proto == ETH_P_IPV6 &&
830 dissector_uses_key(flow_dissector,
831 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
832 key_addrs = skb_flow_dissector_target(flow_dissector,
833 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
834 target_container);
835 memcpy(&key_addrs->v6addrs.src, &flow_keys->ipv6_src,
836 sizeof(key_addrs->v6addrs.src));
837 memcpy(&key_addrs->v6addrs.dst, &flow_keys->ipv6_dst,
838 sizeof(key_addrs->v6addrs.dst));
839 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
840 }
841
842 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
843 key_ports = skb_flow_dissector_target(flow_dissector,
844 FLOW_DISSECTOR_KEY_PORTS,
845 target_container);
846 else if (dissector_uses_key(flow_dissector,
847 FLOW_DISSECTOR_KEY_PORTS_RANGE))
848 key_ports = skb_flow_dissector_target(flow_dissector,
849 FLOW_DISSECTOR_KEY_PORTS_RANGE,
850 target_container);
851
852 if (key_ports) {
853 key_ports->src = flow_keys->sport;
854 key_ports->dst = flow_keys->dport;
855 }
856
857 if (dissector_uses_key(flow_dissector,
858 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
859 key_tags = skb_flow_dissector_target(flow_dissector,
860 FLOW_DISSECTOR_KEY_FLOW_LABEL,
861 target_container);
862 key_tags->flow_label = ntohl(flow_keys->flow_label);
863 }
864}
865
866bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
867 __be16 proto, int nhoff, int hlen, unsigned int flags)
868{
869 struct bpf_flow_keys *flow_keys = ctx->flow_keys;
870 u32 result;
871
872
873 memset(flow_keys, 0, sizeof(*flow_keys));
874 flow_keys->n_proto = proto;
875 flow_keys->nhoff = nhoff;
876 flow_keys->thoff = flow_keys->nhoff;
877
878 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG !=
879 (int)FLOW_DISSECTOR_F_PARSE_1ST_FRAG);
880 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL !=
881 (int)FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
882 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP !=
883 (int)FLOW_DISSECTOR_F_STOP_AT_ENCAP);
884 flow_keys->flags = flags;
885
886 result = bpf_prog_run_pin_on_cpu(prog, ctx);
887
888 flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen);
889 flow_keys->thoff = clamp_t(u16, flow_keys->thoff,
890 flow_keys->nhoff, hlen);
891
892 return result == BPF_OK;
893}
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914bool __skb_flow_dissect(const struct net *net,
915 const struct sk_buff *skb,
916 struct flow_dissector *flow_dissector,
917 void *target_container, const void *data,
918 __be16 proto, int nhoff, int hlen, unsigned int flags)
919{
920 struct flow_dissector_key_control *key_control;
921 struct flow_dissector_key_basic *key_basic;
922 struct flow_dissector_key_addrs *key_addrs;
923 struct flow_dissector_key_tags *key_tags;
924 struct flow_dissector_key_vlan *key_vlan;
925 enum flow_dissect_ret fdret;
926 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX;
927 bool mpls_el = false;
928 int mpls_lse = 0;
929 int num_hdrs = 0;
930 u8 ip_proto = 0;
931 bool ret;
932
933 if (!data) {
934 data = skb->data;
935 proto = skb_vlan_tag_present(skb) ?
936 skb->vlan_proto : skb->protocol;
937 nhoff = skb_network_offset(skb);
938 hlen = skb_headlen(skb);
939#if IS_ENABLED(CONFIG_NET_DSA)
940 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev) &&
941 proto == htons(ETH_P_XDSA))) {
942 const struct dsa_device_ops *ops;
943 int offset = 0;
944
945 ops = skb->dev->dsa_ptr->tag_ops;
946
947 if (ops->needed_headroom) {
948 if (ops->flow_dissect)
949 ops->flow_dissect(skb, &proto, &offset);
950 else
951 dsa_tag_generic_flow_dissect(skb,
952 &proto,
953 &offset);
954 hlen -= offset;
955 nhoff += offset;
956 }
957 }
958#endif
959 }
960
961
962
963
964 key_control = skb_flow_dissector_target(flow_dissector,
965 FLOW_DISSECTOR_KEY_CONTROL,
966 target_container);
967
968
969
970
971 key_basic = skb_flow_dissector_target(flow_dissector,
972 FLOW_DISSECTOR_KEY_BASIC,
973 target_container);
974
975 if (skb) {
976 if (!net) {
977 if (skb->dev)
978 net = dev_net(skb->dev);
979 else if (skb->sk)
980 net = sock_net(skb->sk);
981 }
982 }
983
984 WARN_ON_ONCE(!net);
985 if (net) {
986 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
987 struct bpf_prog_array *run_array;
988
989 rcu_read_lock();
990 run_array = rcu_dereference(init_net.bpf.run_array[type]);
991 if (!run_array)
992 run_array = rcu_dereference(net->bpf.run_array[type]);
993
994 if (run_array) {
995 struct bpf_flow_keys flow_keys;
996 struct bpf_flow_dissector ctx = {
997 .flow_keys = &flow_keys,
998 .data = data,
999 .data_end = data + hlen,
1000 };
1001 __be16 n_proto = proto;
1002 struct bpf_prog *prog;
1003
1004 if (skb) {
1005 ctx.skb = skb;
1006
1007
1008
1009
1010 n_proto = skb->protocol;
1011 }
1012
1013 prog = READ_ONCE(run_array->items[0].prog);
1014 ret = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
1015 hlen, flags);
1016 __skb_flow_bpf_to_target(&flow_keys, flow_dissector,
1017 target_container);
1018 rcu_read_unlock();
1019 return ret;
1020 }
1021 rcu_read_unlock();
1022 }
1023
1024 if (dissector_uses_key(flow_dissector,
1025 FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
1026 struct ethhdr *eth = eth_hdr(skb);
1027 struct flow_dissector_key_eth_addrs *key_eth_addrs;
1028
1029 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
1030 FLOW_DISSECTOR_KEY_ETH_ADDRS,
1031 target_container);
1032 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs));
1033 }
1034
1035proto_again:
1036 fdret = FLOW_DISSECT_RET_CONTINUE;
1037
1038 switch (proto) {
1039 case htons(ETH_P_IP): {
1040 const struct iphdr *iph;
1041 struct iphdr _iph;
1042
1043 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1044 if (!iph || iph->ihl < 5) {
1045 fdret = FLOW_DISSECT_RET_OUT_BAD;
1046 break;
1047 }
1048
1049 nhoff += iph->ihl * 4;
1050
1051 ip_proto = iph->protocol;
1052
1053 if (dissector_uses_key(flow_dissector,
1054 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
1055 key_addrs = skb_flow_dissector_target(flow_dissector,
1056 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1057 target_container);
1058
1059 memcpy(&key_addrs->v4addrs, &iph->saddr,
1060 sizeof(key_addrs->v4addrs));
1061 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1062 }
1063
1064 __skb_flow_dissect_ipv4(skb, flow_dissector,
1065 target_container, data, iph);
1066
1067 if (ip_is_fragment(iph)) {
1068 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1069
1070 if (iph->frag_off & htons(IP_OFFSET)) {
1071 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1072 break;
1073 } else {
1074 key_control->flags |= FLOW_DIS_FIRST_FRAG;
1075 if (!(flags &
1076 FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) {
1077 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1078 break;
1079 }
1080 }
1081 }
1082
1083 break;
1084 }
1085 case htons(ETH_P_IPV6): {
1086 const struct ipv6hdr *iph;
1087 struct ipv6hdr _iph;
1088
1089 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1090 if (!iph) {
1091 fdret = FLOW_DISSECT_RET_OUT_BAD;
1092 break;
1093 }
1094
1095 ip_proto = iph->nexthdr;
1096 nhoff += sizeof(struct ipv6hdr);
1097
1098 if (dissector_uses_key(flow_dissector,
1099 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
1100 key_addrs = skb_flow_dissector_target(flow_dissector,
1101 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1102 target_container);
1103
1104 memcpy(&key_addrs->v6addrs, &iph->saddr,
1105 sizeof(key_addrs->v6addrs));
1106 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1107 }
1108
1109 if ((dissector_uses_key(flow_dissector,
1110 FLOW_DISSECTOR_KEY_FLOW_LABEL) ||
1111 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) &&
1112 ip6_flowlabel(iph)) {
1113 __be32 flow_label = ip6_flowlabel(iph);
1114
1115 if (dissector_uses_key(flow_dissector,
1116 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
1117 key_tags = skb_flow_dissector_target(flow_dissector,
1118 FLOW_DISSECTOR_KEY_FLOW_LABEL,
1119 target_container);
1120 key_tags->flow_label = ntohl(flow_label);
1121 }
1122 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) {
1123 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1124 break;
1125 }
1126 }
1127
1128 __skb_flow_dissect_ipv6(skb, flow_dissector,
1129 target_container, data, iph);
1130
1131 break;
1132 }
1133 case htons(ETH_P_8021AD):
1134 case htons(ETH_P_8021Q): {
1135 const struct vlan_hdr *vlan = NULL;
1136 struct vlan_hdr _vlan;
1137 __be16 saved_vlan_tpid = proto;
1138
1139 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX &&
1140 skb && skb_vlan_tag_present(skb)) {
1141 proto = skb->protocol;
1142 } else {
1143 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan),
1144 data, hlen, &_vlan);
1145 if (!vlan) {
1146 fdret = FLOW_DISSECT_RET_OUT_BAD;
1147 break;
1148 }
1149
1150 proto = vlan->h_vlan_encapsulated_proto;
1151 nhoff += sizeof(*vlan);
1152 }
1153
1154 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) {
1155 dissector_vlan = FLOW_DISSECTOR_KEY_VLAN;
1156 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) {
1157 dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN;
1158 } else {
1159 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1160 break;
1161 }
1162
1163 if (dissector_uses_key(flow_dissector, dissector_vlan)) {
1164 key_vlan = skb_flow_dissector_target(flow_dissector,
1165 dissector_vlan,
1166 target_container);
1167
1168 if (!vlan) {
1169 key_vlan->vlan_id = skb_vlan_tag_get_id(skb);
1170 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb);
1171 } else {
1172 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) &
1173 VLAN_VID_MASK;
1174 key_vlan->vlan_priority =
1175 (ntohs(vlan->h_vlan_TCI) &
1176 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1177 }
1178 key_vlan->vlan_tpid = saved_vlan_tpid;
1179 }
1180
1181 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1182 break;
1183 }
1184 case htons(ETH_P_PPP_SES): {
1185 struct {
1186 struct pppoe_hdr hdr;
1187 __be16 proto;
1188 } *hdr, _hdr;
1189 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
1190 if (!hdr) {
1191 fdret = FLOW_DISSECT_RET_OUT_BAD;
1192 break;
1193 }
1194
1195 proto = hdr->proto;
1196 nhoff += PPPOE_SES_HLEN;
1197 switch (proto) {
1198 case htons(PPP_IP):
1199 proto = htons(ETH_P_IP);
1200 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1201 break;
1202 case htons(PPP_IPV6):
1203 proto = htons(ETH_P_IPV6);
1204 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1205 break;
1206 default:
1207 fdret = FLOW_DISSECT_RET_OUT_BAD;
1208 break;
1209 }
1210 break;
1211 }
1212 case htons(ETH_P_TIPC): {
1213 struct tipc_basic_hdr *hdr, _hdr;
1214
1215 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr),
1216 data, hlen, &_hdr);
1217 if (!hdr) {
1218 fdret = FLOW_DISSECT_RET_OUT_BAD;
1219 break;
1220 }
1221
1222 if (dissector_uses_key(flow_dissector,
1223 FLOW_DISSECTOR_KEY_TIPC)) {
1224 key_addrs = skb_flow_dissector_target(flow_dissector,
1225 FLOW_DISSECTOR_KEY_TIPC,
1226 target_container);
1227 key_addrs->tipckey.key = tipc_hdr_rps_key(hdr);
1228 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC;
1229 }
1230 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1231 break;
1232 }
1233
1234 case htons(ETH_P_MPLS_UC):
1235 case htons(ETH_P_MPLS_MC):
1236 fdret = __skb_flow_dissect_mpls(skb, flow_dissector,
1237 target_container, data,
1238 nhoff, hlen, mpls_lse,
1239 &mpls_el);
1240 nhoff += sizeof(struct mpls_label);
1241 mpls_lse++;
1242 break;
1243 case htons(ETH_P_FCOE):
1244 if ((hlen - nhoff) < FCOE_HEADER_LEN) {
1245 fdret = FLOW_DISSECT_RET_OUT_BAD;
1246 break;
1247 }
1248
1249 nhoff += FCOE_HEADER_LEN;
1250 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1251 break;
1252
1253 case htons(ETH_P_ARP):
1254 case htons(ETH_P_RARP):
1255 fdret = __skb_flow_dissect_arp(skb, flow_dissector,
1256 target_container, data,
1257 nhoff, hlen);
1258 break;
1259
1260 case htons(ETH_P_BATMAN):
1261 fdret = __skb_flow_dissect_batadv(skb, key_control, data,
1262 &proto, &nhoff, hlen, flags);
1263 break;
1264
1265 case htons(ETH_P_1588): {
1266 struct ptp_header *hdr, _hdr;
1267
1268 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
1269 hlen, &_hdr);
1270 if (!hdr) {
1271 fdret = FLOW_DISSECT_RET_OUT_BAD;
1272 break;
1273 }
1274
1275 nhoff += ntohs(hdr->message_length);
1276 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1277 break;
1278 }
1279
1280 default:
1281 fdret = FLOW_DISSECT_RET_OUT_BAD;
1282 break;
1283 }
1284
1285
1286 switch (fdret) {
1287 case FLOW_DISSECT_RET_OUT_GOOD:
1288 goto out_good;
1289 case FLOW_DISSECT_RET_PROTO_AGAIN:
1290 if (skb_flow_dissect_allowed(&num_hdrs))
1291 goto proto_again;
1292 goto out_good;
1293 case FLOW_DISSECT_RET_CONTINUE:
1294 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1295 break;
1296 case FLOW_DISSECT_RET_OUT_BAD:
1297 default:
1298 goto out_bad;
1299 }
1300
1301ip_proto_again:
1302 fdret = FLOW_DISSECT_RET_CONTINUE;
1303
1304 switch (ip_proto) {
1305 case IPPROTO_GRE:
1306 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector,
1307 target_container, data,
1308 &proto, &nhoff, &hlen, flags);
1309 break;
1310
1311 case NEXTHDR_HOP:
1312 case NEXTHDR_ROUTING:
1313 case NEXTHDR_DEST: {
1314 u8 _opthdr[2], *opthdr;
1315
1316 if (proto != htons(ETH_P_IPV6))
1317 break;
1318
1319 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr),
1320 data, hlen, &_opthdr);
1321 if (!opthdr) {
1322 fdret = FLOW_DISSECT_RET_OUT_BAD;
1323 break;
1324 }
1325
1326 ip_proto = opthdr[0];
1327 nhoff += (opthdr[1] + 1) << 3;
1328
1329 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1330 break;
1331 }
1332 case NEXTHDR_FRAGMENT: {
1333 struct frag_hdr _fh, *fh;
1334
1335 if (proto != htons(ETH_P_IPV6))
1336 break;
1337
1338 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh),
1339 data, hlen, &_fh);
1340
1341 if (!fh) {
1342 fdret = FLOW_DISSECT_RET_OUT_BAD;
1343 break;
1344 }
1345
1346 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1347
1348 nhoff += sizeof(_fh);
1349 ip_proto = fh->nexthdr;
1350
1351 if (!(fh->frag_off & htons(IP6_OFFSET))) {
1352 key_control->flags |= FLOW_DIS_FIRST_FRAG;
1353 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) {
1354 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1355 break;
1356 }
1357 }
1358
1359 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1360 break;
1361 }
1362 case IPPROTO_IPIP:
1363 proto = htons(ETH_P_IP);
1364
1365 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1366 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1367 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1368 break;
1369 }
1370
1371 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1372 break;
1373
1374 case IPPROTO_IPV6:
1375 proto = htons(ETH_P_IPV6);
1376
1377 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1378 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1379 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1380 break;
1381 }
1382
1383 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1384 break;
1385
1386
1387 case IPPROTO_MPLS:
1388 proto = htons(ETH_P_MPLS_UC);
1389 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1390 break;
1391
1392 case IPPROTO_TCP:
1393 __skb_flow_dissect_tcp(skb, flow_dissector, target_container,
1394 data, nhoff, hlen);
1395 break;
1396
1397 case IPPROTO_ICMP:
1398 case IPPROTO_ICMPV6:
1399 __skb_flow_dissect_icmp(skb, flow_dissector, target_container,
1400 data, nhoff, hlen);
1401 break;
1402
1403 default:
1404 break;
1405 }
1406
1407 if (!(key_control->flags & FLOW_DIS_IS_FRAGMENT))
1408 __skb_flow_dissect_ports(skb, flow_dissector, target_container,
1409 data, nhoff, ip_proto, hlen);
1410
1411
1412 switch (fdret) {
1413 case FLOW_DISSECT_RET_PROTO_AGAIN:
1414 if (skb_flow_dissect_allowed(&num_hdrs))
1415 goto proto_again;
1416 break;
1417 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1418 if (skb_flow_dissect_allowed(&num_hdrs))
1419 goto ip_proto_again;
1420 break;
1421 case FLOW_DISSECT_RET_OUT_GOOD:
1422 case FLOW_DISSECT_RET_CONTINUE:
1423 break;
1424 case FLOW_DISSECT_RET_OUT_BAD:
1425 default:
1426 goto out_bad;
1427 }
1428
1429out_good:
1430 ret = true;
1431
1432out:
1433 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
1434 key_basic->n_proto = proto;
1435 key_basic->ip_proto = ip_proto;
1436
1437 return ret;
1438
1439out_bad:
1440 ret = false;
1441 goto out;
1442}
1443EXPORT_SYMBOL(__skb_flow_dissect);
1444
1445static siphash_key_t hashrnd __read_mostly;
1446static __always_inline void __flow_hash_secret_init(void)
1447{
1448 net_get_random_once(&hashrnd, sizeof(hashrnd));
1449}
1450
1451static const void *flow_keys_hash_start(const struct flow_keys *flow)
1452{
1453 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % SIPHASH_ALIGNMENT);
1454 return &flow->FLOW_KEYS_HASH_START_FIELD;
1455}
1456
1457static inline size_t flow_keys_hash_length(const struct flow_keys *flow)
1458{
1459 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
1460
1461 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
1462
1463 switch (flow->control.addr_type) {
1464 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1465 diff -= sizeof(flow->addrs.v4addrs);
1466 break;
1467 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1468 diff -= sizeof(flow->addrs.v6addrs);
1469 break;
1470 case FLOW_DISSECTOR_KEY_TIPC:
1471 diff -= sizeof(flow->addrs.tipckey);
1472 break;
1473 }
1474 return sizeof(*flow) - diff;
1475}
1476
1477__be32 flow_get_u32_src(const struct flow_keys *flow)
1478{
1479 switch (flow->control.addr_type) {
1480 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1481 return flow->addrs.v4addrs.src;
1482 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1483 return (__force __be32)ipv6_addr_hash(
1484 &flow->addrs.v6addrs.src);
1485 case FLOW_DISSECTOR_KEY_TIPC:
1486 return flow->addrs.tipckey.key;
1487 default:
1488 return 0;
1489 }
1490}
1491EXPORT_SYMBOL(flow_get_u32_src);
1492
1493__be32 flow_get_u32_dst(const struct flow_keys *flow)
1494{
1495 switch (flow->control.addr_type) {
1496 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1497 return flow->addrs.v4addrs.dst;
1498 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1499 return (__force __be32)ipv6_addr_hash(
1500 &flow->addrs.v6addrs.dst);
1501 default:
1502 return 0;
1503 }
1504}
1505EXPORT_SYMBOL(flow_get_u32_dst);
1506
1507
1508
1509
1510static inline void __flow_hash_consistentify(struct flow_keys *keys)
1511{
1512 int addr_diff, i;
1513
1514 switch (keys->control.addr_type) {
1515 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1516 addr_diff = (__force u32)keys->addrs.v4addrs.dst -
1517 (__force u32)keys->addrs.v4addrs.src;
1518 if (addr_diff < 0)
1519 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
1520
1521 if ((__force u16)keys->ports.dst <
1522 (__force u16)keys->ports.src) {
1523 swap(keys->ports.src, keys->ports.dst);
1524 }
1525 break;
1526 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1527 addr_diff = memcmp(&keys->addrs.v6addrs.dst,
1528 &keys->addrs.v6addrs.src,
1529 sizeof(keys->addrs.v6addrs.dst));
1530 if (addr_diff < 0) {
1531 for (i = 0; i < 4; i++)
1532 swap(keys->addrs.v6addrs.src.s6_addr32[i],
1533 keys->addrs.v6addrs.dst.s6_addr32[i]);
1534 }
1535 if ((__force u16)keys->ports.dst <
1536 (__force u16)keys->ports.src) {
1537 swap(keys->ports.src, keys->ports.dst);
1538 }
1539 break;
1540 }
1541}
1542
1543static inline u32 __flow_hash_from_keys(struct flow_keys *keys,
1544 const siphash_key_t *keyval)
1545{
1546 u32 hash;
1547
1548 __flow_hash_consistentify(keys);
1549
1550 hash = siphash(flow_keys_hash_start(keys),
1551 flow_keys_hash_length(keys), keyval);
1552 if (!hash)
1553 hash = 1;
1554
1555 return hash;
1556}
1557
1558u32 flow_hash_from_keys(struct flow_keys *keys)
1559{
1560 __flow_hash_secret_init();
1561 return __flow_hash_from_keys(keys, &hashrnd);
1562}
1563EXPORT_SYMBOL(flow_hash_from_keys);
1564
1565static inline u32 ___skb_get_hash(const struct sk_buff *skb,
1566 struct flow_keys *keys,
1567 const siphash_key_t *keyval)
1568{
1569 skb_flow_dissect_flow_keys(skb, keys,
1570 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1571
1572 return __flow_hash_from_keys(keys, keyval);
1573}
1574
1575struct _flow_keys_digest_data {
1576 __be16 n_proto;
1577 u8 ip_proto;
1578 u8 padding;
1579 __be32 ports;
1580 __be32 src;
1581 __be32 dst;
1582};
1583
1584void make_flow_keys_digest(struct flow_keys_digest *digest,
1585 const struct flow_keys *flow)
1586{
1587 struct _flow_keys_digest_data *data =
1588 (struct _flow_keys_digest_data *)digest;
1589
1590 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
1591
1592 memset(digest, 0, sizeof(*digest));
1593
1594 data->n_proto = flow->basic.n_proto;
1595 data->ip_proto = flow->basic.ip_proto;
1596 data->ports = flow->ports.ports;
1597 data->src = flow->addrs.v4addrs.src;
1598 data->dst = flow->addrs.v4addrs.dst;
1599}
1600EXPORT_SYMBOL(make_flow_keys_digest);
1601
1602static struct flow_dissector flow_keys_dissector_symmetric __read_mostly;
1603
1604u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
1605{
1606 struct flow_keys keys;
1607
1608 __flow_hash_secret_init();
1609
1610 memset(&keys, 0, sizeof(keys));
1611 __skb_flow_dissect(NULL, skb, &flow_keys_dissector_symmetric,
1612 &keys, NULL, 0, 0, 0,
1613 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1614
1615 return __flow_hash_from_keys(&keys, &hashrnd);
1616}
1617EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric);
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628void __skb_get_hash(struct sk_buff *skb)
1629{
1630 struct flow_keys keys;
1631 u32 hash;
1632
1633 __flow_hash_secret_init();
1634
1635 hash = ___skb_get_hash(skb, &keys, &hashrnd);
1636
1637 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1638}
1639EXPORT_SYMBOL(__skb_get_hash);
1640
1641__u32 skb_get_hash_perturb(const struct sk_buff *skb,
1642 const siphash_key_t *perturb)
1643{
1644 struct flow_keys keys;
1645
1646 return ___skb_get_hash(skb, &keys, perturb);
1647}
1648EXPORT_SYMBOL(skb_get_hash_perturb);
1649
1650u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
1651 const struct flow_keys_basic *keys, int hlen)
1652{
1653 u32 poff = keys->control.thoff;
1654
1655
1656 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) &&
1657 !(keys->control.flags & FLOW_DIS_FIRST_FRAG))
1658 return poff;
1659
1660 switch (keys->basic.ip_proto) {
1661 case IPPROTO_TCP: {
1662
1663 const u8 *doff;
1664 u8 _doff;
1665
1666 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
1667 data, hlen, &_doff);
1668 if (!doff)
1669 return poff;
1670
1671 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
1672 break;
1673 }
1674 case IPPROTO_UDP:
1675 case IPPROTO_UDPLITE:
1676 poff += sizeof(struct udphdr);
1677 break;
1678
1679
1680
1681 case IPPROTO_ICMP:
1682 poff += sizeof(struct icmphdr);
1683 break;
1684 case IPPROTO_ICMPV6:
1685 poff += sizeof(struct icmp6hdr);
1686 break;
1687 case IPPROTO_IGMP:
1688 poff += sizeof(struct igmphdr);
1689 break;
1690 case IPPROTO_DCCP:
1691 poff += sizeof(struct dccp_hdr);
1692 break;
1693 case IPPROTO_SCTP:
1694 poff += sizeof(struct sctphdr);
1695 break;
1696 }
1697
1698 return poff;
1699}
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710u32 skb_get_poff(const struct sk_buff *skb)
1711{
1712 struct flow_keys_basic keys;
1713
1714 if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
1715 NULL, 0, 0, 0, 0))
1716 return 0;
1717
1718 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
1719}
1720
1721__u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys)
1722{
1723 memset(keys, 0, sizeof(*keys));
1724
1725 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr,
1726 sizeof(keys->addrs.v6addrs.src));
1727 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr,
1728 sizeof(keys->addrs.v6addrs.dst));
1729 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1730 keys->ports.src = fl6->fl6_sport;
1731 keys->ports.dst = fl6->fl6_dport;
1732 keys->keyid.keyid = fl6->fl6_gre_key;
1733 keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
1734 keys->basic.ip_proto = fl6->flowi6_proto;
1735
1736 return flow_hash_from_keys(keys);
1737}
1738EXPORT_SYMBOL(__get_hash_from_flowi6);
1739
1740static const struct flow_dissector_key flow_keys_dissector_keys[] = {
1741 {
1742 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1743 .offset = offsetof(struct flow_keys, control),
1744 },
1745 {
1746 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1747 .offset = offsetof(struct flow_keys, basic),
1748 },
1749 {
1750 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1751 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1752 },
1753 {
1754 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1755 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1756 },
1757 {
1758 .key_id = FLOW_DISSECTOR_KEY_TIPC,
1759 .offset = offsetof(struct flow_keys, addrs.tipckey),
1760 },
1761 {
1762 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1763 .offset = offsetof(struct flow_keys, ports),
1764 },
1765 {
1766 .key_id = FLOW_DISSECTOR_KEY_VLAN,
1767 .offset = offsetof(struct flow_keys, vlan),
1768 },
1769 {
1770 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL,
1771 .offset = offsetof(struct flow_keys, tags),
1772 },
1773 {
1774 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
1775 .offset = offsetof(struct flow_keys, keyid),
1776 },
1777};
1778
1779static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = {
1780 {
1781 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1782 .offset = offsetof(struct flow_keys, control),
1783 },
1784 {
1785 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1786 .offset = offsetof(struct flow_keys, basic),
1787 },
1788 {
1789 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1790 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1791 },
1792 {
1793 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1794 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1795 },
1796 {
1797 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1798 .offset = offsetof(struct flow_keys, ports),
1799 },
1800};
1801
1802static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = {
1803 {
1804 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1805 .offset = offsetof(struct flow_keys, control),
1806 },
1807 {
1808 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1809 .offset = offsetof(struct flow_keys, basic),
1810 },
1811};
1812
1813struct flow_dissector flow_keys_dissector __read_mostly;
1814EXPORT_SYMBOL(flow_keys_dissector);
1815
1816struct flow_dissector flow_keys_basic_dissector __read_mostly;
1817EXPORT_SYMBOL(flow_keys_basic_dissector);
1818
1819static int __init init_default_flow_dissectors(void)
1820{
1821 skb_flow_dissector_init(&flow_keys_dissector,
1822 flow_keys_dissector_keys,
1823 ARRAY_SIZE(flow_keys_dissector_keys));
1824 skb_flow_dissector_init(&flow_keys_dissector_symmetric,
1825 flow_keys_dissector_symmetric_keys,
1826 ARRAY_SIZE(flow_keys_dissector_symmetric_keys));
1827 skb_flow_dissector_init(&flow_keys_basic_dissector,
1828 flow_keys_basic_dissector_keys,
1829 ARRAY_SIZE(flow_keys_basic_dissector_keys));
1830 return 0;
1831}
1832core_initcall(init_default_flow_dissectors);
1833
