1
2#include <linux/kernel.h>
3#include <linux/skbuff.h>
4#include <linux/export.h>
5#include <linux/ip.h>
6#include <linux/ipv6.h>
7#include <linux/if_vlan.h>
8#include <net/dsa.h>
9#include <net/dst_metadata.h>
10#include <net/ip.h>
11#include <net/ipv6.h>
12#include <net/gre.h>
13#include <net/pptp.h>
14#include <net/tipc.h>
15#include <linux/igmp.h>
16#include <linux/icmp.h>
17#include <linux/sctp.h>
18#include <linux/dccp.h>
19#include <linux/if_tunnel.h>
20#include <linux/if_pppox.h>
21#include <linux/ppp_defs.h>
22#include <linux/stddef.h>
23#include <linux/if_ether.h>
24#include <linux/mpls.h>
25#include <linux/tcp.h>
26#include <linux/ptp_classify.h>
27#include <net/flow_dissector.h>
28#include <scsi/fc/fc_fcoe.h>
29#include <uapi/linux/batadv_packet.h>
30#include <linux/bpf.h>
31#if IS_ENABLED(CONFIG_NF_CONNTRACK)
32#include <net/netfilter/nf_conntrack_core.h>
33#include <net/netfilter/nf_conntrack_labels.h>
34#endif
35#include <linux/bpf-netns.h>
36
37static void dissector_set_key(struct flow_dissector *flow_dissector,
38 enum flow_dissector_key_id key_id)
39{
40 flow_dissector->used_keys |= (1 << key_id);
41}
42
43void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
44 const struct flow_dissector_key *key,
45 unsigned int key_count)
46{
47 unsigned int i;
48
49 memset(flow_dissector, 0, sizeof(*flow_dissector));
50
51 for (i = 0; i < key_count; i++, key++) {
52
53
54
55 BUG_ON(key->offset > USHRT_MAX);
56 BUG_ON(dissector_uses_key(flow_dissector,
57 key->key_id));
58
59 dissector_set_key(flow_dissector, key->key_id);
60 flow_dissector->offset[key->key_id] = key->offset;
61 }
62
63
64
65
66 BUG_ON(!dissector_uses_key(flow_dissector,
67 FLOW_DISSECTOR_KEY_CONTROL));
68 BUG_ON(!dissector_uses_key(flow_dissector,
69 FLOW_DISSECTOR_KEY_BASIC));
70}
71EXPORT_SYMBOL(skb_flow_dissector_init);
72
73#ifdef CONFIG_BPF_SYSCALL
74int flow_dissector_bpf_prog_attach_check(struct net *net,
75 struct bpf_prog *prog)
76{
77 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
78
79 if (net == &init_net) {
80
81
82
83
84
85 struct net *ns;
86
87 for_each_net(ns) {
88 if (ns == &init_net)
89 continue;
90 if (rcu_access_pointer(ns->bpf.run_array[type]))
91 return -EEXIST;
92 }
93 } else {
94
95
96
97 if (rcu_access_pointer(init_net.bpf.run_array[type]))
98 return -EEXIST;
99 }
100
101 return 0;
102}
103#endif
104
105
106
107
108
109
110
111
112
113
114
115
116__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
117 const void *data, int hlen)
118{
119 int poff = proto_ports_offset(ip_proto);
120
121 if (!data) {
122 data = skb->data;
123 hlen = skb_headlen(skb);
124 }
125
126 if (poff >= 0) {
127 __be32 *ports, _ports;
128
129 ports = __skb_header_pointer(skb, thoff + poff,
130 sizeof(_ports), data, hlen, &_ports);
131 if (ports)
132 return *ports;
133 }
134
135 return 0;
136}
137EXPORT_SYMBOL(__skb_flow_get_ports);
138
139static bool icmp_has_id(u8 type)
140{
141 switch (type) {
142 case ICMP_ECHO:
143 case ICMP_ECHOREPLY:
144 case ICMP_TIMESTAMP:
145 case ICMP_TIMESTAMPREPLY:
146 case ICMPV6_ECHO_REQUEST:
147 case ICMPV6_ECHO_REPLY:
148 return true;
149 }
150
151 return false;
152}
153
154
155
156
157
158
159
160
161
162void skb_flow_get_icmp_tci(const struct sk_buff *skb,
163 struct flow_dissector_key_icmp *key_icmp,
164 const void *data, int thoff, int hlen)
165{
166 struct icmphdr *ih, _ih;
167
168 ih = __skb_header_pointer(skb, thoff, sizeof(_ih), data, hlen, &_ih);
169 if (!ih)
170 return;
171
172 key_icmp->type = ih->type;
173 key_icmp->code = ih->code;
174
175
176
177
178 if (icmp_has_id(ih->type))
179 key_icmp->id = ih->un.echo.id ? ntohs(ih->un.echo.id) : 1;
180 else
181 key_icmp->id = 0;
182}
183EXPORT_SYMBOL(skb_flow_get_icmp_tci);
184
185
186
187
188static void __skb_flow_dissect_icmp(const struct sk_buff *skb,
189 struct flow_dissector *flow_dissector,
190 void *target_container, const void *data,
191 int thoff, int hlen)
192{
193 struct flow_dissector_key_icmp *key_icmp;
194
195 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ICMP))
196 return;
197
198 key_icmp = skb_flow_dissector_target(flow_dissector,
199 FLOW_DISSECTOR_KEY_ICMP,
200 target_container);
201
202 skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen);
203}
204
205void skb_flow_dissect_meta(const struct sk_buff *skb,
206 struct flow_dissector *flow_dissector,
207 void *target_container)
208{
209 struct flow_dissector_key_meta *meta;
210
211 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_META))
212 return;
213
214 meta = skb_flow_dissector_target(flow_dissector,
215 FLOW_DISSECTOR_KEY_META,
216 target_container);
217 meta->ingress_ifindex = skb->skb_iif;
218}
219EXPORT_SYMBOL(skb_flow_dissect_meta);
220
221static void
222skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type,
223 struct flow_dissector *flow_dissector,
224 void *target_container)
225{
226 struct flow_dissector_key_control *ctrl;
227
228 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL))
229 return;
230
231 ctrl = skb_flow_dissector_target(flow_dissector,
232 FLOW_DISSECTOR_KEY_ENC_CONTROL,
233 target_container);
234 ctrl->addr_type = type;
235}
236
237void
238skb_flow_dissect_ct(const struct sk_buff *skb,
239 struct flow_dissector *flow_dissector,
240 void *target_container, u16 *ctinfo_map,
241 size_t mapsize, bool post_ct)
242{
243#if IS_ENABLED(CONFIG_NF_CONNTRACK)
244 struct flow_dissector_key_ct *key;
245 enum ip_conntrack_info ctinfo;
246 struct nf_conn_labels *cl;
247 struct nf_conn *ct;
248
249 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CT))
250 return;
251
252 ct = nf_ct_get(skb, &ctinfo);
253 if (!ct && !post_ct)
254 return;
255
256 key = skb_flow_dissector_target(flow_dissector,
257 FLOW_DISSECTOR_KEY_CT,
258 target_container);
259
260 if (!ct) {
261 key->ct_state = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
262 TCA_FLOWER_KEY_CT_FLAGS_INVALID;
263 return;
264 }
265
266 if (ctinfo < mapsize)
267 key->ct_state = ctinfo_map[ctinfo];
268#if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)
269 key->ct_zone = ct->zone.id;
270#endif
271#if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
272 key->ct_mark = ct->mark;
273#endif
274
275 cl = nf_ct_labels_find(ct);
276 if (cl)
277 memcpy(key->ct_labels, cl->bits, sizeof(key->ct_labels));
278#endif
279}
280EXPORT_SYMBOL(skb_flow_dissect_ct);
281
282void
283skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
284 struct flow_dissector *flow_dissector,
285 void *target_container)
286{
287 struct ip_tunnel_info *info;
288 struct ip_tunnel_key *key;
289
290
291 if (!dissector_uses_key(flow_dissector,
292 FLOW_DISSECTOR_KEY_ENC_KEYID) &&
293 !dissector_uses_key(flow_dissector,
294 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) &&
295 !dissector_uses_key(flow_dissector,
296 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) &&
297 !dissector_uses_key(flow_dissector,
298 FLOW_DISSECTOR_KEY_ENC_CONTROL) &&
299 !dissector_uses_key(flow_dissector,
300 FLOW_DISSECTOR_KEY_ENC_PORTS) &&
301 !dissector_uses_key(flow_dissector,
302 FLOW_DISSECTOR_KEY_ENC_IP) &&
303 !dissector_uses_key(flow_dissector,
304 FLOW_DISSECTOR_KEY_ENC_OPTS))
305 return;
306
307 info = skb_tunnel_info(skb);
308 if (!info)
309 return;
310
311 key = &info->key;
312
313 switch (ip_tunnel_info_af(info)) {
314 case AF_INET:
315 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS,
316 flow_dissector,
317 target_container);
318 if (dissector_uses_key(flow_dissector,
319 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
320 struct flow_dissector_key_ipv4_addrs *ipv4;
321
322 ipv4 = skb_flow_dissector_target(flow_dissector,
323 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
324 target_container);
325 ipv4->src = key->u.ipv4.src;
326 ipv4->dst = key->u.ipv4.dst;
327 }
328 break;
329 case AF_INET6:
330 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS,
331 flow_dissector,
332 target_container);
333 if (dissector_uses_key(flow_dissector,
334 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
335 struct flow_dissector_key_ipv6_addrs *ipv6;
336
337 ipv6 = skb_flow_dissector_target(flow_dissector,
338 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
339 target_container);
340 ipv6->src = key->u.ipv6.src;
341 ipv6->dst = key->u.ipv6.dst;
342 }
343 break;
344 }
345
346 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
347 struct flow_dissector_key_keyid *keyid;
348
349 keyid = skb_flow_dissector_target(flow_dissector,
350 FLOW_DISSECTOR_KEY_ENC_KEYID,
351 target_container);
352 keyid->keyid = tunnel_id_to_key32(key->tun_id);
353 }
354
355 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
356 struct flow_dissector_key_ports *tp;
357
358 tp = skb_flow_dissector_target(flow_dissector,
359 FLOW_DISSECTOR_KEY_ENC_PORTS,
360 target_container);
361 tp->src = key->tp_src;
362 tp->dst = key->tp_dst;
363 }
364
365 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) {
366 struct flow_dissector_key_ip *ip;
367
368 ip = skb_flow_dissector_target(flow_dissector,
369 FLOW_DISSECTOR_KEY_ENC_IP,
370 target_container);
371 ip->tos = key->tos;
372 ip->ttl = key->ttl;
373 }
374
375 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) {
376 struct flow_dissector_key_enc_opts *enc_opt;
377
378 enc_opt = skb_flow_dissector_target(flow_dissector,
379 FLOW_DISSECTOR_KEY_ENC_OPTS,
380 target_container);
381
382 if (info->options_len) {
383 enc_opt->len = info->options_len;
384 ip_tunnel_info_opts_get(enc_opt->data, info);
385 enc_opt->dst_opt_type = info->key.tun_flags &
386 TUNNEL_OPTIONS_PRESENT;
387 }
388 }
389}
390EXPORT_SYMBOL(skb_flow_dissect_tunnel_info);
391
392void skb_flow_dissect_hash(const struct sk_buff *skb,
393 struct flow_dissector *flow_dissector,
394 void *target_container)
395{
396 struct flow_dissector_key_hash *key;
397
398 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_HASH))
399 return;
400
401 key = skb_flow_dissector_target(flow_dissector,
402 FLOW_DISSECTOR_KEY_HASH,
403 target_container);
404
405 key->hash = skb_get_hash_raw(skb);
406}
407EXPORT_SYMBOL(skb_flow_dissect_hash);
408
409static enum flow_dissect_ret
410__skb_flow_dissect_mpls(const struct sk_buff *skb,
411 struct flow_dissector *flow_dissector,
412 void *target_container, const void *data, int nhoff,
413 int hlen, int lse_index, bool *entropy_label)
414{
415 struct mpls_label *hdr, _hdr;
416 u32 entry, label, bos;
417
418 if (!dissector_uses_key(flow_dissector,
419 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) &&
420 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS))
421 return FLOW_DISSECT_RET_OUT_GOOD;
422
423 if (lse_index >= FLOW_DIS_MPLS_MAX)
424 return FLOW_DISSECT_RET_OUT_GOOD;
425
426 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
427 hlen, &_hdr);
428 if (!hdr)
429 return FLOW_DISSECT_RET_OUT_BAD;
430
431 entry = ntohl(hdr->entry);
432 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT;
433 bos = (entry & MPLS_LS_S_MASK) >> MPLS_LS_S_SHIFT;
434
435 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) {
436 struct flow_dissector_key_mpls *key_mpls;
437 struct flow_dissector_mpls_lse *lse;
438
439 key_mpls = skb_flow_dissector_target(flow_dissector,
440 FLOW_DISSECTOR_KEY_MPLS,
441 target_container);
442 lse = &key_mpls->ls[lse_index];
443
444 lse->mpls_ttl = (entry & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
445 lse->mpls_bos = bos;
446 lse->mpls_tc = (entry & MPLS_LS_TC_MASK) >> MPLS_LS_TC_SHIFT;
447 lse->mpls_label = label;
448 dissector_set_mpls_lse(key_mpls, lse_index);
449 }
450
451 if (*entropy_label &&
452 dissector_uses_key(flow_dissector,
453 FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) {
454 struct flow_dissector_key_keyid *key_keyid;
455
456 key_keyid = skb_flow_dissector_target(flow_dissector,
457 FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
458 target_container);
459 key_keyid->keyid = cpu_to_be32(label);
460 }
461
462 *entropy_label = label == MPLS_LABEL_ENTROPY;
463
464 return bos ? FLOW_DISSECT_RET_OUT_GOOD : FLOW_DISSECT_RET_PROTO_AGAIN;
465}
466
467static enum flow_dissect_ret
468__skb_flow_dissect_arp(const struct sk_buff *skb,
469 struct flow_dissector *flow_dissector,
470 void *target_container, const void *data,
471 int nhoff, int hlen)
472{
473 struct flow_dissector_key_arp *key_arp;
474 struct {
475 unsigned char ar_sha[ETH_ALEN];
476 unsigned char ar_sip[4];
477 unsigned char ar_tha[ETH_ALEN];
478 unsigned char ar_tip[4];
479 } *arp_eth, _arp_eth;
480 const struct arphdr *arp;
481 struct arphdr _arp;
482
483 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP))
484 return FLOW_DISSECT_RET_OUT_GOOD;
485
486 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
487 hlen, &_arp);
488 if (!arp)
489 return FLOW_DISSECT_RET_OUT_BAD;
490
491 if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
492 arp->ar_pro != htons(ETH_P_IP) ||
493 arp->ar_hln != ETH_ALEN ||
494 arp->ar_pln != 4 ||
495 (arp->ar_op != htons(ARPOP_REPLY) &&
496 arp->ar_op != htons(ARPOP_REQUEST)))
497 return FLOW_DISSECT_RET_OUT_BAD;
498
499 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
500 sizeof(_arp_eth), data,
501 hlen, &_arp_eth);
502 if (!arp_eth)
503 return FLOW_DISSECT_RET_OUT_BAD;
504
505 key_arp = skb_flow_dissector_target(flow_dissector,
506 FLOW_DISSECTOR_KEY_ARP,
507 target_container);
508
509 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip));
510 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip));
511
512
513
514
515 key_arp->op = ntohs(arp->ar_op) & 0xff;
516
517 ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
518 ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
519
520 return FLOW_DISSECT_RET_OUT_GOOD;
521}
522
523static enum flow_dissect_ret
524__skb_flow_dissect_gre(const struct sk_buff *skb,
525 struct flow_dissector_key_control *key_control,
526 struct flow_dissector *flow_dissector,
527 void *target_container, const void *data,
528 __be16 *p_proto, int *p_nhoff, int *p_hlen,
529 unsigned int flags)
530{
531 struct flow_dissector_key_keyid *key_keyid;
532 struct gre_base_hdr *hdr, _hdr;
533 int offset = 0;
534 u16 gre_ver;
535
536 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr),
537 data, *p_hlen, &_hdr);
538 if (!hdr)
539 return FLOW_DISSECT_RET_OUT_BAD;
540
541
542 if (hdr->flags & GRE_ROUTING)
543 return FLOW_DISSECT_RET_OUT_GOOD;
544
545
546 gre_ver = ntohs(hdr->flags & GRE_VERSION);
547 if (gre_ver > 1)
548 return FLOW_DISSECT_RET_OUT_GOOD;
549
550 *p_proto = hdr->protocol;
551 if (gre_ver) {
552
553 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
554 return FLOW_DISSECT_RET_OUT_GOOD;
555 }
556
557 offset += sizeof(struct gre_base_hdr);
558
559 if (hdr->flags & GRE_CSUM)
560 offset += sizeof_field(struct gre_full_hdr, csum) +
561 sizeof_field(struct gre_full_hdr, reserved1);
562
563 if (hdr->flags & GRE_KEY) {
564 const __be32 *keyid;
565 __be32 _keyid;
566
567 keyid = __skb_header_pointer(skb, *p_nhoff + offset,
568 sizeof(_keyid),
569 data, *p_hlen, &_keyid);
570 if (!keyid)
571 return FLOW_DISSECT_RET_OUT_BAD;
572
573 if (dissector_uses_key(flow_dissector,
574 FLOW_DISSECTOR_KEY_GRE_KEYID)) {
575 key_keyid = skb_flow_dissector_target(flow_dissector,
576 FLOW_DISSECTOR_KEY_GRE_KEYID,
577 target_container);
578 if (gre_ver == 0)
579 key_keyid->keyid = *keyid;
580 else
581 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
582 }
583 offset += sizeof_field(struct gre_full_hdr, key);
584 }
585
586 if (hdr->flags & GRE_SEQ)
587 offset += sizeof_field(struct pptp_gre_header, seq);
588
589 if (gre_ver == 0) {
590 if (*p_proto == htons(ETH_P_TEB)) {
591 const struct ethhdr *eth;
592 struct ethhdr _eth;
593
594 eth = __skb_header_pointer(skb, *p_nhoff + offset,
595 sizeof(_eth),
596 data, *p_hlen, &_eth);
597 if (!eth)
598 return FLOW_DISSECT_RET_OUT_BAD;
599 *p_proto = eth->h_proto;
600 offset += sizeof(*eth);
601
602
603
604
605
606 if (NET_IP_ALIGN)
607 *p_hlen = *p_nhoff + offset;
608 }
609 } else {
610 u8 _ppp_hdr[PPP_HDRLEN];
611 u8 *ppp_hdr;
612
613 if (hdr->flags & GRE_ACK)
614 offset += sizeof_field(struct pptp_gre_header, ack);
615
616 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset,
617 sizeof(_ppp_hdr),
618 data, *p_hlen, _ppp_hdr);
619 if (!ppp_hdr)
620 return FLOW_DISSECT_RET_OUT_BAD;
621
622 switch (PPP_PROTOCOL(ppp_hdr)) {
623 case PPP_IP:
624 *p_proto = htons(ETH_P_IP);
625 break;
626 case PPP_IPV6:
627 *p_proto = htons(ETH_P_IPV6);
628 break;
629 default:
630
631 break;
632 }
633
634 offset += PPP_HDRLEN;
635 }
636
637 *p_nhoff += offset;
638 key_control->flags |= FLOW_DIS_ENCAPSULATION;
639 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
640 return FLOW_DISSECT_RET_OUT_GOOD;
641
642 return FLOW_DISSECT_RET_PROTO_AGAIN;
643}
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664static enum flow_dissect_ret
665__skb_flow_dissect_batadv(const struct sk_buff *skb,
666 struct flow_dissector_key_control *key_control,
667 const void *data, __be16 *p_proto, int *p_nhoff,
668 int hlen, unsigned int flags)
669{
670 struct {
671 struct batadv_unicast_packet batadv_unicast;
672 struct ethhdr eth;
673 } *hdr, _hdr;
674
675 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen,
676 &_hdr);
677 if (!hdr)
678 return FLOW_DISSECT_RET_OUT_BAD;
679
680 if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION)
681 return FLOW_DISSECT_RET_OUT_BAD;
682
683 if (hdr->batadv_unicast.packet_type != BATADV_UNICAST)
684 return FLOW_DISSECT_RET_OUT_BAD;
685
686 *p_proto = hdr->eth.h_proto;
687 *p_nhoff += sizeof(*hdr);
688
689 key_control->flags |= FLOW_DIS_ENCAPSULATION;
690 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
691 return FLOW_DISSECT_RET_OUT_GOOD;
692
693 return FLOW_DISSECT_RET_PROTO_AGAIN;
694}
695
696static void
697__skb_flow_dissect_tcp(const struct sk_buff *skb,
698 struct flow_dissector *flow_dissector,
699 void *target_container, const void *data,
700 int thoff, int hlen)
701{
702 struct flow_dissector_key_tcp *key_tcp;
703 struct tcphdr *th, _th;
704
705 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP))
706 return;
707
708 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th);
709 if (!th)
710 return;
711
712 if (unlikely(__tcp_hdrlen(th) < sizeof(_th)))
713 return;
714
715 key_tcp = skb_flow_dissector_target(flow_dissector,
716 FLOW_DISSECTOR_KEY_TCP,
717 target_container);
718 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF));
719}
720
721static void
722__skb_flow_dissect_ports(const struct sk_buff *skb,
723 struct flow_dissector *flow_dissector,
724 void *target_container, const void *data,
725 int nhoff, u8 ip_proto, int hlen)
726{
727 enum flow_dissector_key_id dissector_ports = FLOW_DISSECTOR_KEY_MAX;
728 struct flow_dissector_key_ports *key_ports;
729
730 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
731 dissector_ports = FLOW_DISSECTOR_KEY_PORTS;
732 else if (dissector_uses_key(flow_dissector,
733 FLOW_DISSECTOR_KEY_PORTS_RANGE))
734 dissector_ports = FLOW_DISSECTOR_KEY_PORTS_RANGE;
735
736 if (dissector_ports == FLOW_DISSECTOR_KEY_MAX)
737 return;
738
739 key_ports = skb_flow_dissector_target(flow_dissector,
740 dissector_ports,
741 target_container);
742 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
743 data, hlen);
744}
745
746static void
747__skb_flow_dissect_ipv4(const struct sk_buff *skb,
748 struct flow_dissector *flow_dissector,
749 void *target_container, const void *data,
750 const struct iphdr *iph)
751{
752 struct flow_dissector_key_ip *key_ip;
753
754 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
755 return;
756
757 key_ip = skb_flow_dissector_target(flow_dissector,
758 FLOW_DISSECTOR_KEY_IP,
759 target_container);
760 key_ip->tos = iph->tos;
761 key_ip->ttl = iph->ttl;
762}
763
764static void
765__skb_flow_dissect_ipv6(const struct sk_buff *skb,
766 struct flow_dissector *flow_dissector,
767 void *target_container, const void *data,
768 const struct ipv6hdr *iph)
769{
770 struct flow_dissector_key_ip *key_ip;
771
772 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
773 return;
774
775 key_ip = skb_flow_dissector_target(flow_dissector,
776 FLOW_DISSECTOR_KEY_IP,
777 target_container);
778 key_ip->tos = ipv6_get_dsfield(iph);
779 key_ip->ttl = iph->hop_limit;
780}
781
782
783
784
785#define MAX_FLOW_DISSECT_HDRS 15
786
787static bool skb_flow_dissect_allowed(int *num_hdrs)
788{
789 ++*num_hdrs;
790
791 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS);
792}
793
794static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys,
795 struct flow_dissector *flow_dissector,
796 void *target_container)
797{
798 struct flow_dissector_key_ports *key_ports = NULL;
799 struct flow_dissector_key_control *key_control;
800 struct flow_dissector_key_basic *key_basic;
801 struct flow_dissector_key_addrs *key_addrs;
802 struct flow_dissector_key_tags *key_tags;
803
804 key_control = skb_flow_dissector_target(flow_dissector,
805 FLOW_DISSECTOR_KEY_CONTROL,
806 target_container);
807 key_control->thoff = flow_keys->thoff;
808 if (flow_keys->is_frag)
809 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
810 if (flow_keys->is_first_frag)
811 key_control->flags |= FLOW_DIS_FIRST_FRAG;
812 if (flow_keys->is_encap)
813 key_control->flags |= FLOW_DIS_ENCAPSULATION;
814
815 key_basic = skb_flow_dissector_target(flow_dissector,
816 FLOW_DISSECTOR_KEY_BASIC,
817 target_container);
818 key_basic->n_proto = flow_keys->n_proto;
819 key_basic->ip_proto = flow_keys->ip_proto;
820
821 if (flow_keys->addr_proto == ETH_P_IP &&
822 dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
823 key_addrs = skb_flow_dissector_target(flow_dissector,
824 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
825 target_container);
826 key_addrs->v4addrs.src = flow_keys->ipv4_src;
827 key_addrs->v4addrs.dst = flow_keys->ipv4_dst;
828 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
829 } else if (flow_keys->addr_proto == ETH_P_IPV6 &&
830 dissector_uses_key(flow_dissector,
831 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
832 key_addrs = skb_flow_dissector_target(flow_dissector,
833 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
834 target_container);
835 memcpy(&key_addrs->v6addrs.src, &flow_keys->ipv6_src,
836 sizeof(key_addrs->v6addrs.src));
837 memcpy(&key_addrs->v6addrs.dst, &flow_keys->ipv6_dst,
838 sizeof(key_addrs->v6addrs.dst));
839 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
840 }
841
842 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
843 key_ports = skb_flow_dissector_target(flow_dissector,
844 FLOW_DISSECTOR_KEY_PORTS,
845 target_container);
846 else if (dissector_uses_key(flow_dissector,
847 FLOW_DISSECTOR_KEY_PORTS_RANGE))
848 key_ports = skb_flow_dissector_target(flow_dissector,
849 FLOW_DISSECTOR_KEY_PORTS_RANGE,
850 target_container);
851
852 if (key_ports) {
853 key_ports->src = flow_keys->sport;
854 key_ports->dst = flow_keys->dport;
855 }
856
857 if (dissector_uses_key(flow_dissector,
858 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
859 key_tags = skb_flow_dissector_target(flow_dissector,
860 FLOW_DISSECTOR_KEY_FLOW_LABEL,
861 target_container);
862 key_tags->flow_label = ntohl(flow_keys->flow_label);
863 }
864}
865
866bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
867 __be16 proto, int nhoff, int hlen, unsigned int flags)
868{
869 struct bpf_flow_keys *flow_keys = ctx->flow_keys;
870 u32 result;
871
872
873 memset(flow_keys, 0, sizeof(*flow_keys));
874 flow_keys->n_proto = proto;
875 flow_keys->nhoff = nhoff;
876 flow_keys->thoff = flow_keys->nhoff;
877
878 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG !=
879 (int)FLOW_DISSECTOR_F_PARSE_1ST_FRAG);
880 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL !=
881 (int)FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
882 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP !=
883 (int)FLOW_DISSECTOR_F_STOP_AT_ENCAP);
884 flow_keys->flags = flags;
885
886 result = bpf_prog_run_pin_on_cpu(prog, ctx);
887
888 flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen);
889 flow_keys->thoff = clamp_t(u16, flow_keys->thoff,
890 flow_keys->nhoff, hlen);
891
892 return result == BPF_OK;
893}
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914bool __skb_flow_dissect(const struct net *net,
915 const struct sk_buff *skb,
916 struct flow_dissector *flow_dissector,
917 void *target_container, const void *data,
918 __be16 proto, int nhoff, int hlen, unsigned int flags)
919{
920 struct flow_dissector_key_control *key_control;
921 struct flow_dissector_key_basic *key_basic;
922 struct flow_dissector_key_addrs *key_addrs;
923 struct flow_dissector_key_tags *key_tags;
924 struct flow_dissector_key_vlan *key_vlan;
925 enum flow_dissect_ret fdret;
926 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX;
927 bool mpls_el = false;
928 int mpls_lse = 0;
929 int num_hdrs = 0;
930 u8 ip_proto = 0;
931 bool ret;
932
933 if (!data) {
934 data = skb->data;
935 proto = skb_vlan_tag_present(skb) ?
936 skb->vlan_proto : skb->protocol;
937 nhoff = skb_network_offset(skb);
938 hlen = skb_headlen(skb);
939#if IS_ENABLED(CONFIG_NET_DSA)
940 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev) &&
941 proto == htons(ETH_P_XDSA))) {
942 const struct dsa_device_ops *ops;
943 int offset = 0;
944
945 ops = skb->dev->dsa_ptr->tag_ops;
946
947 if (ops->needed_headroom) {
948 if (ops->flow_dissect)
949 ops->flow_dissect(skb, &proto, &offset);
950 else
951 dsa_tag_generic_flow_dissect(skb,
952 &proto,
953 &offset);
954 hlen -= offset;
955 nhoff += offset;
956 }
957 }
958#endif
959 }
960
961
962
963
964 key_control = skb_flow_dissector_target(flow_dissector,
965 FLOW_DISSECTOR_KEY_CONTROL,
966 target_container);
967
968
969
970
971 key_basic = skb_flow_dissector_target(flow_dissector,
972 FLOW_DISSECTOR_KEY_BASIC,
973 target_container);
974
975 if (skb) {
976 if (!net) {
977 if (skb->dev)
978 net = dev_net(skb->dev);
979 else if (skb->sk)
980 net = sock_net(skb->sk);
981 }
982 }
983
984 WARN_ON_ONCE(!net);
985 if (net) {
986 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
987 struct bpf_prog_array *run_array;
988
989 rcu_read_lock();
990 run_array = rcu_dereference(init_net.bpf.run_array[type]);
991 if (!run_array)
992 run_array = rcu_dereference(net->bpf.run_array[type]);
993
994 if (run_array) {
995 struct bpf_flow_keys flow_keys;
996 struct bpf_flow_dissector ctx = {
997 .flow_keys = &flow_keys,
998 .data = data,
999 .data_end = data + hlen,
1000 };
1001 __be16 n_proto = proto;
1002 struct bpf_prog *prog;
1003
1004 if (skb) {
1005 ctx.skb = skb;
1006
1007
1008
1009
1010 n_proto = skb->protocol;
1011 }
1012
1013 prog = READ_ONCE(run_array->items[0].prog);
1014 ret = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
1015 hlen, flags);
1016 __skb_flow_bpf_to_target(&flow_keys, flow_dissector,
1017 target_container);
1018 rcu_read_unlock();
1019 return ret;
1020 }
1021 rcu_read_unlock();
1022 }
1023
1024 if (dissector_uses_key(flow_dissector,
1025 FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
1026 struct ethhdr *eth = eth_hdr(skb);
1027 struct flow_dissector_key_eth_addrs *key_eth_addrs;
1028
1029 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
1030 FLOW_DISSECTOR_KEY_ETH_ADDRS,
1031 target_container);
1032 memcpy(key_eth_addrs, ð->h_dest, sizeof(*key_eth_addrs));
1033 }
1034
1035proto_again:
1036 fdret = FLOW_DISSECT_RET_CONTINUE;
1037
1038 switch (proto) {
1039 case htons(ETH_P_IP): {
1040 const struct iphdr *iph;
1041 struct iphdr _iph;
1042
1043 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1044 if (!iph || iph->ihl < 5) {
1045 fdret = FLOW_DISSECT_RET_OUT_BAD;
1046 break;
1047 }
1048
1049 nhoff += iph->ihl * 4;
1050
1051 ip_proto = iph->protocol;
1052
1053 if (dissector_uses_key(flow_dissector,
1054 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
1055 key_addrs = skb_flow_dissector_target(flow_dissector,
1056 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1057 target_container);
1058
1059 memcpy(&key_addrs->v4addrs.src, &iph->saddr,
1060 sizeof(key_addrs->v4addrs.src));
1061 memcpy(&key_addrs->v4addrs.dst, &iph->daddr,
1062 sizeof(key_addrs->v4addrs.dst));
1063 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1064 }
1065
1066 __skb_flow_dissect_ipv4(skb, flow_dissector,
1067 target_container, data, iph);
1068
1069 if (ip_is_fragment(iph)) {
1070 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1071
1072 if (iph->frag_off & htons(IP_OFFSET)) {
1073 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1074 break;
1075 } else {
1076 key_control->flags |= FLOW_DIS_FIRST_FRAG;
1077 if (!(flags &
1078 FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) {
1079 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1080 break;
1081 }
1082 }
1083 }
1084
1085 break;
1086 }
1087 case htons(ETH_P_IPV6): {
1088 const struct ipv6hdr *iph;
1089 struct ipv6hdr _iph;
1090
1091 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1092 if (!iph) {
1093 fdret = FLOW_DISSECT_RET_OUT_BAD;
1094 break;
1095 }
1096
1097 ip_proto = iph->nexthdr;
1098 nhoff += sizeof(struct ipv6hdr);
1099
1100 if (dissector_uses_key(flow_dissector,
1101 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
1102 key_addrs = skb_flow_dissector_target(flow_dissector,
1103 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1104 target_container);
1105
1106 memcpy(&key_addrs->v6addrs.src, &iph->saddr,
1107 sizeof(key_addrs->v6addrs.src));
1108 memcpy(&key_addrs->v6addrs.dst, &iph->daddr,
1109 sizeof(key_addrs->v6addrs.dst));
1110 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1111 }
1112
1113 if ((dissector_uses_key(flow_dissector,
1114 FLOW_DISSECTOR_KEY_FLOW_LABEL) ||
1115 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) &&
1116 ip6_flowlabel(iph)) {
1117 __be32 flow_label = ip6_flowlabel(iph);
1118
1119 if (dissector_uses_key(flow_dissector,
1120 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
1121 key_tags = skb_flow_dissector_target(flow_dissector,
1122 FLOW_DISSECTOR_KEY_FLOW_LABEL,
1123 target_container);
1124 key_tags->flow_label = ntohl(flow_label);
1125 }
1126 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) {
1127 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1128 break;
1129 }
1130 }
1131
1132 __skb_flow_dissect_ipv6(skb, flow_dissector,
1133 target_container, data, iph);
1134
1135 break;
1136 }
1137 case htons(ETH_P_8021AD):
1138 case htons(ETH_P_8021Q): {
1139 const struct vlan_hdr *vlan = NULL;
1140 struct vlan_hdr _vlan;
1141 __be16 saved_vlan_tpid = proto;
1142
1143 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX &&
1144 skb && skb_vlan_tag_present(skb)) {
1145 proto = skb->protocol;
1146 } else {
1147 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan),
1148 data, hlen, &_vlan);
1149 if (!vlan) {
1150 fdret = FLOW_DISSECT_RET_OUT_BAD;
1151 break;
1152 }
1153
1154 proto = vlan->h_vlan_encapsulated_proto;
1155 nhoff += sizeof(*vlan);
1156 }
1157
1158 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) {
1159 dissector_vlan = FLOW_DISSECTOR_KEY_VLAN;
1160 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) {
1161 dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN;
1162 } else {
1163 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1164 break;
1165 }
1166
1167 if (dissector_uses_key(flow_dissector, dissector_vlan)) {
1168 key_vlan = skb_flow_dissector_target(flow_dissector,
1169 dissector_vlan,
1170 target_container);
1171
1172 if (!vlan) {
1173 key_vlan->vlan_id = skb_vlan_tag_get_id(skb);
1174 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb);
1175 } else {
1176 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) &
1177 VLAN_VID_MASK;
1178 key_vlan->vlan_priority =
1179 (ntohs(vlan->h_vlan_TCI) &
1180 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1181 }
1182 key_vlan->vlan_tpid = saved_vlan_tpid;
1183 }
1184
1185 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1186 break;
1187 }
1188 case htons(ETH_P_PPP_SES): {
1189 struct {
1190 struct pppoe_hdr hdr;
1191 __be16 proto;
1192 } *hdr, _hdr;
1193 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
1194 if (!hdr) {
1195 fdret = FLOW_DISSECT_RET_OUT_BAD;
1196 break;
1197 }
1198
1199 proto = hdr->proto;
1200 nhoff += PPPOE_SES_HLEN;
1201 switch (proto) {
1202 case htons(PPP_IP):
1203 proto = htons(ETH_P_IP);
1204 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1205 break;
1206 case htons(PPP_IPV6):
1207 proto = htons(ETH_P_IPV6);
1208 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1209 break;
1210 default:
1211 fdret = FLOW_DISSECT_RET_OUT_BAD;
1212 break;
1213 }
1214 break;
1215 }
1216 case htons(ETH_P_TIPC): {
1217 struct tipc_basic_hdr *hdr, _hdr;
1218
1219 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr),
1220 data, hlen, &_hdr);
1221 if (!hdr) {
1222 fdret = FLOW_DISSECT_RET_OUT_BAD;
1223 break;
1224 }
1225
1226 if (dissector_uses_key(flow_dissector,
1227 FLOW_DISSECTOR_KEY_TIPC)) {
1228 key_addrs = skb_flow_dissector_target(flow_dissector,
1229 FLOW_DISSECTOR_KEY_TIPC,
1230 target_container);
1231 key_addrs->tipckey.key = tipc_hdr_rps_key(hdr);
1232 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC;
1233 }
1234 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1235 break;
1236 }
1237
1238 case htons(ETH_P_MPLS_UC):
1239 case htons(ETH_P_MPLS_MC):
1240 fdret = __skb_flow_dissect_mpls(skb, flow_dissector,
1241 target_container, data,
1242 nhoff, hlen, mpls_lse,
1243 &mpls_el);
1244 nhoff += sizeof(struct mpls_label);
1245 mpls_lse++;
1246 break;
1247 case htons(ETH_P_FCOE):
1248 if ((hlen - nhoff) < FCOE_HEADER_LEN) {
1249 fdret = FLOW_DISSECT_RET_OUT_BAD;
1250 break;
1251 }
1252
1253 nhoff += FCOE_HEADER_LEN;
1254 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1255 break;
1256
1257 case htons(ETH_P_ARP):
1258 case htons(ETH_P_RARP):
1259 fdret = __skb_flow_dissect_arp(skb, flow_dissector,
1260 target_container, data,
1261 nhoff, hlen);
1262 break;
1263
1264 case htons(ETH_P_BATMAN):
1265 fdret = __skb_flow_dissect_batadv(skb, key_control, data,
1266 &proto, &nhoff, hlen, flags);
1267 break;
1268
1269 case htons(ETH_P_1588): {
1270 struct ptp_header *hdr, _hdr;
1271
1272 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
1273 hlen, &_hdr);
1274 if (!hdr) {
1275 fdret = FLOW_DISSECT_RET_OUT_BAD;
1276 break;
1277 }
1278
1279 nhoff += ntohs(hdr->message_length);
1280 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1281 break;
1282 }
1283
1284 default:
1285 fdret = FLOW_DISSECT_RET_OUT_BAD;
1286 break;
1287 }
1288
1289
1290 switch (fdret) {
1291 case FLOW_DISSECT_RET_OUT_GOOD:
1292 goto out_good;
1293 case FLOW_DISSECT_RET_PROTO_AGAIN:
1294 if (skb_flow_dissect_allowed(&num_hdrs))
1295 goto proto_again;
1296 goto out_good;
1297 case FLOW_DISSECT_RET_CONTINUE:
1298 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1299 break;
1300 case FLOW_DISSECT_RET_OUT_BAD:
1301 default:
1302 goto out_bad;
1303 }
1304
1305ip_proto_again:
1306 fdret = FLOW_DISSECT_RET_CONTINUE;
1307
1308 switch (ip_proto) {
1309 case IPPROTO_GRE:
1310 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector,
1311 target_container, data,
1312 &proto, &nhoff, &hlen, flags);
1313 break;
1314
1315 case NEXTHDR_HOP:
1316 case NEXTHDR_ROUTING:
1317 case NEXTHDR_DEST: {
1318 u8 _opthdr[2], *opthdr;
1319
1320 if (proto != htons(ETH_P_IPV6))
1321 break;
1322
1323 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr),
1324 data, hlen, &_opthdr);
1325 if (!opthdr) {
1326 fdret = FLOW_DISSECT_RET_OUT_BAD;
1327 break;
1328 }
1329
1330 ip_proto = opthdr[0];
1331 nhoff += (opthdr[1] + 1) << 3;
1332
1333 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1334 break;
1335 }
1336 case NEXTHDR_FRAGMENT: {
1337 struct frag_hdr _fh, *fh;
1338
1339 if (proto != htons(ETH_P_IPV6))
1340 break;
1341
1342 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh),
1343 data, hlen, &_fh);
1344
1345 if (!fh) {
1346 fdret = FLOW_DISSECT_RET_OUT_BAD;
1347 break;
1348 }
1349
1350 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1351
1352 nhoff += sizeof(_fh);
1353 ip_proto = fh->nexthdr;
1354
1355 if (!(fh->frag_off & htons(IP6_OFFSET))) {
1356 key_control->flags |= FLOW_DIS_FIRST_FRAG;
1357 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) {
1358 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1359 break;
1360 }
1361 }
1362
1363 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1364 break;
1365 }
1366 case IPPROTO_IPIP:
1367 proto = htons(ETH_P_IP);
1368
1369 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1370 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1371 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1372 break;
1373 }
1374
1375 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1376 break;
1377
1378 case IPPROTO_IPV6:
1379 proto = htons(ETH_P_IPV6);
1380
1381 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1382 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1383 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1384 break;
1385 }
1386
1387 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1388 break;
1389
1390
1391 case IPPROTO_MPLS:
1392 proto = htons(ETH_P_MPLS_UC);
1393 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1394 break;
1395
1396 case IPPROTO_TCP:
1397 __skb_flow_dissect_tcp(skb, flow_dissector, target_container,
1398 data, nhoff, hlen);
1399 break;
1400
1401 case IPPROTO_ICMP:
1402 case IPPROTO_ICMPV6:
1403 __skb_flow_dissect_icmp(skb, flow_dissector, target_container,
1404 data, nhoff, hlen);
1405 break;
1406
1407 default:
1408 break;
1409 }
1410
1411 if (!(key_control->flags & FLOW_DIS_IS_FRAGMENT))
1412 __skb_flow_dissect_ports(skb, flow_dissector, target_container,
1413 data, nhoff, ip_proto, hlen);
1414
1415
1416 switch (fdret) {
1417 case FLOW_DISSECT_RET_PROTO_AGAIN:
1418 if (skb_flow_dissect_allowed(&num_hdrs))
1419 goto proto_again;
1420 break;
1421 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1422 if (skb_flow_dissect_allowed(&num_hdrs))
1423 goto ip_proto_again;
1424 break;
1425 case FLOW_DISSECT_RET_OUT_GOOD:
1426 case FLOW_DISSECT_RET_CONTINUE:
1427 break;
1428 case FLOW_DISSECT_RET_OUT_BAD:
1429 default:
1430 goto out_bad;
1431 }
1432
1433out_good:
1434 ret = true;
1435
1436out:
1437 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
1438 key_basic->n_proto = proto;
1439 key_basic->ip_proto = ip_proto;
1440
1441 return ret;
1442
1443out_bad:
1444 ret = false;
1445 goto out;
1446}
1447EXPORT_SYMBOL(__skb_flow_dissect);
1448
1449static siphash_key_t hashrnd __read_mostly;
1450static __always_inline void __flow_hash_secret_init(void)
1451{
1452 net_get_random_once(&hashrnd, sizeof(hashrnd));
1453}
1454
1455static const void *flow_keys_hash_start(const struct flow_keys *flow)
1456{
1457 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % SIPHASH_ALIGNMENT);
1458 return &flow->FLOW_KEYS_HASH_START_FIELD;
1459}
1460
1461static inline size_t flow_keys_hash_length(const struct flow_keys *flow)
1462{
1463 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
1464
1465 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
1466
1467 switch (flow->control.addr_type) {
1468 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1469 diff -= sizeof(flow->addrs.v4addrs);
1470 break;
1471 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1472 diff -= sizeof(flow->addrs.v6addrs);
1473 break;
1474 case FLOW_DISSECTOR_KEY_TIPC:
1475 diff -= sizeof(flow->addrs.tipckey);
1476 break;
1477 }
1478 return sizeof(*flow) - diff;
1479}
1480
1481__be32 flow_get_u32_src(const struct flow_keys *flow)
1482{
1483 switch (flow->control.addr_type) {
1484 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1485 return flow->addrs.v4addrs.src;
1486 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1487 return (__force __be32)ipv6_addr_hash(
1488 &flow->addrs.v6addrs.src);
1489 case FLOW_DISSECTOR_KEY_TIPC:
1490 return flow->addrs.tipckey.key;
1491 default:
1492 return 0;
1493 }
1494}
1495EXPORT_SYMBOL(flow_get_u32_src);
1496
1497__be32 flow_get_u32_dst(const struct flow_keys *flow)
1498{
1499 switch (flow->control.addr_type) {
1500 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1501 return flow->addrs.v4addrs.dst;
1502 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1503 return (__force __be32)ipv6_addr_hash(
1504 &flow->addrs.v6addrs.dst);
1505 default:
1506 return 0;
1507 }
1508}
1509EXPORT_SYMBOL(flow_get_u32_dst);
1510
1511
1512
1513
1514static inline void __flow_hash_consistentify(struct flow_keys *keys)
1515{
1516 int addr_diff, i;
1517
1518 switch (keys->control.addr_type) {
1519 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1520 addr_diff = (__force u32)keys->addrs.v4addrs.dst -
1521 (__force u32)keys->addrs.v4addrs.src;
1522 if (addr_diff < 0)
1523 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
1524
1525 if ((__force u16)keys->ports.dst <
1526 (__force u16)keys->ports.src) {
1527 swap(keys->ports.src, keys->ports.dst);
1528 }
1529 break;
1530 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1531 addr_diff = memcmp(&keys->addrs.v6addrs.dst,
1532 &keys->addrs.v6addrs.src,
1533 sizeof(keys->addrs.v6addrs.dst));
1534 if (addr_diff < 0) {
1535 for (i = 0; i < 4; i++)
1536 swap(keys->addrs.v6addrs.src.s6_addr32[i],
1537 keys->addrs.v6addrs.dst.s6_addr32[i]);
1538 }
1539 if ((__force u16)keys->ports.dst <
1540 (__force u16)keys->ports.src) {
1541 swap(keys->ports.src, keys->ports.dst);
1542 }
1543 break;
1544 }
1545}
1546
1547static inline u32 __flow_hash_from_keys(struct flow_keys *keys,
1548 const siphash_key_t *keyval)
1549{
1550 u32 hash;
1551
1552 __flow_hash_consistentify(keys);
1553
1554 hash = siphash(flow_keys_hash_start(keys),
1555 flow_keys_hash_length(keys), keyval);
1556 if (!hash)
1557 hash = 1;
1558
1559 return hash;
1560}
1561
1562u32 flow_hash_from_keys(struct flow_keys *keys)
1563{
1564 __flow_hash_secret_init();
1565 return __flow_hash_from_keys(keys, &hashrnd);
1566}
1567EXPORT_SYMBOL(flow_hash_from_keys);
1568
1569static inline u32 ___skb_get_hash(const struct sk_buff *skb,
1570 struct flow_keys *keys,
1571 const siphash_key_t *keyval)
1572{
1573 skb_flow_dissect_flow_keys(skb, keys,
1574 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1575
1576 return __flow_hash_from_keys(keys, keyval);
1577}
1578
1579struct _flow_keys_digest_data {
1580 __be16 n_proto;
1581 u8 ip_proto;
1582 u8 padding;
1583 __be32 ports;
1584 __be32 src;
1585 __be32 dst;
1586};
1587
1588void make_flow_keys_digest(struct flow_keys_digest *digest,
1589 const struct flow_keys *flow)
1590{
1591 struct _flow_keys_digest_data *data =
1592 (struct _flow_keys_digest_data *)digest;
1593
1594 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
1595
1596 memset(digest, 0, sizeof(*digest));
1597
1598 data->n_proto = flow->basic.n_proto;
1599 data->ip_proto = flow->basic.ip_proto;
1600 data->ports = flow->ports.ports;
1601 data->src = flow->addrs.v4addrs.src;
1602 data->dst = flow->addrs.v4addrs.dst;
1603}
1604EXPORT_SYMBOL(make_flow_keys_digest);
1605
1606static struct flow_dissector flow_keys_dissector_symmetric __read_mostly;
1607
1608u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
1609{
1610 struct flow_keys keys;
1611
1612 __flow_hash_secret_init();
1613
1614 memset(&keys, 0, sizeof(keys));
1615 __skb_flow_dissect(NULL, skb, &flow_keys_dissector_symmetric,
1616 &keys, NULL, 0, 0, 0,
1617 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1618
1619 return __flow_hash_from_keys(&keys, &hashrnd);
1620}
1621EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric);
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632void __skb_get_hash(struct sk_buff *skb)
1633{
1634 struct flow_keys keys;
1635 u32 hash;
1636
1637 __flow_hash_secret_init();
1638
1639 hash = ___skb_get_hash(skb, &keys, &hashrnd);
1640
1641 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1642}
1643EXPORT_SYMBOL(__skb_get_hash);
1644
1645__u32 skb_get_hash_perturb(const struct sk_buff *skb,
1646 const siphash_key_t *perturb)
1647{
1648 struct flow_keys keys;
1649
1650 return ___skb_get_hash(skb, &keys, perturb);
1651}
1652EXPORT_SYMBOL(skb_get_hash_perturb);
1653
1654u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
1655 const struct flow_keys_basic *keys, int hlen)
1656{
1657 u32 poff = keys->control.thoff;
1658
1659
1660 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) &&
1661 !(keys->control.flags & FLOW_DIS_FIRST_FRAG))
1662 return poff;
1663
1664 switch (keys->basic.ip_proto) {
1665 case IPPROTO_TCP: {
1666
1667 const u8 *doff;
1668 u8 _doff;
1669
1670 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
1671 data, hlen, &_doff);
1672 if (!doff)
1673 return poff;
1674
1675 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
1676 break;
1677 }
1678 case IPPROTO_UDP:
1679 case IPPROTO_UDPLITE:
1680 poff += sizeof(struct udphdr);
1681 break;
1682
1683
1684
1685 case IPPROTO_ICMP:
1686 poff += sizeof(struct icmphdr);
1687 break;
1688 case IPPROTO_ICMPV6:
1689 poff += sizeof(struct icmp6hdr);
1690 break;
1691 case IPPROTO_IGMP:
1692 poff += sizeof(struct igmphdr);
1693 break;
1694 case IPPROTO_DCCP:
1695 poff += sizeof(struct dccp_hdr);
1696 break;
1697 case IPPROTO_SCTP:
1698 poff += sizeof(struct sctphdr);
1699 break;
1700 }
1701
1702 return poff;
1703}
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714u32 skb_get_poff(const struct sk_buff *skb)
1715{
1716 struct flow_keys_basic keys;
1717
1718 if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
1719 NULL, 0, 0, 0, 0))
1720 return 0;
1721
1722 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
1723}
1724
1725__u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys)
1726{
1727 memset(keys, 0, sizeof(*keys));
1728
1729 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr,
1730 sizeof(keys->addrs.v6addrs.src));
1731 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr,
1732 sizeof(keys->addrs.v6addrs.dst));
1733 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1734 keys->ports.src = fl6->fl6_sport;
1735 keys->ports.dst = fl6->fl6_dport;
1736 keys->keyid.keyid = fl6->fl6_gre_key;
1737 keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
1738 keys->basic.ip_proto = fl6->flowi6_proto;
1739
1740 return flow_hash_from_keys(keys);
1741}
1742EXPORT_SYMBOL(__get_hash_from_flowi6);
1743
1744static const struct flow_dissector_key flow_keys_dissector_keys[] = {
1745 {
1746 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1747 .offset = offsetof(struct flow_keys, control),
1748 },
1749 {
1750 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1751 .offset = offsetof(struct flow_keys, basic),
1752 },
1753 {
1754 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1755 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1756 },
1757 {
1758 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1759 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1760 },
1761 {
1762 .key_id = FLOW_DISSECTOR_KEY_TIPC,
1763 .offset = offsetof(struct flow_keys, addrs.tipckey),
1764 },
1765 {
1766 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1767 .offset = offsetof(struct flow_keys, ports),
1768 },
1769 {
1770 .key_id = FLOW_DISSECTOR_KEY_VLAN,
1771 .offset = offsetof(struct flow_keys, vlan),
1772 },
1773 {
1774 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL,
1775 .offset = offsetof(struct flow_keys, tags),
1776 },
1777 {
1778 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
1779 .offset = offsetof(struct flow_keys, keyid),
1780 },
1781};
1782
1783static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = {
1784 {
1785 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1786 .offset = offsetof(struct flow_keys, control),
1787 },
1788 {
1789 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1790 .offset = offsetof(struct flow_keys, basic),
1791 },
1792 {
1793 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1794 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1795 },
1796 {
1797 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1798 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1799 },
1800 {
1801 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1802 .offset = offsetof(struct flow_keys, ports),
1803 },
1804};
1805
1806static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = {
1807 {
1808 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1809 .offset = offsetof(struct flow_keys, control),
1810 },
1811 {
1812 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1813 .offset = offsetof(struct flow_keys, basic),
1814 },
1815};
1816
1817struct flow_dissector flow_keys_dissector __read_mostly;
1818EXPORT_SYMBOL(flow_keys_dissector);
1819
1820struct flow_dissector flow_keys_basic_dissector __read_mostly;
1821EXPORT_SYMBOL(flow_keys_basic_dissector);
1822
1823static int __init init_default_flow_dissectors(void)
1824{
1825 skb_flow_dissector_init(&flow_keys_dissector,
1826 flow_keys_dissector_keys,
1827 ARRAY_SIZE(flow_keys_dissector_keys));
1828 skb_flow_dissector_init(&flow_keys_dissector_symmetric,
1829 flow_keys_dissector_symmetric_keys,
1830 ARRAY_SIZE(flow_keys_dissector_symmetric_keys));
1831 skb_flow_dissector_init(&flow_keys_basic_dissector,
1832 flow_keys_basic_dissector_keys,
1833 ARRAY_SIZE(flow_keys_basic_dissector_keys));
1834 return 0;
1835}
1836core_initcall(init_default_flow_dissectors);
1837
