1
2#include <linux/jhash.h>
3#include <linux/netfilter.h>
4#include <linux/rcupdate.h>
5#include <linux/rhashtable.h>
6#include <linux/vmalloc.h>
7#include <net/genetlink.h>
8#include <net/ila.h>
9#include <net/netns/generic.h>
10#include <uapi/linux/genetlink.h>
11#include "ila.h"
12
13struct ila_xlat_params {
14 struct ila_params ip;
15 int ifindex;
16};
17
18struct ila_map {
19 struct ila_xlat_params xp;
20 struct rhash_head node;
21 struct ila_map __rcu *next;
22 struct rcu_head rcu;
23};
24
25static unsigned int ila_net_id;
26
27struct ila_net {
28 struct rhashtable rhash_table;
29 spinlock_t *locks;
30 unsigned int locks_mask;
31 bool hooks_registered;
32};
33
34#define LOCKS_PER_CPU 10
35
36static int alloc_ila_locks(struct ila_net *ilan)
37{
38 unsigned int i, size;
39 unsigned int nr_pcpus = num_possible_cpus();
40
41 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
42 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
43
44 if (sizeof(spinlock_t) != 0) {
45 ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL);
46 if (!ilan->locks)
47 return -ENOMEM;
48 for (i = 0; i < size; i++)
49 spin_lock_init(&ilan->locks[i]);
50 }
51 ilan->locks_mask = size - 1;
52
53 return 0;
54}
55
56static u32 hashrnd __read_mostly;
57static __always_inline void __ila_hash_secret_init(void)
58{
59 net_get_random_once(&hashrnd, sizeof(hashrnd));
60}
61
62static inline u32 ila_locator_hash(struct ila_locator loc)
63{
64 u32 *v = (u32 *)loc.v32;
65
66 __ila_hash_secret_init();
67 return jhash_2words(v[0], v[1], hashrnd);
68}
69
70static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
71 struct ila_locator loc)
72{
73 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
74}
75
76static inline int ila_cmp_wildcards(struct ila_map *ila,
77 struct ila_addr *iaddr, int ifindex)
78{
79 return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
80}
81
82static inline int ila_cmp_params(struct ila_map *ila,
83 struct ila_xlat_params *xp)
84{
85 return (ila->xp.ifindex != xp->ifindex);
86}
87
88static int ila_cmpfn(struct rhashtable_compare_arg *arg,
89 const void *obj)
90{
91 const struct ila_map *ila = obj;
92
93 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
94}
95
96static inline int ila_order(struct ila_map *ila)
97{
98 int score = 0;
99
100 if (ila->xp.ifindex)
101 score += 1 << 1;
102
103 return score;
104}
105
106static const struct rhashtable_params rht_params = {
107 .nelem_hint = 1024,
108 .head_offset = offsetof(struct ila_map, node),
109 .key_offset = offsetof(struct ila_map, xp.ip.locator_match),
110 .key_len = sizeof(u64),
111 .max_size = 1048576,
112 .min_size = 256,
113 .automatic_shrinking = true,
114 .obj_cmpfn = ila_cmpfn,
115};
116
117static struct genl_family ila_nl_family;
118
119static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
120 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
121 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
122 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
123 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
124 [ILA_ATTR_IDENT_TYPE] = { .type = NLA_U8, },
125};
126
127static int parse_nl_config(struct genl_info *info,
128 struct ila_xlat_params *xp)
129{
130 memset(xp, 0, sizeof(*xp));
131
132 if (info->attrs[ILA_ATTR_LOCATOR])
133 xp->ip.locator.v64 = (__force __be64)nla_get_u64(
134 info->attrs[ILA_ATTR_LOCATOR]);
135
136 if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
137 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
138 info->attrs[ILA_ATTR_LOCATOR_MATCH]);
139
140 if (info->attrs[ILA_ATTR_CSUM_MODE])
141 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
142 else
143 xp->ip.csum_mode = ILA_CSUM_NO_ACTION;
144
145 if (info->attrs[ILA_ATTR_IDENT_TYPE])
146 xp->ip.ident_type = nla_get_u8(
147 info->attrs[ILA_ATTR_IDENT_TYPE]);
148 else
149 xp->ip.ident_type = ILA_ATYPE_USE_FORMAT;
150
151 if (info->attrs[ILA_ATTR_IFINDEX])
152 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
153
154 return 0;
155}
156
157
158static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
159 int ifindex,
160 struct ila_net *ilan)
161{
162 struct ila_map *ila;
163
164 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
165 rht_params);
166 while (ila) {
167 if (!ila_cmp_wildcards(ila, iaddr, ifindex))
168 return ila;
169 ila = rcu_access_pointer(ila->next);
170 }
171
172 return NULL;
173}
174
175
176static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
177 struct ila_net *ilan)
178{
179 struct ila_map *ila;
180
181 ila = rhashtable_lookup_fast(&ilan->rhash_table,
182 &xp->ip.locator_match,
183 rht_params);
184 while (ila) {
185 if (!ila_cmp_params(ila, xp))
186 return ila;
187 ila = rcu_access_pointer(ila->next);
188 }
189
190 return NULL;
191}
192
193static inline void ila_release(struct ila_map *ila)
194{
195 kfree_rcu(ila, rcu);
196}
197
198static void ila_free_cb(void *ptr, void *arg)
199{
200 struct ila_map *ila = (struct ila_map *)ptr, *next;
201
202
203 while (ila) {
204 next = rcu_access_pointer(ila->next);
205 ila_release(ila);
206 ila = next;
207 }
208}
209
210static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila);
211
212static unsigned int
213ila_nf_input(void *priv,
214 struct sk_buff *skb,
215 const struct nf_hook_state *state)
216{
217 ila_xlat_addr(skb, false);
218 return NF_ACCEPT;
219}
220
221static const struct nf_hook_ops ila_nf_hook_ops[] = {
222 {
223 .hook = ila_nf_input,
224 .pf = NFPROTO_IPV6,
225 .hooknum = NF_INET_PRE_ROUTING,
226 .priority = -1,
227 },
228};
229
230static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
231{
232 struct ila_net *ilan = net_generic(net, ila_net_id);
233 struct ila_map *ila, *head;
234 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
235 int err = 0, order;
236
237 if (!ilan->hooks_registered) {
238
239
240
241 err = nf_register_net_hooks(net, ila_nf_hook_ops,
242 ARRAY_SIZE(ila_nf_hook_ops));
243 if (err)
244 return err;
245
246 ilan->hooks_registered = true;
247 }
248
249 ila = kzalloc(sizeof(*ila), GFP_KERNEL);
250 if (!ila)
251 return -ENOMEM;
252
253 ila_init_saved_csum(&xp->ip);
254
255 ila->xp = *xp;
256
257 order = ila_order(ila);
258
259 spin_lock(lock);
260
261 head = rhashtable_lookup_fast(&ilan->rhash_table,
262 &xp->ip.locator_match,
263 rht_params);
264 if (!head) {
265
266 err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
267 &ila->node, rht_params);
268 } else {
269 struct ila_map *tila = head, *prev = NULL;
270
271 do {
272 if (!ila_cmp_params(tila, xp)) {
273 err = -EEXIST;
274 goto out;
275 }
276
277 if (order > ila_order(tila))
278 break;
279
280 prev = tila;
281 tila = rcu_dereference_protected(tila->next,
282 lockdep_is_held(lock));
283 } while (tila);
284
285 if (prev) {
286
287 RCU_INIT_POINTER(ila->next, tila);
288 rcu_assign_pointer(prev->next, ila);
289 } else {
290
291 RCU_INIT_POINTER(ila->next, head);
292 err = rhashtable_replace_fast(&ilan->rhash_table,
293 &head->node,
294 &ila->node, rht_params);
295 if (err)
296 goto out;
297 }
298 }
299
300out:
301 spin_unlock(lock);
302
303 if (err)
304 kfree(ila);
305
306 return err;
307}
308
309static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
310{
311 struct ila_net *ilan = net_generic(net, ila_net_id);
312 struct ila_map *ila, *head, *prev;
313 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
314 int err = -ENOENT;
315
316 spin_lock(lock);
317
318 head = rhashtable_lookup_fast(&ilan->rhash_table,
319 &xp->ip.locator_match, rht_params);
320 ila = head;
321
322 prev = NULL;
323
324 while (ila) {
325 if (ila_cmp_params(ila, xp)) {
326 prev = ila;
327 ila = rcu_dereference_protected(ila->next,
328 lockdep_is_held(lock));
329 continue;
330 }
331
332 err = 0;
333
334 if (prev) {
335
336 rcu_assign_pointer(prev->next, ila->next);
337 } else {
338
339
340
341 head = rcu_dereference_protected(ila->next,
342 lockdep_is_held(lock));
343 if (head) {
344
345
346
347 err = rhashtable_replace_fast(
348 &ilan->rhash_table, &ila->node,
349 &head->node, rht_params);
350 if (err)
351 goto out;
352 } else {
353
354 err = rhashtable_remove_fast(&ilan->rhash_table,
355 &ila->node,
356 rht_params);
357 }
358 }
359
360 ila_release(ila);
361
362 break;
363 }
364
365out:
366 spin_unlock(lock);
367
368 return err;
369}
370
371static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
372{
373 struct net *net = genl_info_net(info);
374 struct ila_xlat_params p;
375 int err;
376
377 err = parse_nl_config(info, &p);
378 if (err)
379 return err;
380
381 return ila_add_mapping(net, &p);
382}
383
384static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
385{
386 struct net *net = genl_info_net(info);
387 struct ila_xlat_params xp;
388 int err;
389
390 err = parse_nl_config(info, &xp);
391 if (err)
392 return err;
393
394 ila_del_mapping(net, &xp);
395
396 return 0;
397}
398
399static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
400{
401 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
402 (__force u64)ila->xp.ip.locator.v64,
403 ILA_ATTR_PAD) ||
404 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
405 (__force u64)ila->xp.ip.locator_match.v64,
406 ILA_ATTR_PAD) ||
407 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
408 nla_put_u8(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode) ||
409 nla_put_u8(msg, ILA_ATTR_IDENT_TYPE, ila->xp.ip.ident_type))
410 return -1;
411
412 return 0;
413}
414
415static int ila_dump_info(struct ila_map *ila,
416 u32 portid, u32 seq, u32 flags,
417 struct sk_buff *skb, u8 cmd)
418{
419 void *hdr;
420
421 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
422 if (!hdr)
423 return -ENOMEM;
424
425 if (ila_fill_info(ila, skb) < 0)
426 goto nla_put_failure;
427
428 genlmsg_end(skb, hdr);
429 return 0;
430
431nla_put_failure:
432 genlmsg_cancel(skb, hdr);
433 return -EMSGSIZE;
434}
435
436static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
437{
438 struct net *net = genl_info_net(info);
439 struct ila_net *ilan = net_generic(net, ila_net_id);
440 struct sk_buff *msg;
441 struct ila_xlat_params xp;
442 struct ila_map *ila;
443 int ret;
444
445 ret = parse_nl_config(info, &xp);
446 if (ret)
447 return ret;
448
449 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
450 if (!msg)
451 return -ENOMEM;
452
453 rcu_read_lock();
454
455 ila = ila_lookup_by_params(&xp, ilan);
456 if (ila) {
457 ret = ila_dump_info(ila,
458 info->snd_portid,
459 info->snd_seq, 0, msg,
460 info->genlhdr->cmd);
461 }
462
463 rcu_read_unlock();
464
465 if (ret < 0)
466 goto out_free;
467
468 return genlmsg_reply(msg, info);
469
470out_free:
471 nlmsg_free(msg);
472 return ret;
473}
474
475struct ila_dump_iter {
476 struct rhashtable_iter rhiter;
477};
478
479static int ila_nl_dump_start(struct netlink_callback *cb)
480{
481 struct net *net = sock_net(cb->skb->sk);
482 struct ila_net *ilan = net_generic(net, ila_net_id);
483 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
484
485 if (!iter) {
486 iter = kmalloc(sizeof(*iter), GFP_KERNEL);
487 if (!iter)
488 return -ENOMEM;
489
490 cb->args[0] = (long)iter;
491 }
492
493 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
494 GFP_KERNEL);
495}
496
497static int ila_nl_dump_done(struct netlink_callback *cb)
498{
499 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
500
501 rhashtable_walk_exit(&iter->rhiter);
502
503 kfree(iter);
504
505 return 0;
506}
507
508static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
509{
510 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
511 struct rhashtable_iter *rhiter = &iter->rhiter;
512 struct ila_map *ila;
513 int ret;
514
515 rhashtable_walk_start(rhiter);
516
517 for (;;) {
518 ila = rhashtable_walk_next(rhiter);
519
520 if (IS_ERR(ila)) {
521 if (PTR_ERR(ila) == -EAGAIN)
522 continue;
523 ret = PTR_ERR(ila);
524 goto done;
525 } else if (!ila) {
526 break;
527 }
528
529 while (ila) {
530 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
531 cb->nlh->nlmsg_seq, NLM_F_MULTI,
532 skb, ILA_CMD_GET);
533 if (ret)
534 goto done;
535
536 ila = rcu_access_pointer(ila->next);
537 }
538 }
539
540 ret = skb->len;
541
542done:
543 rhashtable_walk_stop(rhiter);
544 return ret;
545}
546
547static const struct genl_ops ila_nl_ops[] = {
548 {
549 .cmd = ILA_CMD_ADD,
550 .doit = ila_nl_cmd_add_mapping,
551 .policy = ila_nl_policy,
552 .flags = GENL_ADMIN_PERM,
553 },
554 {
555 .cmd = ILA_CMD_DEL,
556 .doit = ila_nl_cmd_del_mapping,
557 .policy = ila_nl_policy,
558 .flags = GENL_ADMIN_PERM,
559 },
560 {
561 .cmd = ILA_CMD_GET,
562 .doit = ila_nl_cmd_get_mapping,
563 .start = ila_nl_dump_start,
564 .dumpit = ila_nl_dump,
565 .done = ila_nl_dump_done,
566 .policy = ila_nl_policy,
567 },
568};
569
570static struct genl_family ila_nl_family __ro_after_init = {
571 .hdrsize = 0,
572 .name = ILA_GENL_NAME,
573 .version = ILA_GENL_VERSION,
574 .maxattr = ILA_ATTR_MAX,
575 .netnsok = true,
576 .parallel_ops = true,
577 .module = THIS_MODULE,
578 .ops = ila_nl_ops,
579 .n_ops = ARRAY_SIZE(ila_nl_ops),
580};
581
582#define ILA_HASH_TABLE_SIZE 1024
583
584static __net_init int ila_init_net(struct net *net)
585{
586 int err;
587 struct ila_net *ilan = net_generic(net, ila_net_id);
588
589 err = alloc_ila_locks(ilan);
590 if (err)
591 return err;
592
593 rhashtable_init(&ilan->rhash_table, &rht_params);
594
595 return 0;
596}
597
598static __net_exit void ila_exit_net(struct net *net)
599{
600 struct ila_net *ilan = net_generic(net, ila_net_id);
601
602 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
603
604 kvfree(ilan->locks);
605
606 if (ilan->hooks_registered)
607 nf_unregister_net_hooks(net, ila_nf_hook_ops,
608 ARRAY_SIZE(ila_nf_hook_ops));
609}
610
611static struct pernet_operations ila_net_ops = {
612 .init = ila_init_net,
613 .exit = ila_exit_net,
614 .id = &ila_net_id,
615 .size = sizeof(struct ila_net),
616};
617
618static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila)
619{
620 struct ila_map *ila;
621 struct ipv6hdr *ip6h = ipv6_hdr(skb);
622 struct net *net = dev_net(skb->dev);
623 struct ila_net *ilan = net_generic(net, ila_net_id);
624 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
625
626
627
628
629
630
631
632
633 rcu_read_lock();
634
635 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
636 if (ila)
637 ila_update_ipv6_locator(skb, &ila->xp.ip, sir2ila);
638
639 rcu_read_unlock();
640
641 return 0;
642}
643
644int __init ila_xlat_init(void)
645{
646 int ret;
647
648 ret = register_pernet_device(&ila_net_ops);
649 if (ret)
650 goto exit;
651
652 ret = genl_register_family(&ila_nl_family);
653 if (ret < 0)
654 goto unregister;
655
656 return 0;
657
658unregister:
659 unregister_pernet_device(&ila_net_ops);
660exit:
661 return ret;
662}
663
664void ila_xlat_fini(void)
665{
666 genl_unregister_family(&ila_nl_family);
667 unregister_pernet_device(&ila_net_ops);
668}
669
