1#include <linux/jhash.h>
2#include <linux/netfilter.h>
3#include <linux/rcupdate.h>
4#include <linux/rhashtable.h>
5#include <linux/vmalloc.h>
6#include <net/genetlink.h>
7#include <net/ila.h>
8#include <net/netns/generic.h>
9#include <uapi/linux/genetlink.h>
10#include "ila.h"
11
12struct ila_xlat_params {
13 struct ila_params ip;
14 int ifindex;
15};
16
17struct ila_map {
18 struct ila_xlat_params xp;
19 struct rhash_head node;
20 struct ila_map __rcu *next;
21 struct rcu_head rcu;
22};
23
24static unsigned int ila_net_id;
25
26struct ila_net {
27 struct rhashtable rhash_table;
28 spinlock_t *locks;
29 unsigned int locks_mask;
30 bool hooks_registered;
31};
32
33#define LOCKS_PER_CPU 10
34
35static int alloc_ila_locks(struct ila_net *ilan)
36{
37 unsigned int i, size;
38 unsigned int nr_pcpus = num_possible_cpus();
39
40 nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
41 size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
42
43 if (sizeof(spinlock_t) != 0) {
44#ifdef CONFIG_NUMA
45 if (size * sizeof(spinlock_t) > PAGE_SIZE)
46 ilan->locks = vmalloc(size * sizeof(spinlock_t));
47 else
48#endif
49 ilan->locks = kmalloc_array(size, sizeof(spinlock_t),
50 GFP_KERNEL);
51 if (!ilan->locks)
52 return -ENOMEM;
53 for (i = 0; i < size; i++)
54 spin_lock_init(&ilan->locks[i]);
55 }
56 ilan->locks_mask = size - 1;
57
58 return 0;
59}
60
61static u32 hashrnd __read_mostly;
62static __always_inline void __ila_hash_secret_init(void)
63{
64 net_get_random_once(&hashrnd, sizeof(hashrnd));
65}
66
67static inline u32 ila_locator_hash(struct ila_locator loc)
68{
69 u32 *v = (u32 *)loc.v32;
70
71 return jhash_2words(v[0], v[1], hashrnd);
72}
73
74static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
75 struct ila_locator loc)
76{
77 return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
78}
79
80static inline int ila_cmp_wildcards(struct ila_map *ila,
81 struct ila_addr *iaddr, int ifindex)
82{
83 return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
84}
85
86static inline int ila_cmp_params(struct ila_map *ila,
87 struct ila_xlat_params *xp)
88{
89 return (ila->xp.ifindex != xp->ifindex);
90}
91
92static int ila_cmpfn(struct rhashtable_compare_arg *arg,
93 const void *obj)
94{
95 const struct ila_map *ila = obj;
96
97 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
98}
99
100static inline int ila_order(struct ila_map *ila)
101{
102 int score = 0;
103
104 if (ila->xp.ifindex)
105 score += 1 << 1;
106
107 return score;
108}
109
110static const struct rhashtable_params rht_params = {
111 .nelem_hint = 1024,
112 .head_offset = offsetof(struct ila_map, node),
113 .key_offset = offsetof(struct ila_map, xp.ip.locator_match),
114 .key_len = sizeof(u64),
115 .max_size = 1048576,
116 .min_size = 256,
117 .automatic_shrinking = true,
118 .obj_cmpfn = ila_cmpfn,
119};
120
121static struct genl_family ila_nl_family = {
122 .id = GENL_ID_GENERATE,
123 .hdrsize = 0,
124 .name = ILA_GENL_NAME,
125 .version = ILA_GENL_VERSION,
126 .maxattr = ILA_ATTR_MAX,
127 .netnsok = true,
128 .parallel_ops = true,
129};
130
131static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
132 [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
133 [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
134 [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
135 [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
136};
137
138static int parse_nl_config(struct genl_info *info,
139 struct ila_xlat_params *xp)
140{
141 memset(xp, 0, sizeof(*xp));
142
143 if (info->attrs[ILA_ATTR_LOCATOR])
144 xp->ip.locator.v64 = (__force __be64)nla_get_u64(
145 info->attrs[ILA_ATTR_LOCATOR]);
146
147 if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
148 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
149 info->attrs[ILA_ATTR_LOCATOR_MATCH]);
150
151 if (info->attrs[ILA_ATTR_CSUM_MODE])
152 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
153
154 if (info->attrs[ILA_ATTR_IFINDEX])
155 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
156
157 return 0;
158}
159
160
161static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
162 int ifindex,
163 struct ila_net *ilan)
164{
165 struct ila_map *ila;
166
167 ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
168 rht_params);
169 while (ila) {
170 if (!ila_cmp_wildcards(ila, iaddr, ifindex))
171 return ila;
172 ila = rcu_access_pointer(ila->next);
173 }
174
175 return NULL;
176}
177
178
179static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
180 struct ila_net *ilan)
181{
182 struct ila_map *ila;
183
184 ila = rhashtable_lookup_fast(&ilan->rhash_table,
185 &xp->ip.locator_match,
186 rht_params);
187 while (ila) {
188 if (!ila_cmp_params(ila, xp))
189 return ila;
190 ila = rcu_access_pointer(ila->next);
191 }
192
193 return NULL;
194}
195
196static inline void ila_release(struct ila_map *ila)
197{
198 kfree_rcu(ila, rcu);
199}
200
201static void ila_free_cb(void *ptr, void *arg)
202{
203 struct ila_map *ila = (struct ila_map *)ptr, *next;
204
205
206 while (ila) {
207 next = rcu_access_pointer(ila->next);
208 ila_release(ila);
209 ila = next;
210 }
211}
212
213static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);
214
215static unsigned int
216ila_nf_input(void *priv,
217 struct sk_buff *skb,
218 const struct nf_hook_state *state)
219{
220 ila_xlat_addr(skb, false);
221 return NF_ACCEPT;
222}
223
224static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
225 {
226 .hook = ila_nf_input,
227 .pf = NFPROTO_IPV6,
228 .hooknum = NF_INET_PRE_ROUTING,
229 .priority = -1,
230 },
231};
232
233static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
234{
235 struct ila_net *ilan = net_generic(net, ila_net_id);
236 struct ila_map *ila, *head;
237 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
238 int err = 0, order;
239
240 if (!ilan->hooks_registered) {
241
242
243
244 err = nf_register_net_hooks(net, ila_nf_hook_ops,
245 ARRAY_SIZE(ila_nf_hook_ops));
246 if (err)
247 return err;
248
249 ilan->hooks_registered = true;
250 }
251
252 ila = kzalloc(sizeof(*ila), GFP_KERNEL);
253 if (!ila)
254 return -ENOMEM;
255
256 ila_init_saved_csum(&xp->ip);
257
258 ila->xp = *xp;
259
260 order = ila_order(ila);
261
262 spin_lock(lock);
263
264 head = rhashtable_lookup_fast(&ilan->rhash_table,
265 &xp->ip.locator_match,
266 rht_params);
267 if (!head) {
268
269 err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
270 &ila->node, rht_params);
271 } else {
272 struct ila_map *tila = head, *prev = NULL;
273
274 do {
275 if (!ila_cmp_params(tila, xp)) {
276 err = -EEXIST;
277 goto out;
278 }
279
280 if (order > ila_order(tila))
281 break;
282
283 prev = tila;
284 tila = rcu_dereference_protected(tila->next,
285 lockdep_is_held(lock));
286 } while (tila);
287
288 if (prev) {
289
290 RCU_INIT_POINTER(ila->next, tila);
291 rcu_assign_pointer(prev->next, ila);
292 } else {
293
294 RCU_INIT_POINTER(ila->next, head);
295 err = rhashtable_replace_fast(&ilan->rhash_table,
296 &head->node,
297 &ila->node, rht_params);
298 if (err)
299 goto out;
300 }
301 }
302
303out:
304 spin_unlock(lock);
305
306 if (err)
307 kfree(ila);
308
309 return err;
310}
311
312static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
313{
314 struct ila_net *ilan = net_generic(net, ila_net_id);
315 struct ila_map *ila, *head, *prev;
316 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
317 int err = -ENOENT;
318
319 spin_lock(lock);
320
321 head = rhashtable_lookup_fast(&ilan->rhash_table,
322 &xp->ip.locator_match, rht_params);
323 ila = head;
324
325 prev = NULL;
326
327 while (ila) {
328 if (ila_cmp_params(ila, xp)) {
329 prev = ila;
330 ila = rcu_dereference_protected(ila->next,
331 lockdep_is_held(lock));
332 continue;
333 }
334
335 err = 0;
336
337 if (prev) {
338
339 rcu_assign_pointer(prev->next, ila->next);
340 } else {
341
342
343
344 head = rcu_dereference_protected(ila->next,
345 lockdep_is_held(lock));
346 if (head) {
347
348
349
350 err = rhashtable_replace_fast(
351 &ilan->rhash_table, &ila->node,
352 &head->node, rht_params);
353 if (err)
354 goto out;
355 } else {
356
357 err = rhashtable_remove_fast(&ilan->rhash_table,
358 &ila->node,
359 rht_params);
360 }
361 }
362
363 ila_release(ila);
364
365 break;
366 }
367
368out:
369 spin_unlock(lock);
370
371 return err;
372}
373
374static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
375{
376 struct net *net = genl_info_net(info);
377 struct ila_xlat_params p;
378 int err;
379
380 err = parse_nl_config(info, &p);
381 if (err)
382 return err;
383
384 return ila_add_mapping(net, &p);
385}
386
387static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
388{
389 struct net *net = genl_info_net(info);
390 struct ila_xlat_params xp;
391 int err;
392
393 err = parse_nl_config(info, &xp);
394 if (err)
395 return err;
396
397 ila_del_mapping(net, &xp);
398
399 return 0;
400}
401
402static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
403{
404 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
405 (__force u64)ila->xp.ip.locator.v64,
406 ILA_ATTR_PAD) ||
407 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
408 (__force u64)ila->xp.ip.locator_match.v64,
409 ILA_ATTR_PAD) ||
410 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
411 nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
412 return -1;
413
414 return 0;
415}
416
417static int ila_dump_info(struct ila_map *ila,
418 u32 portid, u32 seq, u32 flags,
419 struct sk_buff *skb, u8 cmd)
420{
421 void *hdr;
422
423 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
424 if (!hdr)
425 return -ENOMEM;
426
427 if (ila_fill_info(ila, skb) < 0)
428 goto nla_put_failure;
429
430 genlmsg_end(skb, hdr);
431 return 0;
432
433nla_put_failure:
434 genlmsg_cancel(skb, hdr);
435 return -EMSGSIZE;
436}
437
438static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
439{
440 struct net *net = genl_info_net(info);
441 struct ila_net *ilan = net_generic(net, ila_net_id);
442 struct sk_buff *msg;
443 struct ila_xlat_params xp;
444 struct ila_map *ila;
445 int ret;
446
447 ret = parse_nl_config(info, &xp);
448 if (ret)
449 return ret;
450
451 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
452 if (!msg)
453 return -ENOMEM;
454
455 rcu_read_lock();
456
457 ila = ila_lookup_by_params(&xp, ilan);
458 if (ila) {
459 ret = ila_dump_info(ila,
460 info->snd_portid,
461 info->snd_seq, 0, msg,
462 info->genlhdr->cmd);
463 }
464
465 rcu_read_unlock();
466
467 if (ret < 0)
468 goto out_free;
469
470 return genlmsg_reply(msg, info);
471
472out_free:
473 nlmsg_free(msg);
474 return ret;
475}
476
477struct ila_dump_iter {
478 struct rhashtable_iter rhiter;
479};
480
481static int ila_nl_dump_start(struct netlink_callback *cb)
482{
483 struct net *net = sock_net(cb->skb->sk);
484 struct ila_net *ilan = net_generic(net, ila_net_id);
485 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
486
487 return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
488 GFP_KERNEL);
489}
490
491static int ila_nl_dump_done(struct netlink_callback *cb)
492{
493 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
494
495 rhashtable_walk_exit(&iter->rhiter);
496
497 return 0;
498}
499
500static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
501{
502 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
503 struct rhashtable_iter *rhiter = &iter->rhiter;
504 struct ila_map *ila;
505 int ret;
506
507 ret = rhashtable_walk_start(rhiter);
508 if (ret && ret != -EAGAIN)
509 goto done;
510
511 for (;;) {
512 ila = rhashtable_walk_next(rhiter);
513
514 if (IS_ERR(ila)) {
515 if (PTR_ERR(ila) == -EAGAIN)
516 continue;
517 ret = PTR_ERR(ila);
518 goto done;
519 } else if (!ila) {
520 break;
521 }
522
523 while (ila) {
524 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
525 cb->nlh->nlmsg_seq, NLM_F_MULTI,
526 skb, ILA_CMD_GET);
527 if (ret)
528 goto done;
529
530 ila = rcu_access_pointer(ila->next);
531 }
532 }
533
534 ret = skb->len;
535
536done:
537 rhashtable_walk_stop(rhiter);
538 return ret;
539}
540
541static const struct genl_ops ila_nl_ops[] = {
542 {
543 .cmd = ILA_CMD_ADD,
544 .doit = ila_nl_cmd_add_mapping,
545 .policy = ila_nl_policy,
546 .flags = GENL_ADMIN_PERM,
547 },
548 {
549 .cmd = ILA_CMD_DEL,
550 .doit = ila_nl_cmd_del_mapping,
551 .policy = ila_nl_policy,
552 .flags = GENL_ADMIN_PERM,
553 },
554 {
555 .cmd = ILA_CMD_GET,
556 .doit = ila_nl_cmd_get_mapping,
557 .start = ila_nl_dump_start,
558 .dumpit = ila_nl_dump,
559 .done = ila_nl_dump_done,
560 .policy = ila_nl_policy,
561 },
562};
563
564#define ILA_HASH_TABLE_SIZE 1024
565
566static __net_init int ila_init_net(struct net *net)
567{
568 int err;
569 struct ila_net *ilan = net_generic(net, ila_net_id);
570
571 err = alloc_ila_locks(ilan);
572 if (err)
573 return err;
574
575 rhashtable_init(&ilan->rhash_table, &rht_params);
576
577 return 0;
578}
579
580static __net_exit void ila_exit_net(struct net *net)
581{
582 struct ila_net *ilan = net_generic(net, ila_net_id);
583
584 rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
585
586 kvfree(ilan->locks);
587
588 if (ilan->hooks_registered)
589 nf_unregister_net_hooks(net, ila_nf_hook_ops,
590 ARRAY_SIZE(ila_nf_hook_ops));
591}
592
593static struct pernet_operations ila_net_ops = {
594 .init = ila_init_net,
595 .exit = ila_exit_net,
596 .id = &ila_net_id,
597 .size = sizeof(struct ila_net),
598};
599
600static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
601{
602 struct ila_map *ila;
603 struct ipv6hdr *ip6h = ipv6_hdr(skb);
604 struct net *net = dev_net(skb->dev);
605 struct ila_net *ilan = net_generic(net, ila_net_id);
606 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
607
608
609
610 if (!ila_addr_is_ila(iaddr)) {
611
612 return 0;
613 }
614
615 rcu_read_lock();
616
617 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
618 if (ila)
619 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);
620
621 rcu_read_unlock();
622
623 return 0;
624}
625
626int ila_xlat_init(void)
627{
628 int ret;
629
630 ret = register_pernet_device(&ila_net_ops);
631 if (ret)
632 goto exit;
633
634 ret = genl_register_family_with_ops(&ila_nl_family,
635 ila_nl_ops);
636 if (ret < 0)
637 goto unregister;
638
639 return 0;
640
641unregister:
642 unregister_pernet_device(&ila_net_ops);
643exit:
644 return ret;
645}
646
647void ila_xlat_fini(void)
648{
649 genl_unregister_family(&ila_nl_family);
650 unregister_pernet_device(&ila_net_ops);
651}
652
