LXR linux/kernel/bpf/net

   1// SPDX-License-Identifier: GPL-2.0
   2
   3#include <linux/bpf.h>
   4#include <linux/bpf-netns.h>
   5#include <linux/filter.h>
   6#include <net/net_namespace.h>
   7
   8/*
   9 * Functions to manage BPF programs attached to netns
  10 */
  11
  12struct bpf_netns_link {
  13        struct bpf_link link;
  14        enum bpf_attach_type type;
  15        enum netns_bpf_attach_type netns_type;
  16
  17        /* We don't hold a ref to net in order to auto-detach the link
  18         * when netns is going away. Instead we rely on pernet
  19         * pre_exit callback to clear this pointer. Must be accessed
  20         * with netns_bpf_mutex held.
  21         */
  22        struct net *net;
  23        struct list_head node; /* node in list of links attached to net */
  24};
  25
  26/* Protects updates to netns_bpf */
  27DEFINE_MUTEX(netns_bpf_mutex);
  28
  29static void netns_bpf_attach_type_unneed(enum netns_bpf_attach_type type)
  30{
  31        switch (type) {
  32#ifdef CONFIG_INET
  33        case NETNS_BPF_SK_LOOKUP:
  34                static_branch_dec(&bpf_sk_lookup_enabled);
  35                break;
  36#endif
  37        default:
  38                break;
  39        }
  40}
  41
  42static void netns_bpf_attach_type_need(enum netns_bpf_attach_type type)
  43{
  44        switch (type) {
  45#ifdef CONFIG_INET
  46        case NETNS_BPF_SK_LOOKUP:
  47                static_branch_inc(&bpf_sk_lookup_enabled);
  48                break;
  49#endif
  50        default:
  51                break;
  52        }
  53}
  54
  55/* Must be called with netns_bpf_mutex held. */
  56static void netns_bpf_run_array_detach(struct net *net,
  57                                       enum netns_bpf_attach_type type)
  58{
  59        struct bpf_prog_array *run_array;
  60
  61        run_array = rcu_replace_pointer(net->bpf.run_array[type], NULL,
  62                                        lockdep_is_held(&netns_bpf_mutex));
  63        bpf_prog_array_free(run_array);
  64}
  65
  66static int link_index(struct net *net, enum netns_bpf_attach_type type,
  67                      struct bpf_netns_link *link)
  68{
  69        struct bpf_netns_link *pos;
  70        int i = 0;
  71
  72        list_for_each_entry(pos, &net->bpf.links[type], node) {
  73                if (pos == link)
  74                        return i;
  75                i++;
  76        }
  77        return -ENOENT;
  78}
  79
  80static int link_count(struct net *net, enum netns_bpf_attach_type type)
  81{
  82        struct list_head *pos;
  83        int i = 0;
  84
  85        list_for_each(pos, &net->bpf.links[type])
  86                i++;
  87        return i;
  88}
  89
  90static void fill_prog_array(struct net *net, enum netns_bpf_attach_type type,
  91                            struct bpf_prog_array *prog_array)
  92{
  93        struct bpf_netns_link *pos;
  94        unsigned int i = 0;
  95
  96        list_for_each_entry(pos, &net->bpf.links[type], node) {
  97                prog_array->items[i].prog = pos->link.prog;
  98                i++;
  99        }
 100}
 101
 102static void bpf_netns_link_release(struct bpf_link *link)
 103{
 104        struct bpf_netns_link *net_link =
 105                container_of(link, struct bpf_netns_link, link);
 106        enum netns_bpf_attach_type type = net_link->netns_type;
 107        struct bpf_prog_array *old_array, *new_array;
 108        struct net *net;
 109        int cnt, idx;
 110
 111        mutex_lock(&netns_bpf_mutex);
 112
 113        /* We can race with cleanup_net, but if we see a non-NULL
 114         * struct net pointer, pre_exit has not run yet and wait for
 115         * netns_bpf_mutex.
 116         */
 117        net = net_link->net;
 118        if (!net)
 119                goto out_unlock;
 120
 121        /* Mark attach point as unused */
 122        netns_bpf_attach_type_unneed(type);
 123
 124        /* Remember link position in case of safe delete */
 125        idx = link_index(net, type, net_link);
 126        list_del(&net_link->node);
 127
 128        cnt = link_count(net, type);
 129        if (!cnt) {
 130                netns_bpf_run_array_detach(net, type);
 131                goto out_unlock;
 132        }
 133
 134        old_array = rcu_dereference_protected(net->bpf.run_array[type],
 135                                              lockdep_is_held(&netns_bpf_mutex));
 136        new_array = bpf_prog_array_alloc(cnt, GFP_KERNEL);
 137        if (!new_array) {
 138                WARN_ON(bpf_prog_array_delete_safe_at(old_array, idx));
 139                goto out_unlock;
 140        }
 141        fill_prog_array(net, type, new_array);
 142        rcu_assign_pointer(net->bpf.run_array[type], new_array);
 143        bpf_prog_array_free(old_array);
 144
 145out_unlock:
 146        net_link->net = NULL;
 147        mutex_unlock(&netns_bpf_mutex);
 148}
 149
 150static int bpf_netns_link_detach(struct bpf_link *link)
 151{
 152        bpf_netns_link_release(link);
 153        return 0;
 154}
 155
 156static void bpf_netns_link_dealloc(struct bpf_link *link)
 157{
 158        struct bpf_netns_link *net_link =
 159                container_of(link, struct bpf_netns_link, link);
 160
 161        kfree(net_link);
 162}
 163
 164static int bpf_netns_link_update_prog(struct bpf_link *link,
 165                                      struct bpf_prog *new_prog,
 166                                      struct bpf_prog *old_prog)
 167{
 168        struct bpf_netns_link *net_link =
 169                container_of(link, struct bpf_netns_link, link);
 170        enum netns_bpf_attach_type type = net_link->netns_type;
 171        struct bpf_prog_array *run_array;
 172        struct net *net;
 173        int idx, ret;
 174
 175        if (old_prog && old_prog != link->prog)
 176                return -EPERM;
 177        if (new_prog->type != link->prog->type)
 178                return -EINVAL;
 179
 180        mutex_lock(&netns_bpf_mutex);
 181
 182        net = net_link->net;
 183        if (!net || !check_net(net)) {
 184                /* Link auto-detached or netns dying */
 185                ret = -ENOLINK;
 186                goto out_unlock;
 187        }
 188
 189        run_array = rcu_dereference_protected(net->bpf.run_array[type],
 190                                              lockdep_is_held(&netns_bpf_mutex));
 191        idx = link_index(net, type, net_link);
 192        ret = bpf_prog_array_update_at(run_array, idx, new_prog);
 193        if (ret)
 194                goto out_unlock;
 195
 196        old_prog = xchg(&link->prog, new_prog);
 197        bpf_prog_put(old_prog);
 198
 199out_unlock:
 200        mutex_unlock(&netns_bpf_mutex);
 201        return ret;
 202}
 203
 204static int bpf_netns_link_fill_info(const struct bpf_link *link,
 205                                    struct bpf_link_info *info)
 206{
 207        const struct bpf_netns_link *net_link =
 208                container_of(link, struct bpf_netns_link, link);
 209        unsigned int inum = 0;
 210        struct net *net;
 211
 212        mutex_lock(&netns_bpf_mutex);
 213        net = net_link->net;
 214        if (net && check_net(net))
 215                inum = net->ns.inum;
 216        mutex_unlock(&netns_bpf_mutex);
 217
 218        info->netns.netns_ino = inum;
 219        info->netns.attach_type = net_link->type;
 220        return 0;
 221}
 222
 223static void bpf_netns_link_show_fdinfo(const struct bpf_link *link,
 224                                       struct seq_file *seq)
 225{
 226        struct bpf_link_info info = {};
 227
 228        bpf_netns_link_fill_info(link, &info);
 229        seq_printf(seq,
 230                   "netns_ino:\t%u\n"
 231                   "attach_type:\t%u\n",
 232                   info.netns.netns_ino,
 233                   info.netns.attach_type);
 234}
 235
 236static const struct bpf_link_ops bpf_netns_link_ops = {
 237        .release = bpf_netns_link_release,
 238        .dealloc = bpf_netns_link_dealloc,
 239        .detach = bpf_netns_link_detach,
 240        .update_prog = bpf_netns_link_update_prog,
 241        .fill_link_info = bpf_netns_link_fill_info,
 242        .show_fdinfo = bpf_netns_link_show_fdinfo,
 243};
 244
 245/* Must be called with netns_bpf_mutex held. */
 246static int __netns_bpf_prog_query(const union bpf_attr *attr,
 247                                  union bpf_attr __user *uattr,
 248                                  struct net *net,
 249                                  enum netns_bpf_attach_type type)
 250{
 251        __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
 252        struct bpf_prog_array *run_array;
 253        u32 prog_cnt = 0, flags = 0;
 254
 255        run_array = rcu_dereference_protected(net->bpf.run_array[type],
 256                                              lockdep_is_held(&netns_bpf_mutex));
 257        if (run_array)
 258                prog_cnt = bpf_prog_array_length(run_array);
 259
 260        if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
 261                return -EFAULT;
 262        if (copy_to_user(&uattr->query.prog_cnt, &prog_cnt, sizeof(prog_cnt)))
 263                return -EFAULT;
 264        if (!attr->query.prog_cnt || !prog_ids || !prog_cnt)
 265                return 0;
 266
 267        return bpf_prog_array_copy_to_user(run_array, prog_ids,
 268                                           attr->query.prog_cnt);
 269}
 270
 271int netns_bpf_prog_query(const union bpf_attr *attr,
 272                         union bpf_attr __user *uattr)
 273{
 274        enum netns_bpf_attach_type type;
 275        struct net *net;
 276        int ret;
 277
 278        if (attr->query.query_flags)
 279                return -EINVAL;
 280
 281        type = to_netns_bpf_attach_type(attr->query.attach_type);
 282        if (type < 0)
 283                return -EINVAL;
 284
 285        net = get_net_ns_by_fd(attr->query.target_fd);
 286        if (IS_ERR(net))
 287                return PTR_ERR(net);
 288
 289        mutex_lock(&netns_bpf_mutex);
 290        ret = __netns_bpf_prog_query(attr, uattr, net, type);
 291        mutex_unlock(&netns_bpf_mutex);
 292
 293        put_net(net);
 294        return ret;
 295}
 296
 297int netns_bpf_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 298{
 299        struct bpf_prog_array *run_array;
 300        enum netns_bpf_attach_type type;
 301        struct bpf_prog *attached;
 302        struct net *net;
 303        int ret;
 304
 305        if (attr->target_fd || attr->attach_flags || attr->replace_bpf_fd)
 306                return -EINVAL;
 307
 308        type = to_netns_bpf_attach_type(attr->attach_type);
 309        if (type < 0)
 310                return -EINVAL;
 311
 312        net = current->nsproxy->net_ns;
 313        mutex_lock(&netns_bpf_mutex);
 314
 315        /* Attaching prog directly is not compatible with links */
 316        if (!list_empty(&net->bpf.links[type])) {
 317                ret = -EEXIST;
 318                goto out_unlock;
 319        }
 320
 321        switch (type) {
 322        case NETNS_BPF_FLOW_DISSECTOR:
 323                ret = flow_dissector_bpf_prog_attach_check(net, prog);
 324                break;
 325        default:
 326                ret = -EINVAL;
 327                break;
 328        }
 329        if (ret)
 330                goto out_unlock;
 331
 332        attached = net->bpf.progs[type];
 333        if (attached == prog) {
 334                /* The same program cannot be attached twice */
 335                ret = -EINVAL;
 336                goto out_unlock;
 337        }
 338
 339        run_array = rcu_dereference_protected(net->bpf.run_array[type],
 340                                              lockdep_is_held(&netns_bpf_mutex));
 341        if (run_array) {
 342                WRITE_ONCE(run_array->items[0].prog, prog);
 343        } else {
 344                run_array = bpf_prog_array_alloc(1, GFP_KERNEL);
 345                if (!run_array) {
 346                        ret = -ENOMEM;
 347                        goto out_unlock;
 348                }
 349                run_array->items[0].prog = prog;
 350                rcu_assign_pointer(net->bpf.run_array[type], run_array);
 351        }
 352
 353        net->bpf.progs[type] = prog;
 354        if (attached)
 355                bpf_prog_put(attached);
 356
 357out_unlock:
 358        mutex_unlock(&netns_bpf_mutex);
 359
 360        return ret;
 361}
 362
 363/* Must be called with netns_bpf_mutex held. */
 364static int __netns_bpf_prog_detach(struct net *net,
 365                                   enum netns_bpf_attach_type type,
 366                                   struct bpf_prog *old)
 367{
 368        struct bpf_prog *attached;
 369
 370        /* Progs attached via links cannot be detached */
 371        if (!list_empty(&net->bpf.links[type]))
 372                return -EINVAL;
 373
 374        attached = net->bpf.progs[type];
 375        if (!attached || attached != old)
 376                return -ENOENT;
 377        netns_bpf_run_array_detach(net, type);
 378        net->bpf.progs[type] = NULL;
 379        bpf_prog_put(attached);
 380        return 0;
 381}
 382
 383int netns_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
 384{
 385        enum netns_bpf_attach_type type;
 386        struct bpf_prog *prog;
 387        int ret;
 388
 389        if (attr->target_fd)
 390                return -EINVAL;
 391
 392        type = to_netns_bpf_attach_type(attr->attach_type);
 393        if (type < 0)
 394                return -EINVAL;
 395
 396        prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
 397        if (IS_ERR(prog))
 398                return PTR_ERR(prog);
 399
 400        mutex_lock(&netns_bpf_mutex);
 401        ret = __netns_bpf_prog_detach(current->nsproxy->net_ns, type, prog);
 402        mutex_unlock(&netns_bpf_mutex);
 403
 404        bpf_prog_put(prog);
 405
 406        return ret;
 407}
 408
 409static int netns_bpf_max_progs(enum netns_bpf_attach_type type)
 410{
 411        switch (type) {
 412        case NETNS_BPF_FLOW_DISSECTOR:
 413                return 1;
 414        case NETNS_BPF_SK_LOOKUP:
 415                return 64;
 416        default:
 417                return 0;
 418        }
 419}
 420
 421static int netns_bpf_link_attach(struct net *net, struct bpf_link *link,
 422                                 enum netns_bpf_attach_type type)
 423{
 424        struct bpf_netns_link *net_link =
 425                container_of(link, struct bpf_netns_link, link);
 426        struct bpf_prog_array *run_array;
 427        int cnt, err;
 428
 429        mutex_lock(&netns_bpf_mutex);
 430
 431        cnt = link_count(net, type);
 432        if (cnt >= netns_bpf_max_progs(type)) {
 433                err = -E2BIG;
 434                goto out_unlock;
 435        }
 436        /* Links are not compatible with attaching prog directly */
 437        if (net->bpf.progs[type]) {
 438                err = -EEXIST;
 439                goto out_unlock;
 440        }
 441
 442        switch (type) {
 443        case NETNS_BPF_FLOW_DISSECTOR:
 444                err = flow_dissector_bpf_prog_attach_check(net, link->prog);
 445                break;
 446        case NETNS_BPF_SK_LOOKUP:
 447                err = 0; /* nothing to check */
 448                break;
 449        default:
 450                err = -EINVAL;
 451                break;
 452        }
 453        if (err)
 454                goto out_unlock;
 455
 456        run_array = bpf_prog_array_alloc(cnt + 1, GFP_KERNEL);
 457        if (!run_array) {
 458                err = -ENOMEM;
 459                goto out_unlock;
 460        }
 461
 462        list_add_tail(&net_link->node, &net->bpf.links[type]);
 463
 464        fill_prog_array(net, type, run_array);
 465        run_array = rcu_replace_pointer(net->bpf.run_array[type], run_array,
 466                                        lockdep_is_held(&netns_bpf_mutex));
 467        bpf_prog_array_free(run_array);
 468
 469        /* Mark attach point as used */
 470        netns_bpf_attach_type_need(type);
 471
 472out_unlock:
 473        mutex_unlock(&netns_bpf_mutex);
 474        return err;
 475}
 476
 477int netns_bpf_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
 478{
 479        enum netns_bpf_attach_type netns_type;
 480        struct bpf_link_primer link_primer;
 481        struct bpf_netns_link *net_link;
 482        enum bpf_attach_type type;
 483        struct net *net;
 484        int err;
 485
 486        if (attr->link_create.flags)
 487                return -EINVAL;
 488
 489        type = attr->link_create.attach_type;
 490        netns_type = to_netns_bpf_attach_type(type);
 491        if (netns_type < 0)
 492                return -EINVAL;
 493
 494        net = get_net_ns_by_fd(attr->link_create.target_fd);
 495        if (IS_ERR(net))
 496                return PTR_ERR(net);
 497
 498        net_link = kzalloc(sizeof(*net_link), GFP_USER);
 499        if (!net_link) {
 500                err = -ENOMEM;
 501                goto out_put_net;
 502        }
 503        bpf_link_init(&net_link->link, BPF_LINK_TYPE_NETNS,
 504                      &bpf_netns_link_ops, prog);
 505        net_link->net = net;
 506        net_link->type = type;
 507        net_link->netns_type = netns_type;
 508
 509        err = bpf_link_prime(&net_link->link, &link_primer);
 510        if (err) {
 511                kfree(net_link);
 512                goto out_put_net;
 513        }
 514
 515        err = netns_bpf_link_attach(net, &net_link->link, netns_type);
 516        if (err) {
 517                bpf_link_cleanup(&link_primer);
 518                goto out_put_net;
 519        }
 520
 521        put_net(net);
 522        return bpf_link_settle(&link_primer);
 523
 524out_put_net:
 525        put_net(net);
 526        return err;
 527}
 528
 529static int __net_init netns_bpf_pernet_init(struct net *net)
 530{
 531        int type;
 532
 533        for (type = 0; type < MAX_NETNS_BPF_ATTACH_TYPE; type++)
 534                INIT_LIST_HEAD(&net->bpf.links[type]);
 535
 536        return 0;
 537}
 538
 539static void __net_exit netns_bpf_pernet_pre_exit(struct net *net)
 540{
 541        enum netns_bpf_attach_type type;
 542        struct bpf_netns_link *net_link;
 543
 544        mutex_lock(&netns_bpf_mutex);
 545        for (type = 0; type < MAX_NETNS_BPF_ATTACH_TYPE; type++) {
 546                netns_bpf_run_array_detach(net, type);
 547                list_for_each_entry(net_link, &net->bpf.links[type], node) {
 548                        net_link->net = NULL; /* auto-detach link */
 549                        netns_bpf_attach_type_unneed(type);
 550                }
 551                if (net->bpf.progs[type])
 552                        bpf_prog_put(net->bpf.progs[type]);
 553        }
 554        mutex_unlock(&netns_bpf_mutex);
 555}
 556
 557static struct pernet_operations netns_bpf_pernet_ops __net_initdata = {
 558        .init = netns_bpf_pernet_init,
 559        .pre_exit = netns_bpf_pernet_pre_exit,
 560};
 561
 562static int __init netns_bpf_init(void)
 563{
 564        return register_pernet_subsys(&netns_bpf_pernet_ops);
 565}
 566
 567subsys_initcall(netns_bpf_init);
 568