linux/net/openvswitch/datapath.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2007-2014 Nicira, Inc.
   4 */
   5
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8#include <linux/init.h>
   9#include <linux/module.h>
  10#include <linux/if_arp.h>
  11#include <linux/if_vlan.h>
  12#include <linux/in.h>
  13#include <linux/ip.h>
  14#include <linux/jhash.h>
  15#include <linux/delay.h>
  16#include <linux/time.h>
  17#include <linux/etherdevice.h>
  18#include <linux/genetlink.h>
  19#include <linux/kernel.h>
  20#include <linux/kthread.h>
  21#include <linux/mutex.h>
  22#include <linux/percpu.h>
  23#include <linux/rcupdate.h>
  24#include <linux/tcp.h>
  25#include <linux/udp.h>
  26#include <linux/ethtool.h>
  27#include <linux/wait.h>
  28#include <asm/div64.h>
  29#include <linux/highmem.h>
  30#include <linux/netfilter_bridge.h>
  31#include <linux/netfilter_ipv4.h>
  32#include <linux/inetdevice.h>
  33#include <linux/list.h>
  34#include <linux/openvswitch.h>
  35#include <linux/rculist.h>
  36#include <linux/dmi.h>
  37#include <net/genetlink.h>
  38#include <net/net_namespace.h>
  39#include <net/netns/generic.h>
  40
  41#include "datapath.h"
  42#include "flow.h"
  43#include "flow_table.h"
  44#include "flow_netlink.h"
  45#include "meter.h"
  46#include "openvswitch_trace.h"
  47#include "vport-internal_dev.h"
  48#include "vport-netdev.h"
  49
  50unsigned int ovs_net_id __read_mostly;
  51
  52static struct genl_family dp_packet_genl_family;
  53static struct genl_family dp_flow_genl_family;
  54static struct genl_family dp_datapath_genl_family;
  55
  56static const struct nla_policy flow_policy[];
  57
  58static const struct genl_multicast_group ovs_dp_flow_multicast_group = {
  59        .name = OVS_FLOW_MCGROUP,
  60};
  61
  62static const struct genl_multicast_group ovs_dp_datapath_multicast_group = {
  63        .name = OVS_DATAPATH_MCGROUP,
  64};
  65
  66static const struct genl_multicast_group ovs_dp_vport_multicast_group = {
  67        .name = OVS_VPORT_MCGROUP,
  68};
  69
  70/* Check if need to build a reply message.
  71 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
  72static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
  73                            unsigned int group)
  74{
  75        return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
  76               genl_has_listeners(family, genl_info_net(info), group);
  77}
  78
  79static void ovs_notify(struct genl_family *family,
  80                       struct sk_buff *skb, struct genl_info *info)
  81{
  82        genl_notify(family, skb, info, 0, GFP_KERNEL);
  83}
  84
  85/**
  86 * DOC: Locking:
  87 *
  88 * All writes e.g. Writes to device state (add/remove datapath, port, set
  89 * operations on vports, etc.), Writes to other state (flow table
  90 * modifications, set miscellaneous datapath parameters, etc.) are protected
  91 * by ovs_lock.
  92 *
  93 * Reads are protected by RCU.
  94 *
  95 * There are a few special cases (mostly stats) that have their own
  96 * synchronization but they nest under all of above and don't interact with
  97 * each other.
  98 *
  99 * The RTNL lock nests inside ovs_mutex.
 100 */
 101
 102static DEFINE_MUTEX(ovs_mutex);
 103
 104void ovs_lock(void)
 105{
 106        mutex_lock(&ovs_mutex);
 107}
 108
 109void ovs_unlock(void)
 110{
 111        mutex_unlock(&ovs_mutex);
 112}
 113
 114#ifdef CONFIG_LOCKDEP
 115int lockdep_ovsl_is_held(void)
 116{
 117        if (debug_locks)
 118                return lockdep_is_held(&ovs_mutex);
 119        else
 120                return 1;
 121}
 122#endif
 123
 124static struct vport *new_vport(const struct vport_parms *);
 125static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
 126                             const struct sw_flow_key *,
 127                             const struct dp_upcall_info *,
 128                             uint32_t cutlen);
 129static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
 130                                  const struct sw_flow_key *,
 131                                  const struct dp_upcall_info *,
 132                                  uint32_t cutlen);
 133
 134static void ovs_dp_masks_rebalance(struct work_struct *work);
 135
 136static int ovs_dp_set_upcall_portids(struct datapath *, const struct nlattr *);
 137
 138/* Must be called with rcu_read_lock or ovs_mutex. */
 139const char *ovs_dp_name(const struct datapath *dp)
 140{
 141        struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
 142        return ovs_vport_name(vport);
 143}
 144
 145static int get_dpifindex(const struct datapath *dp)
 146{
 147        struct vport *local;
 148        int ifindex;
 149
 150        rcu_read_lock();
 151
 152        local = ovs_vport_rcu(dp, OVSP_LOCAL);
 153        if (local)
 154                ifindex = local->dev->ifindex;
 155        else
 156                ifindex = 0;
 157
 158        rcu_read_unlock();
 159
 160        return ifindex;
 161}
 162
 163static void destroy_dp_rcu(struct rcu_head *rcu)
 164{
 165        struct datapath *dp = container_of(rcu, struct datapath, rcu);
 166
 167        ovs_flow_tbl_destroy(&dp->table);
 168        free_percpu(dp->stats_percpu);
 169        kfree(dp->ports);
 170        ovs_meters_exit(dp);
 171        kfree(rcu_dereference_raw(dp->upcall_portids));
 172        kfree(dp);
 173}
 174
 175static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 176                                            u16 port_no)
 177{
 178        return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 179}
 180
 181/* Called with ovs_mutex or RCU read lock. */
 182struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 183{
 184        struct vport *vport;
 185        struct hlist_head *head;
 186
 187        head = vport_hash_bucket(dp, port_no);
 188        hlist_for_each_entry_rcu(vport, head, dp_hash_node,
 189                                 lockdep_ovsl_is_held()) {
 190                if (vport->port_no == port_no)
 191                        return vport;
 192        }
 193        return NULL;
 194}
 195
 196/* Called with ovs_mutex. */
 197static struct vport *new_vport(const struct vport_parms *parms)
 198{
 199        struct vport *vport;
 200
 201        vport = ovs_vport_add(parms);
 202        if (!IS_ERR(vport)) {
 203                struct datapath *dp = parms->dp;
 204                struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 205
 206                hlist_add_head_rcu(&vport->dp_hash_node, head);
 207        }
 208        return vport;
 209}
 210
 211void ovs_dp_detach_port(struct vport *p)
 212{
 213        ASSERT_OVSL();
 214
 215        /* First drop references to device. */
 216        hlist_del_rcu(&p->dp_hash_node);
 217
 218        /* Then destroy it. */
 219        ovs_vport_del(p);
 220}
 221
 222/* Must be called with rcu_read_lock. */
 223void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
 224{
 225        const struct vport *p = OVS_CB(skb)->input_vport;
 226        struct datapath *dp = p->dp;
 227        struct sw_flow *flow;
 228        struct sw_flow_actions *sf_acts;
 229        struct dp_stats_percpu *stats;
 230        u64 *stats_counter;
 231        u32 n_mask_hit;
 232        u32 n_cache_hit;
 233        int error;
 234
 235        stats = this_cpu_ptr(dp->stats_percpu);
 236
 237        /* Look up flow. */
 238        flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
 239                                         &n_mask_hit, &n_cache_hit);
 240        if (unlikely(!flow)) {
 241                struct dp_upcall_info upcall;
 242
 243                memset(&upcall, 0, sizeof(upcall));
 244                upcall.cmd = OVS_PACKET_CMD_MISS;
 245
 246                if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU)
 247                        upcall.portid =
 248                            ovs_dp_get_upcall_portid(dp, smp_processor_id());
 249                else
 250                        upcall.portid = ovs_vport_find_upcall_portid(p, skb);
 251
 252                upcall.mru = OVS_CB(skb)->mru;
 253                error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
 254                if (unlikely(error))
 255                        kfree_skb(skb);
 256                else
 257                        consume_skb(skb);
 258                stats_counter = &stats->n_missed;
 259                goto out;
 260        }
 261
 262        ovs_flow_stats_update(flow, key->tp.flags, skb);
 263        sf_acts = rcu_dereference(flow->sf_acts);
 264        error = ovs_execute_actions(dp, skb, sf_acts, key);
 265        if (unlikely(error))
 266                net_dbg_ratelimited("ovs: action execution error on datapath %s: %d\n",
 267                                    ovs_dp_name(dp), error);
 268
 269        stats_counter = &stats->n_hit;
 270
 271out:
 272        /* Update datapath statistics. */
 273        u64_stats_update_begin(&stats->syncp);
 274        (*stats_counter)++;
 275        stats->n_mask_hit += n_mask_hit;
 276        stats->n_cache_hit += n_cache_hit;
 277        u64_stats_update_end(&stats->syncp);
 278}
 279
 280int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 281                  const struct sw_flow_key *key,
 282                  const struct dp_upcall_info *upcall_info,
 283                  uint32_t cutlen)
 284{
 285        struct dp_stats_percpu *stats;
 286        int err;
 287
 288        if (trace_ovs_dp_upcall_enabled())
 289                trace_ovs_dp_upcall(dp, skb, key, upcall_info);
 290
 291        if (upcall_info->portid == 0) {
 292                err = -ENOTCONN;
 293                goto err;
 294        }
 295
 296        if (!skb_is_gso(skb))
 297                err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
 298        else
 299                err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
 300        if (err)
 301                goto err;
 302
 303        return 0;
 304
 305err:
 306        stats = this_cpu_ptr(dp->stats_percpu);
 307
 308        u64_stats_update_begin(&stats->syncp);
 309        stats->n_lost++;
 310        u64_stats_update_end(&stats->syncp);
 311
 312        return err;
 313}
 314
 315static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
 316                             const struct sw_flow_key *key,
 317                             const struct dp_upcall_info *upcall_info,
 318                             uint32_t cutlen)
 319{
 320        unsigned int gso_type = skb_shinfo(skb)->gso_type;
 321        struct sw_flow_key later_key;
 322        struct sk_buff *segs, *nskb;
 323        int err;
 324
 325        BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET);
 326        segs = __skb_gso_segment(skb, NETIF_F_SG, false);
 327        if (IS_ERR(segs))
 328                return PTR_ERR(segs);
 329        if (segs == NULL)
 330                return -EINVAL;
 331
 332        if (gso_type & SKB_GSO_UDP) {
 333                /* The initial flow key extracted by ovs_flow_key_extract()
 334                 * in this case is for a first fragment, so we need to
 335                 * properly mark later fragments.
 336                 */
 337                later_key = *key;
 338                later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 339        }
 340
 341        /* Queue all of the segments. */
 342        skb_list_walk_safe(segs, skb, nskb) {
 343                if (gso_type & SKB_GSO_UDP && skb != segs)
 344                        key = &later_key;
 345
 346                err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
 347                if (err)
 348                        break;
 349
 350        }
 351
 352        /* Free all of the segments. */
 353        skb_list_walk_safe(segs, skb, nskb) {
 354                if (err)
 355                        kfree_skb(skb);
 356                else
 357                        consume_skb(skb);
 358        }
 359        return err;
 360}
 361
 362static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
 363                              unsigned int hdrlen, int actions_attrlen)
 364{
 365        size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 366                + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
 367                + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
 368                + nla_total_size(sizeof(unsigned int)) /* OVS_PACKET_ATTR_LEN */
 369                + nla_total_size(sizeof(u64)); /* OVS_PACKET_ATTR_HASH */
 370
 371        /* OVS_PACKET_ATTR_USERDATA */
 372        if (upcall_info->userdata)
 373                size += NLA_ALIGN(upcall_info->userdata->nla_len);
 374
 375        /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
 376        if (upcall_info->egress_tun_info)
 377                size += nla_total_size(ovs_tun_key_attr_size());
 378
 379        /* OVS_PACKET_ATTR_ACTIONS */
 380        if (upcall_info->actions_len)
 381                size += nla_total_size(actions_attrlen);
 382
 383        /* OVS_PACKET_ATTR_MRU */
 384        if (upcall_info->mru)
 385                size += nla_total_size(sizeof(upcall_info->mru));
 386
 387        return size;
 388}
 389
 390static void pad_packet(struct datapath *dp, struct sk_buff *skb)
 391{
 392        if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
 393                size_t plen = NLA_ALIGN(skb->len) - skb->len;
 394
 395                if (plen > 0)
 396                        skb_put_zero(skb, plen);
 397        }
 398}
 399
 400static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 401                                  const struct sw_flow_key *key,
 402                                  const struct dp_upcall_info *upcall_info,
 403                                  uint32_t cutlen)
 404{
 405        struct ovs_header *upcall;
 406        struct sk_buff *nskb = NULL;
 407        struct sk_buff *user_skb = NULL; /* to be queued to userspace */
 408        struct nlattr *nla;
 409        size_t len;
 410        unsigned int hlen;
 411        int err, dp_ifindex;
 412        u64 hash;
 413
 414        dp_ifindex = get_dpifindex(dp);
 415        if (!dp_ifindex)
 416                return -ENODEV;
 417
 418        if (skb_vlan_tag_present(skb)) {
 419                nskb = skb_clone(skb, GFP_ATOMIC);
 420                if (!nskb)
 421                        return -ENOMEM;
 422
 423                nskb = __vlan_hwaccel_push_inside(nskb);
 424                if (!nskb)
 425                        return -ENOMEM;
 426
 427                skb = nskb;
 428        }
 429
 430        if (nla_attr_size(skb->len) > USHRT_MAX) {
 431                err = -EFBIG;
 432                goto out;
 433        }
 434
 435        /* Complete checksum if needed */
 436        if (skb->ip_summed == CHECKSUM_PARTIAL &&
 437            (err = skb_csum_hwoffload_help(skb, 0)))
 438                goto out;
 439
 440        /* Older versions of OVS user space enforce alignment of the last
 441         * Netlink attribute to NLA_ALIGNTO which would require extensive
 442         * padding logic. Only perform zerocopy if padding is not required.
 443         */
 444        if (dp->user_features & OVS_DP_F_UNALIGNED)
 445                hlen = skb_zerocopy_headlen(skb);
 446        else
 447                hlen = skb->len;
 448
 449        len = upcall_msg_size(upcall_info, hlen - cutlen,
 450                              OVS_CB(skb)->acts_origlen);
 451        user_skb = genlmsg_new(len, GFP_ATOMIC);
 452        if (!user_skb) {
 453                err = -ENOMEM;
 454                goto out;
 455        }
 456
 457        upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 458                             0, upcall_info->cmd);
 459        if (!upcall) {
 460                err = -EINVAL;
 461                goto out;
 462        }
 463        upcall->dp_ifindex = dp_ifindex;
 464
 465        err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
 466        if (err)
 467                goto out;
 468
 469        if (upcall_info->userdata)
 470                __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 471                          nla_len(upcall_info->userdata),
 472                          nla_data(upcall_info->userdata));
 473
 474        if (upcall_info->egress_tun_info) {
 475                nla = nla_nest_start_noflag(user_skb,
 476                                            OVS_PACKET_ATTR_EGRESS_TUN_KEY);
 477                if (!nla) {
 478                        err = -EMSGSIZE;
 479                        goto out;
 480                }
 481                err = ovs_nla_put_tunnel_info(user_skb,
 482                                              upcall_info->egress_tun_info);
 483                if (err)
 484                        goto out;
 485
 486                nla_nest_end(user_skb, nla);
 487        }
 488
 489        if (upcall_info->actions_len) {
 490                nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS);
 491                if (!nla) {
 492                        err = -EMSGSIZE;
 493                        goto out;
 494                }
 495                err = ovs_nla_put_actions(upcall_info->actions,
 496                                          upcall_info->actions_len,
 497                                          user_skb);
 498                if (!err)
 499                        nla_nest_end(user_skb, nla);
 500                else
 501                        nla_nest_cancel(user_skb, nla);
 502        }
 503
 504        /* Add OVS_PACKET_ATTR_MRU */
 505        if (upcall_info->mru &&
 506            nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU, upcall_info->mru)) {
 507                err = -ENOBUFS;
 508                goto out;
 509        }
 510
 511        /* Add OVS_PACKET_ATTR_LEN when packet is truncated */
 512        if (cutlen > 0 &&
 513            nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) {
 514                err = -ENOBUFS;
 515                goto out;
 516        }
 517
 518        /* Add OVS_PACKET_ATTR_HASH */
 519        hash = skb_get_hash_raw(skb);
 520        if (skb->sw_hash)
 521                hash |= OVS_PACKET_HASH_SW_BIT;
 522
 523        if (skb->l4_hash)
 524                hash |= OVS_PACKET_HASH_L4_BIT;
 525
 526        if (nla_put(user_skb, OVS_PACKET_ATTR_HASH, sizeof (u64), &hash)) {
 527                err = -ENOBUFS;
 528                goto out;
 529        }
 530
 531        /* Only reserve room for attribute header, packet data is added
 532         * in skb_zerocopy() */
 533        if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
 534                err = -ENOBUFS;
 535                goto out;
 536        }
 537        nla->nla_len = nla_attr_size(skb->len - cutlen);
 538
 539        err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen);
 540        if (err)
 541                goto out;
 542
 543        /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
 544        pad_packet(dp, user_skb);
 545
 546        ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
 547
 548        err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
 549        user_skb = NULL;
 550out:
 551        if (err)
 552                skb_tx_error(skb);
 553        kfree_skb(user_skb);
 554        kfree_skb(nskb);
 555        return err;
 556}
 557
 558static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 559{
 560        struct ovs_header *ovs_header = info->userhdr;
 561        struct net *net = sock_net(skb->sk);
 562        struct nlattr **a = info->attrs;
 563        struct sw_flow_actions *acts;
 564        struct sk_buff *packet;
 565        struct sw_flow *flow;
 566        struct sw_flow_actions *sf_acts;
 567        struct datapath *dp;
 568        struct vport *input_vport;
 569        u16 mru = 0;
 570        u64 hash;
 571        int len;
 572        int err;
 573        bool log = !a[OVS_PACKET_ATTR_PROBE];
 574
 575        err = -EINVAL;
 576        if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 577            !a[OVS_PACKET_ATTR_ACTIONS])
 578                goto err;
 579
 580        len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 581        packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 582        err = -ENOMEM;
 583        if (!packet)
 584                goto err;
 585        skb_reserve(packet, NET_IP_ALIGN);
 586
 587        nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 588
 589        /* Set packet's mru */
 590        if (a[OVS_PACKET_ATTR_MRU]) {
 591                mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
 592                packet->ignore_df = 1;
 593        }
 594        OVS_CB(packet)->mru = mru;
 595
 596        if (a[OVS_PACKET_ATTR_HASH]) {
 597                hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]);
 598
 599                __skb_set_hash(packet, hash & 0xFFFFFFFFULL,
 600                               !!(hash & OVS_PACKET_HASH_SW_BIT),
 601                               !!(hash & OVS_PACKET_HASH_L4_BIT));
 602        }
 603
 604        /* Build an sw_flow for sending this packet. */
 605        flow = ovs_flow_alloc();
 606        err = PTR_ERR(flow);
 607        if (IS_ERR(flow))
 608                goto err_kfree_skb;
 609
 610        err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
 611                                             packet, &flow->key, log);
 612        if (err)
 613                goto err_flow_free;
 614
 615        err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
 616                                   &flow->key, &acts, log);
 617        if (err)
 618                goto err_flow_free;
 619
 620        rcu_assign_pointer(flow->sf_acts, acts);
 621        packet->priority = flow->key.phy.priority;
 622        packet->mark = flow->key.phy.skb_mark;
 623
 624        rcu_read_lock();
 625        dp = get_dp_rcu(net, ovs_header->dp_ifindex);
 626        err = -ENODEV;
 627        if (!dp)
 628                goto err_unlock;
 629
 630        input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
 631        if (!input_vport)
 632                input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
 633
 634        if (!input_vport)
 635                goto err_unlock;
 636
 637        packet->dev = input_vport->dev;
 638        OVS_CB(packet)->input_vport = input_vport;
 639        sf_acts = rcu_dereference(flow->sf_acts);
 640
 641        local_bh_disable();
 642        err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
 643        local_bh_enable();
 644        rcu_read_unlock();
 645
 646        ovs_flow_free(flow, false);
 647        return err;
 648
 649err_unlock:
 650        rcu_read_unlock();
 651err_flow_free:
 652        ovs_flow_free(flow, false);
 653err_kfree_skb:
 654        kfree_skb(packet);
 655err:
 656        return err;
 657}
 658
 659static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 660        [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 661        [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 662        [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 663        [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
 664        [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
 665        [OVS_PACKET_ATTR_HASH] = { .type = NLA_U64 },
 666};
 667
 668static const struct genl_small_ops dp_packet_genl_ops[] = {
 669        { .cmd = OVS_PACKET_CMD_EXECUTE,
 670          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
 671          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 672          .doit = ovs_packet_cmd_execute
 673        }
 674};
 675
 676static struct genl_family dp_packet_genl_family __ro_after_init = {
 677        .hdrsize = sizeof(struct ovs_header),
 678        .name = OVS_PACKET_FAMILY,
 679        .version = OVS_PACKET_VERSION,
 680        .maxattr = OVS_PACKET_ATTR_MAX,
 681        .policy = packet_policy,
 682        .netnsok = true,
 683        .parallel_ops = true,
 684        .small_ops = dp_packet_genl_ops,
 685        .n_small_ops = ARRAY_SIZE(dp_packet_genl_ops),
 686        .module = THIS_MODULE,
 687};
 688
 689static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
 690                         struct ovs_dp_megaflow_stats *mega_stats)
 691{
 692        int i;
 693
 694        memset(mega_stats, 0, sizeof(*mega_stats));
 695
 696        stats->n_flows = ovs_flow_tbl_count(&dp->table);
 697        mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 698
 699        stats->n_hit = stats->n_missed = stats->n_lost = 0;
 700
 701        for_each_possible_cpu(i) {
 702                const struct dp_stats_percpu *percpu_stats;
 703                struct dp_stats_percpu local_stats;
 704                unsigned int start;
 705
 706                percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 707
 708                do {
 709                        start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
 710                        local_stats = *percpu_stats;
 711                } while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
 712
 713                stats->n_hit += local_stats.n_hit;
 714                stats->n_missed += local_stats.n_missed;
 715                stats->n_lost += local_stats.n_lost;
 716                mega_stats->n_mask_hit += local_stats.n_mask_hit;
 717                mega_stats->n_cache_hit += local_stats.n_cache_hit;
 718        }
 719}
 720
 721static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
 722{
 723        return ovs_identifier_is_ufid(sfid) &&
 724               !(ufid_flags & OVS_UFID_F_OMIT_KEY);
 725}
 726
 727static bool should_fill_mask(uint32_t ufid_flags)
 728{
 729        return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
 730}
 731
 732static bool should_fill_actions(uint32_t ufid_flags)
 733{
 734        return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
 735}
 736
 737static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
 738                                    const struct sw_flow_id *sfid,
 739                                    uint32_t ufid_flags)
 740{
 741        size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
 742
 743        /* OVS_FLOW_ATTR_UFID, or unmasked flow key as fallback
 744         * see ovs_nla_put_identifier()
 745         */
 746        if (sfid && ovs_identifier_is_ufid(sfid))
 747                len += nla_total_size(sfid->ufid_len);
 748        else
 749                len += nla_total_size(ovs_key_attr_size());
 750
 751        /* OVS_FLOW_ATTR_KEY */
 752        if (!sfid || should_fill_key(sfid, ufid_flags))
 753                len += nla_total_size(ovs_key_attr_size());
 754
 755        /* OVS_FLOW_ATTR_MASK */
 756        if (should_fill_mask(ufid_flags))
 757                len += nla_total_size(ovs_key_attr_size());
 758
 759        /* OVS_FLOW_ATTR_ACTIONS */
 760        if (should_fill_actions(ufid_flags))
 761                len += nla_total_size(acts->orig_len);
 762
 763        return len
 764                + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
 765                + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
 766                + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
 767}
 768
 769/* Called with ovs_mutex or RCU read lock. */
 770static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
 771                                   struct sk_buff *skb)
 772{
 773        struct ovs_flow_stats stats;
 774        __be16 tcp_flags;
 775        unsigned long used;
 776
 777        ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
 778
 779        if (used &&
 780            nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
 781                              OVS_FLOW_ATTR_PAD))
 782                return -EMSGSIZE;
 783
 784        if (stats.n_packets &&
 785            nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
 786                          sizeof(struct ovs_flow_stats), &stats,
 787                          OVS_FLOW_ATTR_PAD))
 788                return -EMSGSIZE;
 789
 790        if ((u8)ntohs(tcp_flags) &&
 791             nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
 792                return -EMSGSIZE;
 793
 794        return 0;
 795}
 796
 797/* Called with ovs_mutex or RCU read lock. */
 798static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
 799                                     struct sk_buff *skb, int skb_orig_len)
 800{
 801        struct nlattr *start;
 802        int err;
 803
 804        /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
 805         * this is the first flow to be dumped into 'skb'.  This is unusual for
 806         * Netlink but individual action lists can be longer than
 807         * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
 808         * The userspace caller can always fetch the actions separately if it
 809         * really wants them.  (Most userspace callers in fact don't care.)
 810         *
 811         * This can only fail for dump operations because the skb is always
 812         * properly sized for single flows.
 813         */
 814        start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS);
 815        if (start) {
 816                const struct sw_flow_actions *sf_acts;
 817
 818                sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 819                err = ovs_nla_put_actions(sf_acts->actions,
 820                                          sf_acts->actions_len, skb);
 821
 822                if (!err)
 823                        nla_nest_end(skb, start);
 824                else {
 825                        if (skb_orig_len)
 826                                return err;
 827
 828                        nla_nest_cancel(skb, start);
 829                }
 830        } else if (skb_orig_len) {
 831                return -EMSGSIZE;
 832        }
 833
 834        return 0;
 835}
 836
 837/* Called with ovs_mutex or RCU read lock. */
 838static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
 839                                  struct sk_buff *skb, u32 portid,
 840                                  u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
 841{
 842        const int skb_orig_len = skb->len;
 843        struct ovs_header *ovs_header;
 844        int err;
 845
 846        ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
 847                                 flags, cmd);
 848        if (!ovs_header)
 849                return -EMSGSIZE;
 850
 851        ovs_header->dp_ifindex = dp_ifindex;
 852
 853        err = ovs_nla_put_identifier(flow, skb);
 854        if (err)
 855                goto error;
 856
 857        if (should_fill_key(&flow->id, ufid_flags)) {
 858                err = ovs_nla_put_masked_key(flow, skb);
 859                if (err)
 860                        goto error;
 861        }
 862
 863        if (should_fill_mask(ufid_flags)) {
 864                err = ovs_nla_put_mask(flow, skb);
 865                if (err)
 866                        goto error;
 867        }
 868
 869        err = ovs_flow_cmd_fill_stats(flow, skb);
 870        if (err)
 871                goto error;
 872
 873        if (should_fill_actions(ufid_flags)) {
 874                err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
 875                if (err)
 876                        goto error;
 877        }
 878
 879        genlmsg_end(skb, ovs_header);
 880        return 0;
 881
 882error:
 883        genlmsg_cancel(skb, ovs_header);
 884        return err;
 885}
 886
 887/* May not be called with RCU read lock. */
 888static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
 889                                               const struct sw_flow_id *sfid,
 890                                               struct genl_info *info,
 891                                               bool always,
 892                                               uint32_t ufid_flags)
 893{
 894        struct sk_buff *skb;
 895        size_t len;
 896
 897        if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0))
 898                return NULL;
 899
 900        len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
 901        skb = genlmsg_new(len, GFP_KERNEL);
 902        if (!skb)
 903                return ERR_PTR(-ENOMEM);
 904
 905        return skb;
 906}
 907
 908/* Called with ovs_mutex. */
 909static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
 910                                               int dp_ifindex,
 911                                               struct genl_info *info, u8 cmd,
 912                                               bool always, u32 ufid_flags)
 913{
 914        struct sk_buff *skb;
 915        int retval;
 916
 917        skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
 918                                      &flow->id, info, always, ufid_flags);
 919        if (IS_ERR_OR_NULL(skb))
 920                return skb;
 921
 922        retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
 923                                        info->snd_portid, info->snd_seq, 0,
 924                                        cmd, ufid_flags);
 925        if (WARN_ON_ONCE(retval < 0)) {
 926                kfree_skb(skb);
 927                skb = ERR_PTR(retval);
 928        }
 929        return skb;
 930}
 931
 932static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
 933{
 934        struct net *net = sock_net(skb->sk);
 935        struct nlattr **a = info->attrs;
 936        struct ovs_header *ovs_header = info->userhdr;
 937        struct sw_flow *flow = NULL, *new_flow;
 938        struct sw_flow_mask mask;
 939        struct sk_buff *reply;
 940        struct datapath *dp;
 941        struct sw_flow_actions *acts;
 942        struct sw_flow_match match;
 943        u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
 944        int error;
 945        bool log = !a[OVS_FLOW_ATTR_PROBE];
 946
 947        /* Must have key and actions. */
 948        error = -EINVAL;
 949        if (!a[OVS_FLOW_ATTR_KEY]) {
 950                OVS_NLERR(log, "Flow key attr not present in new flow.");
 951                goto error;
 952        }
 953        if (!a[OVS_FLOW_ATTR_ACTIONS]) {
 954                OVS_NLERR(log, "Flow actions attr not present in new flow.");
 955                goto error;
 956        }
 957
 958        /* Most of the time we need to allocate a new flow, do it before
 959         * locking.
 960         */
 961        new_flow = ovs_flow_alloc();
 962        if (IS_ERR(new_flow)) {
 963                error = PTR_ERR(new_flow);
 964                goto error;
 965        }
 966
 967        /* Extract key. */
 968        ovs_match_init(&match, &new_flow->key, false, &mask);
 969        error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
 970                                  a[OVS_FLOW_ATTR_MASK], log);
 971        if (error)
 972                goto err_kfree_flow;
 973
 974        /* Extract flow identifier. */
 975        error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
 976                                       &new_flow->key, log);
 977        if (error)
 978                goto err_kfree_flow;
 979
 980        /* unmasked key is needed to match when ufid is not used. */
 981        if (ovs_identifier_is_key(&new_flow->id))
 982                match.key = new_flow->id.unmasked_key;
 983
 984        ovs_flow_mask_key(&new_flow->key, &new_flow->key, true, &mask);
 985
 986        /* Validate actions. */
 987        error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
 988                                     &new_flow->key, &acts, log);
 989        if (error) {
 990                OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
 991                goto err_kfree_flow;
 992        }
 993
 994        reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
 995                                        ufid_flags);
 996        if (IS_ERR(reply)) {
 997                error = PTR_ERR(reply);
 998                goto err_kfree_acts;
 999        }
1000
1001        ovs_lock();
1002        dp = get_dp(net, ovs_header->dp_ifindex);
1003        if (unlikely(!dp)) {
1004                error = -ENODEV;
1005                goto err_unlock_ovs;
1006        }
1007
1008        /* Check if this is a duplicate flow */
1009        if (ovs_identifier_is_ufid(&new_flow->id))
1010                flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
1011        if (!flow)
1012                flow = ovs_flow_tbl_lookup(&dp->table, &new_flow->key);
1013        if (likely(!flow)) {
1014                rcu_assign_pointer(new_flow->sf_acts, acts);
1015
1016                /* Put flow in bucket. */
1017                error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
1018                if (unlikely(error)) {
1019                        acts = NULL;
1020                        goto err_unlock_ovs;
1021                }
1022
1023                if (unlikely(reply)) {
1024                        error = ovs_flow_cmd_fill_info(new_flow,
1025                                                       ovs_header->dp_ifindex,
1026                                                       reply, info->snd_portid,
1027                                                       info->snd_seq, 0,
1028                                                       OVS_FLOW_CMD_NEW,
1029                                                       ufid_flags);
1030                        BUG_ON(error < 0);
1031                }
1032                ovs_unlock();
1033        } else {
1034                struct sw_flow_actions *old_acts;
1035
1036                /* Bail out if we're not allowed to modify an existing flow.
1037                 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1038                 * because Generic Netlink treats the latter as a dump
1039                 * request.  We also accept NLM_F_EXCL in case that bug ever
1040                 * gets fixed.
1041                 */
1042                if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
1043                                                         | NLM_F_EXCL))) {
1044                        error = -EEXIST;
1045                        goto err_unlock_ovs;
1046                }
1047                /* The flow identifier has to be the same for flow updates.
1048                 * Look for any overlapping flow.
1049                 */
1050                if (unlikely(!ovs_flow_cmp(flow, &match))) {
1051                        if (ovs_identifier_is_key(&flow->id))
1052                                flow = ovs_flow_tbl_lookup_exact(&dp->table,
1053                                                                 &match);
1054                        else /* UFID matches but key is different */
1055                                flow = NULL;
1056                        if (!flow) {
1057                                error = -ENOENT;
1058                                goto err_unlock_ovs;
1059                        }
1060                }
1061                /* Update actions. */
1062                old_acts = ovsl_dereference(flow->sf_acts);
1063                rcu_assign_pointer(flow->sf_acts, acts);
1064
1065                if (unlikely(reply)) {
1066                        error = ovs_flow_cmd_fill_info(flow,
1067                                                       ovs_header->dp_ifindex,
1068                                                       reply, info->snd_portid,
1069                                                       info->snd_seq, 0,
1070                                                       OVS_FLOW_CMD_NEW,
1071                                                       ufid_flags);
1072                        BUG_ON(error < 0);
1073                }
1074                ovs_unlock();
1075
1076                ovs_nla_free_flow_actions_rcu(old_acts);
1077                ovs_flow_free(new_flow, false);
1078        }
1079
1080        if (reply)
1081                ovs_notify(&dp_flow_genl_family, reply, info);
1082        return 0;
1083
1084err_unlock_ovs:
1085        ovs_unlock();
1086        kfree_skb(reply);
1087err_kfree_acts:
1088        ovs_nla_free_flow_actions(acts);
1089err_kfree_flow:
1090        ovs_flow_free(new_flow, false);
1091error:
1092        return error;
1093}
1094
1095/* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1096static noinline_for_stack
1097struct sw_flow_actions *get_flow_actions(struct net *net,
1098                                         const struct nlattr *a,
1099                                         const struct sw_flow_key *key,
1100                                         const struct sw_flow_mask *mask,
1101                                         bool log)
1102{
1103        struct sw_flow_actions *acts;
1104        struct sw_flow_key masked_key;
1105        int error;
1106
1107        ovs_flow_mask_key(&masked_key, key, true, mask);
1108        error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
1109        if (error) {
1110                OVS_NLERR(log,
1111                          "Actions may not be safe on all matching packets");
1112                return ERR_PTR(error);
1113        }
1114
1115        return acts;
1116}
1117
1118/* Factor out match-init and action-copy to avoid
1119 * "Wframe-larger-than=1024" warning. Because mask is only
1120 * used to get actions, we new a function to save some
1121 * stack space.
1122 *
1123 * If there are not key and action attrs, we return 0
1124 * directly. In the case, the caller will also not use the
1125 * match as before. If there is action attr, we try to get
1126 * actions and save them to *acts. Before returning from
1127 * the function, we reset the match->mask pointer. Because
1128 * we should not to return match object with dangling reference
1129 * to mask.
1130 * */
1131static noinline_for_stack int
1132ovs_nla_init_match_and_action(struct net *net,
1133                              struct sw_flow_match *match,
1134                              struct sw_flow_key *key,
1135                              struct nlattr **a,
1136                              struct sw_flow_actions **acts,
1137                              bool log)
1138{
1139        struct sw_flow_mask mask;
1140        int error = 0;
1141
1142        if (a[OVS_FLOW_ATTR_KEY]) {
1143                ovs_match_init(match, key, true, &mask);
1144                error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
1145                                          a[OVS_FLOW_ATTR_MASK], log);
1146                if (error)
1147                        goto error;
1148        }
1149
1150        if (a[OVS_FLOW_ATTR_ACTIONS]) {
1151                if (!a[OVS_FLOW_ATTR_KEY]) {
1152                        OVS_NLERR(log,
1153                                  "Flow key attribute not present in set flow.");
1154                        error = -EINVAL;
1155                        goto error;
1156                }
1157
1158                *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
1159                                         &mask, log);
1160                if (IS_ERR(*acts)) {
1161                        error = PTR_ERR(*acts);
1162                        goto error;
1163                }
1164        }
1165
1166        /* On success, error is 0. */
1167error:
1168        match->mask = NULL;
1169        return error;
1170}
1171
1172static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1173{
1174        struct net *net = sock_net(skb->sk);
1175        struct nlattr **a = info->attrs;
1176        struct ovs_header *ovs_header = info->userhdr;
1177        struct sw_flow_key key;
1178        struct sw_flow *flow;
1179        struct sk_buff *reply = NULL;
1180        struct datapath *dp;
1181        struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1182        struct sw_flow_match match;
1183        struct sw_flow_id sfid;
1184        u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1185        int error = 0;
1186        bool log = !a[OVS_FLOW_ATTR_PROBE];
1187        bool ufid_present;
1188
1189        ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1190        if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
1191                OVS_NLERR(log,
1192                          "Flow set message rejected, Key attribute missing.");
1193                return -EINVAL;
1194        }
1195
1196        error = ovs_nla_init_match_and_action(net, &match, &key, a,
1197                                              &acts, log);
1198        if (error)
1199                goto error;
1200
1201        if (acts) {
1202                /* Can allocate before locking if have acts. */
1203                reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1204                                                ufid_flags);
1205                if (IS_ERR(reply)) {
1206                        error = PTR_ERR(reply);
1207                        goto err_kfree_acts;
1208                }
1209        }
1210
1211        ovs_lock();
1212        dp = get_dp(net, ovs_header->dp_ifindex);
1213        if (unlikely(!dp)) {
1214                error = -ENODEV;
1215                goto err_unlock_ovs;
1216        }
1217        /* Check that the flow exists. */
1218        if (ufid_present)
1219                flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1220        else
1221                flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1222        if (unlikely(!flow)) {
1223                error = -ENOENT;
1224                goto err_unlock_ovs;
1225        }
1226
1227        /* Update actions, if present. */
1228        if (likely(acts)) {
1229                old_acts = ovsl_dereference(flow->sf_acts);
1230                rcu_assign_pointer(flow->sf_acts, acts);
1231
1232                if (unlikely(reply)) {
1233                        error = ovs_flow_cmd_fill_info(flow,
1234                                                       ovs_header->dp_ifindex,
1235                                                       reply, info->snd_portid,
1236                                                       info->snd_seq, 0,
1237                                                       OVS_FLOW_CMD_SET,
1238                                                       ufid_flags);
1239                        BUG_ON(error < 0);
1240                }
1241        } else {
1242                /* Could not alloc without acts before locking. */
1243                reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1244                                                info, OVS_FLOW_CMD_SET, false,
1245                                                ufid_flags);
1246
1247                if (IS_ERR(reply)) {
1248                        error = PTR_ERR(reply);
1249                        goto err_unlock_ovs;
1250                }
1251        }
1252
1253        /* Clear stats. */
1254        if (a[OVS_FLOW_ATTR_CLEAR])
1255                ovs_flow_stats_clear(flow);
1256        ovs_unlock();
1257
1258        if (reply)
1259                ovs_notify(&dp_flow_genl_family, reply, info);
1260        if (old_acts)
1261                ovs_nla_free_flow_actions_rcu(old_acts);
1262
1263        return 0;
1264
1265err_unlock_ovs:
1266        ovs_unlock();
1267        kfree_skb(reply);
1268err_kfree_acts:
1269        ovs_nla_free_flow_actions(acts);
1270error:
1271        return error;
1272}
1273
1274static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1275{
1276        struct nlattr **a = info->attrs;
1277        struct ovs_header *ovs_header = info->userhdr;
1278        struct net *net = sock_net(skb->sk);
1279        struct sw_flow_key key;
1280        struct sk_buff *reply;
1281        struct sw_flow *flow;
1282        struct datapath *dp;
1283        struct sw_flow_match match;
1284        struct sw_flow_id ufid;
1285        u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1286        int err = 0;
1287        bool log = !a[OVS_FLOW_ATTR_PROBE];
1288        bool ufid_present;
1289
1290        ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1291        if (a[OVS_FLOW_ATTR_KEY]) {
1292                ovs_match_init(&match, &key, true, NULL);
1293                err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
1294                                        log);
1295        } else if (!ufid_present) {
1296                OVS_NLERR(log,
1297                          "Flow get message rejected, Key attribute missing.");
1298                err = -EINVAL;
1299        }
1300        if (err)
1301                return err;
1302
1303        ovs_lock();
1304        dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1305        if (!dp) {
1306                err = -ENODEV;
1307                goto unlock;
1308        }
1309
1310        if (ufid_present)
1311                flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1312        else
1313                flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1314        if (!flow) {
1315                err = -ENOENT;
1316                goto unlock;
1317        }
1318
1319        reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1320                                        OVS_FLOW_CMD_GET, true, ufid_flags);
1321        if (IS_ERR(reply)) {
1322                err = PTR_ERR(reply);
1323                goto unlock;
1324        }
1325
1326        ovs_unlock();
1327        return genlmsg_reply(reply, info);
1328unlock:
1329        ovs_unlock();
1330        return err;
1331}
1332
1333static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1334{
1335        struct nlattr **a = info->attrs;
1336        struct ovs_header *ovs_header = info->userhdr;
1337        struct net *net = sock_net(skb->sk);
1338        struct sw_flow_key key;
1339        struct sk_buff *reply;
1340        struct sw_flow *flow = NULL;
1341        struct datapath *dp;
1342        struct sw_flow_match match;
1343        struct sw_flow_id ufid;
1344        u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1345        int err;
1346        bool log = !a[OVS_FLOW_ATTR_PROBE];
1347        bool ufid_present;
1348
1349        ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1350        if (a[OVS_FLOW_ATTR_KEY]) {
1351                ovs_match_init(&match, &key, true, NULL);
1352                err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1353                                        NULL, log);
1354                if (unlikely(err))
1355                        return err;
1356        }
1357
1358        ovs_lock();
1359        dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1360        if (unlikely(!dp)) {
1361                err = -ENODEV;
1362                goto unlock;
1363        }
1364
1365        if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1366                err = ovs_flow_tbl_flush(&dp->table);
1367                goto unlock;
1368        }
1369
1370        if (ufid_present)
1371                flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1372        else
1373                flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1374        if (unlikely(!flow)) {
1375                err = -ENOENT;
1376                goto unlock;
1377        }
1378
1379        ovs_flow_tbl_remove(&dp->table, flow);
1380        ovs_unlock();
1381
1382        reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts,
1383                                        &flow->id, info, false, ufid_flags);
1384        if (likely(reply)) {
1385                if (!IS_ERR(reply)) {
1386                        rcu_read_lock();        /*To keep RCU checker happy. */
1387                        err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
1388                                                     reply, info->snd_portid,
1389                                                     info->snd_seq, 0,
1390                                                     OVS_FLOW_CMD_DEL,
1391                                                     ufid_flags);
1392                        rcu_read_unlock();
1393                        if (WARN_ON_ONCE(err < 0)) {
1394                                kfree_skb(reply);
1395                                goto out_free;
1396                        }
1397
1398                        ovs_notify(&dp_flow_genl_family, reply, info);
1399                } else {
1400                        netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0,
1401                                        PTR_ERR(reply));
1402                }
1403        }
1404
1405out_free:
1406        ovs_flow_free(flow, true);
1407        return 0;
1408unlock:
1409        ovs_unlock();
1410        return err;
1411}
1412
1413static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1414{
1415        struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1416        struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1417        struct table_instance *ti;
1418        struct datapath *dp;
1419        u32 ufid_flags;
1420        int err;
1421
1422        err = genlmsg_parse_deprecated(cb->nlh, &dp_flow_genl_family, a,
1423                                       OVS_FLOW_ATTR_MAX, flow_policy, NULL);
1424        if (err)
1425                return err;
1426        ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1427
1428        rcu_read_lock();
1429        dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1430        if (!dp) {
1431                rcu_read_unlock();
1432                return -ENODEV;
1433        }
1434
1435        ti = rcu_dereference(dp->table.ti);
1436        for (;;) {
1437                struct sw_flow *flow;
1438                u32 bucket, obj;
1439
1440                bucket = cb->args[0];
1441                obj = cb->args[1];
1442                flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1443                if (!flow)
1444                        break;
1445
1446                if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1447                                           NETLINK_CB(cb->skb).portid,
1448                                           cb->nlh->nlmsg_seq, NLM_F_MULTI,
1449                                           OVS_FLOW_CMD_GET, ufid_flags) < 0)
1450                        break;
1451
1452                cb->args[0] = bucket;
1453                cb->args[1] = obj;
1454        }
1455        rcu_read_unlock();
1456        return skb->len;
1457}
1458
1459static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1460        [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1461        [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1462        [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1463        [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1464        [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1465        [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1466        [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1467};
1468
1469static const struct genl_small_ops dp_flow_genl_ops[] = {
1470        { .cmd = OVS_FLOW_CMD_NEW,
1471          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1472          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1473          .doit = ovs_flow_cmd_new
1474        },
1475        { .cmd = OVS_FLOW_CMD_DEL,
1476          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1477          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1478          .doit = ovs_flow_cmd_del
1479        },
1480        { .cmd = OVS_FLOW_CMD_GET,
1481          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1482          .flags = 0,               /* OK for unprivileged users. */
1483          .doit = ovs_flow_cmd_get,
1484          .dumpit = ovs_flow_cmd_dump
1485        },
1486        { .cmd = OVS_FLOW_CMD_SET,
1487          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1488          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1489          .doit = ovs_flow_cmd_set,
1490        },
1491};
1492
1493static struct genl_family dp_flow_genl_family __ro_after_init = {
1494        .hdrsize = sizeof(struct ovs_header),
1495        .name = OVS_FLOW_FAMILY,
1496        .version = OVS_FLOW_VERSION,
1497        .maxattr = OVS_FLOW_ATTR_MAX,
1498        .policy = flow_policy,
1499        .netnsok = true,
1500        .parallel_ops = true,
1501        .small_ops = dp_flow_genl_ops,
1502        .n_small_ops = ARRAY_SIZE(dp_flow_genl_ops),
1503        .mcgrps = &ovs_dp_flow_multicast_group,
1504        .n_mcgrps = 1,
1505        .module = THIS_MODULE,
1506};
1507
1508static size_t ovs_dp_cmd_msg_size(void)
1509{
1510        size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1511
1512        msgsize += nla_total_size(IFNAMSIZ);
1513        msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
1514        msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
1515        msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1516        msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
1517
1518        return msgsize;
1519}
1520
1521/* Called with ovs_mutex. */
1522static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1523                                u32 portid, u32 seq, u32 flags, u8 cmd)
1524{
1525        struct ovs_header *ovs_header;
1526        struct ovs_dp_stats dp_stats;
1527        struct ovs_dp_megaflow_stats dp_megaflow_stats;
1528        int err;
1529
1530        ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1531                                 flags, cmd);
1532        if (!ovs_header)
1533                goto error;
1534
1535        ovs_header->dp_ifindex = get_dpifindex(dp);
1536
1537        err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1538        if (err)
1539                goto nla_put_failure;
1540
1541        get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1542        if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1543                          &dp_stats, OVS_DP_ATTR_PAD))
1544                goto nla_put_failure;
1545
1546        if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1547                          sizeof(struct ovs_dp_megaflow_stats),
1548                          &dp_megaflow_stats, OVS_DP_ATTR_PAD))
1549                goto nla_put_failure;
1550
1551        if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1552                goto nla_put_failure;
1553
1554        if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE,
1555                        ovs_flow_tbl_masks_cache_size(&dp->table)))
1556                goto nla_put_failure;
1557
1558        genlmsg_end(skb, ovs_header);
1559        return 0;
1560
1561nla_put_failure:
1562        genlmsg_cancel(skb, ovs_header);
1563error:
1564        return -EMSGSIZE;
1565}
1566
1567static struct sk_buff *ovs_dp_cmd_alloc_info(void)
1568{
1569        return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
1570}
1571
1572/* Called with rcu_read_lock or ovs_mutex. */
1573static struct datapath *lookup_datapath(struct net *net,
1574                                        const struct ovs_header *ovs_header,
1575                                        struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1576{
1577        struct datapath *dp;
1578
1579        if (!a[OVS_DP_ATTR_NAME])
1580                dp = get_dp(net, ovs_header->dp_ifindex);
1581        else {
1582                struct vport *vport;
1583
1584                vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1585                dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1586        }
1587        return dp ? dp : ERR_PTR(-ENODEV);
1588}
1589
1590static void ovs_dp_reset_user_features(struct sk_buff *skb,
1591                                       struct genl_info *info)
1592{
1593        struct datapath *dp;
1594
1595        dp = lookup_datapath(sock_net(skb->sk), info->userhdr,
1596                             info->attrs);
1597        if (IS_ERR(dp))
1598                return;
1599
1600        WARN(dp->user_features, "Dropping previously announced user features\n");
1601        dp->user_features = 0;
1602}
1603
1604DEFINE_STATIC_KEY_FALSE(tc_recirc_sharing_support);
1605
1606static int ovs_dp_set_upcall_portids(struct datapath *dp,
1607                              const struct nlattr *ids)
1608{
1609        struct dp_nlsk_pids *old, *dp_nlsk_pids;
1610
1611        if (!nla_len(ids) || nla_len(ids) % sizeof(u32))
1612                return -EINVAL;
1613
1614        old = ovsl_dereference(dp->upcall_portids);
1615
1616        dp_nlsk_pids = kmalloc(sizeof(*dp_nlsk_pids) + nla_len(ids),
1617                               GFP_KERNEL);
1618        if (!dp_nlsk_pids)
1619                return -ENOMEM;
1620
1621        dp_nlsk_pids->n_pids = nla_len(ids) / sizeof(u32);
1622        nla_memcpy(dp_nlsk_pids->pids, ids, nla_len(ids));
1623
1624        rcu_assign_pointer(dp->upcall_portids, dp_nlsk_pids);
1625
1626        kfree_rcu(old, rcu);
1627
1628        return 0;
1629}
1630
1631u32 ovs_dp_get_upcall_portid(const struct datapath *dp, uint32_t cpu_id)
1632{
1633        struct dp_nlsk_pids *dp_nlsk_pids;
1634
1635        dp_nlsk_pids = rcu_dereference(dp->upcall_portids);
1636
1637        if (dp_nlsk_pids) {
1638                if (cpu_id < dp_nlsk_pids->n_pids) {
1639                        return dp_nlsk_pids->pids[cpu_id];
1640                } else if (dp_nlsk_pids->n_pids > 0 &&
1641                           cpu_id >= dp_nlsk_pids->n_pids) {
1642                        /* If the number of netlink PIDs is mismatched with
1643                         * the number of CPUs as seen by the kernel, log this
1644                         * and send the upcall to an arbitrary socket (0) in
1645                         * order to not drop packets
1646                         */
1647                        pr_info_ratelimited("cpu_id mismatch with handler threads");
1648                        return dp_nlsk_pids->pids[cpu_id %
1649                                                  dp_nlsk_pids->n_pids];
1650                } else {
1651                        return 0;
1652                }
1653        } else {
1654                return 0;
1655        }
1656}
1657
1658static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1659{
1660        u32 user_features = 0;
1661        int err;
1662
1663        if (a[OVS_DP_ATTR_USER_FEATURES]) {
1664                user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1665
1666                if (user_features & ~(OVS_DP_F_VPORT_PIDS |
1667                                      OVS_DP_F_UNALIGNED |
1668                                      OVS_DP_F_TC_RECIRC_SHARING |
1669                                      OVS_DP_F_DISPATCH_UPCALL_PER_CPU))
1670                        return -EOPNOTSUPP;
1671
1672#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1673                if (user_features & OVS_DP_F_TC_RECIRC_SHARING)
1674                        return -EOPNOTSUPP;
1675#endif
1676        }
1677
1678        if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) {
1679                int err;
1680                u32 cache_size;
1681
1682                cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]);
1683                err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size);
1684                if (err)
1685                        return err;
1686        }
1687
1688        dp->user_features = user_features;
1689
1690        if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU &&
1691            a[OVS_DP_ATTR_PER_CPU_PIDS]) {
1692                /* Upcall Netlink Port IDs have been updated */
1693                err = ovs_dp_set_upcall_portids(dp,
1694                                                a[OVS_DP_ATTR_PER_CPU_PIDS]);
1695                if (err)
1696                        return err;
1697        }
1698
1699        if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING)
1700                static_branch_enable(&tc_recirc_sharing_support);
1701        else
1702                static_branch_disable(&tc_recirc_sharing_support);
1703
1704        return 0;
1705}
1706
1707static int ovs_dp_stats_init(struct datapath *dp)
1708{
1709        dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1710        if (!dp->stats_percpu)
1711                return -ENOMEM;
1712
1713        return 0;
1714}
1715
1716static int ovs_dp_vport_init(struct datapath *dp)
1717{
1718        int i;
1719
1720        dp->ports = kmalloc_array(DP_VPORT_HASH_BUCKETS,
1721                                  sizeof(struct hlist_head),
1722                                  GFP_KERNEL);
1723        if (!dp->ports)
1724                return -ENOMEM;
1725
1726        for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1727                INIT_HLIST_HEAD(&dp->ports[i]);
1728
1729        return 0;
1730}
1731
1732static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1733{
1734        struct nlattr **a = info->attrs;
1735        struct vport_parms parms;
1736        struct sk_buff *reply;
1737        struct datapath *dp;
1738        struct vport *vport;
1739        struct ovs_net *ovs_net;
1740        int err;
1741
1742        err = -EINVAL;
1743        if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1744                goto err;
1745
1746        reply = ovs_dp_cmd_alloc_info();
1747        if (!reply)
1748                return -ENOMEM;
1749
1750        err = -ENOMEM;
1751        dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1752        if (dp == NULL)
1753                goto err_destroy_reply;
1754
1755        ovs_dp_set_net(dp, sock_net(skb->sk));
1756
1757        /* Allocate table. */
1758        err = ovs_flow_tbl_init(&dp->table);
1759        if (err)
1760                goto err_destroy_dp;
1761
1762        err = ovs_dp_stats_init(dp);
1763        if (err)
1764                goto err_destroy_table;
1765
1766        err = ovs_dp_vport_init(dp);
1767        if (err)
1768                goto err_destroy_stats;
1769
1770        err = ovs_meters_init(dp);
1771        if (err)
1772                goto err_destroy_ports;
1773
1774        /* Set up our datapath device. */
1775        parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1776        parms.type = OVS_VPORT_TYPE_INTERNAL;
1777        parms.options = NULL;
1778        parms.dp = dp;
1779        parms.port_no = OVSP_LOCAL;
1780        parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1781
1782        /* So far only local changes have been made, now need the lock. */
1783        ovs_lock();
1784
1785        err = ovs_dp_change(dp, a);
1786        if (err)
1787                goto err_unlock_and_destroy_meters;
1788
1789        vport = new_vport(&parms);
1790        if (IS_ERR(vport)) {
1791                err = PTR_ERR(vport);
1792                if (err == -EBUSY)
1793                        err = -EEXIST;
1794
1795                if (err == -EEXIST) {
1796                        /* An outdated user space instance that does not understand
1797                         * the concept of user_features has attempted to create a new
1798                         * datapath and is likely to reuse it. Drop all user features.
1799                         */
1800                        if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1801                                ovs_dp_reset_user_features(skb, info);
1802                }
1803
1804                goto err_unlock_and_destroy_meters;
1805        }
1806
1807        err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1808                                   info->snd_seq, 0, OVS_DP_CMD_NEW);
1809        BUG_ON(err < 0);
1810
1811        ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1812        list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1813
1814        ovs_unlock();
1815
1816        ovs_notify(&dp_datapath_genl_family, reply, info);
1817        return 0;
1818
1819err_unlock_and_destroy_meters:
1820        ovs_unlock();
1821        ovs_meters_exit(dp);
1822err_destroy_ports:
1823        kfree(dp->ports);
1824err_destroy_stats:
1825        free_percpu(dp->stats_percpu);
1826err_destroy_table:
1827        ovs_flow_tbl_destroy(&dp->table);
1828err_destroy_dp:
1829        kfree(dp);
1830err_destroy_reply:
1831        kfree_skb(reply);
1832err:
1833        return err;
1834}
1835
1836/* Called with ovs_mutex. */
1837static void __dp_destroy(struct datapath *dp)
1838{
1839        struct flow_table *table = &dp->table;
1840        int i;
1841
1842        for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1843                struct vport *vport;
1844                struct hlist_node *n;
1845
1846                hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1847                        if (vport->port_no != OVSP_LOCAL)
1848                                ovs_dp_detach_port(vport);
1849        }
1850
1851        list_del_rcu(&dp->list_node);
1852
1853        /* OVSP_LOCAL is datapath internal port. We need to make sure that
1854         * all ports in datapath are destroyed first before freeing datapath.
1855         */
1856        ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1857
1858        /* Flush sw_flow in the tables. RCU cb only releases resource
1859         * such as dp, ports and tables. That may avoid some issues
1860         * such as RCU usage warning.
1861         */
1862        table_instance_flow_flush(table, ovsl_dereference(table->ti),
1863                                  ovsl_dereference(table->ufid_ti));
1864
1865        /* RCU destroy the ports, meters and flow tables. */
1866        call_rcu(&dp->rcu, destroy_dp_rcu);
1867}
1868
1869static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1870{
1871        struct sk_buff *reply;
1872        struct datapath *dp;
1873        int err;
1874
1875        reply = ovs_dp_cmd_alloc_info();
1876        if (!reply)
1877                return -ENOMEM;
1878
1879        ovs_lock();
1880        dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1881        err = PTR_ERR(dp);
1882        if (IS_ERR(dp))
1883                goto err_unlock_free;
1884
1885        err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1886                                   info->snd_seq, 0, OVS_DP_CMD_DEL);
1887        BUG_ON(err < 0);
1888
1889        __dp_destroy(dp);
1890        ovs_unlock();
1891
1892        ovs_notify(&dp_datapath_genl_family, reply, info);
1893
1894        return 0;
1895
1896err_unlock_free:
1897        ovs_unlock();
1898        kfree_skb(reply);
1899        return err;
1900}
1901
1902static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1903{
1904        struct sk_buff *reply;
1905        struct datapath *dp;
1906        int err;
1907
1908        reply = ovs_dp_cmd_alloc_info();
1909        if (!reply)
1910                return -ENOMEM;
1911
1912        ovs_lock();
1913        dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1914        err = PTR_ERR(dp);
1915        if (IS_ERR(dp))
1916                goto err_unlock_free;
1917
1918        err = ovs_dp_change(dp, info->attrs);
1919        if (err)
1920                goto err_unlock_free;
1921
1922        err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1923                                   info->snd_seq, 0, OVS_DP_CMD_SET);
1924        BUG_ON(err < 0);
1925
1926        ovs_unlock();
1927        ovs_notify(&dp_datapath_genl_family, reply, info);
1928
1929        return 0;
1930
1931err_unlock_free:
1932        ovs_unlock();
1933        kfree_skb(reply);
1934        return err;
1935}
1936
1937static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1938{
1939        struct sk_buff *reply;
1940        struct datapath *dp;
1941        int err;
1942
1943        reply = ovs_dp_cmd_alloc_info();
1944        if (!reply)
1945                return -ENOMEM;
1946
1947        ovs_lock();
1948        dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1949        if (IS_ERR(dp)) {
1950                err = PTR_ERR(dp);
1951                goto err_unlock_free;
1952        }
1953        err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1954                                   info->snd_seq, 0, OVS_DP_CMD_GET);
1955        BUG_ON(err < 0);
1956        ovs_unlock();
1957
1958        return genlmsg_reply(reply, info);
1959
1960err_unlock_free:
1961        ovs_unlock();
1962        kfree_skb(reply);
1963        return err;
1964}
1965
1966static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1967{
1968        struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1969        struct datapath *dp;
1970        int skip = cb->args[0];
1971        int i = 0;
1972
1973        ovs_lock();
1974        list_for_each_entry(dp, &ovs_net->dps, list_node) {
1975                if (i >= skip &&
1976                    ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1977                                         cb->nlh->nlmsg_seq, NLM_F_MULTI,
1978                                         OVS_DP_CMD_GET) < 0)
1979                        break;
1980                i++;
1981        }
1982        ovs_unlock();
1983
1984        cb->args[0] = i;
1985
1986        return skb->len;
1987}
1988
1989static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1990        [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1991        [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1992        [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1993        [OVS_DP_ATTR_MASKS_CACHE_SIZE] =  NLA_POLICY_RANGE(NLA_U32, 0,
1994                PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)),
1995};
1996
1997static const struct genl_small_ops dp_datapath_genl_ops[] = {
1998        { .cmd = OVS_DP_CMD_NEW,
1999          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2000          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2001          .doit = ovs_dp_cmd_new
2002        },
2003        { .cmd = OVS_DP_CMD_DEL,
2004          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2005          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2006          .doit = ovs_dp_cmd_del
2007        },
2008        { .cmd = OVS_DP_CMD_GET,
2009          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2010          .flags = 0,               /* OK for unprivileged users. */
2011          .doit = ovs_dp_cmd_get,
2012          .dumpit = ovs_dp_cmd_dump
2013        },
2014        { .cmd = OVS_DP_CMD_SET,
2015          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2016          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2017          .doit = ovs_dp_cmd_set,
2018        },
2019};
2020
2021static struct genl_family dp_datapath_genl_family __ro_after_init = {
2022        .hdrsize = sizeof(struct ovs_header),
2023        .name = OVS_DATAPATH_FAMILY,
2024        .version = OVS_DATAPATH_VERSION,
2025        .maxattr = OVS_DP_ATTR_MAX,
2026        .policy = datapath_policy,
2027        .netnsok = true,
2028        .parallel_ops = true,
2029        .small_ops = dp_datapath_genl_ops,
2030        .n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops),
2031        .mcgrps = &ovs_dp_datapath_multicast_group,
2032        .n_mcgrps = 1,
2033        .module = THIS_MODULE,
2034};
2035
2036/* Called with ovs_mutex or RCU read lock. */
2037static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
2038                                   struct net *net, u32 portid, u32 seq,
2039                                   u32 flags, u8 cmd, gfp_t gfp)
2040{
2041        struct ovs_header *ovs_header;
2042        struct ovs_vport_stats vport_stats;
2043        int err;
2044
2045        ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
2046                                 flags, cmd);
2047        if (!ovs_header)
2048                return -EMSGSIZE;
2049
2050        ovs_header->dp_ifindex = get_dpifindex(vport->dp);
2051
2052        if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
2053            nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
2054            nla_put_string(skb, OVS_VPORT_ATTR_NAME,
2055                           ovs_vport_name(vport)) ||
2056            nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
2057                goto nla_put_failure;
2058
2059        if (!net_eq(net, dev_net(vport->dev))) {
2060                int id = peernet2id_alloc(net, dev_net(vport->dev), gfp);
2061
2062                if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
2063                        goto nla_put_failure;
2064        }
2065
2066        ovs_vport_get_stats(vport, &vport_stats);
2067        if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
2068                          sizeof(struct ovs_vport_stats), &vport_stats,
2069                          OVS_VPORT_ATTR_PAD))
2070                goto nla_put_failure;
2071
2072        if (ovs_vport_get_upcall_portids(vport, skb))
2073                goto nla_put_failure;
2074
2075        err = ovs_vport_get_options(vport, skb);
2076        if (err == -EMSGSIZE)
2077                goto error;
2078
2079        genlmsg_end(skb, ovs_header);
2080        return 0;
2081
2082nla_put_failure:
2083        err = -EMSGSIZE;
2084error:
2085        genlmsg_cancel(skb, ovs_header);
2086        return err;
2087}
2088
2089static struct sk_buff *ovs_vport_cmd_alloc_info(void)
2090{
2091        return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2092}
2093
2094/* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
2095struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
2096                                         u32 portid, u32 seq, u8 cmd)
2097{
2098        struct sk_buff *skb;
2099        int retval;
2100
2101        skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2102        if (!skb)
2103                return ERR_PTR(-ENOMEM);
2104
2105        retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd,
2106                                         GFP_KERNEL);
2107        BUG_ON(retval < 0);
2108
2109        return skb;
2110}
2111
2112/* Called with ovs_mutex or RCU read lock. */
2113static struct vport *lookup_vport(struct net *net,
2114                                  const struct ovs_header *ovs_header,
2115                                  struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
2116{
2117        struct datapath *dp;
2118        struct vport *vport;
2119
2120        if (a[OVS_VPORT_ATTR_IFINDEX])
2121                return ERR_PTR(-EOPNOTSUPP);
2122        if (a[OVS_VPORT_ATTR_NAME]) {
2123                vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
2124                if (!vport)
2125                        return ERR_PTR(-ENODEV);
2126                if (ovs_header->dp_ifindex &&
2127                    ovs_header->dp_ifindex != get_dpifindex(vport->dp))
2128                        return ERR_PTR(-ENODEV);
2129                return vport;
2130        } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
2131                u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
2132
2133                if (port_no >= DP_MAX_PORTS)
2134                        return ERR_PTR(-EFBIG);
2135
2136                dp = get_dp(net, ovs_header->dp_ifindex);
2137                if (!dp)
2138                        return ERR_PTR(-ENODEV);
2139
2140                vport = ovs_vport_ovsl_rcu(dp, port_no);
2141                if (!vport)
2142                        return ERR_PTR(-ENODEV);
2143                return vport;
2144        } else
2145                return ERR_PTR(-EINVAL);
2146
2147}
2148
2149static unsigned int ovs_get_max_headroom(struct datapath *dp)
2150{
2151        unsigned int dev_headroom, max_headroom = 0;
2152        struct net_device *dev;
2153        struct vport *vport;
2154        int i;
2155
2156        for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2157                hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2158                                         lockdep_ovsl_is_held()) {
2159                        dev = vport->dev;
2160                        dev_headroom = netdev_get_fwd_headroom(dev);
2161                        if (dev_headroom > max_headroom)
2162                                max_headroom = dev_headroom;
2163                }
2164        }
2165
2166        return max_headroom;
2167}
2168
2169/* Called with ovs_mutex */
2170static void ovs_update_headroom(struct datapath *dp, unsigned int new_headroom)
2171{
2172        struct vport *vport;
2173        int i;
2174
2175        dp->max_headroom = new_headroom;
2176        for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2177                hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2178                                         lockdep_ovsl_is_held())
2179                        netdev_set_rx_headroom(vport->dev, new_headroom);
2180        }
2181}
2182
2183static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
2184{
2185        struct nlattr **a = info->attrs;
2186        struct ovs_header *ovs_header = info->userhdr;
2187        struct vport_parms parms;
2188        struct sk_buff *reply;
2189        struct vport *vport;
2190        struct datapath *dp;
2191        unsigned int new_headroom;
2192        u32 port_no;
2193        int err;
2194
2195        if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
2196            !a[OVS_VPORT_ATTR_UPCALL_PID])
2197                return -EINVAL;
2198        if (a[OVS_VPORT_ATTR_IFINDEX])
2199                return -EOPNOTSUPP;
2200
2201        port_no = a[OVS_VPORT_ATTR_PORT_NO]
2202                ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
2203        if (port_no >= DP_MAX_PORTS)
2204                return -EFBIG;
2205
2206        reply = ovs_vport_cmd_alloc_info();
2207        if (!reply)
2208                return -ENOMEM;
2209
2210        ovs_lock();
2211restart:
2212        dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
2213        err = -ENODEV;
2214        if (!dp)
2215                goto exit_unlock_free;
2216
2217        if (port_no) {
2218                vport = ovs_vport_ovsl(dp, port_no);
2219                err = -EBUSY;
2220                if (vport)
2221                        goto exit_unlock_free;
2222        } else {
2223                for (port_no = 1; ; port_no++) {
2224                        if (port_no >= DP_MAX_PORTS) {
2225                                err = -EFBIG;
2226                                goto exit_unlock_free;
2227                        }
2228                        vport = ovs_vport_ovsl(dp, port_no);
2229                        if (!vport)
2230                                break;
2231                }
2232        }
2233
2234        parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
2235        parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
2236        parms.options = a[OVS_VPORT_ATTR_OPTIONS];
2237        parms.dp = dp;
2238        parms.port_no = port_no;
2239        parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
2240
2241        vport = new_vport(&parms);
2242        err = PTR_ERR(vport);
2243        if (IS_ERR(vport)) {
2244                if (err == -EAGAIN)
2245                        goto restart;
2246                goto exit_unlock_free;
2247        }
2248
2249        err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2250                                      info->snd_portid, info->snd_seq, 0,
2251                                      OVS_VPORT_CMD_NEW, GFP_KERNEL);
2252
2253        new_headroom = netdev_get_fwd_headroom(vport->dev);
2254
2255        if (new_headroom > dp->max_headroom)
2256                ovs_update_headroom(dp, new_headroom);
2257        else
2258                netdev_set_rx_headroom(vport->dev, dp->max_headroom);
2259
2260        BUG_ON(err < 0);
2261        ovs_unlock();
2262
2263        ovs_notify(&dp_vport_genl_family, reply, info);
2264        return 0;
2265
2266exit_unlock_free:
2267        ovs_unlock();
2268        kfree_skb(reply);
2269        return err;
2270}
2271
2272static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
2273{
2274        struct nlattr **a = info->attrs;
2275        struct sk_buff *reply;
2276        struct vport *vport;
2277        int err;
2278
2279        reply = ovs_vport_cmd_alloc_info();
2280        if (!reply)
2281                return -ENOMEM;
2282
2283        ovs_lock();
2284        vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2285        err = PTR_ERR(vport);
2286        if (IS_ERR(vport))
2287                goto exit_unlock_free;
2288
2289        if (a[OVS_VPORT_ATTR_TYPE] &&
2290            nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
2291                err = -EINVAL;
2292                goto exit_unlock_free;
2293        }
2294
2295        if (a[OVS_VPORT_ATTR_OPTIONS]) {
2296                err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
2297                if (err)
2298                        goto exit_unlock_free;
2299        }
2300
2301
2302        if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
2303                struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2304
2305                err = ovs_vport_set_upcall_portids(vport, ids);
2306                if (err)
2307                        goto exit_unlock_free;
2308        }
2309
2310        err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2311                                      info->snd_portid, info->snd_seq, 0,
2312                                      OVS_VPORT_CMD_SET, GFP_KERNEL);
2313        BUG_ON(err < 0);
2314
2315        ovs_unlock();
2316        ovs_notify(&dp_vport_genl_family, reply, info);
2317        return 0;
2318
2319exit_unlock_free:
2320        ovs_unlock();
2321        kfree_skb(reply);
2322        return err;
2323}
2324
2325static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2326{
2327        bool update_headroom = false;
2328        struct nlattr **a = info->attrs;
2329        struct sk_buff *reply;
2330        struct datapath *dp;
2331        struct vport *vport;
2332        unsigned int new_headroom;
2333        int err;
2334
2335        reply = ovs_vport_cmd_alloc_info();
2336        if (!reply)
2337                return -ENOMEM;
2338
2339        ovs_lock();
2340        vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2341        err = PTR_ERR(vport);
2342        if (IS_ERR(vport))
2343                goto exit_unlock_free;
2344
2345        if (vport->port_no == OVSP_LOCAL) {
2346                err = -EINVAL;
2347                goto exit_unlock_free;
2348        }
2349
2350        err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2351                                      info->snd_portid, info->snd_seq, 0,
2352                                      OVS_VPORT_CMD_DEL, GFP_KERNEL);
2353        BUG_ON(err < 0);
2354
2355        /* the vport deletion may trigger dp headroom update */
2356        dp = vport->dp;
2357        if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
2358                update_headroom = true;
2359
2360        netdev_reset_rx_headroom(vport->dev);
2361        ovs_dp_detach_port(vport);
2362
2363        if (update_headroom) {
2364                new_headroom = ovs_get_max_headroom(dp);
2365
2366                if (new_headroom < dp->max_headroom)
2367                        ovs_update_headroom(dp, new_headroom);
2368        }
2369        ovs_unlock();
2370
2371        ovs_notify(&dp_vport_genl_family, reply, info);
2372        return 0;
2373
2374exit_unlock_free:
2375        ovs_unlock();
2376        kfree_skb(reply);
2377        return err;
2378}
2379
2380static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2381{
2382        struct nlattr **a = info->attrs;
2383        struct ovs_header *ovs_header = info->userhdr;
2384        struct sk_buff *reply;
2385        struct vport *vport;
2386        int err;
2387
2388        reply = ovs_vport_cmd_alloc_info();
2389        if (!reply)
2390                return -ENOMEM;
2391
2392        rcu_read_lock();
2393        vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2394        err = PTR_ERR(vport);
2395        if (IS_ERR(vport))
2396                goto exit_unlock_free;
2397        err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2398                                      info->snd_portid, info->snd_seq, 0,
2399                                      OVS_VPORT_CMD_GET, GFP_ATOMIC);
2400        BUG_ON(err < 0);
2401        rcu_read_unlock();
2402
2403        return genlmsg_reply(reply, info);
2404
2405exit_unlock_free:
2406        rcu_read_unlock();
2407        kfree_skb(reply);
2408        return err;
2409}
2410
2411static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2412{
2413        struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2414        struct datapath *dp;
2415        int bucket = cb->args[0], skip = cb->args[1];
2416        int i, j = 0;
2417
2418        rcu_read_lock();
2419        dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2420        if (!dp) {
2421                rcu_read_unlock();
2422                return -ENODEV;
2423        }
2424        for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2425                struct vport *vport;
2426
2427                j = 0;
2428                hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2429                        if (j >= skip &&
2430                            ovs_vport_cmd_fill_info(vport, skb,
2431                                                    sock_net(skb->sk),
2432                                                    NETLINK_CB(cb->skb).portid,
2433                                                    cb->nlh->nlmsg_seq,
2434                                                    NLM_F_MULTI,
2435                                                    OVS_VPORT_CMD_GET,
2436                                                    GFP_ATOMIC) < 0)
2437                                goto out;
2438
2439                        j++;
2440                }
2441                skip = 0;
2442        }
2443out:
2444        rcu_read_unlock();
2445
2446        cb->args[0] = i;
2447        cb->args[1] = j;
2448
2449        return skb->len;
2450}
2451
2452static void ovs_dp_masks_rebalance(struct work_struct *work)
2453{
2454        struct ovs_net *ovs_net = container_of(work, struct ovs_net,
2455                                               masks_rebalance.work);
2456        struct datapath *dp;
2457
2458        ovs_lock();
2459
2460        list_for_each_entry(dp, &ovs_net->dps, list_node)
2461                ovs_flow_masks_rebalance(&dp->table);
2462
2463        ovs_unlock();
2464
2465        schedule_delayed_work(&ovs_net->masks_rebalance,
2466                              msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2467}
2468
2469static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2470        [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2471        [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2472        [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2473        [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2474        [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC },
2475        [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
2476        [OVS_VPORT_ATTR_IFINDEX] = { .type = NLA_U32 },
2477        [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
2478};
2479
2480static const struct genl_small_ops dp_vport_genl_ops[] = {
2481        { .cmd = OVS_VPORT_CMD_NEW,
2482          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2483          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2484          .doit = ovs_vport_cmd_new
2485        },
2486        { .cmd = OVS_VPORT_CMD_DEL,
2487          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2488          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2489          .doit = ovs_vport_cmd_del
2490        },
2491        { .cmd = OVS_VPORT_CMD_GET,
2492          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2493          .flags = 0,               /* OK for unprivileged users. */
2494          .doit = ovs_vport_cmd_get,
2495          .dumpit = ovs_vport_cmd_dump
2496        },
2497        { .cmd = OVS_VPORT_CMD_SET,
2498          .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2499          .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2500          .doit = ovs_vport_cmd_set,
2501        },
2502};
2503
2504struct genl_family dp_vport_genl_family __ro_after_init = {
2505        .hdrsize = sizeof(struct ovs_header),
2506        .name = OVS_VPORT_FAMILY,
2507        .version = OVS_VPORT_VERSION,
2508        .maxattr = OVS_VPORT_ATTR_MAX,
2509        .policy = vport_policy,
2510        .netnsok = true,
2511        .parallel_ops = true,
2512        .small_ops = dp_vport_genl_ops,
2513        .n_small_ops = ARRAY_SIZE(dp_vport_genl_ops),
2514        .mcgrps = &ovs_dp_vport_multicast_group,
2515        .n_mcgrps = 1,
2516        .module = THIS_MODULE,
2517};
2518
2519static struct genl_family * const dp_genl_families[] = {
2520        &dp_datapath_genl_family,
2521        &dp_vport_genl_family,
2522        &dp_flow_genl_family,
2523        &dp_packet_genl_family,
2524        &dp_meter_genl_family,
2525#if     IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
2526        &dp_ct_limit_genl_family,
2527#endif
2528};
2529
2530static void dp_unregister_genl(int n_families)
2531{
2532        int i;
2533
2534        for (i = 0; i < n_families; i++)
2535                genl_unregister_family(dp_genl_families[i]);
2536}
2537
2538static int __init dp_register_genl(void)
2539{
2540        int err;
2541        int i;
2542
2543        for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2544
2545                err = genl_register_family(dp_genl_families[i]);
2546                if (err)
2547                        goto error;
2548        }
2549
2550        return 0;
2551
2552error:
2553        dp_unregister_genl(i);
2554        return err;
2555}
2556
2557static int __net_init ovs_init_net(struct net *net)
2558{
2559        struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2560        int err;
2561
2562        INIT_LIST_HEAD(&ovs_net->dps);
2563        INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2564        INIT_DELAYED_WORK(&ovs_net->masks_rebalance, ovs_dp_masks_rebalance);
2565
2566        err = ovs_ct_init(net);
2567        if (err)
2568                return err;
2569
2570        schedule_delayed_work(&ovs_net->masks_rebalance,
2571                              msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2572        return 0;
2573}
2574
2575static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2576                                            struct list_head *head)
2577{
2578        struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2579        struct datapath *dp;
2580
2581        list_for_each_entry(dp, &ovs_net->dps, list_node) {
2582                int i;
2583
2584                for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2585                        struct vport *vport;
2586
2587                        hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
2588                                if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2589                                        continue;
2590
2591                                if (dev_net(vport->dev) == dnet)
2592                                        list_add(&vport->detach_list, head);
2593                        }
2594                }
2595        }
2596}
2597
2598static void __net_exit ovs_exit_net(struct net *dnet)
2599{
2600        struct datapath *dp, *dp_next;
2601        struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2602        struct vport *vport, *vport_next;
2603        struct net *net;
2604        LIST_HEAD(head);
2605
2606        ovs_lock();
2607
2608        ovs_ct_exit(dnet);
2609
2610        list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2611                __dp_destroy(dp);
2612
2613        down_read(&net_rwsem);
2614        for_each_net(net)
2615                list_vports_from_net(net, dnet, &head);
2616        up_read(&net_rwsem);
2617
2618        /* Detach all vports from given namespace. */
2619        list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2620                list_del(&vport->detach_list);
2621                ovs_dp_detach_port(vport);
2622        }
2623
2624        ovs_unlock();
2625
2626        cancel_delayed_work_sync(&ovs_net->masks_rebalance);
2627        cancel_work_sync(&ovs_net->dp_notify_work);
2628}
2629
2630static struct pernet_operations ovs_net_ops = {
2631        .init = ovs_init_net,
2632        .exit = ovs_exit_net,
2633        .id   = &ovs_net_id,
2634        .size = sizeof(struct ovs_net),
2635};
2636
2637static int __init dp_init(void)
2638{
2639        int err;
2640
2641        BUILD_BUG_ON(sizeof(struct ovs_skb_cb) >
2642                     sizeof_field(struct sk_buff, cb));
2643
2644        pr_info("Open vSwitch switching datapath\n");
2645
2646        err = action_fifos_init();
2647        if (err)
2648                goto error;
2649
2650        err = ovs_internal_dev_rtnl_link_register();
2651        if (err)
2652                goto error_action_fifos_exit;
2653
2654        err = ovs_flow_init();
2655        if (err)
2656                goto error_unreg_rtnl_link;
2657
2658        err = ovs_vport_init();
2659        if (err)
2660                goto error_flow_exit;
2661
2662        err = register_pernet_device(&ovs_net_ops);
2663        if (err)
2664                goto error_vport_exit;
2665
2666        err = register_netdevice_notifier(&ovs_dp_device_notifier);
2667        if (err)
2668                goto error_netns_exit;
2669
2670        err = ovs_netdev_init();
2671        if (err)
2672                goto error_unreg_notifier;
2673
2674        err = dp_register_genl();
2675        if (err < 0)
2676                goto error_unreg_netdev;
2677
2678        return 0;
2679
2680error_unreg_netdev:
2681        ovs_netdev_exit();
2682error_unreg_notifier:
2683        unregister_netdevice_notifier(&ovs_dp_device_notifier);
2684error_netns_exit:
2685        unregister_pernet_device(&ovs_net_ops);
2686error_vport_exit:
2687        ovs_vport_exit();
2688error_flow_exit:
2689        ovs_flow_exit();
2690error_unreg_rtnl_link:
2691        ovs_internal_dev_rtnl_link_unregister();
2692error_action_fifos_exit:
2693        action_fifos_exit();
2694error:
2695        return err;
2696}
2697
2698static void dp_cleanup(void)
2699{
2700        dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2701        ovs_netdev_exit();
2702        unregister_netdevice_notifier(&ovs_dp_device_notifier);
2703        unregister_pernet_device(&ovs_net_ops);
2704        rcu_barrier();
2705        ovs_vport_exit();
2706        ovs_flow_exit();
2707        ovs_internal_dev_rtnl_link_unregister();
2708        action_fifos_exit();
2709}
2710
2711module_init(dp_init);
2712module_exit(dp_cleanup);
2713
2714MODULE_DESCRIPTION("Open vSwitch switching datapath");
2715MODULE_LICENSE("GPL");
2716MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
2717MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
2718MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
2719MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
2720MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
2721MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);
2722