/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
 */

#include <errno.h>
#include <stdint.h>
#include <rte_log.h>
#include <ethdev_driver.h>
#include <rte_flow_driver.h>
#include <rte_ether.h>
#include <rte_geneve.h>
#include <rte_hash.h>
#include <rte_jhash.h>
#include <rte_ip.h>
#include <rte_udp.h>
#include <rte_memzone.h>

#include "enic_compat.h"
#include "enic.h"
#include "vnic_dev.h"
#include "vnic_nic.h"

#define IP_DEFTTL  64   /* from RFC 1340. */
#define IP6_VTC_FLOW 0x60000000

/* Up to 1024 TCAM entries */
#define FM_MAX_TCAM_TABLE_SIZE 1024

/* Up to 4096 entries per exact match table */
#define FM_MAX_EXACT_TABLE_SIZE 4096

/* Number of counters to increase on for each increment */
#define FM_COUNTERS_EXPAND  100

#define FM_INVALID_HANDLE 0

/* Low priority used for implicit VF -> representor flow */
#define FM_LOWEST_PRIORITY 100000

/* High priority used for implicit representor -> VF flow */
#define FM_HIGHEST_PRIORITY 0

/* Tag used for implicit VF <-> representor flows */
#define FM_VF_REP_TAG 1

/* Max number of actions supported by VIC is 2K. Make hash table double that. */
#define FM_MAX_ACTION_TABLE_SIZE 4096

/*
 * Flow exact match tables (FET) in the VIC and rte_flow groups.
 * Use a simple scheme to map groups to tables.
 * Group 0 uses the single TCAM tables, one for each direction.
 * Group 1, 2, ... uses its own exact match table.
 *
 * The TCAM tables are allocated upfront during init.
 *
 * Exact match tables are allocated on demand. 3 paths that lead allocations.
 *
 * 1. Add a flow that jumps from group 0 to group N.
 *
 * If N does not exist, we allocate an exact match table for it, using
 * a dummy key. A key is required for the table.
 *
 * 2. Add a flow that uses group N.
 *
 * If N does not exist, we allocate an exact match table for it, using
 * the flow's key. Subsequent flows to the same group all should have
 * the same key.
 *
 * Without a jump flow to N, N is not reachable in hardware. No packets
 * reach N and match.
 *
 * 3. Add a flow to an empty group N.
 *
 * N has been created via (1) and the dummy key. We free that table, allocate
 * a new table using the new flow's key. Also re-do the existing jump flow to
 * point to the new table.
 */
#define FM_TCAM_RTE_GROUP 0

struct enic_fm_fet {
        TAILQ_ENTRY(enic_fm_fet) list;
        uint32_t group; /* rte_flow group ID */
        uint64_t handle; /* Exact match table handle from flowman */
        uint8_t ingress;
        uint8_t default_key;
        int ref; /* Reference count via get/put */
        struct fm_key_template key; /* Key associated with the table */
};

struct enic_fm_counter {
        SLIST_ENTRY(enic_fm_counter) next;
        uint32_t handle;
};

struct enic_fm_action {
        int ref;
        uint64_t handle;
        struct fm_action key;
};

/* rte_flow.fm */
struct enic_fm_flow {
        bool counter_valid;
        uint64_t entry_handle;
        struct enic_fm_action  *action;
        struct enic_fm_counter *counter;
        struct enic_fm_fet *fet;
        /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
        struct enic_fm_flow *hairpin_steer_flow;
};

struct enic_fm_jump_flow {
        TAILQ_ENTRY(enic_fm_jump_flow) list;
        struct rte_flow *flow;
        uint32_t group;
        struct fm_tcam_match_entry match;
        struct fm_action action;
};

/*
 * Flowman uses host memory for commands. This structure is allocated
 * in DMA-able memory.
 */
union enic_flowman_cmd_mem {
        struct fm_tcam_match_table fm_tcam_match_table;
        struct fm_exact_match_table fm_exact_match_table;
        struct fm_tcam_match_entry fm_tcam_match_entry;
        struct fm_exact_match_entry fm_exact_match_entry;
        struct fm_action fm_action;
};

/*
 * PF has a flowman instance, and VF representors share it with PF.
 * PF allocates this structure and owns it. VF representors borrow
 * the PF's structure during API calls (e.g. create, query).
 */
struct enic_flowman {
        struct enic *owner_enic; /* PF */
        struct enic *user_enic;  /* API caller (PF or representor) */
        /*
         * Representors and PF share the same underlying flowman.
         * Lock API calls to serialize accesses from them. Only used
         * when VF representors are present.
         */
        rte_spinlock_t lock;
        /* Command buffer */
        struct {
                union enic_flowman_cmd_mem *va;
                dma_addr_t pa;
        } cmd;
        /* TCAM tables allocated upfront, used for group 0 */
        uint64_t ig_tcam_hndl;
        uint64_t eg_tcam_hndl;
        /* Counters */
        SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
        void *counter_stack;
        uint32_t counters_alloced;
        /* Exact match tables for groups != 0, dynamically allocated */
        TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
        /*
         * Default exact match tables used for jump actions to
         * non-existent groups.
         */
        struct enic_fm_fet *default_eg_fet;
        struct enic_fm_fet *default_ig_fet;
        /* hash table for Action reuse */
        struct rte_hash *action_hash;
        /* Flows that jump to the default table above */
        TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
        /*
         * Scratch data used during each invocation of flow_create
         * and flow_validate.
         */
        struct enic_fm_fet *fet;
        struct fm_tcam_match_entry tcam_entry;
        struct fm_action action;
        struct fm_action action_tmp; /* enic_fm_reorder_action_op */
        int action_op_count;
        /* Tags used for representor flows */
        uint8_t vf_rep_tag;
        /* For auto-added steer action for hairpin */
        int need_hairpin_steer;
        uint64_t hairpin_steer_vnic_h;
};

static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
/*
 * API functions (create, destroy, validate, flush) call begin_fm()
 * upon entering to save the caller enic (PF or VF representor) and
 * lock. Upon exit, they call end_fm() to unlock.
 */
static struct enic_flowman *begin_fm(struct enic *enic);
static void end_fm(struct enic_flowman *fm);
/* Delete internal flows created for representor paths */
static void delete_rep_flows(struct enic *enic);

/*
 * Common arguments passed to copy_item functions. Use this structure
 * so we can easily add new arguments.
 * item: Item specification.
 * fm_tcam_entry: Flowman TCAM match entry.
 * header_level: 0 for outer header, 1 for inner header.
 */
struct copy_item_args {
        const struct rte_flow_item *item;
        struct fm_tcam_match_entry *fm_tcam_entry;
        uint8_t header_level;
        struct rte_flow_error *error;
};

/* functions for copying items into flowman match */
typedef int (enic_copy_item_fn)(struct copy_item_args *arg);

/* Info about how to copy items into flowman match */
struct enic_fm_items {
        /* Function for copying and validating an item. */
        enic_copy_item_fn * const copy_item;
        /* List of valid previous items. */
        const enum rte_flow_item_type * const prev_items;
        /*
         * True if it's OK for this item to be the first item. For some NIC
         * versions, it's invalid to start the stack above layer 3.
         */
        const uint8_t valid_start_item;
};

static enic_copy_item_fn enic_fm_copy_item_eth;
static enic_copy_item_fn enic_fm_copy_item_ipv4;
static enic_copy_item_fn enic_fm_copy_item_ipv6;
static enic_copy_item_fn enic_fm_copy_item_raw;
static enic_copy_item_fn enic_fm_copy_item_sctp;
static enic_copy_item_fn enic_fm_copy_item_tcp;
static enic_copy_item_fn enic_fm_copy_item_udp;
static enic_copy_item_fn enic_fm_copy_item_vlan;
static enic_copy_item_fn enic_fm_copy_item_vxlan;
static enic_copy_item_fn enic_fm_copy_item_gtp;
static enic_copy_item_fn enic_fm_copy_item_geneve;
static enic_copy_item_fn enic_fm_copy_item_geneve_opt;
static enic_copy_item_fn enic_fm_copy_item_ecpri;

/* Ingress actions */
static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
        RTE_FLOW_ACTION_TYPE_COUNT,
        RTE_FLOW_ACTION_TYPE_DROP,
        RTE_FLOW_ACTION_TYPE_FLAG,
        RTE_FLOW_ACTION_TYPE_JUMP,
        RTE_FLOW_ACTION_TYPE_MARK,
        RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
        RTE_FLOW_ACTION_TYPE_PORT_ID,
        RTE_FLOW_ACTION_TYPE_PASSTHRU,
        RTE_FLOW_ACTION_TYPE_QUEUE,
        RTE_FLOW_ACTION_TYPE_RSS,
        RTE_FLOW_ACTION_TYPE_VOID,
        RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
        RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
        RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
};

/* Egress actions */
static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
        RTE_FLOW_ACTION_TYPE_COUNT,
        RTE_FLOW_ACTION_TYPE_DROP,
        RTE_FLOW_ACTION_TYPE_JUMP,
        RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
        RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
        RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
        RTE_FLOW_ACTION_TYPE_PORT_ID,
        RTE_FLOW_ACTION_TYPE_PASSTHRU,
        RTE_FLOW_ACTION_TYPE_VOID,
        RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
        RTE_FLOW_ACTION_TYPE_END,
};

static const struct enic_fm_items enic_fm_items[] = {
        [RTE_FLOW_ITEM_TYPE_RAW] = {
                .copy_item = enic_fm_copy_item_raw,
                .valid_start_item = 0,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_UDP,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_ETH] = {
                .copy_item = enic_fm_copy_item_eth,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_VLAN] = {
                .copy_item = enic_fm_copy_item_vlan,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_ETH,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_IPV4] = {
                .copy_item = enic_fm_copy_item_ipv4,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_ETH,
                               RTE_FLOW_ITEM_TYPE_VLAN,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_IPV6] = {
                .copy_item = enic_fm_copy_item_ipv6,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_ETH,
                               RTE_FLOW_ITEM_TYPE_VLAN,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_UDP] = {
                .copy_item = enic_fm_copy_item_udp,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_IPV4,
                               RTE_FLOW_ITEM_TYPE_IPV6,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_TCP] = {
                .copy_item = enic_fm_copy_item_tcp,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_IPV4,
                               RTE_FLOW_ITEM_TYPE_IPV6,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_SCTP] = {
                .copy_item = enic_fm_copy_item_sctp,
                .valid_start_item = 0,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_IPV4,
                               RTE_FLOW_ITEM_TYPE_IPV6,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_VXLAN] = {
                .copy_item = enic_fm_copy_item_vxlan,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_UDP,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_GTP] = {
                .copy_item = enic_fm_copy_item_gtp,
                .valid_start_item = 0,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_UDP,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_GTPC] = {
                .copy_item = enic_fm_copy_item_gtp,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_UDP,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_GTPU] = {
                .copy_item = enic_fm_copy_item_gtp,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_UDP,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_GENEVE] = {
                .copy_item = enic_fm_copy_item_geneve,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_ETH,
                               RTE_FLOW_ITEM_TYPE_IPV4,
                               RTE_FLOW_ITEM_TYPE_IPV6,
                               RTE_FLOW_ITEM_TYPE_UDP,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_GENEVE_OPT] = {
                .copy_item = enic_fm_copy_item_geneve_opt,
                .valid_start_item = 1,
                /* Can match at most 1 option */
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_GENEVE,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
        [RTE_FLOW_ITEM_TYPE_ECPRI] = {
                .copy_item = enic_fm_copy_item_ecpri,
                .valid_start_item = 1,
                .prev_items = (const enum rte_flow_item_type[]) {
                               RTE_FLOW_ITEM_TYPE_ETH,
                               RTE_FLOW_ITEM_TYPE_UDP,
                               RTE_FLOW_ITEM_TYPE_END,
                },
        },
};

static int
enic_fm_copy_item_eth(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_eth *spec = item->spec;
        const struct rte_flow_item_eth *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;

        ENICPMD_FUNC_TRACE();
        /* Match all if no spec */
        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_eth_mask;
        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        fm_data->fk_header_select |= FKH_ETHER;
        fm_mask->fk_header_select |= FKH_ETHER;
        memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
        memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
        return 0;
}

static int
enic_fm_copy_item_vlan(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_vlan *spec = item->spec;
        const struct rte_flow_item_vlan *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;
        struct rte_ether_hdr *eth_mask;
        struct rte_ether_hdr *eth_val;
        uint32_t meta;

        ENICPMD_FUNC_TRACE();
        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        /* Outer and inner packet vlans need different flags */
        meta = FKM_VLAN_PRES;
        if (lvl > 0)
                meta = FKM_QTAG;
        fm_data->fk_metadata |= meta;
        fm_mask->fk_metadata |= meta;

        /* Match all if no spec */
        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_vlan_mask;

        eth_mask = (void *)&fm_mask->l2.eth;
        eth_val = (void *)&fm_data->l2.eth;

        /*
         * Outer TPID cannot be matched. If inner_type is 0, use what is
         * in the eth header.
         */
        if (eth_mask->ether_type && mask->inner_type)
                return -ENOTSUP;

        /*
         * When packet matching, the VIC always compares vlan-stripped
         * L2, regardless of vlan stripping settings. So, the inner type
         * from vlan becomes the ether type of the eth header.
         */
        if (mask->inner_type) {
                eth_mask->ether_type = mask->inner_type;
                eth_val->ether_type = spec->inner_type;
        }
        fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
        fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
        fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
        fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
        return 0;
}

static int
enic_fm_copy_item_ipv4(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_ipv4 *spec = item->spec;
        const struct rte_flow_item_ipv4 *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;

        ENICPMD_FUNC_TRACE();
        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        fm_data->fk_metadata |= FKM_IPV4;
        fm_mask->fk_metadata |= FKM_IPV4;

        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_ipv4_mask;

        fm_data->fk_header_select |= FKH_IPV4;
        fm_mask->fk_header_select |= FKH_IPV4;
        memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
        memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
        return 0;
}

static int
enic_fm_copy_item_ipv6(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_ipv6 *spec = item->spec;
        const struct rte_flow_item_ipv6 *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;

        ENICPMD_FUNC_TRACE();
        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        fm_data->fk_metadata |= FKM_IPV6;
        fm_mask->fk_metadata |= FKM_IPV6;

        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_ipv6_mask;

        fm_data->fk_header_select |= FKH_IPV6;
        fm_mask->fk_header_select |= FKH_IPV6;
        memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
        memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
        return 0;
}

static int
enic_fm_copy_item_udp(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_udp *spec = item->spec;
        const struct rte_flow_item_udp *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;

        ENICPMD_FUNC_TRACE();
        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        fm_data->fk_metadata |= FKM_UDP;
        fm_mask->fk_metadata |= FKM_UDP;

        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_udp_mask;

        fm_data->fk_header_select |= FKH_UDP;
        fm_mask->fk_header_select |= FKH_UDP;
        memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
        memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
        return 0;
}

static int
enic_fm_copy_item_tcp(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_tcp *spec = item->spec;
        const struct rte_flow_item_tcp *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;

        ENICPMD_FUNC_TRACE();
        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        fm_data->fk_metadata |= FKM_TCP;
        fm_mask->fk_metadata |= FKM_TCP;

        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_tcp_mask;

        fm_data->fk_header_select |= FKH_TCP;
        fm_mask->fk_header_select |= FKH_TCP;
        memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
        memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
        return 0;
}

static int
enic_fm_copy_item_sctp(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_sctp *spec = item->spec;
        const struct rte_flow_item_sctp *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;
        uint8_t *ip_proto_mask = NULL;
        uint8_t *ip_proto = NULL;
        uint32_t l3_fkh;

        ENICPMD_FUNC_TRACE();
        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        /*
         * The NIC filter API has no flags for "match sctp", so explicitly
         * set the protocol number in the IP pattern.
         */
        if (fm_data->fk_metadata & FKM_IPV4) {
                struct rte_ipv4_hdr *ip;
                ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
                ip_proto_mask = &ip->next_proto_id;
                ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
                ip_proto = &ip->next_proto_id;
                l3_fkh = FKH_IPV4;
        } else if (fm_data->fk_metadata & FKM_IPV6) {
                struct rte_ipv6_hdr *ip;
                ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
                ip_proto_mask = &ip->proto;
                ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
                ip_proto = &ip->proto;
                l3_fkh = FKH_IPV6;
        } else {
                /* Need IPv4/IPv6 pattern first */
                return -EINVAL;
        }
        *ip_proto = IPPROTO_SCTP;
        *ip_proto_mask = 0xff;
        fm_data->fk_header_select |= l3_fkh;
        fm_mask->fk_header_select |= l3_fkh;

        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_sctp_mask;

        fm_data->fk_header_select |= FKH_L4RAW;
        fm_mask->fk_header_select |= FKH_L4RAW;
        memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
        memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
        return 0;
}

static int
enic_fm_copy_item_vxlan(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_vxlan *spec = item->spec;
        const struct rte_flow_item_vxlan *mask = item->mask;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;

        ENICPMD_FUNC_TRACE();
        /* Only 2 header levels (outer and inner) allowed */
        if (arg->header_level > 0)
                return -EINVAL;

        fm_data = &entry->ftm_data.fk_hdrset[0];
        fm_mask = &entry->ftm_mask.fk_hdrset[0];
        fm_data->fk_metadata |= FKM_VXLAN;
        fm_mask->fk_metadata |= FKM_VXLAN;
        /* items from here on out are inner header items */
        arg->header_level = 1;

        /* Match all if no spec */
        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_vxlan_mask;

        fm_data->fk_header_select |= FKH_VXLAN;
        fm_mask->fk_header_select |= FKH_VXLAN;
        memcpy(&fm_data->vxlan, spec, sizeof(*spec));
        memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
        return 0;
}

static int
enic_fm_copy_item_gtp(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_gtp *spec = item->spec;
        const struct rte_flow_item_gtp *mask = item->mask;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;
        int off;
        uint16_t udp_gtp_uc_port_be = 0;

        ENICPMD_FUNC_TRACE();
        /* Only 2 header levels (outer and inner) allowed */
        if (arg->header_level > 0)
                return -EINVAL;

        fm_data = &entry->ftm_data.fk_hdrset[0];
        fm_mask = &entry->ftm_mask.fk_hdrset[0];

        switch (item->type) {
        case RTE_FLOW_ITEM_TYPE_GTP:
        {
                /* For vanilla GTP, the UDP destination port must be specified
                 * but value of the port is not enforced here.
                 */
                if (!(fm_data->fk_metadata & FKM_UDP) ||
                    !(fm_data->fk_header_select & FKH_UDP) ||
                    fm_data->l4.udp.fk_dest == 0)
                        return -EINVAL;
                if (!(fm_mask->fk_metadata & FKM_UDP) ||
                    !(fm_mask->fk_header_select & FKH_UDP) ||
                    fm_mask->l4.udp.fk_dest != 0xFFFF)
                        return -EINVAL;
                break;
        }
        case RTE_FLOW_ITEM_TYPE_GTPC:
        {
                udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPC_UDP_PORT);
                break;
        }
        case RTE_FLOW_ITEM_TYPE_GTPU:
        {
                udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPU_UDP_PORT);
                break;
        }
        default:
                RTE_ASSERT(0);
        }

        /* The GTP-C or GTP-U UDP destination port must be matched. */
        if (udp_gtp_uc_port_be) {
                if (fm_data->fk_metadata & FKM_UDP &&
                    fm_data->fk_header_select & FKH_UDP &&
                    fm_data->l4.udp.fk_dest != udp_gtp_uc_port_be)
                        return -EINVAL;
                if (fm_mask->fk_metadata & FKM_UDP &&
                    fm_mask->fk_header_select & FKH_UDP &&
                    fm_mask->l4.udp.fk_dest != 0xFFFF)
                        return -EINVAL;

                /* In any case, add match for GTP-C GTP-U UDP dst port */
                fm_data->fk_metadata |= FKM_UDP;
                fm_data->fk_header_select |= FKH_UDP;
                fm_data->l4.udp.fk_dest = udp_gtp_uc_port_be;
                fm_mask->fk_metadata |= FKM_UDP;
                fm_mask->fk_header_select |= FKH_UDP;
                fm_mask->l4.udp.fk_dest = 0xFFFF;
        }

        /* NIC does not support GTP tunnels. No Items are allowed after this.
         * This prevents the specification of further items.
         */
        arg->header_level = 0;

        /* Match all if no spec */
        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_gtp_mask;

        /*
         * Use the raw L4 buffer to match GTP as fm_header_set does not have
         * GTP header. UDP dst port must be specific. Using the raw buffer
         * does not affect such UDP item, since we skip UDP in the raw buffer.
         */
        fm_data->fk_header_select |= FKH_L4RAW;
        fm_mask->fk_header_select |= FKH_L4RAW;
        off = sizeof(fm_data->l4.udp);
        memcpy(&fm_data->l4.rawdata[off], spec, sizeof(*spec));
        memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(*mask));
        return 0;
}

static int
enic_fm_copy_item_geneve(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_geneve *spec = item->spec;
        const struct rte_flow_item_geneve *mask = item->mask;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;
        int off;

        ENICPMD_FUNC_TRACE();
        /* Only 2 header levels (outer and inner) allowed */
        if (arg->header_level > 0)
                return -EINVAL;

        fm_data = &entry->ftm_data.fk_hdrset[0];
        fm_mask = &entry->ftm_mask.fk_hdrset[0];
        fm_data->fk_metadata |= FKM_GENEVE;
        fm_mask->fk_metadata |= FKM_GENEVE;
        /* items from here on out are inner header items, except options */
        arg->header_level = 1;

        /* Match all if no spec */
        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_geneve_mask;

        /*
         * Use the raw L4 buffer to match geneve as fm_header_set does
         * not have geneve header. A UDP item may precede the geneve
         * item. Using the raw buffer does not affect such UDP item,
         * since we skip UDP in the raw buffer.
         */
        fm_data->fk_header_select |= FKH_L4RAW;
        fm_mask->fk_header_select |= FKH_L4RAW;
        off = sizeof(fm_data->l4.udp);
        memcpy(&fm_data->l4.rawdata[off], spec, sizeof(struct rte_geneve_hdr));
        memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(struct rte_geneve_hdr));
        return 0;
}

static int
enic_fm_copy_item_geneve_opt(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_geneve_opt *spec = item->spec;
        const struct rte_flow_item_geneve_opt *mask = item->mask;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;
        struct rte_geneve_hdr *geneve;
        int off, len;

        ENICPMD_FUNC_TRACE();
        fm_data = &entry->ftm_data.fk_hdrset[0];
        fm_mask = &entry->ftm_mask.fk_hdrset[0];
        /* Match all if no spec */
        if (!spec)
                return 0;
        if (!mask)
                mask = &rte_flow_item_geneve_opt_mask;

        if (spec->option_len > 0 &&
            (spec->data == NULL || mask->data == NULL)) {
                return rte_flow_error_set(arg->error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ITEM,
                        NULL, "enic: geneve_opt unexpected null data");
        }
        /*
         * Geneve item must already be in the raw buffer. Append the
         * option pattern to it. There are two limitations.
         * (1) Can match only the 1st option, the first one following Geneve
         * (2) Geneve header must specify option length, as HW does not
         *     have "has Geneve option" flag.
         */
        RTE_ASSERT((fm_data->fk_header_select & FKH_L4RAW) != 0);
        RTE_ASSERT((fm_mask->fk_header_select & FKH_L4RAW) != 0);
        off = sizeof(fm_data->l4.udp);
        geneve = (struct rte_geneve_hdr *)&fm_data->l4.rawdata[off];
        if (geneve->opt_len == 0) {
                return rte_flow_error_set(arg->error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ITEM,
                        NULL, "enic: geneve_opt requires non-zero geneve option length");
        }
        geneve = (struct rte_geneve_hdr *)&fm_mask->l4.rawdata[off];
        if (geneve->opt_len == 0) {
                return rte_flow_error_set(arg->error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ITEM,
                        NULL, "enic: geneve_opt requires non-zero geneve option length mask");
        }
        off = sizeof(fm_data->l4.udp) + sizeof(struct rte_geneve_hdr);
        if (off + (spec->option_len + 1) * 4 > FM_LAYER_SIZE) {
                return rte_flow_error_set(arg->error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ITEM,
                        NULL, "enic: geneve_opt too large");
        }
        /* Copy option header */
        memcpy(&fm_data->l4.rawdata[off], spec, 4);
        memcpy(&fm_mask->l4.rawdata[off], mask, 4);
        /* Copy option data */
        if (spec->option_len > 0) {
                off += 4;
                len = spec->option_len * 4;
                memcpy(&fm_data->l4.rawdata[off], spec->data, len);
                memcpy(&fm_mask->l4.rawdata[off], mask->data, len);
        }
        return 0;
}

/* Match eCPRI combined message header */
static int
enic_fm_copy_item_ecpri(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_ecpri *spec = item->spec;
        const struct rte_flow_item_ecpri *mask = item->mask;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;
        uint8_t *fm_data_to, *fm_mask_to;

        ENICPMD_FUNC_TRACE();

        /* Tunneling not supported- only matching on inner eCPRI fields. */
        if (arg->header_level > 0)
                return -EINVAL;

        /* Need both spec and mask */
        if (!spec || !mask)
                return -EINVAL;

        fm_data = &entry->ftm_data.fk_hdrset[0];
        fm_mask = &entry->ftm_mask.fk_hdrset[0];

        /* eCPRI can only follow L2/VLAN layer if ethernet type is 0xAEFE. */
        if (!(fm_data->fk_metadata & FKM_UDP) &&
            (fm_mask->l2.eth.fk_ethtype != UINT16_MAX ||
            rte_cpu_to_be_16(fm_data->l2.eth.fk_ethtype) !=
            RTE_ETHER_TYPE_ECPRI))
                return -EINVAL;

        if (fm_data->fk_metadata & FKM_UDP) {
                /* eCPRI on UDP */
                fm_data->fk_header_select |= FKH_L4RAW;
                fm_mask->fk_header_select |= FKH_L4RAW;
                fm_data_to = &fm_data->l4.rawdata[sizeof(fm_data->l4.udp)];
                fm_mask_to = &fm_mask->l4.rawdata[sizeof(fm_data->l4.udp)];
        } else {
                /* eCPRI directly after Etherent header */
                fm_data->fk_header_select |= FKH_L3RAW;
                fm_mask->fk_header_select |= FKH_L3RAW;
                fm_data_to = &fm_data->l3.rawdata[0];
                fm_mask_to = &fm_mask->l3.rawdata[0];
        }

        /*
         * Use the raw L3 or L4 buffer to match eCPRI since fm_header_set does
         * not have eCPRI header. Only 1st message header of PDU can be matched.
         * "C" * bit ignored.
         */
        memcpy(fm_data_to, spec, sizeof(*spec));
        memcpy(fm_mask_to, mask, sizeof(*mask));
        return 0;
}

/*
 * Currently, raw pattern match is very limited. It is intended for matching
 * UDP tunnel header (e.g. vxlan or geneve).
 */
static int
enic_fm_copy_item_raw(struct copy_item_args *arg)
{
        const struct rte_flow_item *item = arg->item;
        const struct rte_flow_item_raw *spec = item->spec;
        const struct rte_flow_item_raw *mask = item->mask;
        const uint8_t lvl = arg->header_level;
        struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
        struct fm_header_set *fm_data, *fm_mask;

        ENICPMD_FUNC_TRACE();
        /* Cannot be used for inner packet */
        if (lvl > 0)
                return -EINVAL;
        /* Need both spec and mask */
        if (!spec || !mask)
                return -EINVAL;
        /* Only supports relative with offset 0 */
        if (!spec->relative || spec->offset != 0 || spec->search ||
            spec->limit)
                return -EINVAL;
        /* Need non-null pattern that fits within the NIC's filter pattern */
        if (spec->length == 0 ||
            spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
            !spec->pattern || !mask->pattern)
                return -EINVAL;
        /*
         * Mask fields, including length, are often set to zero. Assume that
         * means "same as spec" to avoid breaking existing apps. If length
         * is not zero, then it should be >= spec length.
         *
         * No more pattern follows this, so append to the L4 layer instead of
         * L5 to work with both recent and older VICs.
         */
        if (mask->length != 0 && mask->length < spec->length)
                return -EINVAL;

        fm_data = &entry->ftm_data.fk_hdrset[lvl];
        fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
        fm_data->fk_header_select |= FKH_L4RAW;
        fm_mask->fk_header_select |= FKH_L4RAW;
        fm_data->fk_header_select &= ~FKH_UDP;
        fm_mask->fk_header_select &= ~FKH_UDP;
        memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
               spec->pattern, spec->length);
        memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
               mask->pattern, spec->length);
        return 0;
}

static int
flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)

{
        return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
}

static int
enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
               struct fm_key_template *key, int entries,
               struct enic_fm_fet **fet_out)
{
        struct fm_exact_match_table *cmd;
        struct fm_header_set *hdr;
        struct enic_fm_fet *fet;
        uint64_t args[3];
        int ret;

        ENICPMD_FUNC_TRACE();
        fet = calloc(1, sizeof(struct enic_fm_fet));
        if (fet == NULL)
                return -ENOMEM;
        cmd = &fm->cmd.va->fm_exact_match_table;
        memset(cmd, 0, sizeof(*cmd));
        cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
        cmd->fet_stage = FM_STAGE_LAST;
        cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
        if (key == NULL) {
                hdr = &cmd->fet_key.fk_hdrset[0];
                memset(hdr, 0, sizeof(*hdr));
                hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
                hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
                hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
                hdr->l4.udp.fk_source = 0xFFFF;
                hdr->l4.udp.fk_dest = 0xFFFF;
                fet->default_key = 1;
        } else {
                memcpy(&cmd->fet_key, key, sizeof(*key));
                memcpy(&fet->key, key, sizeof(*key));
                fet->default_key = 0;
        }
        cmd->fet_key.fk_packet_tag = 1;

        args[0] = FM_EXACT_TABLE_ALLOC;
        args[1] = fm->cmd.pa;
        ret = flowman_cmd(fm, args, 2);
        if (ret) {
                ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
                free(fet);
                return ret;
        }
        fet->handle = args[0];
        fet->ingress = ingress;
        ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
                    fet->handle);
        *fet_out = fet;
        return 0;
}

static void
enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
{
        ENICPMD_FUNC_TRACE();
        enic_fm_tbl_free(fm, fet->handle);
        if (!fet->default_key)
                TAILQ_REMOVE(&fm->fet_list, fet, list);
        free(fet);
}

/*
 * Get the exact match table for the given combination of
 * <group, ingress, key>. Allocate one on the fly as necessary.
 */
static int
enic_fet_get(struct enic_flowman *fm,
             uint32_t group,
             uint8_t ingress,
             struct fm_key_template *key,
             struct enic_fm_fet **fet_out,
             struct rte_flow_error *error)
{
        struct enic_fm_fet *fet;

        ENICPMD_FUNC_TRACE();
        /* See if we already have this table open */
        TAILQ_FOREACH(fet, &fm->fet_list, list) {
                if (fet->group == group && fet->ingress == ingress)
                        break;
        }
        if (fet == NULL) {
                /* Jumping to a non-existing group? Use the default table */
                if (key == NULL) {
                        fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
                } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
                        return rte_flow_error_set(error, EINVAL,
                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                NULL, "enic: cannot get exact match table");
                }
                fet->group = group;
                /* Default table is never on the open table list */
                if (!fet->default_key)
                        TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
        }
        fet->ref++;
        *fet_out = fet;
        ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
                    fet->default_key ? "default" : "",
                    fet->ingress ? "ingress" : "egress",
                    fet->group, fet->ref);
        return 0;
}

static void
enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
{
        ENICPMD_FUNC_TRACE();
        RTE_ASSERT(fet->ref > 0);
        fet->ref--;
        ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
                    fet->default_key ? "default" : "",
                    fet->ingress ? "ingress" : "egress",
                    fet->group, fet->ref);
        if (fet->ref == 0)
                enic_fet_free(fm, fet);
}

/* Return 1 if current item is valid on top of the previous one. */
static int
fm_item_stacking_valid(enum rte_flow_item_type prev_item,
                       const struct enic_fm_items *item_info,
                       uint8_t is_first_item)
{
        enum rte_flow_item_type const *allowed_items = item_info->prev_items;

        ENICPMD_FUNC_TRACE();
        for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
                if (prev_item == *allowed_items)
                        return 1;
        }

        /* This is the first item in the stack. Check if that's cool */
        if (is_first_item && item_info->valid_start_item)
                return 1;
        return 0;
}

/*
 * Build the flow manager match entry structure from the provided pattern.
 * The pattern is validated as the items are copied.
 */
static int
enic_fm_copy_entry(struct enic_flowman *fm,
                   const struct rte_flow_item pattern[],
                   struct rte_flow_error *error)
{
        const struct enic_fm_items *item_info;
        enum rte_flow_item_type prev_item;
        const struct rte_flow_item *item;
        struct copy_item_args args;
        uint8_t prev_header_level;
        uint8_t is_first_item;
        int ret;

        ENICPMD_FUNC_TRACE();
        item = pattern;
        is_first_item = 1;
        prev_item = RTE_FLOW_ITEM_TYPE_END;

        args.fm_tcam_entry = &fm->tcam_entry;
        args.header_level = 0;
        prev_header_level = 0;
        for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
                /*
                 * Get info about how to validate and copy the item. If NULL
                 * is returned the nic does not support the item.
                 */
                if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
                        continue;

                item_info = &enic_fm_items[item->type];

                if (item->type >= RTE_DIM(enic_fm_items) ||
                    item_info->copy_item == NULL) {
                        return rte_flow_error_set(error, ENOTSUP,
                                RTE_FLOW_ERROR_TYPE_ITEM,
                                NULL, "enic: unsupported item");
                }
                /*
                 * Check vNIC feature dependencies. Geneve item needs
                 * Geneve offload feature
                 */
                if (item->type == RTE_FLOW_ITEM_TYPE_GENEVE &&
                    !fm->user_enic->geneve) {
                        return rte_flow_error_set(error, ENOTSUP,
                                RTE_FLOW_ERROR_TYPE_ITEM,
                                NULL, "enic: geneve not supported");
                }
                /* check to see if item stacking is valid */
                if (!fm_item_stacking_valid(prev_item, item_info,
                                            is_first_item))
                        goto stacking_error;

                args.item = item;
                args.error = error;
                if (error)
                        error->type = RTE_FLOW_ERROR_TYPE_NONE;
                ret = item_info->copy_item(&args);
                if (ret) {
                        /* If copy_item set the error, return that */
                        if (error && error->type != RTE_FLOW_ERROR_TYPE_NONE)
                                return ret;
                        goto item_not_supported;
                }
                /* Going from outer to inner? Treat it as a new packet start */
                if (prev_header_level != args.header_level) {
                        prev_item = RTE_FLOW_ITEM_TYPE_END;
                        is_first_item = 1;
                } else {
                        prev_item = item->type;
                        is_first_item = 0;
                }
                prev_header_level = args.header_level;
        }
        return 0;

item_not_supported:
        return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
                                  NULL, "enic: unsupported item type");

stacking_error:
        return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
                                  item, "enic: unsupported item stack");
}

static void
flow_item_skip_void(const struct rte_flow_item **item)
{
        for ( ; ; (*item)++)
                if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
                        return;
}

static void
append_template(void **template, uint8_t *off, const void *data, int len)
{
        memcpy(*template, data, len);
        *template = (char *)*template + len;
        *off = *off + len;
}

static int
enic_fm_append_action_op(struct enic_flowman *fm,
                         struct fm_action_op *fm_op,
                         struct rte_flow_error *error)
{
        int count;

        count = fm->action_op_count;
        ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
                    count, fm_op->fa_op);
        if (count == FM_ACTION_OP_MAX) {
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                        "too many action operations");
        }
        fm->action.fma_action_ops[count] = *fm_op;
        fm->action_op_count = count + 1;
        return 0;
}

static struct fm_action_op *
find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
{
        struct fm_action_op *op;
        int i;

        for (i = 0; i < fm->action_op_count; i++) {
                op = &fm->action.fma_action_ops[i];
                if (op->fa_op == opcode)
                        return op;
        }
        return NULL;
}

/* NIC requires that 1st steer appear before decap.
 * Correct example: steer, decap, steer, steer, ...
 */
static void
enic_fm_reorder_action_op(struct enic_flowman *fm)
{
        struct fm_action_op *op, *steer, *decap;
        struct fm_action_op tmp_op;

        ENICPMD_FUNC_TRACE();
        /* Find 1st steer and decap */
        op = fm->action.fma_action_ops;
        steer = NULL;
        decap = NULL;
        while (op->fa_op != FMOP_END) {
                if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
                               op->fa_op == FMOP_DECAP_STRIP))
                        decap = op;
                else if (!steer && op->fa_op == FMOP_RQ_STEER)
                        steer = op;
                op++;
        }
        /* If decap is before steer, swap */
        if (steer && decap && decap < steer) {
                op = fm->action.fma_action_ops;
                ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
                            (long)(decap - op), (long)(steer - op));
                tmp_op = *decap;
                *decap = *steer;
                *steer = tmp_op;
        }
}

/* VXLAN decap is done via flowman compound action */
static int
enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
                         struct fm_tcam_match_entry *fmt,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        struct fm_header_set *fm_data;
        struct fm_action_op fm_op;

        ENICPMD_FUNC_TRACE();
        fm_data = &fmt->ftm_data.fk_hdrset[0];
        if (!(fm_data->fk_metadata & FKM_VXLAN)) {
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ACTION, action,
                        "vxlan-decap: vxlan must be in pattern");
        }

        memset(&fm_op, 0, sizeof(fm_op));
        fm_op.fa_op = FMOP_DECAP_NOSTRIP;
        return enic_fm_append_action_op(fm, &fm_op, error);
}

/* Generate a reasonable source port number */
static uint16_t
gen_src_port(void)
{
        /* Min/max below are the default values in OVS-DPDK and Linux */
        uint16_t p = rte_rand();
        p = RTE_MAX(p, 32768);
        p = RTE_MIN(p, 61000);
        return rte_cpu_to_be_16(p);
}

/* VXLAN encap is done via flowman compound action */
static int
enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
                         const struct rte_flow_item *item,
                         struct rte_flow_error *error)
{
        struct fm_action_op fm_op;
        struct rte_ether_hdr *eth;
        struct rte_udp_hdr *udp;
        uint16_t *ethertype;
        void *template;
        uint8_t off;

        ENICPMD_FUNC_TRACE();
        memset(&fm_op, 0, sizeof(fm_op));
        fm_op.fa_op = FMOP_ENCAP;
        template = fm->action.fma_data;
        off = 0;
        /*
         * Copy flow items to the flowman template starting L2.
         * L2 must be ethernet.
         */
        flow_item_skip_void(&item);
        if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ITEM, item,
                        "vxlan-encap: first item should be ethernet");
        eth = (struct rte_ether_hdr *)template;
        ethertype = &eth->ether_type;
        append_template(&template, &off, item->spec,
                        sizeof(struct rte_ether_hdr));
        item++;
        flow_item_skip_void(&item);
        /* Optional VLAN */
        if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
                const struct rte_flow_item_vlan *spec;

                ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
                spec = item->spec;
                fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
                item++;
                flow_item_skip_void(&item);
        }
        /* L3 must be IPv4, IPv6 */
        switch (item->type) {
        case RTE_FLOW_ITEM_TYPE_IPV4:
        {
                struct rte_ipv4_hdr *ip4;

                ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
                *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
                ip4 = (struct rte_ipv4_hdr *)template;
                /*
                 * Offset of IPv4 length field and its initial value
                 * (IP + UDP + VXLAN) are specified in the action. The NIC
                 * will add inner packet length.
                 */
                fm_op.encap.len1_offset = off +
                        offsetof(struct rte_ipv4_hdr, total_length);
                fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
                        sizeof(struct rte_udp_hdr) +
                        sizeof(struct rte_vxlan_hdr);
                append_template(&template, &off, item->spec,
                                sizeof(struct rte_ipv4_hdr));
                ip4->version_ihl = RTE_IPV4_VHL_DEF;
                if (ip4->time_to_live == 0)
                        ip4->time_to_live = IP_DEFTTL;
                ip4->next_proto_id = IPPROTO_UDP;
                break;
        }
        case RTE_FLOW_ITEM_TYPE_IPV6:
        {
                struct rte_ipv6_hdr *ip6;

                ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
                *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
                ip6 = (struct rte_ipv6_hdr *)template;
                fm_op.encap.len1_offset = off +
                        offsetof(struct rte_ipv6_hdr, payload_len);
                fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
                        sizeof(struct rte_vxlan_hdr);
                append_template(&template, &off, item->spec,
                                sizeof(struct rte_ipv6_hdr));
                ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
                if (ip6->hop_limits == 0)
                        ip6->hop_limits = IP_DEFTTL;
                ip6->proto = IPPROTO_UDP;
                break;
        }
        default:
                return rte_flow_error_set(error,
                        EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
                        "vxlan-encap: L3 must be IPv4/IPv6");
        }
        item++;
        flow_item_skip_void(&item);

        /* L4 is UDP */
        if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ITEM, item,
                        "vxlan-encap: UDP must follow IPv4/IPv6");
        /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
        fm_op.encap.len2_offset =
                off + offsetof(struct rte_udp_hdr, dgram_len);
        fm_op.encap.len2_delta =
                sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
        udp = (struct rte_udp_hdr *)template;
        append_template(&template, &off, item->spec,
                        sizeof(struct rte_udp_hdr));
        /*
         * Firmware does not hash/fill source port yet. Generate a
         * random port, as there is *usually* one rte_flow for the
         * given inner packet stream (i.e. a single stream has one
         * random port).
         */
        if (udp->src_port == 0)
                udp->src_port = gen_src_port();
        item++;
        flow_item_skip_void(&item);

        /* Finally VXLAN */
        if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
                return rte_flow_error_set(error,
                        EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
                        "vxlan-encap: VXLAN must follow UDP");
        append_template(&template, &off, item->spec,
                        sizeof(struct rte_flow_item_vxlan));

        /*
         * Fill in the rest of the action structure.
         * Indicate that we want to encap with vxlan at packet start.
         */
        fm_op.encap.template_offset = 0;
        fm_op.encap.template_len = off;
        return enic_fm_append_action_op(fm, &fm_op, error);
}

static int
enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
                  uint64_t *handle)
{
        uint32_t bdf;
        uint64_t args[2];
        int rc;

        ENICPMD_FUNC_TRACE();
        ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
                    addr->function);
        bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
        args[0] = FM_VNIC_FIND;
        args[1] = bdf;
        rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
        if (rc != 0) {
                /* Expected to fail if BDF is not on the adapter */
                ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
                return rc;
        }
        *handle = args[0];
        ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
        return 0;
}

/*
 * Egress: target port should be either PF uplink or VF.
 * Supported cases
 * 1. VF egress -> PF uplink
 *   PF may be this VF's PF, or another PF, as long as they are on the same VIC.
 * 2. VF egress -> VF
 *
 * Unsupported cases
 * 1. PF egress -> VF
 *   App should be using representor to pass packets to VF
 */
static int
vf_egress_port_id_action(struct enic_flowman *fm,
                         struct rte_eth_dev *dst_dev,
                         uint64_t dst_vnic_h,
                         struct fm_action_op *fm_op,
                         struct rte_flow_error *error)
{
        struct enic *src_enic, *dst_enic;
        struct enic_vf_representor *vf;
        uint8_t uif;
        int ret;

        ENICPMD_FUNC_TRACE();
        src_enic = fm->user_enic;
        dst_enic = pmd_priv(dst_dev);
        if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ACTION,
                        NULL, "source port is not VF representor");
        }

        /* VF -> PF uplink. dst is not VF representor */
        if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
                /* PF is the VF's PF? Then nothing to do */
                vf = VF_ENIC_TO_VF_REP(src_enic);
                if (vf->pf == dst_enic) {
                        ENICPMD_LOG(DEBUG, "destination port is VF's PF");
                        return 0;
                }
                /* If not, steer to the remote PF's uplink */
                uif = dst_enic->fm_vnic_uif;
                ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
                memset(fm_op, 0, sizeof(*fm_op));
                fm_op->fa_op = FMOP_SET_EGPORT;
                fm_op->set_egport.egport = uif;
                ret = enic_fm_append_action_op(fm, fm_op, error);
                return ret;
        }

        /* VF -> VF loopback. Hairpin and steer to vnic */
        memset(fm_op, 0, sizeof(*fm_op));
        fm_op->fa_op = FMOP_EG_HAIRPIN;
        ret = enic_fm_append_action_op(fm, fm_op, error);
        if (ret)
                return ret;
        ENICPMD_LOG(DEBUG, "egress hairpin");
        fm->hairpin_steer_vnic_h = dst_vnic_h;
        fm->need_hairpin_steer = 1;
        return 0;
}

static int
enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
                           struct rte_eth_dev **dst_dev,
                           struct rte_flow_error *error)
{
        struct rte_eth_dev *dev;

        ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
        if (!rte_eth_dev_is_valid_port(dst_port_id)) {
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ACTION,
                        NULL, "invalid port_id");
        }
        dev = &rte_eth_devices[dst_port_id];
        if (!dev_is_enic(dev)) {
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ACTION,
                        NULL, "port_id is not enic");
        }
        if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ACTION,
                        NULL, "destination and source ports are not in the same switch domain");
        }

        *dst_dev = dev;
        return 0;
}

/* Translate flow actions to flowman TCAM entry actions */
static int
enic_fm_copy_action(struct enic_flowman *fm,
                    const struct rte_flow_action actions[],
                    uint8_t ingress,
                    struct rte_flow_error *error)
{
        enum {
                FATE = 1 << 0,
                DECAP = 1 << 1,
                PASSTHRU = 1 << 2,
                COUNT = 1 << 3,
                ENCAP = 1 << 4,
                PUSH_VLAN = 1 << 5,
                PORT_ID = 1 << 6,
        };
        struct fm_tcam_match_entry *fmt;
        struct fm_action_op fm_op;
        bool need_ovlan_action;
        struct enic *enic;
        uint32_t overlap;
        uint64_t vnic_h;
        uint16_t ovlan;
        bool first_rq;
        bool steer;
        int ret;

        ENICPMD_FUNC_TRACE();
        fmt = &fm->tcam_entry;
        need_ovlan_action = false;
        ovlan = 0;
        first_rq = true;
        steer = false;
        enic = fm->user_enic;
        overlap = 0;
        vnic_h = enic->fm_vnic_handle;

        for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
                switch (actions->type) {
                case RTE_FLOW_ACTION_TYPE_VOID:
                        continue;
                case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
                        if (overlap & PASSTHRU)
                                goto unsupported;
                        overlap |= PASSTHRU;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_JUMP: {
                        const struct rte_flow_action_jump *jump =
                                actions->conf;
                        struct enic_fm_fet *fet;

                        if (overlap & FATE)
                                goto unsupported;
                        ret = enic_fet_get(fm, jump->group, ingress, NULL,
                                           &fet, error);
                        if (ret)
                                return ret;
                        overlap |= FATE;
                        memset(&fm_op, 0, sizeof(fm_op));
                        fm_op.fa_op = FMOP_EXACT_MATCH;
                        fm_op.exact.handle = fet->handle;
                        fm->fet = fet;
                        ret = enic_fm_append_action_op(fm, &fm_op, error);
                        if (ret)
                                return ret;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_MARK: {
                        const struct rte_flow_action_mark *mark =
                                actions->conf;

                        if (enic->use_noscatter_vec_rx_handler)
                                goto unsupported;
                        if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
                                return rte_flow_error_set(error, EINVAL,
                                        RTE_FLOW_ERROR_TYPE_ACTION,
                                        NULL, "invalid mark id");
                        memset(&fm_op, 0, sizeof(fm_op));
                        fm_op.fa_op = FMOP_MARK;
                        fm_op.mark.mark = mark->id + 1;
                        ret = enic_fm_append_action_op(fm, &fm_op, error);
                        if (ret)
                                return ret;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_FLAG: {
                        if (enic->use_noscatter_vec_rx_handler)
                                goto unsupported;
                        /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
                        memset(&fm_op, 0, sizeof(fm_op));
                        fm_op.fa_op = FMOP_MARK;
                        fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
                        ret = enic_fm_append_action_op(fm, &fm_op, error);
                        if (ret)
                                return ret;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_QUEUE: {
                        const struct rte_flow_action_queue *queue =
                                actions->conf;

                        /*
                         * If fate other than QUEUE or RSS, fail. Multiple
                         * rss and queue actions are ok.
                         */
                        if ((overlap & FATE) && first_rq)
                                goto unsupported;
                        first_rq = false;
                        overlap |= FATE;
                        memset(&fm_op, 0, sizeof(fm_op));
                        fm_op.fa_op = FMOP_RQ_STEER;
                        fm_op.rq_steer.rq_index =
                                enic_rte_rq_idx_to_sop_idx(queue->index);
                        fm_op.rq_steer.rq_count = 1;
                        fm_op.rq_steer.vnic_handle = vnic_h;
                        ret = enic_fm_append_action_op(fm, &fm_op, error);
                        if (ret)
                                return ret;
                        ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
                                    fm_op.rq_steer.rq_index);
                        steer = true;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_DROP: {
                        if (overlap & FATE)
                                goto unsupported;
                        overlap |= FATE;
                        memset(&fm_op, 0, sizeof(fm_op));
                        fm_op.fa_op = FMOP_DROP;
                        ret = enic_fm_append_action_op(fm, &fm_op, error);
                        if (ret)
                                return ret;
                        ENICPMD_LOG(DEBUG, "create DROP action");
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_COUNT: {
                        if (overlap & COUNT)
                                goto unsupported;
                        overlap |= COUNT;
                        /* Count is associated with entry not action on VIC. */
                        fmt->ftm_flags |= FMEF_COUNTER;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_RSS: {
                        const struct rte_flow_action_rss *rss = actions->conf;
                        bool allow;
                        uint16_t i;

                        /*
                         * If fate other than QUEUE or RSS, fail. Multiple
                         * rss and queue actions are ok.
                         */
                        if ((overlap & FATE) && first_rq)
                                goto unsupported;
                        first_rq = false;
                        overlap |= FATE;

                        /*
                         * Hardware only supports RSS actions on outer level
                         * with default type and function. Queues must be
                         * sequential.
                         */
                        allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
                                rss->level == 0 && (rss->types == 0 ||
                                rss->types == enic->rss_hf) &&
                                rss->queue_num <= enic->rq_count &&
                                rss->queue[rss->queue_num - 1] < enic->rq_count;


                        /* Identity queue map needs to be sequential */
                        for (i = 1; i < rss->queue_num; i++)
                                allow = allow && (rss->queue[i] ==
                                        rss->queue[i - 1] + 1);
                        if (!allow)
                                goto unsupported;

                        memset(&fm_op, 0, sizeof(fm_op));
                        fm_op.fa_op = FMOP_RQ_STEER;
                        fm_op.rq_steer.rq_index =
                                enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
                        fm_op.rq_steer.rq_count = rss->queue_num;
                        fm_op.rq_steer.vnic_handle = vnic_h;
                        ret = enic_fm_append_action_op(fm, &fm_op, error);
                        if (ret)
                                return ret;
                        ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
                                    fm_op.rq_steer.rq_index);
                        steer = true;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_PORT_ID: {
                        const struct rte_flow_action_port_id *port;
                        struct rte_eth_dev *dev = NULL;

                        if (!ingress && (overlap & PORT_ID)) {
                                ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
                                goto unsupported;
                        }
                        port = actions->conf;
                        if (port->original) {
                                vnic_h = enic->fm_vnic_handle; /* This port */
                                break;
                        }
                        ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
                                                         error);
                        if (ret)
                                return ret;
                        vnic_h = pmd_priv(dev)->fm_vnic_handle;
                        overlap |= PORT_ID;
                        /*
                         * Ingress. Nothing more to do. We add an implicit
                         * steer at the end if needed.
                         */
                        if (ingress)
                                break;
                        /* Egress */
                        ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
                                error);
                        if (ret)
                                return ret;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
                        if (overlap & DECAP)
                                goto unsupported;
                        overlap |= DECAP;

                        ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
                                error);
                        if (ret != 0)
                                return ret;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
                        const struct rte_flow_action_vxlan_encap *encap;

                        encap = actions->conf;
                        if (overlap & ENCAP)
                                goto unsupported;
                        overlap |= ENCAP;
                        ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
                                error);
                        if (ret != 0)
                                return ret;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
                        struct fm_action_op *decap;

                        /*
                         * If decap-nostrip appears before pop vlan, this pop
                         * applies to the inner packet vlan. Turn it into
                         * decap-strip.
                         */
                        decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
                        if (decap) {
                                ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
                                decap->fa_op = FMOP_DECAP_STRIP;
                                break;
                        }
                        memset(&fm_op, 0, sizeof(fm_op));
                        fm_op.fa_op = FMOP_POP_VLAN;
                        ret = enic_fm_append_action_op(fm, &fm_op, error);
                        if (ret)
                                return ret;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
                        const struct rte_flow_action_of_push_vlan *vlan;

                        if (overlap & PASSTHRU)
                                goto unsupported;
                        vlan = actions->conf;
                        if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
                                return rte_flow_error_set(error, EINVAL,
                                        RTE_FLOW_ERROR_TYPE_ACTION,
                                        NULL, "unexpected push_vlan ethertype");
                        }
                        overlap |= PUSH_VLAN;
                        need_ovlan_action = true;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
                        const struct rte_flow_action_of_set_vlan_pcp *pcp;

                        pcp = actions->conf;
                        if (pcp->vlan_pcp > 7) {
                                return rte_flow_error_set(error, EINVAL,
                                        RTE_FLOW_ERROR_TYPE_ACTION,
                                        NULL, "invalid vlan_pcp");
                        }
                        need_ovlan_action = true;
                        ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
                        const struct rte_flow_action_of_set_vlan_vid *vid;

                        vid = actions->conf;
                        need_ovlan_action = true;
                        ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
                        const struct rte_flow_action_ethdev *ethdev;
                        struct rte_eth_dev *dev = NULL;

                        ethdev = actions->conf;
                        ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
                                                         &dev, error);
                        if (ret)
                                return ret;
                        vnic_h = pmd_priv(dev)->fm_vnic_handle;
                        overlap |= PORT_ID;
                        /*
                         * Action PORT_REPRESENTOR implies ingress destination.
                         * Noting to do. We add an implicit stree at the
                         * end if needed.
                         */
                        ingress = 1;
                        break;
                }
                case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
                        const struct rte_flow_action_ethdev *ethdev;
                        struct rte_eth_dev *dev = NULL;

                        if (overlap & PORT_ID) {
                                ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
                                goto unsupported;
                        }
                        ethdev = actions->conf;
                        ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
                                                         &dev, error);
                        if (ret)
                                return ret;
                        vnic_h = pmd_priv(dev)->fm_vnic_handle;
                        overlap |= PORT_ID;
                        /* Action REPRESENTED_PORT: always egress destination */
                        ingress = 0;
                        ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
                                error);
                        if (ret)
                                return ret;
                        break;
                }
                default:
                        goto unsupported;
                }
        }

        if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
                goto unsupported;
        /* Egress from VF: need implicit WQ match */
        if (enic_is_vf_rep(enic) && !ingress) {
                fmt->ftm_data.fk_wq_id = 0;
                fmt->ftm_mask.fk_wq_id = 0xffff;
                fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
                ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
                            VF_ENIC_TO_VF_REP(enic)->vf_id);
        }
        if (need_ovlan_action) {
                memset(&fm_op, 0, sizeof(fm_op));
                fm_op.fa_op = FMOP_SET_OVLAN;
                fm_op.ovlan.vlan = ovlan;
                ret = enic_fm_append_action_op(fm, &fm_op, error);
                if (ret)
                        return ret;
        }
        /* Add steer op for PORT_ID without QUEUE */
        if ((overlap & PORT_ID) && !steer && ingress) {
                memset(&fm_op, 0, sizeof(fm_op));
                /* Always to queue 0 for now as generic RSS is not available */
                fm_op.fa_op = FMOP_RQ_STEER;
                fm_op.rq_steer.rq_index = 0;
                fm_op.rq_steer.vnic_handle = vnic_h;
                ret = enic_fm_append_action_op(fm, &fm_op, error);
                if (ret)
                        return ret;
                ENICPMD_LOG(DEBUG, "add implicit steer op");
        }
        /* Add required END */
        memset(&fm_op, 0, sizeof(fm_op));
        fm_op.fa_op = FMOP_END;
        ret = enic_fm_append_action_op(fm, &fm_op, error);
        if (ret)
                return ret;
        enic_fm_reorder_action_op(fm);
        return 0;

unsupported:
        return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
                                  NULL, "enic: unsupported action");
}

/** Check if the action is supported */
static int
enic_fm_match_action(const struct rte_flow_action *action,

                     const enum rte_flow_action_type *supported_actions)
{
        for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
             supported_actions++) {
                if (action->type == *supported_actions)
                        return 1;
        }
        return 0;
}

/* Debug function to dump internal NIC action structure. */
static void
enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
{
        /* Manually keep in sync with FMOP commands */
        const char *fmop_str[FMOP_OP_MAX] = {
                [FMOP_END] = "end",
                [FMOP_DROP] = "drop",
                [FMOP_RQ_STEER] = "steer",
                [FMOP_EXACT_MATCH] = "exmatch",
                [FMOP_MARK] = "mark",
                [FMOP_EXT_MARK] = "ext_mark",
                [FMOP_TAG] = "tag",
                [FMOP_EG_HAIRPIN] = "eg_hairpin",
                [FMOP_IG_HAIRPIN] = "ig_hairpin",
                [FMOP_ENCAP_IVLAN] = "encap_ivlan",
                [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
                [FMOP_ENCAP] = "encap",
                [FMOP_SET_OVLAN] = "set_ovlan",
                [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
                [FMOP_DECAP_STRIP] = "decap_strip",
                [FMOP_POP_VLAN] = "pop_vlan",
                [FMOP_SET_EGPORT] = "set_egport",
                [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
                [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
                [FMOP_EMIT] = "emit",
                [FMOP_MODIFY] = "modify",
        };
        const struct fm_action_op *op = &fm_action->fma_action_ops[0];
        char buf[128], *bp = buf;
        const char *op_str;
        int i, n, buf_len;

        buf[0] = '\0';
        buf_len = sizeof(buf);
        for (i = 0; i < FM_ACTION_OP_MAX; i++) {
                if (op->fa_op == FMOP_END)
                        break;
                if (op->fa_op >= FMOP_OP_MAX)
                        op_str = "unknown";
                else
                        op_str = fmop_str[op->fa_op];
                n = snprintf(bp, buf_len, "%s,", op_str);
                if (n > 0 && n < buf_len) {
                        bp += n;
                        buf_len -= n;
                }
                op++;
        }
        /* Remove trailing comma */
        if (buf[0])
                *(bp - 1) = '\0';
        ENICPMD_LOG(DEBUG, "       Actions: %s", buf);
}

static int
bits_to_str(uint32_t bits, const char *strings[], int max,
            char *buf, int buf_len)
{
        int i, n = 0, len = 0;

        for (i = 0; i < max; i++) {
                if (bits & (1 << i)) {
                        n = snprintf(buf, buf_len, "%s,", strings[i]);
                        if (n > 0 && n < buf_len) {
                                buf += n;
                                buf_len -= n;
                                len += n;
                        }
                }
        }
        /* Remove trailing comma */
        if (len) {
                *(buf - 1) = '\0';
                len--;
        }
        return len;
}

/* Debug function to dump internal NIC filter structure. */
static void
__enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
                          int buf_len)
{
        /* Manually keep in sync with FKM_BITS */
        const char *fm_fkm_str[FKM_BIT_COUNT] = {
                [FKM_QTAG_BIT] = "qtag",
                [FKM_CMD_BIT] = "cmd",
                [FKM_IPV4_BIT] = "ip4",
                [FKM_IPV6_BIT] = "ip6",
                [FKM_ROCE_BIT] = "roce",
                [FKM_UDP_BIT] = "udp",
                [FKM_TCP_BIT] = "tcp",
                [FKM_TCPORUDP_BIT] = "tcpportudp",
                [FKM_IPFRAG_BIT] = "ipfrag",
                [FKM_NVGRE_BIT] = "nvgre",
                [FKM_VXLAN_BIT] = "vxlan",
                [FKM_GENEVE_BIT] = "geneve",
                [FKM_NSH_BIT] = "nsh",
                [FKM_ROCEV2_BIT] = "rocev2",
                [FKM_VLAN_PRES_BIT] = "vlan_pres",
                [FKM_IPOK_BIT] = "ipok",
                [FKM_L4OK_BIT] = "l4ok",
                [FKM_ROCEOK_BIT] = "roceok",
                [FKM_FCSOK_BIT] = "fcsok",
                [FKM_EG_SPAN_BIT] = "eg_span",
                [FKM_IG_SPAN_BIT] = "ig_span",
                [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
        };
        /* Manually keep in sync with FKH_BITS */
        const char *fm_fkh_str[FKH_BIT_COUNT] = {
                [FKH_ETHER_BIT] = "eth",
                [FKH_QTAG_BIT] = "qtag",
                [FKH_L2RAW_BIT] = "l2raw",
                [FKH_IPV4_BIT] = "ip4",
                [FKH_IPV6_BIT] = "ip6",
                [FKH_L3RAW_BIT] = "l3raw",
                [FKH_UDP_BIT] = "udp",
                [FKH_TCP_BIT] = "tcp",
                [FKH_ICMP_BIT] = "icmp",
                [FKH_VXLAN_BIT] = "vxlan",
                [FKH_L4RAW_BIT] = "l4raw",
        };
        uint32_t fkh_bits = fk_hdrset->fk_header_select;
        uint32_t fkm_bits = fk_hdrset->fk_metadata;
        int n;

        if (!fkm_bits && !fkh_bits)
                return;
        n = snprintf(buf, buf_len, "metadata(");
        if (n > 0 && n < buf_len) {
                buf += n;
                buf_len -= n;
        }
        n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
        if (n > 0 && n < buf_len) {
                buf += n;
                buf_len -= n;
        }
        n = snprintf(buf, buf_len, ") valid hdr fields(");
        if (n > 0 && n < buf_len) {
                buf += n;
                buf_len -= n;
        }
        n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
        if (n > 0 && n < buf_len) {
                buf += n;
                buf_len -= n;
        }
        snprintf(buf, buf_len, ")");
}

static void
enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
                        uint8_t ingress)
{
        char buf[256];

        memset(buf, 0, sizeof(buf));
        __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
                                  buf, sizeof(buf));
        ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
                    (ingress) ? "IG" : "EG", buf,
                    (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
                    match->ftm_position);
        memset(buf, 0, sizeof(buf));
        __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
                                  buf, sizeof(buf));
        if (buf[0])
                ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
}

/* Debug function to dump internal NIC flow structures. */
static void
enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
                        const struct fm_action *fm_action,
                        uint8_t ingress)
{
        if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
                return;
        enic_fm_dump_tcam_match(fm_match, ingress);
        enic_fm_dump_tcam_actions(fm_action);
}

static int
enic_fm_flow_parse(struct enic_flowman *fm,
                   const struct rte_flow_attr *attrs,
                   const struct rte_flow_item pattern[],
                   const struct rte_flow_action actions[],
                   struct rte_flow_error *error)
{
        const struct rte_flow_action *action;
        unsigned int ret;
        static const enum rte_flow_action_type *sa;

        ENICPMD_FUNC_TRACE();
        ret = 0;
        if (!pattern) {
                rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
                                   NULL, "no pattern specified");
                return -rte_errno;
        }

        if (!actions) {
                rte_flow_error_set(error, EINVAL,
                                   RTE_FLOW_ERROR_TYPE_ACTION_NUM,
                                   NULL, "no action specified");
                return -rte_errno;
        }

        if (attrs) {
                if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
                        rte_flow_error_set(error, ENOTSUP,
                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
                                           NULL,
                                           "priorities are not supported for non-default (0) groups");
                        return -rte_errno;
                } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
                        rte_flow_error_set(error, ENOTSUP,
                                           RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
                                           NULL,
                                           "transfer is not supported");
                        return -rte_errno;
                } else if (attrs->ingress && attrs->egress) {
                        rte_flow_error_set(error, ENOTSUP,
                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
                                           NULL,
                                           "bidirectional rules not supported");
                        return -rte_errno;
                }

        } else {
                rte_flow_error_set(error, EINVAL,
                                   RTE_FLOW_ERROR_TYPE_ATTR,
                                   NULL, "no attribute specified");
                return -rte_errno;
        }

        /* Verify Actions. */
        sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
             enic_fm_supported_eg_actions;
        for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
             action++) {
                if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
                        continue;
                else if (!enic_fm_match_action(action, sa))
                        break;
        }
        if (action->type != RTE_FLOW_ACTION_TYPE_END) {
                rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
                                   action, "invalid action");
                return -rte_errno;
        }
        ret = enic_fm_copy_entry(fm, pattern, error);
        if (ret)
                return ret;
        ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
        return ret;
}

static void
enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
{
        if (!fm_flow->counter_valid)
                return;
        SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
        fm_flow->counter_valid = false;
}

static int
enic_fm_more_counters(struct enic_flowman *fm)
{
        struct enic_fm_counter *new_stack;
        struct enic_fm_counter *ctrs;
        int i, rc;
        uint64_t args[2];

        ENICPMD_FUNC_TRACE();
        new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
                                FM_COUNTERS_EXPAND) *
                                sizeof(struct enic_fm_counter), 0);
        if (new_stack == NULL) {
                ENICPMD_LOG(ERR, "cannot alloc counter memory");
                return -ENOMEM;
        }
        fm->counter_stack = new_stack;

        args[0] = FM_COUNTER_BRK;
        args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
        rc = flowman_cmd(fm, args, 2);
        if (rc != 0) {
                ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
                return rc;
        }
        ctrs = (struct enic_fm_counter *)fm->counter_stack +
                fm->counters_alloced;
        for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
                ctrs->handle = fm->counters_alloced + i;
                SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
        }
        fm->counters_alloced += FM_COUNTERS_EXPAND;
        ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
                    FM_COUNTERS_EXPAND, fm->counters_alloced);
        return 0;
}

static int
enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
{
        uint64_t args[3];
        int ret;

        ENICPMD_FUNC_TRACE();
        args[0] = FM_COUNTER_QUERY;
        args[1] = c->handle;
        args[2] = 1; /* clear */
        ret = flowman_cmd(fm, args, 3);
        if (ret) {
                ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
                            ret, c->handle);
                return ret;
        }
        return 0;
}

static int
enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
                      struct enic_fm_counter **ctr)
{
        struct enic_fm_counter *c;
        int ret;

        ENICPMD_FUNC_TRACE();
        *ctr = NULL;
        if (SLIST_EMPTY(&fm->counters)) {
                ret = enic_fm_more_counters(fm);
                if (ret)
                        return rte_flow_error_set(error, -ret,
                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                NULL, "enic: out of counters");
        }
        c = SLIST_FIRST(&fm->counters);
        SLIST_REMOVE_HEAD(&fm->counters, next);
        *ctr = c;
        return 0;
}

static int
enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
{
        uint64_t args[2];
        int ret = 0;

        ENICPMD_FUNC_TRACE();
        RTE_ASSERT(ah->ref > 0);
        ah->ref--;
        if (ah->ref == 0) {
                args[0] = FM_ACTION_FREE;
                args[1] = ah->handle;
                ret = flowman_cmd(fm, args, 2);
                if (ret)
                        /* This is a "should never happen" error. */
                        ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
                                    PRIx64, ret, ah->handle);
                rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
                free(ah);
        }
        return ret;
}

static int
enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
{
        uint64_t args[2];
        int rc;

        ENICPMD_FUNC_TRACE();
        args[0] = FM_MATCH_ENTRY_REMOVE;
        args[1] = handle;
        rc = flowman_cmd(fm, args, 2);
        if (rc)
                ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
                            " handle=0x%" PRIx64, rc, handle);
        return rc;
}

static struct enic_fm_jump_flow *
find_jump_flow(struct enic_flowman *fm, uint32_t group)
{
        struct enic_fm_jump_flow *j;

        ENICPMD_FUNC_TRACE();
        TAILQ_FOREACH(j, &fm->jump_list, list) {
                if (j->group == group)
                        return j;
        }
        return NULL;
}

static void
remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
{
        struct enic_fm_jump_flow *j;

        ENICPMD_FUNC_TRACE();
        TAILQ_FOREACH(j, &fm->jump_list, list) {
                if (j->flow == flow) {
                        TAILQ_REMOVE(&fm->jump_list, j, list);
                        free(j);
                        return;
                }
        }
}

static int
save_jump_flow(struct enic_flowman *fm,
               struct rte_flow *flow,
               uint32_t group,
               struct fm_tcam_match_entry *match,
               struct fm_action *action)
{
        struct enic_fm_jump_flow *j;

        ENICPMD_FUNC_TRACE();
        j = calloc(1, sizeof(struct enic_fm_jump_flow));
        if (j == NULL)
                return -ENOMEM;
        j->flow = flow;
        j->group = group;
        j->match = *match;
        j->action = *action;
        TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
        ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
        return 0;
}

static void
__enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
{
        if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
                enic_fm_entry_free(fm, fm_flow->entry_handle);
                fm_flow->entry_handle = FM_INVALID_HANDLE;
        }
        if (fm_flow->action != NULL) {
                enic_fm_action_free(fm, fm_flow->action);
                fm_flow->action = NULL;
        }
        enic_fm_counter_free(fm, fm_flow);
        if (fm_flow->fet) {
                enic_fet_put(fm, fm_flow->fet);
                fm_flow->fet = NULL;
        }
}

static void
enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
{
        struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;

        if (flow->fm->fet && flow->fm->fet->default_key)
                remove_jump_flow(fm, flow);
        __enic_fm_flow_free(fm, flow->fm);
        if (steer) {
                __enic_fm_flow_free(fm, steer);
                free(steer);
        }
        free(flow->fm);
        free(flow);
}

static int
enic_fm_add_tcam_entry(struct enic_flowman *fm,
                       struct fm_tcam_match_entry *match_in,
                       uint64_t *entry_handle,
                       uint8_t ingress,
                       struct rte_flow_error *error)
{
        struct fm_tcam_match_entry *ftm;
        uint64_t args[3];
        int ret;

        ENICPMD_FUNC_TRACE();
        /* Copy entry to the command buffer */
        ftm = &fm->cmd.va->fm_tcam_match_entry;
        memcpy(ftm, match_in, sizeof(*ftm));
        /* Add TCAM entry */
        args[0] = FM_TCAM_ENTRY_INSTALL;
        args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
        args[2] = fm->cmd.pa;
        ret = flowman_cmd(fm, args, 3);
        if (ret != 0) {
                ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
                            ingress ? "ingress" : "egress", ret);
                rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                        NULL, "enic: devcmd(tcam-entry-install)");
                return ret;
        }
        ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
                    ingress ? "ingress" : "egress", (uint64_t)args[0]);
        *entry_handle = args[0];
        return 0;
}

static int
enic_fm_add_exact_entry(struct enic_flowman *fm,
                        struct fm_tcam_match_entry *match_in,
                        uint64_t *entry_handle,
                        struct enic_fm_fet *fet,
                        struct rte_flow_error *error)
{
        struct fm_exact_match_entry *fem;
        uint64_t args[3];
        int ret;

        ENICPMD_FUNC_TRACE();
        /* The new entry must have the table's key */
        if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
                   sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
                return rte_flow_error_set(error, EINVAL,
                        RTE_FLOW_ERROR_TYPE_ITEM, NULL,
                        "enic: key does not match group's key");
        }

        /* Copy entry to the command buffer */
        fem = &fm->cmd.va->fm_exact_match_entry;
        /*
         * Translate TCAM entry to exact entry. As is only need to drop
         * position and mask. The mask is part of the exact match table.
         * Position (aka priority) is not supported in the exact match table.
         */
        fem->fem_data = match_in->ftm_data;
        fem->fem_flags = match_in->ftm_flags;
        fem->fem_action = match_in->ftm_action;
        fem->fem_counter = match_in->ftm_counter;

        /* Add exact entry */
        args[0] = FM_EXACT_ENTRY_INSTALL;
        args[1] = fet->handle;
        args[2] = fm->cmd.pa;
        ret = flowman_cmd(fm, args, 3);
        if (ret != 0) {
                ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
                            fet->ingress ? "ingress" : "egress", fet->group);
                rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                        NULL, "enic: devcmd(exact-entry-install)");
                return ret;
        }
        ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
                    " handle=0x%" PRIx64,
                    fet->ingress ? "ingress" : "egress", fet->group,
                    (uint64_t)args[0]);
        *entry_handle = args[0];
        return 0;
}

static int
enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
                       struct rte_flow_error *error,
                       struct enic_fm_action **ah_o)
{
        struct enic_fm_action *ah;
        struct fm_action *fma;
        uint64_t args[2];
        int ret = 0;

        ret = rte_hash_lookup_data(fm->action_hash, action_in,
                                   (void **)&ah);
        if (ret < 0 && ret != -ENOENT)
                return rte_flow_error_set(error, -ret,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                   NULL, "enic: rte_hash_lookup(action)");

        if (ret == -ENOENT) {
                /* Allocate a new action on the NIC. */
                fma = &fm->cmd.va->fm_action;
                memcpy(fma, action_in, sizeof(*fma));

                ah = calloc(1, sizeof(*ah));
                if (ah == NULL)
                        return rte_flow_error_set(error, ENOMEM,
                                           RTE_FLOW_ERROR_TYPE_HANDLE,
                                           NULL, "enic: calloc(fm-action)");
                memcpy(&ah->key, action_in, sizeof(struct fm_action));
                args[0] = FM_ACTION_ALLOC;
                args[1] = fm->cmd.pa;
                ret = flowman_cmd(fm, args, 2);
                if (ret != 0) {
                        rte_flow_error_set(error, -ret,
                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                           NULL, "enic: devcmd(action-alloc)");
                        goto error_with_ah;
                }
                ah->handle = args[0];
                ret = rte_hash_add_key_data(fm->action_hash,
                                            (const void *)action_in,
                                            (void *)ah);
                if (ret != 0) {
                        rte_flow_error_set(error, -ret,
                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                           NULL,
                                           "enic: rte_hash_add_key_data(actn)");
                        goto error_with_action_handle;
                }
                ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
                            ah->handle);
        }

        /* Action handle struct is valid, increment reference count. */
        ah->ref++;
        *ah_o = ah;
        return 0;
error_with_action_handle:
        args[0] = FM_ACTION_FREE;
        args[1] = ah->handle;
        ret = flowman_cmd(fm, args, 2);
        if (ret != 0)
                rte_flow_error_set(error, -ret,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                   NULL, "enic: devcmd(action-free)");
error_with_ah:
        free(ah);
        return ret;
}

/* Push match-action to the NIC. */
static int
__enic_fm_flow_add_entry(struct enic_flowman *fm,
                         struct enic_fm_flow *fm_flow,
                         struct fm_tcam_match_entry *match_in,
                         struct fm_action *action_in,
                         uint32_t group,
                         uint8_t ingress,
                         struct rte_flow_error *error)
{
        struct enic_fm_counter *ctr;
        struct enic_fm_action *ah = NULL;
        uint64_t entry_h;
        int ret;

        ENICPMD_FUNC_TRACE();

        /* Get or create an action handle. */
        ret = enic_action_handle_get(fm, action_in, error, &ah);
        if (ret)
                return ret;
        match_in->ftm_action = ah->handle;
        fm_flow->action = ah;

        /* Allocate counter if requested. */
        if (match_in->ftm_flags & FMEF_COUNTER) {
                ret = enic_fm_counter_alloc(fm, error, &ctr);
                if (ret) /* error has been filled in */
                        return ret;
                fm_flow->counter_valid = true;
                fm_flow->counter = ctr;
                match_in->ftm_counter = ctr->handle;
        }

        /*
         * Get the group's table (either TCAM or exact match table) and
         * add entry to it. If we use the exact match table, the handler
         * will translate the TCAM entry (match_in) to the appropriate
         * exact match entry and use that instead.
         */
        entry_h = FM_INVALID_HANDLE;
        if (group == FM_TCAM_RTE_GROUP) {
                ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
                                             error);
                if (ret)
                        return ret;
                /* Jump action might have a ref to fet */
                fm_flow->fet = fm->fet;
                fm->fet = NULL;
        } else {
                struct enic_fm_fet *fet = NULL;

                ret = enic_fet_get(fm, group, ingress,
                                   &match_in->ftm_mask, &fet, error);
                if (ret)
                        return ret;
                fm_flow->fet = fet;
                ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
                                              error);
                if (ret)
                        return ret;
        }
        /* Clear counter after adding entry, as it requires in-use counter */
        if (fm_flow->counter_valid) {
                ret = enic_fm_counter_zero(fm, fm_flow->counter);
                if (ret)
                        return ret;
        }
        fm_flow->entry_handle = entry_h;
        return 0;
}

/* Push match-action to the NIC. */
static struct rte_flow *
enic_fm_flow_add_entry(struct enic_flowman *fm,
                       struct fm_tcam_match_entry *match_in,
                       struct fm_action *action_in,
                       const struct rte_flow_attr *attrs,
                       struct rte_flow_error *error)
{
        struct enic_fm_flow *fm_flow;
        struct rte_flow *flow;

        ENICPMD_FUNC_TRACE();
        match_in->ftm_position = attrs->priority;
        enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
        flow = calloc(1, sizeof(*flow));
        fm_flow = calloc(1, sizeof(*fm_flow));
        if (flow == NULL || fm_flow == NULL) {
                rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
                        NULL, "enic: cannot allocate rte_flow");
                free(flow);
                free(fm_flow);
                return NULL;
        }
        flow->fm = fm_flow;
        fm_flow->action = NULL;
        fm_flow->entry_handle = FM_INVALID_HANDLE;
        if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
                                     attrs->group, attrs->ingress, error)) {
                enic_fm_flow_free(fm, flow);
                return NULL;
        }
        return flow;
}

static void
convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
                   struct rte_flow_error *error)
{
        struct enic_fm_flow *fm_flow;
        struct enic_fm_jump_flow *j;
        struct fm_action *fma;
        uint32_t group;

        ENICPMD_FUNC_TRACE();
        /*
         * Find the saved flows that should jump to the new table (fet).
         * Then delete the old TCAM entry that jumps to the default table,
         * and add a new one that jumps to the new table.
         */
        group = fet->group;
        j = find_jump_flow(fm, group);
        while (j) {
                ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
                            j->flow, group);
                /* Delete old entry */
                fm_flow = j->flow->fm;
                __enic_fm_flow_free(fm, fm_flow);

                /* Add new entry */
                fma = &j->action;
                fma->fma_action_ops[0].exact.handle = fet->handle;
                if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
                        FM_TCAM_RTE_GROUP, fet->ingress, error)) {
                        /* Cannot roll back changes at the moment */
                        ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
                                    j->flow);
                } else {
                        fm_flow->fet = fet;
                        fet->ref++;
                        ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
                                    fet->group, fet->ref);
                }

                TAILQ_REMOVE(&fm->jump_list, j, list);
                free(j);
                j = find_jump_flow(fm, group);
        }
}

static int
add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
                  struct rte_flow_error *error)
{
        struct fm_tcam_match_entry *fm_tcam_entry;
        struct enic_fm_flow *fm_flow;
        struct fm_action *fm_action;
        struct fm_action_op fm_op;
        int ret;

        ENICPMD_FUNC_TRACE();
        fm_flow = calloc(1, sizeof(*fm_flow));
        if (fm_flow == NULL) {
                rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
                        NULL, "enic: cannot allocate rte_flow");
                return -ENOMEM;
        }
        /* Original egress hairpin flow */
        fm_tcam_entry = &fm->tcam_entry;
        fm_action = &fm->action;
        /* Use the match pattern of the egress flow as is, without counters */
        fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
        /* The only action is steer to vnic */
        fm->action_op_count = 0;
        memset(fm_action, 0, sizeof(*fm_action));
        memset(&fm_op, 0, sizeof(fm_op));
        /* Always to queue 0 for now */
        fm_op.fa_op = FMOP_RQ_STEER;
        fm_op.rq_steer.rq_index = 0;
        fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
        ret = enic_fm_append_action_op(fm, &fm_op, error);
        if (ret)
                goto error_with_flow;
        ENICPMD_LOG(DEBUG, "add steer op");
        /* Add required END */
        memset(&fm_op, 0, sizeof(fm_op));
        fm_op.fa_op = FMOP_END;
        ret = enic_fm_append_action_op(fm, &fm_op, error);
        if (ret)
                goto error_with_flow;
        /* Add the ingress flow */
        fm_flow->action = NULL;
        fm_flow->entry_handle = FM_INVALID_HANDLE;
        ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
                                       FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
        if (ret) {
                ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
                goto error_with_flow;
        }
        /* The new flow is now the egress flow's paired flow */
        flow->fm->hairpin_steer_flow = fm_flow;
        return 0;

error_with_flow:
        free(fm_flow);
        return ret;
}

static void
enic_fm_open_scratch(struct enic_flowman *fm)
{
        fm->action_op_count = 0;
        fm->fet = NULL;
        fm->need_hairpin_steer = 0;
        fm->hairpin_steer_vnic_h = 0;
        memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
        memset(&fm->action, 0, sizeof(fm->action));
}

static void
enic_fm_close_scratch(struct enic_flowman *fm)
{
        if (fm->fet) {
                enic_fet_put(fm, fm->fet);
                fm->fet = NULL;
        }
        fm->action_op_count = 0;
}

static int
enic_fm_flow_validate(struct rte_eth_dev *dev,
                      const struct rte_flow_attr *attrs,
                      const struct rte_flow_item pattern[],
                      const struct rte_flow_action actions[],
                      struct rte_flow_error *error)
{
        struct fm_tcam_match_entry *fm_tcam_entry;
        struct fm_action *fm_action;
        struct enic_flowman *fm;
        int ret;

        ENICPMD_FUNC_TRACE();
        fm = begin_fm(pmd_priv(dev));
        if (fm == NULL)
                return -ENOTSUP;
        enic_fm_open_scratch(fm);
        ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
        if (!ret) {
                fm_tcam_entry = &fm->tcam_entry;
                fm_action = &fm->action;
                enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
                                        attrs->ingress);
        }
        enic_fm_close_scratch(fm);
        end_fm(fm);
        return ret;
}

static int
enic_fm_flow_query_count(struct rte_eth_dev *dev,
                         struct rte_flow *flow, void *data,
                         struct rte_flow_error *error)
{
        struct rte_flow_query_count *query;
        struct enic_fm_flow *fm_flow;
        struct enic_flowman *fm;
        uint64_t args[3];
        int rc;

        ENICPMD_FUNC_TRACE();
        fm = begin_fm(pmd_priv(dev));
        query = data;
        fm_flow = flow->fm;
        if (!fm_flow->counter_valid) {
                rc = rte_flow_error_set(error, ENOTSUP,
                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                        "enic: flow does not have counter");
                goto exit;
        }

        args[0] = FM_COUNTER_QUERY;
        args[1] = fm_flow->counter->handle;
        args[2] = query->reset;
        rc = flowman_cmd(fm, args, 3);
        if (rc) {
                ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
                            rc, fm_flow->counter->handle);
                goto exit;
        }
        query->hits_set = 1;
        query->hits = args[0];
        query->bytes_set = 1;
        query->bytes = args[1];
        rc = 0;
exit:
        end_fm(fm);
        return rc;
}

static int
enic_fm_flow_query(struct rte_eth_dev *dev,
                   struct rte_flow *flow,
                   const struct rte_flow_action *actions,
                   void *data,
                   struct rte_flow_error *error)
{
        int ret = 0;

        ENICPMD_FUNC_TRACE();
        for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
                switch (actions->type) {
                case RTE_FLOW_ACTION_TYPE_VOID:
                        break;
                case RTE_FLOW_ACTION_TYPE_COUNT:
                        ret = enic_fm_flow_query_count(dev, flow, data, error);
                        break;
                default:
                        return rte_flow_error_set(error, ENOTSUP,
                                                  RTE_FLOW_ERROR_TYPE_ACTION,
                                                  actions,
                                                  "action not supported");
                }
                if (ret < 0)
                        return ret;
        }
        return 0;
}

static struct rte_flow *
enic_fm_flow_create(struct rte_eth_dev *dev,
                    const struct rte_flow_attr *attrs,
                    const struct rte_flow_item pattern[],
                    const struct rte_flow_action actions[],
                    struct rte_flow_error *error)
{
        struct fm_tcam_match_entry *fm_tcam_entry;
        struct fm_action *fm_action;
        struct enic_flowman *fm;
        struct enic_fm_fet *fet;
        struct rte_flow *flow;
        struct enic *enic;
        int ret;

        ENICPMD_FUNC_TRACE();
        enic = pmd_priv(dev);
        fm = begin_fm(enic);
        if (fm == NULL) {
                rte_flow_error_set(error, ENOTSUP,
                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                        "flowman is not initialized");
                return NULL;
        }
        enic_fm_open_scratch(fm);
        flow = NULL;
        ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
        if (ret < 0)
                goto error_with_scratch;
        fm_tcam_entry = &fm->tcam_entry;
        fm_action = &fm->action;
        flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
                                      attrs, error);
        if (flow) {
                /* Add ingress rule that pairs with hairpin rule */
                if (fm->need_hairpin_steer) {
                        ret = add_hairpin_steer(fm, flow, error);