/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright 2018 Mellanox Technologies, Ltd
 */

#include <sys/queue.h>
#include <stdalign.h>
#include <stdint.h>
#include <string.h>
#include <unistd.h>

#include <rte_common.h>
#include <rte_ether.h>
#include <ethdev_driver.h>
#include <rte_flow.h>
#include <rte_flow_driver.h>
#include <rte_malloc.h>
#include <rte_cycles.h>
#include <rte_bus_pci.h>
#include <rte_ip.h>
#include <rte_gre.h>
#include <rte_vxlan.h>
#include <rte_gtp.h>
#include <rte_eal_paging.h>
#include <rte_mpls.h>
#include <rte_mtr.h>
#include <rte_mtr_driver.h>
#include <rte_tailq.h>

#include <mlx5_glue.h>
#include <mlx5_devx_cmds.h>
#include <mlx5_prm.h>
#include <mlx5_malloc.h>

#include "mlx5_defs.h"
#include "mlx5.h"
#include "mlx5_common_os.h"
#include "mlx5_flow.h"
#include "mlx5_flow_os.h"
#include "mlx5_rx.h"
#include "mlx5_tx.h"
#include "rte_pmd_mlx5.h"

#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)

#ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
#define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
#endif

#ifndef HAVE_MLX5DV_DR_ESWITCH
#ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
#define MLX5DV_FLOW_TABLE_TYPE_FDB 0
#endif
#endif

#ifndef HAVE_MLX5DV_DR
#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
#endif

/* VLAN header definitions */
#define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
#define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
#define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
#define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
#define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)

union flow_dv_attr {
        struct {
                uint32_t valid:1;
                uint32_t ipv4:1;
                uint32_t ipv6:1;
                uint32_t tcp:1;
                uint32_t udp:1;
                uint32_t reserved:27;
        };
        uint32_t attr;
};

static int
flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
                             struct mlx5_flow_tbl_resource *tbl);

static int
flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
                                     uint32_t encap_decap_idx);

static int
flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
                                        uint32_t port_id);
static void
flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);

static int
flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
                                  uint32_t rix_jump);

/**
 * Initialize flow attributes structure according to flow items' types.
 *
 * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
 * mode. For tunnel mode, the items to be modified are the outermost ones.
 *
 * @param[in] item
 *   Pointer to item specification.
 * @param[out] attr
 *   Pointer to flow attributes structure.
 * @param[in] dev_flow
 *   Pointer to the sub flow.
 * @param[in] tunnel_decap
 *   Whether action is after tunnel decapsulation.
 */
static void
flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
                  struct mlx5_flow *dev_flow, bool tunnel_decap)
{
        uint64_t layers = dev_flow->handle->layers;

        /*
         * If layers is already initialized, it means this dev_flow is the
         * suffix flow, the layers flags is set by the prefix flow. Need to
         * use the layer flags from prefix flow as the suffix flow may not
         * have the user defined items as the flow is split.
         */
        if (layers) {
                if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
                        attr->ipv4 = 1;
                else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
                        attr->ipv6 = 1;
                if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
                        attr->tcp = 1;
                else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
                        attr->udp = 1;
                attr->valid = 1;
                return;
        }
        for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
                uint8_t next_protocol = 0xff;
                switch (item->type) {
                case RTE_FLOW_ITEM_TYPE_GRE:
                case RTE_FLOW_ITEM_TYPE_NVGRE:
                case RTE_FLOW_ITEM_TYPE_VXLAN:
                case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
                case RTE_FLOW_ITEM_TYPE_GENEVE:
                case RTE_FLOW_ITEM_TYPE_MPLS:
                case RTE_FLOW_ITEM_TYPE_GTP:
                        if (tunnel_decap)
                                attr->attr = 0;
                        break;
                case RTE_FLOW_ITEM_TYPE_IPV4:
                        if (!attr->ipv6)
                                attr->ipv4 = 1;
                        if (item->mask != NULL &&
                            ((const struct rte_flow_item_ipv4 *)
                            item->mask)->hdr.next_proto_id)
                                next_protocol =
                                    ((const struct rte_flow_item_ipv4 *)
                                      (item->spec))->hdr.next_proto_id &
                                    ((const struct rte_flow_item_ipv4 *)
                                      (item->mask))->hdr.next_proto_id;
                        if ((next_protocol == IPPROTO_IPIP ||
                            next_protocol == IPPROTO_IPV6) && tunnel_decap)
                                attr->attr = 0;
                        break;
                case RTE_FLOW_ITEM_TYPE_IPV6:
                        if (!attr->ipv4)
                                attr->ipv6 = 1;
                        if (item->mask != NULL &&
                            ((const struct rte_flow_item_ipv6 *)
                            item->mask)->hdr.proto)
                                next_protocol =
                                    ((const struct rte_flow_item_ipv6 *)
                                      (item->spec))->hdr.proto &
                                    ((const struct rte_flow_item_ipv6 *)
                                      (item->mask))->hdr.proto;
                        if ((next_protocol == IPPROTO_IPIP ||
                            next_protocol == IPPROTO_IPV6) && tunnel_decap)
                                attr->attr = 0;
                        break;
                case RTE_FLOW_ITEM_TYPE_UDP:
                        if (!attr->tcp)
                                attr->udp = 1;
                        break;
                case RTE_FLOW_ITEM_TYPE_TCP:
                        if (!attr->udp)
                                attr->tcp = 1;
                        break;
                default:
                        break;
                }
        }
        attr->valid = 1;
}

/*
 * Convert rte_mtr_color to mlx5 color.
 *
 * @param[in] rcol
 *   rte_mtr_color.
 *
 * @return
 *   mlx5 color.
 */
static inline int
rte_col_2_mlx5_col(enum rte_color rcol)
{
        switch (rcol) {
        case RTE_COLOR_GREEN:
                return MLX5_FLOW_COLOR_GREEN;
        case RTE_COLOR_YELLOW:
                return MLX5_FLOW_COLOR_YELLOW;
        case RTE_COLOR_RED:
                return MLX5_FLOW_COLOR_RED;
        default:
                break;
        }
        return MLX5_FLOW_COLOR_UNDEFINED;
}

struct field_modify_info {
        uint32_t size; /* Size of field in protocol header, in bytes. */
        uint32_t offset; /* Offset of field in protocol header, in bytes. */
        enum mlx5_modification_field id;
};

struct field_modify_info modify_eth[] = {
        {4,  0, MLX5_MODI_OUT_DMAC_47_16},
        {2,  4, MLX5_MODI_OUT_DMAC_15_0},
        {4,  6, MLX5_MODI_OUT_SMAC_47_16},
        {2, 10, MLX5_MODI_OUT_SMAC_15_0},
        {0, 0, 0},
};

struct field_modify_info modify_vlan_out_first_vid[] = {
        /* Size in bits !!! */
        {12, 0, MLX5_MODI_OUT_FIRST_VID},
        {0, 0, 0},
};

struct field_modify_info modify_ipv4[] = {
        {1,  1, MLX5_MODI_OUT_IP_DSCP},
        {1,  8, MLX5_MODI_OUT_IPV4_TTL},
        {4, 12, MLX5_MODI_OUT_SIPV4},
        {4, 16, MLX5_MODI_OUT_DIPV4},
        {0, 0, 0},
};

struct field_modify_info modify_ipv6[] = {
        {1,  0, MLX5_MODI_OUT_IP_DSCP},
        {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
        {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
        {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
        {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
        {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
        {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
        {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
        {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
        {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
        {0, 0, 0},
};

struct field_modify_info modify_udp[] = {
        {2, 0, MLX5_MODI_OUT_UDP_SPORT},
        {2, 2, MLX5_MODI_OUT_UDP_DPORT},
        {0, 0, 0},
};

struct field_modify_info modify_tcp[] = {
        {2, 0, MLX5_MODI_OUT_TCP_SPORT},
        {2, 2, MLX5_MODI_OUT_TCP_DPORT},
        {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
        {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
        {0, 0, 0},
};

static void
mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
                          uint8_t next_protocol, uint64_t *item_flags,
                          int *tunnel)
{
        MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
                    item->type == RTE_FLOW_ITEM_TYPE_IPV6);
        if (next_protocol == IPPROTO_IPIP) {
                *item_flags |= MLX5_FLOW_LAYER_IPIP;
                *tunnel = 1;
        }
        if (next_protocol == IPPROTO_IPV6) {
                *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
                *tunnel = 1;
        }
}

static inline struct mlx5_hlist *
flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
                     const char *name, uint32_t size, bool direct_key,
                     bool lcores_share, void *ctx,
                     mlx5_list_create_cb cb_create,
                     mlx5_list_match_cb cb_match,
                     mlx5_list_remove_cb cb_remove,
                     mlx5_list_clone_cb cb_clone,
                     mlx5_list_clone_free_cb cb_clone_free,
                     struct rte_flow_error *error)
{
        struct mlx5_hlist *hl;
        struct mlx5_hlist *expected = NULL;
        char s[MLX5_NAME_SIZE];

        hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
        if (likely(hl))
                return hl;
        snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
        hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
                        ctx, cb_create, cb_match, cb_remove, cb_clone,
                        cb_clone_free);
        if (!hl) {
                DRV_LOG(ERR, "%s hash creation failed", name);
                rte_flow_error_set(error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                   "cannot allocate resource memory");
                return NULL;
        }
        if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
                                         __ATOMIC_SEQ_CST,
                                         __ATOMIC_SEQ_CST)) {
                mlx5_hlist_destroy(hl);
                hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
        }
        return hl;
}

/* Update VLAN's VID/PCP based on input rte_flow_action.
 *
 * @param[in] action
 *   Pointer to struct rte_flow_action.
 * @param[out] vlan
 *   Pointer to struct rte_vlan_hdr.
 */
static void
mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
                         struct rte_vlan_hdr *vlan)
{
        uint16_t vlan_tci;
        if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
                vlan_tci =
                    ((const struct rte_flow_action_of_set_vlan_pcp *)
                                               action->conf)->vlan_pcp;
                vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
                vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
                vlan->vlan_tci |= vlan_tci;
        } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
                vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
                vlan->vlan_tci |= rte_be_to_cpu_16
                    (((const struct rte_flow_action_of_set_vlan_vid *)
                                             action->conf)->vlan_vid);
        }
}

/**
 * Fetch 1, 2, 3 or 4 byte field from the byte array
 * and return as unsigned integer in host-endian format.
 *
 * @param[in] data
 *   Pointer to data array.
 * @param[in] size
 *   Size of field to extract.
 *
 * @return
 *   converted field in host endian format.
 */
static inline uint32_t
flow_dv_fetch_field(const uint8_t *data, uint32_t size)
{
        uint32_t ret;

        switch (size) {
        case 1:
                ret = *data;
                break;
        case 2:
                ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
                break;
        case 3:
                ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
                ret = (ret << 8) | *(data + sizeof(uint16_t));
                break;
        case 4:
                ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
                break;
        default:
                MLX5_ASSERT(false);
                ret = 0;
                break;
        }
        return ret;
}

/**
 * Convert modify-header action to DV specification.
 *
 * Data length of each action is determined by provided field description
 * and the item mask. Data bit offset and width of each action is determined
 * by provided item mask.
 *
 * @param[in] item
 *   Pointer to item specification.
 * @param[in] field
 *   Pointer to field modification information.
 *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 * @param[in] dcopy
 *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 *   Negative offset value sets the same offset as source offset.
 *   size field is ignored, value is taken from source field.
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] type
 *   Type of modification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_modify_action(struct rte_flow_item *item,
                              struct field_modify_info *field,
                              struct field_modify_info *dcopy,
                              struct mlx5_flow_dv_modify_hdr_resource *resource,
                              uint32_t type, struct rte_flow_error *error)
{
        uint32_t i = resource->actions_num;
        struct mlx5_modification_cmd *actions = resource->actions;
        uint32_t carry_b = 0;

        /*
         * The item and mask are provided in big-endian format.
         * The fields should be presented as in big-endian format either.
         * Mask must be always present, it defines the actual field width.
         */
        MLX5_ASSERT(item->mask);
        MLX5_ASSERT(field->size);
        do {
                uint32_t size_b;
                uint32_t off_b;
                uint32_t mask;
                uint32_t data;
                bool next_field = true;
                bool next_dcopy = true;

                if (i >= MLX5_MAX_MODIFY_NUM)
                        return rte_flow_error_set(error, EINVAL,
                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                 "too many items to modify");
                /* Fetch variable byte size mask from the array. */
                mask = flow_dv_fetch_field((const uint8_t *)item->mask +
                                           field->offset, field->size);
                if (!mask) {
                        ++field;
                        continue;
                }
                /* Deduce actual data width in bits from mask value. */
                off_b = rte_bsf32(mask) + carry_b;
                size_b = sizeof(uint32_t) * CHAR_BIT -
                         off_b - __builtin_clz(mask);
                MLX5_ASSERT(size_b);
                actions[i] = (struct mlx5_modification_cmd) {
                        .action_type = type,
                        .field = field->id,
                        .offset = off_b,
                        .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
                                0 : size_b,
                };
                if (type == MLX5_MODIFICATION_TYPE_COPY) {
                        MLX5_ASSERT(dcopy);
                        actions[i].dst_field = dcopy->id;
                        actions[i].dst_offset =
                                (int)dcopy->offset < 0 ? off_b : dcopy->offset;
                        /* Convert entire record to big-endian format. */
                        actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
                        /*
                         * Destination field overflow. Copy leftovers of
                         * a source field to the next destination field.
                         */
                        carry_b = 0;
                        if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
                            dcopy->size != 0) {
                                actions[i].length =
                                        dcopy->size * CHAR_BIT - dcopy->offset;
                                carry_b = actions[i].length;
                                next_field = false;
                        }
                        /*
                         * Not enough bits in a source filed to fill a
                         * destination field. Switch to the next source.
                         */
                        if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
                            (size_b == field->size * CHAR_BIT - off_b)) {
                                actions[i].length =
                                        field->size * CHAR_BIT - off_b;
                                dcopy->offset += actions[i].length;
                                next_dcopy = false;
                        }
                        if (next_dcopy)
                                ++dcopy;
                } else {
                        MLX5_ASSERT(item->spec);
                        data = flow_dv_fetch_field((const uint8_t *)item->spec +
                                                   field->offset, field->size);
                        /* Shift out the trailing masked bits from data. */
                        data = (data & mask) >> off_b;
                        actions[i].data1 = rte_cpu_to_be_32(data);
                }
                /* Convert entire record to expected big-endian format. */
                actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
                if (next_field)
                        ++field;
                ++i;
        } while (field->size);
        if (resource->actions_num == i)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "invalid modification flow item");
        resource->actions_num = i;
        return 0;
}

/**
 * Convert modify-header set IPv4 address action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_ipv4
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const struct rte_flow_action_set_ipv4 *conf =
                (const struct rte_flow_action_set_ipv4 *)(action->conf);
        struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
        struct rte_flow_item_ipv4 ipv4;
        struct rte_flow_item_ipv4 ipv4_mask;

        memset(&ipv4, 0, sizeof(ipv4));
        memset(&ipv4_mask, 0, sizeof(ipv4_mask));
        if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
                ipv4.hdr.src_addr = conf->ipv4_addr;
                ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
        } else {
                ipv4.hdr.dst_addr = conf->ipv4_addr;
                ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
        }
        item.spec = &ipv4;
        item.mask = &ipv4_mask;
        return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert modify-header set IPv6 address action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_ipv6
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const struct rte_flow_action_set_ipv6 *conf =
                (const struct rte_flow_action_set_ipv6 *)(action->conf);
        struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
        struct rte_flow_item_ipv6 ipv6;
        struct rte_flow_item_ipv6 ipv6_mask;

        memset(&ipv6, 0, sizeof(ipv6));
        memset(&ipv6_mask, 0, sizeof(ipv6_mask));
        if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
                memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
                       sizeof(ipv6.hdr.src_addr));
                memcpy(&ipv6_mask.hdr.src_addr,
                       &rte_flow_item_ipv6_mask.hdr.src_addr,
                       sizeof(ipv6.hdr.src_addr));
        } else {
                memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
                       sizeof(ipv6.hdr.dst_addr));
                memcpy(&ipv6_mask.hdr.dst_addr,
                       &rte_flow_item_ipv6_mask.hdr.dst_addr,
                       sizeof(ipv6.hdr.dst_addr));
        }
        item.spec = &ipv6;
        item.mask = &ipv6_mask;
        return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert modify-header set MAC address action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_mac
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const struct rte_flow_action_set_mac *conf =
                (const struct rte_flow_action_set_mac *)(action->conf);
        struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
        struct rte_flow_item_eth eth;
        struct rte_flow_item_eth eth_mask;

        memset(&eth, 0, sizeof(eth));
        memset(&eth_mask, 0, sizeof(eth_mask));
        if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
                memcpy(&eth.src.addr_bytes, &conf->mac_addr,
                       sizeof(eth.src.addr_bytes));
                memcpy(&eth_mask.src.addr_bytes,
                       &rte_flow_item_eth_mask.src.addr_bytes,
                       sizeof(eth_mask.src.addr_bytes));
        } else {
                memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
                       sizeof(eth.dst.addr_bytes));
                memcpy(&eth_mask.dst.addr_bytes,
                       &rte_flow_item_eth_mask.dst.addr_bytes,
                       sizeof(eth_mask.dst.addr_bytes));
        }
        item.spec = &eth;
        item.mask = &eth_mask;
        return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert modify-header set VLAN VID action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_vlan_vid
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const struct rte_flow_action_of_set_vlan_vid *conf =
                (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
        int i = resource->actions_num;
        struct mlx5_modification_cmd *actions = resource->actions;
        struct field_modify_info *field = modify_vlan_out_first_vid;

        if (i >= MLX5_MAX_MODIFY_NUM)
                return rte_flow_error_set(error, EINVAL,
                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                         "too many items to modify");
        actions[i] = (struct mlx5_modification_cmd) {
                .action_type = MLX5_MODIFICATION_TYPE_SET,
                .field = field->id,
                .length = field->size,
                .offset = field->offset,
        };
        actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
        actions[i].data1 = conf->vlan_vid;
        actions[i].data1 = actions[i].data1 << 16;
        resource->actions_num = ++i;
        return 0;
}

/**
 * Convert modify-header set TP action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[in] items
 *   Pointer to rte_flow_item objects list.
 * @param[in] attr
 *   Pointer to flow attributes structure.
 * @param[in] dev_flow
 *   Pointer to the sub flow.
 * @param[in] tunnel_decap
 *   Whether action is after tunnel decapsulation.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_tp
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         const struct rte_flow_item *items,
                         union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
                         bool tunnel_decap, struct rte_flow_error *error)
{
        const struct rte_flow_action_set_tp *conf =
                (const struct rte_flow_action_set_tp *)(action->conf);
        struct rte_flow_item item;
        struct rte_flow_item_udp udp;
        struct rte_flow_item_udp udp_mask;
        struct rte_flow_item_tcp tcp;
        struct rte_flow_item_tcp tcp_mask;
        struct field_modify_info *field;

        if (!attr->valid)
                flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
        if (attr->udp) {
                memset(&udp, 0, sizeof(udp));
                memset(&udp_mask, 0, sizeof(udp_mask));
                if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
                        udp.hdr.src_port = conf->port;
                        udp_mask.hdr.src_port =
                                        rte_flow_item_udp_mask.hdr.src_port;
                } else {
                        udp.hdr.dst_port = conf->port;
                        udp_mask.hdr.dst_port =
                                        rte_flow_item_udp_mask.hdr.dst_port;
                }
                item.type = RTE_FLOW_ITEM_TYPE_UDP;
                item.spec = &udp;
                item.mask = &udp_mask;
                field = modify_udp;
        } else {
                MLX5_ASSERT(attr->tcp);
                memset(&tcp, 0, sizeof(tcp));
                memset(&tcp_mask, 0, sizeof(tcp_mask));
                if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
                        tcp.hdr.src_port = conf->port;
                        tcp_mask.hdr.src_port =
                                        rte_flow_item_tcp_mask.hdr.src_port;
                } else {
                        tcp.hdr.dst_port = conf->port;
                        tcp_mask.hdr.dst_port =
                                        rte_flow_item_tcp_mask.hdr.dst_port;
                }
                item.type = RTE_FLOW_ITEM_TYPE_TCP;
                item.spec = &tcp;
                item.mask = &tcp_mask;
                field = modify_tcp;
        }
        return flow_dv_convert_modify_action(&item, field, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert modify-header set TTL action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[in] items
 *   Pointer to rte_flow_item objects list.
 * @param[in] attr
 *   Pointer to flow attributes structure.
 * @param[in] dev_flow
 *   Pointer to the sub flow.
 * @param[in] tunnel_decap
 *   Whether action is after tunnel decapsulation.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_ttl
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         const struct rte_flow_item *items,
                         union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
                         bool tunnel_decap, struct rte_flow_error *error)
{
        const struct rte_flow_action_set_ttl *conf =
                (const struct rte_flow_action_set_ttl *)(action->conf);
        struct rte_flow_item item;
        struct rte_flow_item_ipv4 ipv4;
        struct rte_flow_item_ipv4 ipv4_mask;
        struct rte_flow_item_ipv6 ipv6;
        struct rte_flow_item_ipv6 ipv6_mask;
        struct field_modify_info *field;

        if (!attr->valid)
                flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
        if (attr->ipv4) {
                memset(&ipv4, 0, sizeof(ipv4));
                memset(&ipv4_mask, 0, sizeof(ipv4_mask));
                ipv4.hdr.time_to_live = conf->ttl_value;
                ipv4_mask.hdr.time_to_live = 0xFF;
                item.type = RTE_FLOW_ITEM_TYPE_IPV4;
                item.spec = &ipv4;
                item.mask = &ipv4_mask;
                field = modify_ipv4;
        } else {
                MLX5_ASSERT(attr->ipv6);
                memset(&ipv6, 0, sizeof(ipv6));
                memset(&ipv6_mask, 0, sizeof(ipv6_mask));
                ipv6.hdr.hop_limits = conf->ttl_value;
                ipv6_mask.hdr.hop_limits = 0xFF;
                item.type = RTE_FLOW_ITEM_TYPE_IPV6;
                item.spec = &ipv6;
                item.mask = &ipv6_mask;
                field = modify_ipv6;
        }
        return flow_dv_convert_modify_action(&item, field, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert modify-header decrement TTL action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[in] items
 *   Pointer to rte_flow_item objects list.
 * @param[in] attr
 *   Pointer to flow attributes structure.
 * @param[in] dev_flow
 *   Pointer to the sub flow.
 * @param[in] tunnel_decap
 *   Whether action is after tunnel decapsulation.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_dec_ttl
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_item *items,
                         union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
                         bool tunnel_decap, struct rte_flow_error *error)
{
        struct rte_flow_item item;
        struct rte_flow_item_ipv4 ipv4;
        struct rte_flow_item_ipv4 ipv4_mask;
        struct rte_flow_item_ipv6 ipv6;
        struct rte_flow_item_ipv6 ipv6_mask;
        struct field_modify_info *field;

        if (!attr->valid)
                flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
        if (attr->ipv4) {
                memset(&ipv4, 0, sizeof(ipv4));
                memset(&ipv4_mask, 0, sizeof(ipv4_mask));
                ipv4.hdr.time_to_live = 0xFF;
                ipv4_mask.hdr.time_to_live = 0xFF;
                item.type = RTE_FLOW_ITEM_TYPE_IPV4;
                item.spec = &ipv4;
                item.mask = &ipv4_mask;
                field = modify_ipv4;
        } else {
                MLX5_ASSERT(attr->ipv6);
                memset(&ipv6, 0, sizeof(ipv6));
                memset(&ipv6_mask, 0, sizeof(ipv6_mask));
                ipv6.hdr.hop_limits = 0xFF;
                ipv6_mask.hdr.hop_limits = 0xFF;
                item.type = RTE_FLOW_ITEM_TYPE_IPV6;
                item.spec = &ipv6;
                item.mask = &ipv6_mask;
                field = modify_ipv6;
        }
        return flow_dv_convert_modify_action(&item, field, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_ADD, error);
}

/**
 * Convert modify-header increment/decrement TCP Sequence number
 * to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_tcp_seq
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
        uint64_t value = rte_be_to_cpu_32(*conf);
        struct rte_flow_item item;
        struct rte_flow_item_tcp tcp;
        struct rte_flow_item_tcp tcp_mask;

        memset(&tcp, 0, sizeof(tcp));
        memset(&tcp_mask, 0, sizeof(tcp_mask));
        if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
                /*
                 * The HW has no decrement operation, only increment operation.
                 * To simulate decrement X from Y using increment operation
                 * we need to add UINT32_MAX X times to Y.
                 * Each adding of UINT32_MAX decrements Y by 1.
                 */
                value *= UINT32_MAX;
        tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
        tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
        item.type = RTE_FLOW_ITEM_TYPE_TCP;
        item.spec = &tcp;
        item.mask = &tcp_mask;
        return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_ADD, error);
}

/**
 * Convert modify-header increment/decrement TCP Acknowledgment number
 * to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_tcp_ack
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
        uint64_t value = rte_be_to_cpu_32(*conf);
        struct rte_flow_item item;
        struct rte_flow_item_tcp tcp;
        struct rte_flow_item_tcp tcp_mask;

        memset(&tcp, 0, sizeof(tcp));
        memset(&tcp_mask, 0, sizeof(tcp_mask));
        if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
                /*
                 * The HW has no decrement operation, only increment operation.
                 * To simulate decrement X from Y using increment operation
                 * we need to add UINT32_MAX X times to Y.
                 * Each adding of UINT32_MAX decrements Y by 1.
                 */
                value *= UINT32_MAX;
        tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
        tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
        item.type = RTE_FLOW_ITEM_TYPE_TCP;
        item.spec = &tcp;
        item.mask = &tcp_mask;
        return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_ADD, error);
}

static enum mlx5_modification_field reg_to_field[] = {
        [REG_NON] = MLX5_MODI_OUT_NONE,
        [REG_A] = MLX5_MODI_META_DATA_REG_A,
        [REG_B] = MLX5_MODI_META_DATA_REG_B,
        [REG_C_0] = MLX5_MODI_META_REG_C_0,
        [REG_C_1] = MLX5_MODI_META_REG_C_1,
        [REG_C_2] = MLX5_MODI_META_REG_C_2,
        [REG_C_3] = MLX5_MODI_META_REG_C_3,

        [REG_C_4] = MLX5_MODI_META_REG_C_4,
        [REG_C_5] = MLX5_MODI_META_REG_C_5,
        [REG_C_6] = MLX5_MODI_META_REG_C_6,
        [REG_C_7] = MLX5_MODI_META_REG_C_7,
};

/**
 * Convert register set to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_set_reg
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
        struct mlx5_modification_cmd *actions = resource->actions;
        uint32_t i = resource->actions_num;

        if (i >= MLX5_MAX_MODIFY_NUM)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "too many items to modify");
        MLX5_ASSERT(conf->id != REG_NON);
        MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
        actions[i] = (struct mlx5_modification_cmd) {
                .action_type = MLX5_MODIFICATION_TYPE_SET,
                .field = reg_to_field[conf->id],
                .offset = conf->offset,
                .length = conf->length,
        };
        actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
        actions[i].data1 = rte_cpu_to_be_32(conf->data);
        ++i;
        resource->actions_num = i;
        return 0;
}

/**
 * Convert SET_TAG action to DV specification.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] conf
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_set_tag
                        (struct rte_eth_dev *dev,
                         struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action_set_tag *conf,
                         struct rte_flow_error *error)
{
        rte_be32_t data = rte_cpu_to_be_32(conf->data);
        rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
        struct rte_flow_item item = {
                .spec = &data,
                .mask = &mask,
        };
        struct field_modify_info reg_c_x[] = {
                [1] = {0, 0, 0},
        };
        enum mlx5_modification_field reg_type;
        int ret;

        ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
        if (ret < 0)
                return ret;
        MLX5_ASSERT(ret != REG_NON);
        MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
        reg_type = reg_to_field[ret];
        MLX5_ASSERT(reg_type > 0);
        reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
        return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert internal COPY_REG action to DV specification.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in,out] res
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
                                 struct mlx5_flow_dv_modify_hdr_resource *res,
                                 const struct rte_flow_action *action,
                                 struct rte_flow_error *error)
{
        const struct mlx5_flow_action_copy_mreg *conf = action->conf;
        rte_be32_t mask = RTE_BE32(UINT32_MAX);
        struct rte_flow_item item = {
                .spec = NULL,
                .mask = &mask,
        };
        struct field_modify_info reg_src[] = {
                {4, 0, reg_to_field[conf->src]},
                {0, 0, 0},
        };
        struct field_modify_info reg_dst = {
                .offset = 0,
                .id = reg_to_field[conf->dst],
        };
        /* Adjust reg_c[0] usage according to reported mask. */
        if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
                struct mlx5_priv *priv = dev->data->dev_private;
                uint32_t reg_c0 = priv->sh->dv_regc0_mask;

                MLX5_ASSERT(reg_c0);
                MLX5_ASSERT(priv->sh->config.dv_xmeta_en !=
                            MLX5_XMETA_MODE_LEGACY);
                if (conf->dst == REG_C_0) {
                        /* Copy to reg_c[0], within mask only. */
                        reg_dst.offset = rte_bsf32(reg_c0);
                        mask = rte_cpu_to_be_32(reg_c0 >> reg_dst.offset);
                } else {
                        reg_dst.offset = 0;
                        mask = rte_cpu_to_be_32(reg_c0);
                }
        }
        return flow_dv_convert_modify_action(&item,
                                             reg_src, &reg_dst, res,
                                             MLX5_MODIFICATION_TYPE_COPY,
                                             error);
}

/**
 * Convert MARK action to DV specification. This routine is used
 * in extensive metadata only and requires metadata register to be
 * handled. In legacy mode hardware tag resource is engaged.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] conf
 *   Pointer to MARK action specification.
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_mark(struct rte_eth_dev *dev,
                            const struct rte_flow_action_mark *conf,
                            struct mlx5_flow_dv_modify_hdr_resource *resource,
                            struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
                                           priv->sh->dv_mark_mask);
        rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
        struct rte_flow_item item = {
                .spec = &data,
                .mask = &mask,
        };
        struct field_modify_info reg_c_x[] = {
                [1] = {0, 0, 0},
        };
        int reg;

        if (!mask)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
                                          NULL, "zero mark action mask");
        reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
        if (reg < 0)
                return reg;
        MLX5_ASSERT(reg > 0);
        if (reg == REG_C_0) {
                uint32_t msk_c0 = priv->sh->dv_regc0_mask;
                uint32_t shl_c0 = rte_bsf32(msk_c0);

                data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
                mask = rte_cpu_to_be_32(mask) & msk_c0;
                mask = rte_cpu_to_be_32(mask << shl_c0);
        }
        reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
        return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Get metadata register index for specified steering domain.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] attr
 *   Attributes of flow to determine steering domain.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   positive index on success, a negative errno value otherwise
 *   and rte_errno is set.
 */
static enum modify_reg
flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
                         const struct rte_flow_attr *attr,
                         struct rte_flow_error *error)
{
        int reg =
                mlx5_flow_get_reg_id(dev, attr->transfer ?
                                          MLX5_METADATA_FDB :
                                            attr->egress ?
                                            MLX5_METADATA_TX :
                                            MLX5_METADATA_RX, 0, error);
        if (reg < 0)
                return rte_flow_error_set(error,
                                          ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
                                          NULL, "unavailable "
                                          "metadata register");
        return reg;
}

/**
 * Convert SET_META action to DV specification.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] attr
 *   Attributes of flow that includes this item.
 * @param[in] conf
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_set_meta
                        (struct rte_eth_dev *dev,
                         struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_attr *attr,
                         const struct rte_flow_action_set_meta *conf,
                         struct rte_flow_error *error)
{
        uint32_t mask = rte_cpu_to_be_32(conf->mask);
        uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
        struct rte_flow_item item = {
                .spec = &data,
                .mask = &mask,
        };
        struct field_modify_info reg_c_x[] = {
                [1] = {0, 0, 0},
        };
        int reg = flow_dv_get_metadata_reg(dev, attr, error);

        if (reg < 0)
                return reg;
        MLX5_ASSERT(reg != REG_NON);
        if (reg == REG_C_0) {
                struct mlx5_priv *priv = dev->data->dev_private;
                uint32_t msk_c0 = priv->sh->dv_regc0_mask;
                uint32_t shl_c0 = rte_bsf32(msk_c0);

                data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
                mask = rte_cpu_to_be_32(mask) & msk_c0;
                mask = rte_cpu_to_be_32(mask << shl_c0);
        }
        reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
        /* The routine expects parameters in memory as big-endian ones. */
        return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert modify-header set IPv4 DSCP action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_ipv4_dscp
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const struct rte_flow_action_set_dscp *conf =
                (const struct rte_flow_action_set_dscp *)(action->conf);
        struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
        struct rte_flow_item_ipv4 ipv4;
        struct rte_flow_item_ipv4 ipv4_mask;

        memset(&ipv4, 0, sizeof(ipv4));
        memset(&ipv4_mask, 0, sizeof(ipv4_mask));
        ipv4.hdr.type_of_service = conf->dscp;
        ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
        item.spec = &ipv4;
        item.mask = &ipv4_mask;
        return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

/**
 * Convert modify-header set IPv6 DSCP action to DV specification.
 *
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_ipv6_dscp
                        (struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         struct rte_flow_error *error)
{
        const struct rte_flow_action_set_dscp *conf =
                (const struct rte_flow_action_set_dscp *)(action->conf);
        struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
        struct rte_flow_item_ipv6 ipv6;
        struct rte_flow_item_ipv6 ipv6_mask;

        memset(&ipv6, 0, sizeof(ipv6));
        memset(&ipv6_mask, 0, sizeof(ipv6_mask));
        /*
         * Even though the DSCP bits offset of IPv6 is not byte aligned,
         * rdma-core only accept the DSCP bits byte aligned start from
         * bit 0 to 5 as to be compatible with IPv4. No need to shift the
         * bits in IPv6 case as rdma-core requires byte aligned value.
         */
        ipv6.hdr.vtc_flow = conf->dscp;
        ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
        item.spec = &ipv6;
        item.mask = &ipv6_mask;
        return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
}

static int
mlx5_flow_item_field_width(struct rte_eth_dev *dev,
                           enum rte_flow_field_id field, int inherit,
                           const struct rte_flow_attr *attr,
                           struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;

        switch (field) {
        case RTE_FLOW_FIELD_START:
                return 32;
        case RTE_FLOW_FIELD_MAC_DST:
        case RTE_FLOW_FIELD_MAC_SRC:
                return 48;
        case RTE_FLOW_FIELD_VLAN_TYPE:
                return 16;
        case RTE_FLOW_FIELD_VLAN_ID:
                return 12;
        case RTE_FLOW_FIELD_MAC_TYPE:
                return 16;
        case RTE_FLOW_FIELD_IPV4_DSCP:
                return 6;
        case RTE_FLOW_FIELD_IPV4_TTL:
                return 8;
        case RTE_FLOW_FIELD_IPV4_SRC:
        case RTE_FLOW_FIELD_IPV4_DST:
                return 32;
        case RTE_FLOW_FIELD_IPV6_DSCP:
                return 6;
        case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
                return 8;
        case RTE_FLOW_FIELD_IPV6_SRC:
        case RTE_FLOW_FIELD_IPV6_DST:
                return 128;
        case RTE_FLOW_FIELD_TCP_PORT_SRC:
        case RTE_FLOW_FIELD_TCP_PORT_DST:
                return 16;
        case RTE_FLOW_FIELD_TCP_SEQ_NUM:
        case RTE_FLOW_FIELD_TCP_ACK_NUM:
                return 32;
        case RTE_FLOW_FIELD_TCP_FLAGS:
                return 9;
        case RTE_FLOW_FIELD_UDP_PORT_SRC:
        case RTE_FLOW_FIELD_UDP_PORT_DST:
                return 16;
        case RTE_FLOW_FIELD_VXLAN_VNI:
        case RTE_FLOW_FIELD_GENEVE_VNI:
                return 24;
        case RTE_FLOW_FIELD_GTP_TEID:
        case RTE_FLOW_FIELD_TAG:
                return 32;
        case RTE_FLOW_FIELD_MARK:
                return __builtin_popcount(priv->sh->dv_mark_mask);
        case RTE_FLOW_FIELD_META:
                return (flow_dv_get_metadata_reg(dev, attr, error) == REG_C_0) ?
                        __builtin_popcount(priv->sh->dv_meta_mask) : 32;
        case RTE_FLOW_FIELD_POINTER:
        case RTE_FLOW_FIELD_VALUE:
                return inherit < 0 ? 0 : inherit;
        case RTE_FLOW_FIELD_IPV4_ECN:
        case RTE_FLOW_FIELD_IPV6_ECN:
                return 2;
        default:
                MLX5_ASSERT(false);
        }
        return 0;
}

static void
mlx5_flow_field_id_to_modify_info
                (const struct rte_flow_action_modify_data *data,
                 struct field_modify_info *info, uint32_t *mask,
                 uint32_t width, struct rte_eth_dev *dev,
                 const struct rte_flow_attr *attr, struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        uint32_t idx = 0;
        uint32_t off = 0;

        switch (data->field) {
        case RTE_FLOW_FIELD_START:
                /* not supported yet */
                MLX5_ASSERT(false);
                break;
        case RTE_FLOW_FIELD_MAC_DST:
                off = data->offset > 16 ? data->offset - 16 : 0;
                if (mask) {
                        if (data->offset < 16) {
                                info[idx] = (struct field_modify_info){2, 4,
                                                MLX5_MODI_OUT_DMAC_15_0};
                                if (width < 16) {
                                        mask[1] = rte_cpu_to_be_16(0xffff >>
                                                                 (16 - width));
                                        width = 0;
                                } else {
                                        mask[1] = RTE_BE16(0xffff);
                                        width -= 16;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        info[idx] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_DMAC_47_16};
                        mask[0] = rte_cpu_to_be_32((0xffffffff >>
                                                    (32 - width)) << off);
                } else {
                        if (data->offset < 16)
                                info[idx++] = (struct field_modify_info){2, 0,
                                                MLX5_MODI_OUT_DMAC_15_0};
                        info[idx] = (struct field_modify_info){4, off,
                                                MLX5_MODI_OUT_DMAC_47_16};
                }
                break;
        case RTE_FLOW_FIELD_MAC_SRC:
                off = data->offset > 16 ? data->offset - 16 : 0;
                if (mask) {
                        if (data->offset < 16) {
                                info[idx] = (struct field_modify_info){2, 4,
                                                MLX5_MODI_OUT_SMAC_15_0};
                                if (width < 16) {
                                        mask[1] = rte_cpu_to_be_16(0xffff >>
                                                                 (16 - width));
                                        width = 0;
                                } else {
                                        mask[1] = RTE_BE16(0xffff);
                                        width -= 16;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        info[idx] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_SMAC_47_16};
                        mask[0] = rte_cpu_to_be_32((0xffffffff >>
                                                    (32 - width)) << off);
                } else {
                        if (data->offset < 16)
                                info[idx++] = (struct field_modify_info){2, 0,
                                                MLX5_MODI_OUT_SMAC_15_0};
                        info[idx] = (struct field_modify_info){4, off,
                                                MLX5_MODI_OUT_SMAC_47_16};
                }
                break;
        case RTE_FLOW_FIELD_VLAN_TYPE:
                /* not supported yet */
                break;
        case RTE_FLOW_FIELD_VLAN_ID:
                info[idx] = (struct field_modify_info){2, 0,
                                        MLX5_MODI_OUT_FIRST_VID};
                if (mask)
                        mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
                break;
        case RTE_FLOW_FIELD_MAC_TYPE:
                info[idx] = (struct field_modify_info){2, 0,
                                        MLX5_MODI_OUT_ETHERTYPE};
                if (mask)
                        mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
                break;
        case RTE_FLOW_FIELD_IPV4_DSCP:
                info[idx] = (struct field_modify_info){1, 0,
                                        MLX5_MODI_OUT_IP_DSCP};
                if (mask)
                        mask[idx] = 0x3f >> (6 - width);
                break;
        case RTE_FLOW_FIELD_IPV4_TTL:
                info[idx] = (struct field_modify_info){1, 0,
                                        MLX5_MODI_OUT_IPV4_TTL};
                if (mask)
                        mask[idx] = 0xff >> (8 - width);
                break;
        case RTE_FLOW_FIELD_IPV4_SRC:
                info[idx] = (struct field_modify_info){4, 0,
                                        MLX5_MODI_OUT_SIPV4};
                if (mask)
                        mask[idx] = rte_cpu_to_be_32(0xffffffff >>
                                                     (32 - width));
                break;
        case RTE_FLOW_FIELD_IPV4_DST:
                info[idx] = (struct field_modify_info){4, 0,
                                        MLX5_MODI_OUT_DIPV4};
                if (mask)
                        mask[idx] = rte_cpu_to_be_32(0xffffffff >>
                                                     (32 - width));
                break;
        case RTE_FLOW_FIELD_IPV6_DSCP:
                info[idx] = (struct field_modify_info){1, 0,
                                        MLX5_MODI_OUT_IP_DSCP};
                if (mask)
                        mask[idx] = 0x3f >> (6 - width);
                break;
        case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
                info[idx] = (struct field_modify_info){1, 0,
                                        MLX5_MODI_OUT_IPV6_HOPLIMIT};
                if (mask)
                        mask[idx] = 0xff >> (8 - width);
                break;
        case RTE_FLOW_FIELD_IPV6_SRC:
                if (mask) {
                        if (data->offset < 32) {
                                info[idx] = (struct field_modify_info){4, 12,
                                                MLX5_MODI_OUT_SIPV6_31_0};
                                if (width < 32) {
                                        mask[3] =
                                                rte_cpu_to_be_32(0xffffffff >>
                                                                 (32 - width));
                                        width = 0;
                                } else {
                                        mask[3] = RTE_BE32(0xffffffff);
                                        width -= 32;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        if (data->offset < 64) {
                                info[idx] = (struct field_modify_info){4, 8,
                                                MLX5_MODI_OUT_SIPV6_63_32};
                                if (width < 32) {
                                        mask[2] =
                                                rte_cpu_to_be_32(0xffffffff >>
                                                                 (32 - width));
                                        width = 0;
                                } else {
                                        mask[2] = RTE_BE32(0xffffffff);
                                        width -= 32;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        if (data->offset < 96) {
                                info[idx] = (struct field_modify_info){4, 4,
                                                MLX5_MODI_OUT_SIPV6_95_64};
                                if (width < 32) {
                                        mask[1] =
                                                rte_cpu_to_be_32(0xffffffff >>
                                                                 (32 - width));
                                        width = 0;
                                } else {
                                        mask[1] = RTE_BE32(0xffffffff);
                                        width -= 32;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        info[idx] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_SIPV6_127_96};
                        mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width));
                } else {
                        if (data->offset < 32)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_SIPV6_31_0};
                        if (data->offset < 64)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_SIPV6_63_32};
                        if (data->offset < 96)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_SIPV6_95_64};
                        if (data->offset < 128)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_SIPV6_127_96};
                }
                break;
        case RTE_FLOW_FIELD_IPV6_DST:
                if (mask) {
                        if (data->offset < 32) {
                                info[idx] = (struct field_modify_info){4, 12,
                                                MLX5_MODI_OUT_DIPV6_31_0};
                                if (width < 32) {
                                        mask[3] =
                                                rte_cpu_to_be_32(0xffffffff >>
                                                                 (32 - width));
                                        width = 0;
                                } else {
                                        mask[3] = RTE_BE32(0xffffffff);
                                        width -= 32;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        if (data->offset < 64) {
                                info[idx] = (struct field_modify_info){4, 8,
                                                MLX5_MODI_OUT_DIPV6_63_32};
                                if (width < 32) {
                                        mask[2] =
                                                rte_cpu_to_be_32(0xffffffff >>
                                                                 (32 - width));
                                        width = 0;
                                } else {
                                        mask[2] = RTE_BE32(0xffffffff);
                                        width -= 32;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        if (data->offset < 96) {
                                info[idx] = (struct field_modify_info){4, 4,
                                                MLX5_MODI_OUT_DIPV6_95_64};
                                if (width < 32) {
                                        mask[1] =
                                                rte_cpu_to_be_32(0xffffffff >>
                                                                 (32 - width));
                                        width = 0;
                                } else {
                                        mask[1] = RTE_BE32(0xffffffff);
                                        width -= 32;
                                }
                                if (!width)
                                        break;
                                ++idx;
                        }
                        info[idx] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_DIPV6_127_96};
                        mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width));
                } else {
                        if (data->offset < 32)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_DIPV6_31_0};
                        if (data->offset < 64)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_DIPV6_63_32};
                        if (data->offset < 96)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_DIPV6_95_64};
                        if (data->offset < 128)
                                info[idx++] = (struct field_modify_info){4, 0,
                                                MLX5_MODI_OUT_DIPV6_127_96};
                }
                break;
        case RTE_FLOW_FIELD_TCP_PORT_SRC:
                info[idx] = (struct field_modify_info){2, 0,
                                        MLX5_MODI_OUT_TCP_SPORT};
                if (mask)
                        mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
                break;
        case RTE_FLOW_FIELD_TCP_PORT_DST:
                info[idx] = (struct field_modify_info){2, 0,
                                        MLX5_MODI_OUT_TCP_DPORT};
                if (mask)
                        mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
                break;
        case RTE_FLOW_FIELD_TCP_SEQ_NUM:
                info[idx] = (struct field_modify_info){4, 0,
                                        MLX5_MODI_OUT_TCP_SEQ_NUM};
                if (mask)
                        mask[idx] = rte_cpu_to_be_32(0xffffffff >>
                                                     (32 - width));
                break;
        case RTE_FLOW_FIELD_TCP_ACK_NUM:
                info[idx] = (struct field_modify_info){4, 0,
                                        MLX5_MODI_OUT_TCP_ACK_NUM};
                if (mask)
                        mask[idx] = rte_cpu_to_be_32(0xffffffff >>
                                                     (32 - width));
                break;
        case RTE_FLOW_FIELD_TCP_FLAGS:
                info[idx] = (struct field_modify_info){2, 0,
                                        MLX5_MODI_OUT_TCP_FLAGS};
                if (mask)
                        mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
                break;
        case RTE_FLOW_FIELD_UDP_PORT_SRC:
                info[idx] = (struct field_modify_info){2, 0,
                                        MLX5_MODI_OUT_UDP_SPORT};
                if (mask)
                        mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
                break;
        case RTE_FLOW_FIELD_UDP_PORT_DST:
                info[idx] = (struct field_modify_info){2, 0,
                                        MLX5_MODI_OUT_UDP_DPORT};
                if (mask)
                        mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
                break;
        case RTE_FLOW_FIELD_VXLAN_VNI:
                /* not supported yet */
                break;
        case RTE_FLOW_FIELD_GENEVE_VNI:
                /* not supported yet*/
                break;
        case RTE_FLOW_FIELD_GTP_TEID:
                info[idx] = (struct field_modify_info){4, 0,
                                        MLX5_MODI_GTP_TEID};
                if (mask)
                        mask[idx] = rte_cpu_to_be_32(0xffffffff >>
                                                     (32 - width));
                break;
        case RTE_FLOW_FIELD_TAG:
                {
                        int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
                                                   data->level, error);
                        if (reg < 0)
                                return;
                        MLX5_ASSERT(reg != REG_NON);
                        MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
                        info[idx] = (struct field_modify_info){4, 0,
                                                reg_to_field[reg]};
                        if (mask)
                                mask[idx] =
                                        rte_cpu_to_be_32(0xffffffff >>
                                                         (32 - width));
                }
                break;
        case RTE_FLOW_FIELD_MARK:
                {
                        uint32_t mark_mask = priv->sh->dv_mark_mask;
                        uint32_t mark_count = __builtin_popcount(mark_mask);
                        int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
                                                       0, error);
                        if (reg < 0)
                                return;
                        MLX5_ASSERT(reg != REG_NON);
                        MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
                        info[idx] = (struct field_modify_info){4, 0,
                                                reg_to_field[reg]};
                        if (mask)
                                mask[idx] = rte_cpu_to_be_32((mark_mask >>
                                         (mark_count - width)) & mark_mask);
                }
                break;
        case RTE_FLOW_FIELD_META:
                {
                        uint32_t meta_mask = priv->sh->dv_meta_mask;
                        uint32_t meta_count = __builtin_popcount(meta_mask);
                        int reg = flow_dv_get_metadata_reg(dev, attr, error);
                        if (reg < 0)
                                return;
                        MLX5_ASSERT(reg != REG_NON);
                        MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
                        info[idx] = (struct field_modify_info){4, 0,
                                                reg_to_field[reg]};
                        if (mask)
                                mask[idx] = rte_cpu_to_be_32((meta_mask >>
                                        (meta_count - width)) & meta_mask);
                }
                break;
        case RTE_FLOW_FIELD_IPV4_ECN:
        case RTE_FLOW_FIELD_IPV6_ECN:
                info[idx] = (struct field_modify_info){1, 0,
                                        MLX5_MODI_OUT_IP_ECN};
                if (mask)
                        mask[idx] = 0x3 >> (2 - width);
                break;
        case RTE_FLOW_FIELD_POINTER:
        case RTE_FLOW_FIELD_VALUE:
        default:
                MLX5_ASSERT(false);
                break;
        }
}

/**
 * Convert modify_field action to DV specification.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in,out] resource
 *   Pointer to the modify-header resource.
 * @param[in] action
 *   Pointer to action specification.
 * @param[in] attr
 *   Attributes of flow that includes this item.
 * @param[out] error
 *   Pointer to the error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_convert_action_modify_field
                        (struct rte_eth_dev *dev,
                         struct mlx5_flow_dv_modify_hdr_resource *resource,
                         const struct rte_flow_action *action,
                         const struct rte_flow_attr *attr,
                         struct rte_flow_error *error)
{
        const struct rte_flow_action_modify_field *conf =
                (const struct rte_flow_action_modify_field *)(action->conf);
        struct rte_flow_item item = {
                .spec = NULL,
                .mask = NULL
        };
        struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
                                                                {0, 0, 0} };
        struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
                                                                {0, 0, 0} };
        uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
        uint32_t type, meta = 0;

        if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
            conf->src.field == RTE_FLOW_FIELD_VALUE) {
                type = MLX5_MODIFICATION_TYPE_SET;
                /** For SET fill the destination field (field) first. */
                mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
                                                  conf->width, dev,
                                                  attr, error);
                item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ?
                                        (void *)(uintptr_t)conf->src.pvalue :
                                        (void *)(uintptr_t)&conf->src.value;
                if (conf->dst.field == RTE_FLOW_FIELD_META) {
                        meta = *(const unaligned_uint32_t *)item.spec;
                        meta = rte_cpu_to_be_32(meta);
                        item.spec = &meta;
                }
        } else {
                type = MLX5_MODIFICATION_TYPE_COPY;
                /** For COPY fill the destination field (dcopy) without mask. */
                mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
                                                  conf->width, dev,
                                                  attr, error);
                /** Then construct the source field (field) with mask. */
                mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
                                                  conf->width, dev,
                                                  attr, error);
        }
        item.mask = &mask;
        return flow_dv_convert_modify_action(&item,
                        field, dcopy, resource, type, error);
}

/**
 * Validate MARK item.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] item
 *   Item specification.
 * @param[in] attr
 *   Attributes of flow that includes this item.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_mark(struct rte_eth_dev *dev,
                           const struct rte_flow_item *item,
                           const struct rte_flow_attr *attr __rte_unused,
                           struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_sh_config *config = &priv->sh->config;
        const struct rte_flow_item_mark *spec = item->spec;
        const struct rte_flow_item_mark *mask = item->mask;
        const struct rte_flow_item_mark nic_mask = {
                .id = priv->sh->dv_mark_mask,
        };
        int ret;

        if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "extended metadata feature"
                                          " isn't enabled");
        if (!mlx5_flow_ext_mreg_supported(dev))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "extended metadata register"
                                          " isn't supported");
        if (!nic_mask.id)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "extended metadata register"
                                          " isn't available");
        ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
        if (ret < 0)
                return ret;
        if (!spec)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
                                          item->spec,
                                          "data cannot be empty");
        if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
                                          &spec->id,
                                          "mark id exceeds the limit");
        if (!mask)
                mask = &nic_mask;
        if (!mask->id)
                return rte_flow_error_set(error, EINVAL,
                                        RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
                                        "mask cannot be zero");

        ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
                                        (const uint8_t *)&nic_mask,
                                        sizeof(struct rte_flow_item_mark),
                                        MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
        if (ret < 0)
                return ret;
        return 0;
}

/**
 * Validate META item.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] item
 *   Item specification.
 * @param[in] attr
 *   Attributes of flow that includes this item.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
                           const struct rte_flow_item *item,
                           const struct rte_flow_attr *attr,
                           struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_sh_config *config = &priv->sh->config;
        const struct rte_flow_item_meta *spec = item->spec;
        const struct rte_flow_item_meta *mask = item->mask;
        struct rte_flow_item_meta nic_mask = {
                .data = UINT32_MAX
        };
        int reg;
        int ret;

        if (!spec)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
                                          item->spec,
                                          "data cannot be empty");
        if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
                if (!mlx5_flow_ext_mreg_supported(dev))
                        return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "extended metadata register"
                                          " isn't supported");
                reg = flow_dv_get_metadata_reg(dev, attr, error);
                if (reg < 0)
                        return reg;
                if (reg == REG_NON)
                        return rte_flow_error_set(error, ENOTSUP,
                                        RTE_FLOW_ERROR_TYPE_ITEM, item,
                                        "unavailable extended metadata register");
                if (reg == REG_B)
                        return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "match on reg_b "
                                          "isn't supported");
                if (reg != REG_A)
                        nic_mask.data = priv->sh->dv_meta_mask;
        } else {
                if (attr->transfer)
                        return rte_flow_error_set(error, ENOTSUP,
                                        RTE_FLOW_ERROR_TYPE_ITEM, item,
                                        "extended metadata feature "
                                        "should be enabled when "
                                        "meta item is requested "
                                        "with e-switch mode ");
                if (attr->ingress)
                        return rte_flow_error_set(error, ENOTSUP,
                                        RTE_FLOW_ERROR_TYPE_ITEM, item,
                                        "match on metadata for ingress "
                                        "is not supported in legacy "
                                        "metadata mode");
        }
        if (!mask)
                mask = &rte_flow_item_meta_mask;
        if (!mask->data)
                return rte_flow_error_set(error, EINVAL,
                                        RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
                                        "mask cannot be zero");

        ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
                                        (const uint8_t *)&nic_mask,
                                        sizeof(struct rte_flow_item_meta),
                                        MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
        return ret;
}

/**
 * Validate TAG item.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] item
 *   Item specification.
 * @param[in] attr
 *   Attributes of flow that includes this item.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_tag(struct rte_eth_dev *dev,
                          const struct rte_flow_item *item,
                          const struct rte_flow_attr *attr __rte_unused,
                          struct rte_flow_error *error)
{
        const struct rte_flow_item_tag *spec = item->spec;
        const struct rte_flow_item_tag *mask = item->mask;
        const struct rte_flow_item_tag nic_mask = {
                .data = RTE_BE32(UINT32_MAX),
                .index = 0xff,
        };
        int ret;

        if (!mlx5_flow_ext_mreg_supported(dev))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "extensive metadata register"
                                          " isn't supported");
        if (!spec)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
                                          item->spec,
                                          "data cannot be empty");
        if (!mask)
                mask = &rte_flow_item_tag_mask;
        if (!mask->data)
                return rte_flow_error_set(error, EINVAL,
                                        RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
                                        "mask cannot be zero");

        ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
                                        (const uint8_t *)&nic_mask,
                                        sizeof(struct rte_flow_item_tag),
                                        MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
        if (ret < 0)
                return ret;
        if (mask->index != 0xff)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
                                          "partial mask for tag index"
                                          " is not supported");
        ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
        if (ret < 0)
                return ret;
        MLX5_ASSERT(ret != REG_NON);
        return 0;
}

/**
 * Validate vport item.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] item
 *   Item specification.
 * @param[in] attr
 *   Attributes of flow that includes this item.
 * @param[in] item_flags
 *   Bit-fields that holds the items detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
                              const struct rte_flow_item *item,
                              const struct rte_flow_attr *attr,
                              uint64_t item_flags,
                              struct rte_flow_error *error)
{
        const struct rte_flow_item_port_id *spec = item->spec;
        const struct rte_flow_item_port_id *mask = item->mask;
        const struct rte_flow_item_port_id switch_mask = {
                        .id = 0xffffffff,
        };
        struct mlx5_priv *esw_priv;
        struct mlx5_priv *dev_priv;
        int ret;

        if (!attr->transfer)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM,
                                          NULL,
                                          "match on port id is valid only"
                                          " when transfer flag is enabled");
        if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "multiple source ports are not"
                                          " supported");
        if (!mask)
                mask = &switch_mask;
        if (mask->id != 0xffffffff)
                return rte_flow_error_set(error, ENOTSUP,
                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
                                           mask,
                                           "no support for partial mask on"
                                           " \"id\" field");
        ret = mlx5_flow_item_acceptable
                                (item, (const uint8_t *)mask,
                                 (const uint8_t *)&rte_flow_item_port_id_mask,
                                 sizeof(struct rte_flow_item_port_id),
                                 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
        if (ret)
                return ret;
        if (!spec)
                return 0;
        if (spec->id == MLX5_PORT_ESW_MGR)
                return 0;
        esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
        if (!esw_priv)
                return rte_flow_error_set(error, rte_errno,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
                                          "failed to obtain E-Switch info for"
                                          " port");
        dev_priv = mlx5_dev_to_eswitch_info(dev);
        if (!dev_priv)
                return rte_flow_error_set(error, rte_errno,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL,
                                          "failed to obtain E-Switch info");
        if (esw_priv->domain_id != dev_priv->domain_id)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
                                          "cannot match on a port from a"
                                          " different E-Switch");
        return 0;
}

/**
 * Validate represented port item.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] item
 *   Item specification.
 * @param[in] attr
 *   Attributes of flow that includes this item.
 * @param[in] item_flags
 *   Bit-fields that holds the items detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_represented_port(struct rte_eth_dev *dev,
                                       const struct rte_flow_item *item,
                                       const struct rte_flow_attr *attr,
                                       uint64_t item_flags,
                                       struct rte_flow_error *error)
{
        const struct rte_flow_item_ethdev *spec = item->spec;
        const struct rte_flow_item_ethdev *mask = item->mask;
        const struct rte_flow_item_ethdev switch_mask = {
                        .port_id = UINT16_MAX,
        };
        struct mlx5_priv *esw_priv;
        struct mlx5_priv *dev_priv;
        int ret;

        if (!attr->transfer)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, NULL,
                                          "match on port id is valid only when transfer flag is enabled");
        if (item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "multiple source ports are not supported");
        if (!mask)
                mask = &switch_mask;
        if (mask->port_id != UINT16_MAX)
                return rte_flow_error_set(error, ENOTSUP,
                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
                                           "no support for partial mask on \"id\" field");
        ret = mlx5_flow_item_acceptable
                                (item, (const uint8_t *)mask,
                                 (const uint8_t *)&rte_flow_item_ethdev_mask,
                                 sizeof(struct rte_flow_item_ethdev),
                                 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
        if (ret)
                return ret;
        if (!spec || spec->port_id == UINT16_MAX)
                return 0;
        esw_priv = mlx5_port_to_eswitch_info(spec->port_id, false);
        if (!esw_priv)
                return rte_flow_error_set(error, rte_errno,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
                                          "failed to obtain E-Switch info for port");
        dev_priv = mlx5_dev_to_eswitch_info(dev);
        if (!dev_priv)
                return rte_flow_error_set(error, rte_errno,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL,
                                          "failed to obtain E-Switch info");
        if (esw_priv->domain_id != dev_priv->domain_id)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
                                          "cannot match on a port from a different E-Switch");
        return 0;
}

/**
 * Validate VLAN item.
 *
 * @param[in] item
 *   Item specification.
 * @param[in] item_flags
 *   Bit-fields that holds the items detected until now.
 * @param[in] dev
 *   Ethernet device flow is being created on.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_vlan(const struct rte_flow_item *item,
                           uint64_t item_flags,
                           struct rte_eth_dev *dev,
                           struct rte_flow_error *error)
{
        const struct rte_flow_item_vlan *mask = item->mask;
        const struct rte_flow_item_vlan nic_mask = {
                .tci = RTE_BE16(UINT16_MAX),
                .inner_type = RTE_BE16(UINT16_MAX),
                .has_more_vlan = 1,
        };
        const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
        int ret;
        const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
                                        MLX5_FLOW_LAYER_INNER_L4) :
                                       (MLX5_FLOW_LAYER_OUTER_L3 |
                                        MLX5_FLOW_LAYER_OUTER_L4);
        const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
                                        MLX5_FLOW_LAYER_OUTER_VLAN;

        if (item_flags & vlanm)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "multiple VLAN layers not supported");
        else if ((item_flags & l34m) != 0)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "VLAN cannot follow L3/L4 layer");
        if (!mask)
                mask = &rte_flow_item_vlan_mask;
        ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
                                        (const uint8_t *)&nic_mask,
                                        sizeof(struct rte_flow_item_vlan),
                                        MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
        if (ret)
                return ret;
        if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
                struct mlx5_priv *priv = dev->data->dev_private;

                if (priv->vmwa_context) {
                        /*
                         * Non-NULL context means we have a virtual machine
                         * and SR-IOV enabled, we have to create VLAN interface
                         * to make hypervisor to setup E-Switch vport
                         * context correctly. We avoid creating the multiple
                         * VLAN interfaces, so we cannot support VLAN tag mask.
                         */
                        return rte_flow_error_set(error, EINVAL,
                                                  RTE_FLOW_ERROR_TYPE_ITEM,
                                                  item,
                                                  "VLAN tag mask is not"
                                                  " supported in virtual"
                                                  " environment");
                }
        }
        return 0;
}

/*
 * GTP flags are contained in 1 byte of the format:
 * -------------------------------------------
 * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
 * |-----------------------------------------|
 * | value | Version | PT | Res | E | S | PN |
 * -------------------------------------------
 *
 * Matching is supported only for GTP flags E, S, PN.
 */
#define MLX5_GTP_FLAGS_MASK     0x07

/**
 * Validate GTP item.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] item
 *   Item specification.
 * @param[in] item_flags
 *   Bit-fields that holds the items detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
                          const struct rte_flow_item *item,
                          uint64_t item_flags,
                          struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        const struct rte_flow_item_gtp *spec = item->spec;
        const struct rte_flow_item_gtp *mask = item->mask;
        const struct rte_flow_item_gtp nic_mask = {
                .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
                .msg_type = 0xff,
                .teid = RTE_BE32(0xffffffff),
        };

        if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_gtp)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "GTP support is not enabled");
        if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "multiple tunnel layers not"
                                          " supported");
        if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "no outer UDP layer found");
        if (!mask)
                mask = &rte_flow_item_gtp_mask;
        if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "Match is supported for GTP"
                                          " flags only");
        return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
                                         (const uint8_t *)&nic_mask,
                                         sizeof(struct rte_flow_item_gtp),
                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
}

/**
 * Validate GTP PSC item.
 *
 * @param[in] item
 *   Item specification.
 * @param[in] last_item
 *   Previous validated item in the pattern items.
 * @param[in] gtp_item
 *   Previous GTP item specification.
 * @param[in] attr
 *   Pointer to flow attributes.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
                              uint64_t last_item,
                              const struct rte_flow_item *gtp_item,
                              const struct rte_flow_attr *attr,
                              struct rte_flow_error *error)
{
        const struct rte_flow_item_gtp *gtp_spec;
        const struct rte_flow_item_gtp *gtp_mask;
        const struct rte_flow_item_gtp_psc *mask;
        const struct rte_flow_item_gtp_psc nic_mask = {
                .hdr.type = 0xF,
                .hdr.qfi = 0x3F,
        };

        if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
                return rte_flow_error_set
                        (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
                         "GTP PSC item must be preceded with GTP item");
        gtp_spec = gtp_item->spec;
        gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
        /* GTP spec and E flag is requested to match zero. */
        if (gtp_spec &&
                (gtp_mask->v_pt_rsv_flags &
                ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
                return rte_flow_error_set
                        (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
                         "GTP E flag must be 1 to match GTP PSC");
        /* Check the flow is not created in group zero. */
        if (!attr->transfer && !attr->group)
                return rte_flow_error_set
                        (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                         "GTP PSC is not supported for group 0");
        /* GTP spec is here and E flag is requested to match zero. */
        if (!item->spec)
                return 0;
        mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
        return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
                                         (const uint8_t *)&nic_mask,
                                         sizeof(struct rte_flow_item_gtp_psc),
                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
}

/**
 * Validate IPV4 item.
 * Use existing validation function mlx5_flow_validate_item_ipv4(), and
 * add specific validation of fragment_offset field,
 *
 * @param[in] item
 *   Item specification.
 * @param[in] item_flags
 *   Bit-fields that holds the items detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_ipv4(struct rte_eth_dev *dev,
                           const struct rte_flow_item *item,
                           uint64_t item_flags, uint64_t last_item,
                           uint16_t ether_type, struct rte_flow_error *error)
{
        int ret;
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_hca_attr *attr = &priv->sh->cdev->config.hca_attr;
        const struct rte_flow_item_ipv4 *spec = item->spec;
        const struct rte_flow_item_ipv4 *last = item->last;
        const struct rte_flow_item_ipv4 *mask = item->mask;
        rte_be16_t fragment_offset_spec = 0;
        rte_be16_t fragment_offset_last = 0;
        struct rte_flow_item_ipv4 nic_ipv4_mask = {
                .hdr = {
                        .src_addr = RTE_BE32(0xffffffff),
                        .dst_addr = RTE_BE32(0xffffffff),
                        .type_of_service = 0xff,
                        .fragment_offset = RTE_BE16(0xffff),
                        .next_proto_id = 0xff,
                        .time_to_live = 0xff,
                },
        };

        if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) {
                int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
                bool ihl_cap = !tunnel ?
                               attr->outer_ipv4_ihl : attr->inner_ipv4_ihl;
                if (!ihl_cap)
                        return rte_flow_error_set(error, ENOTSUP,
                                                  RTE_FLOW_ERROR_TYPE_ITEM,
                                                  item,
                                                  "IPV4 ihl offload not supported");
                nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl;
        }
        ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
                                           ether_type, &nic_ipv4_mask,
                                           MLX5_ITEM_RANGE_ACCEPTED, error);
        if (ret < 0)
                return ret;
        if (spec && mask)
                fragment_offset_spec = spec->hdr.fragment_offset &
                                       mask->hdr.fragment_offset;
        if (!fragment_offset_spec)
                return 0;
        /*
         * spec and mask are valid, enforce using full mask to make sure the
         * complete value is used correctly.
         */
        if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
                        != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
                                          item, "must use full mask for"
                                          " fragment_offset");
        /*
         * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
         * indicating this is 1st fragment of fragmented packet.
         * This is not yet supported in MLX5, return appropriate error message.
         */
        if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "match on first fragment not "
                                          "supported");
        if (fragment_offset_spec && !last)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "specified value not supported");
        /* spec and last are valid, validate the specified range. */
        fragment_offset_last = last->hdr.fragment_offset &
                               mask->hdr.fragment_offset;
        /*
         * Match on fragment_offset spec 0x2001 and last 0x3fff
         * means MF is 1 and frag-offset is > 0.
         * This packet is fragment 2nd and onward, excluding last.
         * This is not yet supported in MLX5, return appropriate
         * error message.
         */
        if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
            fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM_LAST,
                                          last, "match on following "
                                          "fragments not supported");
        /*
         * Match on fragment_offset spec 0x0001 and last 0x1fff
         * means MF is 0 and frag-offset is > 0.
         * This packet is last fragment of fragmented packet.
         * This is not yet supported in MLX5, return appropriate
         * error message.
         */
        if (fragment_offset_spec == RTE_BE16(1) &&
            fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM_LAST,
                                          last, "match on last "
                                          "fragment not supported");
        /*
         * Match on fragment_offset spec 0x0001 and last 0x3fff
         * means MF and/or frag-offset is not 0.
         * This is a fragmented packet.
         * Other range values are invalid and rejected.
         */
        if (!(fragment_offset_spec == RTE_BE16(1) &&
              fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
                                          "specified range not supported");
        return 0;
}

/**
 * Validate IPV6 fragment extension item.
 *
 * @param[in] item
 *   Item specification.
 * @param[in] item_flags
 *   Bit-fields that holds the items detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
                                    uint64_t item_flags,
                                    struct rte_flow_error *error)
{
        const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
        const struct rte_flow_item_ipv6_frag_ext *last = item->last;
        const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
        rte_be16_t frag_data_spec = 0;
        rte_be16_t frag_data_last = 0;
        const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
        const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
                                      MLX5_FLOW_LAYER_OUTER_L4;
        int ret = 0;
        struct rte_flow_item_ipv6_frag_ext nic_mask = {
                .hdr = {
                        .next_header = 0xff,
                        .frag_data = RTE_BE16(0xffff),
                },
        };

        if (item_flags & l4m)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "ipv6 fragment extension item cannot "
                                          "follow L4 item.");
        if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
            (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "ipv6 fragment extension item must "
                                          "follow ipv6 item");
        if (spec && mask)
                frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
        if (!frag_data_spec)
                return 0;
        /*
         * spec and mask are valid, enforce using full mask to make sure the
         * complete value is used correctly.
         */
        if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
                                RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM_MASK,
                                          item, "must use full mask for"
                                          " frag_data");
        /*
         * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
         * This is 1st fragment of fragmented packet.
         */
        if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "match on first fragment not "
                                          "supported");
        if (frag_data_spec && !last)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, item,
                                          "specified value not supported");
        ret = mlx5_flow_item_acceptable
                                (item, (const uint8_t *)mask,
                                 (const uint8_t *)&nic_mask,
                                 sizeof(struct rte_flow_item_ipv6_frag_ext),
                                 MLX5_ITEM_RANGE_ACCEPTED, error);
        if (ret)
                return ret;
        /* spec and last are valid, validate the specified range. */
        frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
        /*
         * Match on frag_data spec 0x0009 and last 0xfff9
         * means M is 1 and frag-offset is > 0.
         * This packet is fragment 2nd and onward, excluding last.
         * This is not yet supported in MLX5, return appropriate
         * error message.
         */
        if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
                                       RTE_IPV6_EHDR_MF_MASK) &&
            frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM_LAST,
                                          last, "match on following "
                                          "fragments not supported");
        /*
         * Match on frag_data spec 0x0008 and last 0xfff8
         * means M is 0 and frag-offset is > 0.
         * This packet is last fragment of fragmented packet.
         * This is not yet supported in MLX5, return appropriate
         * error message.
         */
        if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
            frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ITEM_LAST,
                                          last, "match on last "
                                          "fragment not supported");
        /* Other range values are invalid and rejected. */
        return rte_flow_error_set(error, EINVAL,
                                  RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
                                  "specified range not supported");
}

/*
 * Validate ASO CT item.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] item
 *   Item specification.
 * @param[in] item_flags
 *   Pointer to bit-fields that holds the items detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
                             const struct rte_flow_item *item,
                             uint64_t *item_flags,
                             struct rte_flow_error *error)
{
        const struct rte_flow_item_conntrack *spec = item->spec;
        const struct rte_flow_item_conntrack *mask = item->mask;
        RTE_SET_USED(dev);
        uint32_t flags;

        if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, NULL,
                                          "Only one CT is supported");
        if (!mask)
                mask = &rte_flow_item_conntrack_mask;
        flags = spec->flags & mask->flags;
        if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
            ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
             (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
             (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ITEM, NULL,
                                          "Conflict status bits");
        /* State change also needs to be considered. */
        *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
        return 0;
}

/**
 * Validate the pop VLAN action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] action
 *   Pointer to the pop vlan action.
 * @param[in] item_flags
 *   The items found in this flow rule.
 * @param[in] attr
 *   Pointer to flow attributes.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
                                 uint64_t action_flags,
                                 const struct rte_flow_action *action,
                                 uint64_t item_flags,
                                 const struct rte_flow_attr *attr,
                                 struct rte_flow_error *error)
{
        const struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_dev_ctx_shared *sh = priv->sh;
        bool direction_error = false;

        if (!priv->sh->pop_vlan_action)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL,
                                          "pop vlan action is not supported");
        /* Pop VLAN is not supported in egress except for CX6 FDB mode. */
        if (attr->transfer) {
                bool fdb_tx = priv->representor_id != UINT16_MAX;
                bool is_cx5 = sh->steering_format_version ==
                    MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;

                if (fdb_tx && is_cx5)
                        direction_error = true;
        } else if (attr->egress) {
                direction_error = true;
        }
        if (direction_error)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
                                          NULL,
                                          "pop vlan action not supported for egress");
        if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "no support for multiple VLAN "
                                          "actions");
        /* Pop VLAN with preceding Decap requires inner header with VLAN. */
        if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
            !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL,
                                          "cannot pop vlan after decap without "
                                          "match on inner vlan in the flow");
        /* Pop VLAN without preceding Decap requires outer header with VLAN. */
        if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
            !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL,
                                          "cannot pop vlan without a "
                                          "match on (outer) vlan in the flow");
        if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "wrong action order, port_id should "
                                          "be after pop VLAN action");
        if (!attr->transfer && priv->representor)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                          "pop vlan action for VF representor "
                                          "not supported on NIC table");
        return 0;
}

/**
 * Get VLAN default info from vlan match info.
 *
 * @param[in] items
 *   the list of item specifications.
 * @param[out] vlan
 *   pointer VLAN info to fill to.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static void
flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
                                  struct rte_vlan_hdr *vlan)
{
        const struct rte_flow_item_vlan nic_mask = {
                .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
                                MLX5DV_FLOW_VLAN_VID_MASK),
                .inner_type = RTE_BE16(0xffff),
        };

        if (items == NULL)
                return;
        for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
                int type = items->type;

                if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
                    type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
                        break;
        }
        if (items->type != RTE_FLOW_ITEM_TYPE_END) {
                const struct rte_flow_item_vlan *vlan_m = items->mask;
                const struct rte_flow_item_vlan *vlan_v = items->spec;

                /* If VLAN item in pattern doesn't contain data, return here. */
                if (!vlan_v)
                        return;
                if (!vlan_m)
                        vlan_m = &nic_mask;
                /* Only full match values are accepted */
                if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
                     MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
                        vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
                        vlan->vlan_tci |=
                                rte_be_to_cpu_16(vlan_v->tci &
                                                 MLX5DV_FLOW_VLAN_PCP_MASK_BE);
                }
                if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
                     MLX5DV_FLOW_VLAN_VID_MASK_BE) {
                        vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
                        vlan->vlan_tci |=
                                rte_be_to_cpu_16(vlan_v->tci &
                                                 MLX5DV_FLOW_VLAN_VID_MASK_BE);
                }
                if (vlan_m->inner_type == nic_mask.inner_type)
                        vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
                                                           vlan_m->inner_type);
        }
}

/**
 * Validate the push VLAN action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] item_flags
 *   The items found in this flow rule.
 * @param[in] action
 *   Pointer to the action structure.
 * @param[in] attr
 *   Pointer to flow attributes
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
                                  uint64_t action_flags,
                                  const struct rte_flow_item_vlan *vlan_m,
                                  const struct rte_flow_action *action,
                                  const struct rte_flow_attr *attr,
                                  struct rte_flow_error *error)
{
        const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
        const struct mlx5_priv *priv = dev->data->dev_private;

        if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
            push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "invalid vlan ethertype");
        if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "wrong action order, port_id should "
                                          "be after push VLAN");
        if (!attr->transfer && priv->representor)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                          "push vlan action for VF representor "
                                          "not supported on NIC table");
        if (vlan_m &&
            (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
            (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
                MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
            !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
            !(mlx5_flow_find_action
                (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "not full match mask on VLAN PCP and "
                                          "there is no of_set_vlan_pcp action, "
                                          "push VLAN action cannot figure out "
                                          "PCP value");
        if (vlan_m &&
            (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
            (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
                MLX5DV_FLOW_VLAN_VID_MASK_BE &&
            !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
            !(mlx5_flow_find_action
                (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "not full match mask on VLAN VID and "
                                          "there is no of_set_vlan_vid action, "
                                          "push VLAN action cannot figure out "
                                          "VID value");
        (void)attr;
        return 0;
}

/**
 * Validate the set VLAN PCP.
 *
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] actions
 *   Pointer to the list of actions remaining in the flow rule.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,

                                     const struct rte_flow_action actions[],
                                     struct rte_flow_error *error)
{
        const struct rte_flow_action *action = actions;
        const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;

        if (conf->vlan_pcp > 7)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "VLAN PCP value is too big");
        if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "set VLAN PCP action must follow "
                                          "the push VLAN action");
        if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "Multiple VLAN PCP modification are "
                                          "not supported");
        if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "wrong action order, port_id should "
                                          "be after set VLAN PCP");
        return 0;
}

/**
 * Validate the set VLAN VID.
 *
 * @param[in] item_flags
 *   Holds the items detected in this rule.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] actions
 *   Pointer to the list of actions remaining in the flow rule.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
                                     uint64_t action_flags,
                                     const struct rte_flow_action actions[],
                                     struct rte_flow_error *error)
{
        const struct rte_flow_action *action = actions;
        const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;

        if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "VLAN VID value is too big");
        if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
            !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "set VLAN VID action must follow push"
                                          " VLAN action or match on VLAN item");
        if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "Multiple VLAN VID modifications are "
                                          "not supported");
        if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "wrong action order, port_id should "
                                          "be after set VLAN VID");
        return 0;
}

/*
 * Validate the FLAG action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] attr
 *   Pointer to flow attributes
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_flag(struct rte_eth_dev *dev,
                             uint64_t action_flags,
                             const struct rte_flow_attr *attr,
                             struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_sh_config *config = &priv->sh->config;
        int ret;

        /* Fall back if no extended metadata register support. */
        if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
                return mlx5_flow_validate_action_flag(action_flags, attr,
                                                      error);
        /* Extensive metadata mode requires registers. */
        if (!mlx5_flow_ext_mreg_supported(dev))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "no metadata registers "
                                          "to support flag action");
        if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "extended metadata register"
                                          " isn't available");
        ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
        if (ret < 0)
                return ret;
        MLX5_ASSERT(ret > 0);
        if (action_flags & MLX5_FLOW_ACTION_MARK)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "can't mark and flag in same flow");
        if (action_flags & MLX5_FLOW_ACTION_FLAG)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "can't have 2 flag"
                                          " actions in same flow");
        return 0;
}

/**
 * Validate MARK action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action
 *   Pointer to action.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] attr
 *   Pointer to flow attributes
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_mark(struct rte_eth_dev *dev,
                             const struct rte_flow_action *action,
                             uint64_t action_flags,
                             const struct rte_flow_attr *attr,
                             struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_sh_config *config = &priv->sh->config;
        const struct rte_flow_action_mark *mark = action->conf;
        int ret;

        if (is_tunnel_offload_active(dev))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "no mark action "
                                          "if tunnel offload active");
        /* Fall back if no extended metadata register support. */
        if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
                return mlx5_flow_validate_action_mark(action, action_flags,
                                                      attr, error);
        /* Extensive metadata mode requires registers. */
        if (!mlx5_flow_ext_mreg_supported(dev))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "no metadata registers "
                                          "to support mark action");
        if (!priv->sh->dv_mark_mask)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "extended metadata register"
                                          " isn't available");
        ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
        if (ret < 0)
                return ret;
        MLX5_ASSERT(ret > 0);
        if (!mark)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "configuration cannot be null");
        if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION_CONF,
                                          &mark->id,
                                          "mark id exceeds the limit");
        if (action_flags & MLX5_FLOW_ACTION_FLAG)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "can't flag and mark in same flow");
        if (action_flags & MLX5_FLOW_ACTION_MARK)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "can't have 2 mark actions in same"
                                          " flow");
        return 0;
}

/**
 * Validate SET_META action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the action structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] attr
 *   Pointer to flow attributes
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
                                 const struct rte_flow_action *action,
                                 uint64_t action_flags __rte_unused,
                                 const struct rte_flow_attr *attr,
                                 struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_sh_config *config = &priv->sh->config;
        const struct rte_flow_action_set_meta *conf;
        uint32_t nic_mask = UINT32_MAX;
        int reg;

        if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
            !mlx5_flow_ext_mreg_supported(dev))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "extended metadata register"
                                          " isn't supported");
        reg = flow_dv_get_metadata_reg(dev, attr, error);
        if (reg < 0)
                return reg;
        if (reg == REG_NON)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "unavailable extended metadata register");
        if (reg != REG_A && reg != REG_B) {
                struct mlx5_priv *priv = dev->data->dev_private;

                nic_mask = priv->sh->dv_meta_mask;
        }
        if (!(action->conf))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "configuration cannot be null");
        conf = (const struct rte_flow_action_set_meta *)action->conf;
        if (!conf->mask)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "zero mask doesn't have any effect");
        if (conf->mask & ~nic_mask)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "meta data must be within reg C0");
        return 0;
}

/**
 * Validate SET_TAG action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the action structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] attr
 *   Pointer to flow attributes
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
                                const struct rte_flow_action *action,
                                uint64_t action_flags,
                                const struct rte_flow_attr *attr,
                                struct rte_flow_error *error)
{
        const struct rte_flow_action_set_tag *conf;
        const uint64_t terminal_action_flags =
                MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
                MLX5_FLOW_ACTION_RSS;
        int ret;

        if (!mlx5_flow_ext_mreg_supported(dev))
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "extensive metadata register"
                                          " isn't supported");
        if (!(action->conf))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "configuration cannot be null");
        conf = (const struct rte_flow_action_set_tag *)action->conf;
        if (!conf->mask)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "zero mask doesn't have any effect");
        ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
        if (ret < 0)
                return ret;
        if (!attr->transfer && attr->ingress &&
            (action_flags & terminal_action_flags))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "set_tag has no effect"
                                          " with terminal actions");
        return 0;
}

/**
 * Indicates whether ASO aging is supported.
 *
 * @param[in] sh
 *   Pointer to shared device context structure.
 * @param[in] attr
 *   Attributes of flow that includes AGE action.
 *
 * @return
 *   True when ASO aging is supported, false otherwise.
 */
static inline bool
flow_hit_aso_supported(const struct mlx5_dev_ctx_shared *sh,
                const struct rte_flow_attr *attr)
{
        MLX5_ASSERT(sh && attr);
        return (sh->flow_hit_aso_en && (attr->transfer || attr->group));
}

/**
 * Validate count action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] shared
 *   Indicator if action is shared.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] attr
 *   Attributes of flow that includes this action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
                              uint64_t action_flags,
                              const struct rte_flow_attr *attr,
                              struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;

        if (!priv->sh->cdev->config.devx)
                goto notsup_err;
        if (action_flags & MLX5_FLOW_ACTION_COUNT)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "duplicate count actions set");
        if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
            !flow_hit_aso_supported(priv->sh, attr))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "old age and indirect count combination is not supported");
#ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
        return 0;
#endif
notsup_err:
        return rte_flow_error_set
                      (error, ENOTSUP,
                       RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                       NULL,
                       "count action not supported");
}

/**
 * Validate the L2 encap action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] action
 *   Pointer to the action structure.
 * @param[in] attr
 *   Pointer to flow attributes.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
                                 uint64_t action_flags,
                                 const struct rte_flow_action *action,
                                 const struct rte_flow_attr *attr,
                                 struct rte_flow_error *error)
{
        const struct mlx5_priv *priv = dev->data->dev_private;

        if (!(action->conf))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, action,
                                          "configuration cannot be null");
        if (action_flags & MLX5_FLOW_ACTION_ENCAP)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "can only have a single encap action "
                                          "in a flow");
        if (!attr->transfer && priv->representor)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                          "encap action for VF representor "
                                          "not supported on NIC table");
        return 0;
}

/**
 * Validate a decap action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] action
 *   Pointer to the action structure.
 * @param[in] item_flags
 *   Holds the items detected.
 * @param[in] attr
 *   Pointer to flow attributes
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_decap(struct rte_eth_dev *dev,
                              uint64_t action_flags,
                              const struct rte_flow_action *action,
                              const uint64_t item_flags,
                              const struct rte_flow_attr *attr,
                              struct rte_flow_error *error)
{
        const struct mlx5_priv *priv = dev->data->dev_private;

        if (priv->sh->cdev->config.hca_attr.scatter_fcs_w_decap_disable &&
            !priv->sh->config.decap_en)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "decap is not enabled");
        if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          action_flags &
                                          MLX5_FLOW_ACTION_DECAP ? "can only "
                                          "have a single decap action" : "decap "
                                          "after encap is not supported");
        if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "can't have decap action after"
                                          " modify action");
        if (attr->egress)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
                                          NULL,
                                          "decap action not supported for "
                                          "egress");
        if (!attr->transfer && priv->representor)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                          "decap action for VF representor "
                                          "not supported on NIC table");
        if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
            !(item_flags & MLX5_FLOW_LAYER_VXLAN))
                return rte_flow_error_set(error, ENOTSUP,
                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                "VXLAN item should be present for VXLAN decap");
        return 0;
}

const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};

/**
 * Validate the raw encap and decap actions.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] decap
 *   Pointer to the decap action.
 * @param[in] encap
 *   Pointer to the encap action.
 * @param[in] attr
 *   Pointer to flow attributes
 * @param[in/out] action_flags
 *   Holds the actions detected until now.
 * @param[out] actions_n
 *   pointer to the number of actions counter.
 * @param[in] action
 *   Pointer to the action structure.
 * @param[in] item_flags
 *   Holds the items detected.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_raw_encap_decap
        (struct rte_eth_dev *dev,
         const struct rte_flow_action_raw_decap *decap,
         const struct rte_flow_action_raw_encap *encap,
         const struct rte_flow_attr *attr, uint64_t *action_flags,
         int *actions_n, const struct rte_flow_action *action,
         uint64_t item_flags, struct rte_flow_error *error)
{
        const struct mlx5_priv *priv = dev->data->dev_private;
        int ret;

        if (encap && (!encap->size || !encap->data))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "raw encap data cannot be empty");
        if (decap && encap) {
                if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
                    encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
                        /* L3 encap. */
                        decap = NULL;
                else if (encap->size <=
                           MLX5_ENCAPSULATION_DECISION_SIZE &&
                           decap->size >
                           MLX5_ENCAPSULATION_DECISION_SIZE)
                        /* L3 decap. */
                        encap = NULL;
                else if (encap->size >
                           MLX5_ENCAPSULATION_DECISION_SIZE &&
                           decap->size >
                           MLX5_ENCAPSULATION_DECISION_SIZE)
                        /* 2 L2 actions: encap and decap. */
                        ;
                else
                        return rte_flow_error_set(error,
                                ENOTSUP,
                                RTE_FLOW_ERROR_TYPE_ACTION,
                                NULL, "unsupported too small "
                                "raw decap and too small raw "
                                "encap combination");
        }
        if (decap) {
                ret = flow_dv_validate_action_decap(dev, *action_flags, action,
                                                    item_flags, attr, error);
                if (ret < 0)
                        return ret;
                *action_flags |= MLX5_FLOW_ACTION_DECAP;
                ++(*actions_n);
        }
        if (encap) {
                if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
                        return rte_flow_error_set(error, ENOTSUP,
                                                  RTE_FLOW_ERROR_TYPE_ACTION,
                                                  NULL,
                                                  "small raw encap size");
                if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
                        return rte_flow_error_set(error, EINVAL,
                                                  RTE_FLOW_ERROR_TYPE_ACTION,
                                                  NULL,
                                                  "more than one encap action");
                if (!attr->transfer && priv->representor)
                        return rte_flow_error_set
                                        (error, ENOTSUP,
                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                         "encap action for VF representor "
                                         "not supported on NIC table");
                *action_flags |= MLX5_FLOW_ACTION_ENCAP;
                ++(*actions_n);
        }
        return 0;
}

/*
 * Validate the ASO CT action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] item_flags
 *   The items found in this flow rule.
 * @param[in] attr
 *   Pointer to flow attributes.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
                               uint64_t action_flags,
                               uint64_t item_flags,
                               const struct rte_flow_attr *attr,
                               struct rte_flow_error *error)
{
        RTE_SET_USED(dev);

        if (attr->group == 0 && !attr->transfer)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL,
                                          "Only support non-root table");
        if (action_flags & MLX5_FLOW_FATE_ACTIONS)
                return rte_flow_error_set(error, ENOTSUP,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "CT cannot follow a fate action");
        if ((action_flags & MLX5_FLOW_ACTION_METER) ||
            (action_flags & MLX5_FLOW_ACTION_AGE))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "Only one ASO action is supported");
        if (action_flags & MLX5_FLOW_ACTION_ENCAP)
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
                                          "Encap cannot exist before CT");
        if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
                return rte_flow_error_set(error, EINVAL,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                          "Not a outer TCP packet");
        return 0;
}

int
flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
                             struct mlx5_list_entry *entry, void *cb_ctx)
{
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
        struct mlx5_flow_dv_encap_decap_resource *resource;

        resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
                                entry);
        if (resource->reformat_type == ctx_resource->reformat_type &&
            resource->ft_type == ctx_resource->ft_type &&
            resource->flags == ctx_resource->flags &&
            resource->size == ctx_resource->size &&
            !memcmp((const void *)resource->buf,
                    (const void *)ctx_resource->buf,
                    resource->size))
                return 0;
        return -1;
}

struct mlx5_list_entry *
flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
{
        struct mlx5_dev_ctx_shared *sh = tool_ctx;
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5dv_dr_domain *domain;
        struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
        struct mlx5_flow_dv_encap_decap_resource *resource;
        uint32_t idx;
        int ret;

        if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
                domain = sh->fdb_domain;
        else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
                domain = sh->rx_domain;
        else
                domain = sh->tx_domain;
        /* Register new encap/decap resource. */
        resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
        if (!resource) {
                rte_flow_error_set(ctx->error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                   "cannot allocate resource memory");
                return NULL;
        }
        *resource = *ctx_resource;
        resource->idx = idx;
        ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->cdev->ctx,
                                                              domain, resource,
                                                             &resource->action);
        if (ret) {
                mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
                rte_flow_error_set(ctx->error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                   NULL, "cannot create action");
                return NULL;
        }

        return &resource->entry;
}

struct mlx5_list_entry *
flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
                             void *cb_ctx)
{
        struct mlx5_dev_ctx_shared *sh = tool_ctx;
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5_flow_dv_encap_decap_resource *cache_resource;
        uint32_t idx;

        cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
                                           &idx);
        if (!cache_resource) {
                rte_flow_error_set(ctx->error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                   "cannot allocate resource memory");
                return NULL;
        }
        memcpy(cache_resource, oentry, sizeof(*cache_resource));
        cache_resource->idx = idx;
        return &cache_resource->entry;
}

void
flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
        struct mlx5_dev_ctx_shared *sh = tool_ctx;
        struct mlx5_flow_dv_encap_decap_resource *res =
                                       container_of(entry, typeof(*res), entry);

        mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
}

/**
 * Find existing encap/decap resource or create and register a new one.
 *
 * @param[in, out] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in, out] resource
 *   Pointer to encap/decap resource.
 * @parm[in, out] dev_flow
 *   Pointer to the dev_flow.
 * @param[out] error
 *   pointer to error structure.
 *
 * @return
 *   0 on success otherwise -errno and errno is set.
 */
static int
flow_dv_encap_decap_resource_register
                        (struct rte_eth_dev *dev,
                         struct mlx5_flow_dv_encap_decap_resource *resource,
                         struct mlx5_flow *dev_flow,
                         struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_dev_ctx_shared *sh = priv->sh;
        struct mlx5_list_entry *entry;
        union {
                struct {
                        uint32_t ft_type:8;
                        uint32_t refmt_type:8;
                        /*
                         * Header reformat actions can be shared between
                         * non-root tables. One bit to indicate non-root
                         * table or not.
                         */
                        uint32_t is_root:1;
                        uint32_t reserve:15;
                };
                uint32_t v32;
        } encap_decap_key = {
                {
                        .ft_type = resource->ft_type,
                        .refmt_type = resource->reformat_type,
                        .is_root = !!dev_flow->dv.group,
                        .reserve = 0,
                }
        };
        struct mlx5_flow_cb_ctx ctx = {
                .error = error,
                .data = resource,
        };
        struct mlx5_hlist *encaps_decaps;
        uint64_t key64;

        encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
                                "encaps_decaps",
                                MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
                                true, true, sh,
                                flow_dv_encap_decap_create_cb,
                                flow_dv_encap_decap_match_cb,
                                flow_dv_encap_decap_remove_cb,
                                flow_dv_encap_decap_clone_cb,
                                flow_dv_encap_decap_clone_free_cb,
                                error);
        if (unlikely(!encaps_decaps))
                return -rte_errno;
        resource->flags = dev_flow->dv.group ? 0 : 1;
        key64 =  __rte_raw_cksum(&encap_decap_key.v32,
                                 sizeof(encap_decap_key.v32), 0);
        if (resource->reformat_type !=
            MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
            resource->size)
                key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
        entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
        if (!entry)
                return -rte_errno;
        resource = container_of(entry, typeof(*resource), entry);
        dev_flow->dv.encap_decap = resource;
        dev_flow->handle->dvh.rix_encap_decap = resource->idx;
        return 0;
}

/**
 * Find existing table jump resource or create and register a new one.
 *
 * @param[in, out] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in, out] tbl
 *   Pointer to flow table resource.
 * @parm[in, out] dev_flow
 *   Pointer to the dev_flow.
 * @param[out] error
 *   pointer to error structure.
 *
 * @return
 *   0 on success otherwise -errno and errno is set.
 */
static int
flow_dv_jump_tbl_resource_register
                        (struct rte_eth_dev *dev __rte_unused,
                         struct mlx5_flow_tbl_resource *tbl,
                         struct mlx5_flow *dev_flow,
                         struct rte_flow_error *error __rte_unused)
{
        struct mlx5_flow_tbl_data_entry *tbl_data =
                container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);

        MLX5_ASSERT(tbl);
        MLX5_ASSERT(tbl_data->jump.action);
        dev_flow->handle->rix_jump = tbl_data->idx;
        dev_flow->dv.jump = &tbl_data->jump;
        return 0;
}

int
flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
                         struct mlx5_list_entry *entry, void *cb_ctx)
{
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
        struct mlx5_flow_dv_port_id_action_resource *res =
                                       container_of(entry, typeof(*res), entry);

        return ref->port_id != res->port_id;
}

struct mlx5_list_entry *
flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
{
        struct mlx5_dev_ctx_shared *sh = tool_ctx;
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
        struct mlx5_flow_dv_port_id_action_resource *resource;
        uint32_t idx;
        int ret;

        /* Register new port id action resource. */
        resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
        if (!resource) {
                rte_flow_error_set(ctx->error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                   "cannot allocate port_id action memory");
                return NULL;
        }
        *resource = *ref;
        ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
                                                        ref->port_id,
                                                        &resource->action);
        if (ret) {
                mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
                rte_flow_error_set(ctx->error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                   "cannot create action");
                return NULL;
        }
        resource->idx = idx;
        return &resource->entry;
}

struct mlx5_list_entry *
flow_dv_port_id_clone_cb(void *tool_ctx,
                         struct mlx5_list_entry *entry __rte_unused,
                         void *cb_ctx)
{
        struct mlx5_dev_ctx_shared *sh = tool_ctx;
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5_flow_dv_port_id_action_resource *resource;
        uint32_t idx;

        resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
        if (!resource) {
                rte_flow_error_set(ctx->error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                   "cannot allocate port_id action memory");
                return NULL;
        }
        memcpy(resource, entry, sizeof(*resource));
        resource->idx = idx;
        return &resource->entry;
}

void
flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
        struct mlx5_dev_ctx_shared *sh = tool_ctx;
        struct mlx5_flow_dv_port_id_action_resource *resource =
                                  container_of(entry, typeof(*resource), entry);

        mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
}

/**
 * Find existing table port ID resource or create and register a new one.
 *
 * @param[in, out] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in, out] ref
 *   Pointer to port ID action resource reference.
 * @parm[in, out] dev_flow
 *   Pointer to the dev_flow.
 * @param[out] error
 *   pointer to error structure.
 *
 * @return
 *   0 on success otherwise -errno and errno is set.
 */
static int
flow_dv_port_id_action_resource_register
                        (struct rte_eth_dev *dev,
                         struct mlx5_flow_dv_port_id_action_resource *ref,
                         struct mlx5_flow *dev_flow,
                         struct rte_flow_error *error)
{
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_list_entry *entry;
        struct mlx5_flow_dv_port_id_action_resource *resource;
        struct mlx5_flow_cb_ctx ctx = {
                .error = error,
                .data = ref,
        };

        entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
        if (!entry)
                return -rte_errno;
        resource = container_of(entry, typeof(*resource), entry);
        dev_flow->dv.port_id_action = resource;
        dev_flow->handle->rix_port_id_action = resource->idx;
        return 0;
}

int
flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
                           struct mlx5_list_entry *entry, void *cb_ctx)
{
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
        struct mlx5_flow_dv_push_vlan_action_resource *res =
                                       container_of(entry, typeof(*res), entry);

        return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
}

struct mlx5_list_entry *
flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
{
        struct mlx5_dev_ctx_shared *sh = tool_ctx;
        struct mlx5_flow_cb_ctx *ctx = cb_ctx;
        struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
        struct mlx5_flow_dv_push_vlan_action_resource *resource;
        struct mlx5dv_dr_domain *domain;
        uint32_t idx;
        int ret;

        /* Register new port id action resource. */
        resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
        if (!resource) {
                rte_flow_error_set(ctx->error, ENOMEM,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,