/*
 * drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c
 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2017 Jiri Pirko <jiri@mellanox.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/bitops.h>
#include <linux/list.h>
#include <linux/rhashtable.h>
#include <linux/netdevice.h>
#include <linux/parman.h>

#include "reg.h"
#include "core.h"
#include "resources.h"
#include "spectrum.h"
#include "core_acl_flex_keys.h"

struct mlxsw_sp_acl_tcam {
        unsigned long *used_regions; /* bit array */
        unsigned int max_regions;
        unsigned long *used_groups;  /* bit array */
        unsigned int max_groups;
        unsigned int max_group_size;
};

static int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp, void *priv)
{
        struct mlxsw_sp_acl_tcam *tcam = priv;
        u64 max_tcam_regions;
        u64 max_regions;
        u64 max_groups;
        size_t alloc_size;
        int err;

        max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core,
                                              ACL_MAX_TCAM_REGIONS);
        max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);

        /* Use 1:1 mapping between ACL region and TCAM region */
        if (max_tcam_regions < max_regions)
                max_regions = max_tcam_regions;

        alloc_size = sizeof(tcam->used_regions[0]) * BITS_TO_LONGS(max_regions);
        tcam->used_regions = kzalloc(alloc_size, GFP_KERNEL);
        if (!tcam->used_regions)
                return -ENOMEM;
        tcam->max_regions = max_regions;

        max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS);
        alloc_size = sizeof(tcam->used_groups[0]) * BITS_TO_LONGS(max_groups);
        tcam->used_groups = kzalloc(alloc_size, GFP_KERNEL);
        if (!tcam->used_groups) {
                err = -ENOMEM;
                goto err_alloc_used_groups;
        }
        tcam->max_groups = max_groups;
        tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
                                                 ACL_MAX_GROUP_SIZE);
        return 0;

err_alloc_used_groups:
        kfree(tcam->used_regions);
        return err;
}

static void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp, void *priv)
{
        struct mlxsw_sp_acl_tcam *tcam = priv;

        kfree(tcam->used_groups);
        kfree(tcam->used_regions);
}

static int mlxsw_sp_acl_tcam_region_id_get(struct mlxsw_sp_acl_tcam *tcam,
                                           u16 *p_id)
{
        u16 id;

        id = find_first_zero_bit(tcam->used_regions, tcam->max_regions);
        if (id < tcam->max_regions) {
                __set_bit(id, tcam->used_regions);
                *p_id = id;
                return 0;
        }
        return -ENOBUFS;
}

static void mlxsw_sp_acl_tcam_region_id_put(struct mlxsw_sp_acl_tcam *tcam,
                                            u16 id)
{
        __clear_bit(id, tcam->used_regions);
}

static int mlxsw_sp_acl_tcam_group_id_get(struct mlxsw_sp_acl_tcam *tcam,
                                          u16 *p_id)
{
        u16 id;

        id = find_first_zero_bit(tcam->used_groups, tcam->max_groups);
        if (id < tcam->max_groups) {
                __set_bit(id, tcam->used_groups);
                *p_id = id;
                return 0;
        }
        return -ENOBUFS;
}

static void mlxsw_sp_acl_tcam_group_id_put(struct mlxsw_sp_acl_tcam *tcam,
                                           u16 id)
{
        __clear_bit(id, tcam->used_groups);
}

struct mlxsw_sp_acl_tcam_pattern {
        const enum mlxsw_afk_element *elements;
        unsigned int elements_count;
};

struct mlxsw_sp_acl_tcam_group {
        struct mlxsw_sp_acl_tcam *tcam;
        u16 id;
        struct list_head region_list;
        unsigned int region_count;
        struct rhashtable chunk_ht;
        struct {
                u16 local_port;
                bool ingress;
        } bound;
        struct mlxsw_sp_acl_tcam_group_ops *ops;
        const struct mlxsw_sp_acl_tcam_pattern *patterns;
        unsigned int patterns_count;
};

struct mlxsw_sp_acl_tcam_region {
        struct list_head list; /* Member of a TCAM group */
        struct list_head chunk_list; /* List of chunks under this region */
        struct parman *parman;
        struct mlxsw_sp *mlxsw_sp;
        struct mlxsw_sp_acl_tcam_group *group;
        u16 id; /* ACL ID and region ID - they are same */
        char tcam_region_info[MLXSW_REG_PXXX_TCAM_REGION_INFO_LEN];
        struct mlxsw_afk_key_info *key_info;
        struct {
                struct parman_prio parman_prio;
                struct parman_item parman_item;
                struct mlxsw_sp_acl_rule_info *rulei;
        } catchall;
};

struct mlxsw_sp_acl_tcam_chunk {
        struct list_head list; /* Member of a TCAM region */
        struct rhash_head ht_node; /* Member of a chunk HT */
        unsigned int priority; /* Priority within the region and group */
        struct parman_prio parman_prio;
        struct mlxsw_sp_acl_tcam_group *group;
        struct mlxsw_sp_acl_tcam_region *region;
        unsigned int ref_count;
};

struct mlxsw_sp_acl_tcam_entry {
        struct parman_item parman_item;
        struct mlxsw_sp_acl_tcam_chunk *chunk;
};

static const struct rhashtable_params mlxsw_sp_acl_tcam_chunk_ht_params = {
        .key_len = sizeof(unsigned int),
        .key_offset = offsetof(struct mlxsw_sp_acl_tcam_chunk, priority),
        .head_offset = offsetof(struct mlxsw_sp_acl_tcam_chunk, ht_node),
        .automatic_shrinking = true,
};

static int mlxsw_sp_acl_tcam_group_update(struct mlxsw_sp *mlxsw_sp,
                                          struct mlxsw_sp_acl_tcam_group *group)
{
        struct mlxsw_sp_acl_tcam_region *region;
        char pagt_pl[MLXSW_REG_PAGT_LEN];
        int acl_index = 0;

        mlxsw_reg_pagt_pack(pagt_pl, group->id);
        list_for_each_entry(region, &group->region_list, list)
                mlxsw_reg_pagt_acl_id_pack(pagt_pl, acl_index++, region->id);
        mlxsw_reg_pagt_size_set(pagt_pl, acl_index);
        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pagt), pagt_pl);
}

static int
mlxsw_sp_acl_tcam_group_add(struct mlxsw_sp *mlxsw_sp,
                            struct mlxsw_sp_acl_tcam *tcam,
                            struct mlxsw_sp_acl_tcam_group *group,
                            const struct mlxsw_sp_acl_tcam_pattern *patterns,
                            unsigned int patterns_count)
{
        int err;

        group->tcam = tcam;
        group->patterns = patterns;
        group->patterns_count = patterns_count;
        INIT_LIST_HEAD(&group->region_list);
        err = mlxsw_sp_acl_tcam_group_id_get(tcam, &group->id);
        if (err)
                return err;

        err = mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
        if (err)
                goto err_group_update;

        err = rhashtable_init(&group->chunk_ht,
                              &mlxsw_sp_acl_tcam_chunk_ht_params);
        if (err)
                goto err_rhashtable_init;

        return 0;

err_rhashtable_init:
err_group_update:
        mlxsw_sp_acl_tcam_group_id_put(tcam, group->id);
        return err;
}

static void mlxsw_sp_acl_tcam_group_del(struct mlxsw_sp *mlxsw_sp,
                                        struct mlxsw_sp_acl_tcam_group *group)
{
        struct mlxsw_sp_acl_tcam *tcam = group->tcam;

        rhashtable_destroy(&group->chunk_ht);
        mlxsw_sp_acl_tcam_group_id_put(tcam, group->id);
        WARN_ON(!list_empty(&group->region_list));
}

static int
mlxsw_sp_acl_tcam_group_bind(struct mlxsw_sp *mlxsw_sp,
                             struct mlxsw_sp_acl_tcam_group *group,
                             struct net_device *dev, bool ingress)
{
        struct mlxsw_sp_port *mlxsw_sp_port;
        char ppbt_pl[MLXSW_REG_PPBT_LEN];

        if (!mlxsw_sp_port_dev_check(dev))
                return -EINVAL;

        mlxsw_sp_port = netdev_priv(dev);
        group->bound.local_port = mlxsw_sp_port->local_port;
        group->bound.ingress = ingress;
        mlxsw_reg_ppbt_pack(ppbt_pl,
                            group->bound.ingress ? MLXSW_REG_PXBT_E_IACL :
                                                   MLXSW_REG_PXBT_E_EACL,
                            MLXSW_REG_PXBT_OP_BIND, group->bound.local_port,
                            group->id);
        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
}

static void
mlxsw_sp_acl_tcam_group_unbind(struct mlxsw_sp *mlxsw_sp,
                               struct mlxsw_sp_acl_tcam_group *group)
{
        char ppbt_pl[MLXSW_REG_PPBT_LEN];

        mlxsw_reg_ppbt_pack(ppbt_pl,
                            group->bound.ingress ? MLXSW_REG_PXBT_E_IACL :
                                                   MLXSW_REG_PXBT_E_EACL,
                            MLXSW_REG_PXBT_OP_UNBIND, group->bound.local_port,
                            group->id);
        mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
}

static u16
mlxsw_sp_acl_tcam_group_id(struct mlxsw_sp_acl_tcam_group *group)
{
        return group->id;
}

static unsigned int
mlxsw_sp_acl_tcam_region_prio(struct mlxsw_sp_acl_tcam_region *region)
{
        struct mlxsw_sp_acl_tcam_chunk *chunk;

        if (list_empty(&region->chunk_list))
                return 0;
        /* As a priority of a region, return priority of the first chunk */
        chunk = list_first_entry(&region->chunk_list, typeof(*chunk), list);
        return chunk->priority;
}

static unsigned int
mlxsw_sp_acl_tcam_region_max_prio(struct mlxsw_sp_acl_tcam_region *region)
{
        struct mlxsw_sp_acl_tcam_chunk *chunk;

        if (list_empty(&region->chunk_list))
                return 0;
        chunk = list_last_entry(&region->chunk_list, typeof(*chunk), list);
        return chunk->priority;
}

static void
mlxsw_sp_acl_tcam_group_list_add(struct mlxsw_sp_acl_tcam_group *group,
                                 struct mlxsw_sp_acl_tcam_region *region)
{
        struct mlxsw_sp_acl_tcam_region *region2;
        struct list_head *pos;

        /* Position the region inside the list according to priority */
        list_for_each(pos, &group->region_list) {
                region2 = list_entry(pos, typeof(*region2), list);
                if (mlxsw_sp_acl_tcam_region_prio(region2) >
                    mlxsw_sp_acl_tcam_region_prio(region))
                        break;
        }
        list_add_tail(&region->list, pos);
        group->region_count++;
}

static void
mlxsw_sp_acl_tcam_group_list_del(struct mlxsw_sp_acl_tcam_group *group,
                                 struct mlxsw_sp_acl_tcam_region *region)
{
        group->region_count--;
        list_del(&region->list);
}

static int
mlxsw_sp_acl_tcam_group_region_attach(struct mlxsw_sp *mlxsw_sp,
                                      struct mlxsw_sp_acl_tcam_group *group,
                                      struct mlxsw_sp_acl_tcam_region *region)
{
        int err;

        if (group->region_count == group->tcam->max_group_size)
                return -ENOBUFS;

        mlxsw_sp_acl_tcam_group_list_add(group, region);

        err = mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
        if (err)
                goto err_group_update;
        region->group = group;

        return 0;

err_group_update:
        mlxsw_sp_acl_tcam_group_list_del(group, region);
        mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
        return err;
}

static void
mlxsw_sp_acl_tcam_group_region_detach(struct mlxsw_sp *mlxsw_sp,
                                      struct mlxsw_sp_acl_tcam_region *region)
{
        struct mlxsw_sp_acl_tcam_group *group = region->group;

        mlxsw_sp_acl_tcam_group_list_del(group, region);
        mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
}

static struct mlxsw_sp_acl_tcam_region *
mlxsw_sp_acl_tcam_group_region_find(struct mlxsw_sp_acl_tcam_group *group,
                                    unsigned int priority,
                                    struct mlxsw_afk_element_usage *elusage,
                                    bool *p_need_split)
{
        struct mlxsw_sp_acl_tcam_region *region, *region2;
        struct list_head *pos;
        bool issubset;

        list_for_each(pos, &group->region_list) {
                region = list_entry(pos, typeof(*region), list);

                /* First, check if the requested priority does not rather belong
                 * under some of the next regions.
                 */
                if (pos->next != &group->region_list) { /* not last */
                        region2 = list_entry(pos->next, typeof(*region2), list);
                        if (priority >= mlxsw_sp_acl_tcam_region_prio(region2))
                                continue;
                }

                issubset = mlxsw_afk_key_info_subset(region->key_info, elusage);

                /* If requested element usage would not fit and the priority
                 * is lower than the currently inspected region we cannot
                 * use this region, so return NULL to indicate new region has
                 * to be created.
                 */
                if (!issubset &&
                    priority < mlxsw_sp_acl_tcam_region_prio(region))
                        return NULL;

                /* If requested element usage would not fit and the priority
                 * is higher than the currently inspected region we cannot
                 * use this region. There is still some hope that the next
                 * region would be the fit. So let it be processed and
                 * eventually break at the check right above this.
                 */
                if (!issubset &&
                    priority > mlxsw_sp_acl_tcam_region_max_prio(region))
                        continue;

                /* Indicate if the region needs to be split in order to add
                 * the requested priority. Split is needed when requested
                 * element usage won't fit into the found region.
                 */
                *p_need_split = !issubset;
                return region;
        }
        return NULL; /* New region has to be created. */
}

static void
mlxsw_sp_acl_tcam_group_use_patterns(struct mlxsw_sp_acl_tcam_group *group,
                                     struct mlxsw_afk_element_usage *elusage,
                                     struct mlxsw_afk_element_usage *out)
{
        const struct mlxsw_sp_acl_tcam_pattern *pattern;
        int i;

        for (i = 0; i < group->patterns_count; i++) {
                pattern = &group->patterns[i];
                mlxsw_afk_element_usage_fill(out, pattern->elements,
                                             pattern->elements_count);
                if (mlxsw_afk_element_usage_subset(elusage, out))
                        return;
        }
        memcpy(out, elusage, sizeof(*out));
}

#define MLXSW_SP_ACL_TCAM_REGION_BASE_COUNT 16
#define MLXSW_SP_ACL_TCAM_REGION_RESIZE_STEP 16

static int
mlxsw_sp_acl_tcam_region_alloc(struct mlxsw_sp *mlxsw_sp,
                               struct mlxsw_sp_acl_tcam_region *region)
{
        struct mlxsw_afk_key_info *key_info = region->key_info;
        char ptar_pl[MLXSW_REG_PTAR_LEN];
        unsigned int encodings_count;
        int i;
        int err;

        mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_ALLOC,
                            MLXSW_SP_ACL_TCAM_REGION_BASE_COUNT,
                            region->id, region->tcam_region_info);
        encodings_count = mlxsw_afk_key_info_blocks_count_get(key_info);
        for (i = 0; i < encodings_count; i++) {
                u16 encoding;

                encoding = mlxsw_afk_key_info_block_encoding_get(key_info, i);
                mlxsw_reg_ptar_key_id_pack(ptar_pl, i, encoding);
        }
        err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
        if (err)
                return err;
        mlxsw_reg_ptar_unpack(ptar_pl, region->tcam_region_info);
        return 0;
}

static void
mlxsw_sp_acl_tcam_region_free(struct mlxsw_sp *mlxsw_sp,
                              struct mlxsw_sp_acl_tcam_region *region)
{
        char ptar_pl[MLXSW_REG_PTAR_LEN];

        mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_FREE, 0, region->id,
                            region->tcam_region_info);
        mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
}

static int
mlxsw_sp_acl_tcam_region_resize(struct mlxsw_sp *mlxsw_sp,
                                struct mlxsw_sp_acl_tcam_region *region,
                                u16 new_size)
{
        char ptar_pl[MLXSW_REG_PTAR_LEN];

        mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_RESIZE,
                            new_size, region->id, region->tcam_region_info);
        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
}

static int
mlxsw_sp_acl_tcam_region_enable(struct mlxsw_sp *mlxsw_sp,
                                struct mlxsw_sp_acl_tcam_region *region)
{
        char pacl_pl[MLXSW_REG_PACL_LEN];

        mlxsw_reg_pacl_pack(pacl_pl, region->id, true,
                            region->tcam_region_info);
        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
}

static void
mlxsw_sp_acl_tcam_region_disable(struct mlxsw_sp *mlxsw_sp,
                                 struct mlxsw_sp_acl_tcam_region *region)
{
        char pacl_pl[MLXSW_REG_PACL_LEN];

        mlxsw_reg_pacl_pack(pacl_pl, region->id, false,
                            region->tcam_region_info);
        mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
}

static int
mlxsw_sp_acl_tcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
                                      struct mlxsw_sp_acl_tcam_region *region,
                                      unsigned int offset,
                                      struct mlxsw_sp_acl_rule_info *rulei)
{
        char ptce2_pl[MLXSW_REG_PTCE2_LEN];
        char *act_set;
        char *mask;
        char *key;

        mlxsw_reg_ptce2_pack(ptce2_pl, true, MLXSW_REG_PTCE2_OP_WRITE_WRITE,
                             region->tcam_region_info, offset);
        key = mlxsw_reg_ptce2_flex_key_blocks_data(ptce2_pl);
        mask = mlxsw_reg_ptce2_mask_data(ptce2_pl);
        mlxsw_afk_encode(region->key_info, &rulei->values, key, mask);

        /* Only the first action set belongs here, the rest is in KVD */
        act_set = mlxsw_afa_block_first_set(rulei->act_block);
        mlxsw_reg_ptce2_flex_action_set_memcpy_to(ptce2_pl, act_set);

        return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce2), ptce2_pl);
}

static void
mlxsw_sp_acl_tcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
                                      struct mlxsw_sp_acl_tcam_region *region,
                                      unsigned int offset)
{
        char ptce2_pl[MLXSW_REG_PTCE2_LEN];

        mlxsw_reg_ptce2_pack(ptce2_pl, false, MLXSW_REG_PTCE2_OP_WRITE_WRITE,
                             region->tcam_region_info, offset);
        mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce2), ptce2_pl);
}

static int
mlxsw_sp_acl_tcam_region_entry_activity_get(struct mlxsw_sp *mlxsw_sp,
                                            struct mlxsw_sp_acl_tcam_region *region,
                                            unsigned int offset,
                                            bool *activity)
{
        char ptce2_pl[MLXSW_REG_PTCE2_LEN];
        int err;

        mlxsw_reg_ptce2_pack(ptce2_pl, true, MLXSW_REG_PTCE2_OP_QUERY_CLEAR_ON_READ,
                             region->tcam_region_info, offset);
        err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptce2), ptce2_pl);
        if (err)
                return err;
        *activity = mlxsw_reg_ptce2_a_get(ptce2_pl);
        return 0;
}

#define MLXSW_SP_ACL_TCAM_CATCHALL_PRIO (~0U)

static int
mlxsw_sp_acl_tcam_region_catchall_add(struct mlxsw_sp *mlxsw_sp,
                                      struct mlxsw_sp_acl_tcam_region *region)
{
        struct parman_prio *parman_prio = &region->catchall.parman_prio;
        struct parman_item *parman_item = &region->catchall.parman_item;
        struct mlxsw_sp_acl_rule_info *rulei;
        int err;

        parman_prio_init(region->parman, parman_prio,
                         MLXSW_SP_ACL_TCAM_CATCHALL_PRIO);
        err = parman_item_add(region->parman, parman_prio, parman_item);
        if (err)
                goto err_parman_item_add;

        rulei = mlxsw_sp_acl_rulei_create(mlxsw_sp->acl);
        if (IS_ERR(rulei)) {
                err = PTR_ERR(rulei);
                goto err_rulei_create;
        }

        mlxsw_sp_acl_rulei_act_continue(rulei);
        err = mlxsw_sp_acl_rulei_commit(rulei);
        if (err)
                goto err_rulei_commit;

        err = mlxsw_sp_acl_tcam_region_entry_insert(mlxsw_sp, region,
                                                    parman_item->index, rulei);
        region->catchall.rulei = rulei;
        if (err)
                goto err_rule_insert;

        return 0;

err_rule_insert:
err_rulei_commit:
        mlxsw_sp_acl_rulei_destroy(rulei);
err_rulei_create:
        parman_item_remove(region->parman, parman_prio, parman_item);
err_parman_item_add:
        parman_prio_fini(parman_prio);
        return err;
}

static void
mlxsw_sp_acl_tcam_region_catchall_del(struct mlxsw_sp *mlxsw_sp,
                                      struct mlxsw_sp_acl_tcam_region *region)
{
        struct parman_prio *parman_prio = &region->catchall.parman_prio;
        struct parman_item *parman_item = &region->catchall.parman_item;
        struct mlxsw_sp_acl_rule_info *rulei = region->catchall.rulei;

        mlxsw_sp_acl_tcam_region_entry_remove(mlxsw_sp, region,
                                              parman_item->index);
        mlxsw_sp_acl_rulei_destroy(rulei);
        parman_item_remove(region->parman, parman_prio, parman_item);
        parman_prio_fini(parman_prio);
}

static void
mlxsw_sp_acl_tcam_region_move(struct mlxsw_sp *mlxsw_sp,
                              struct mlxsw_sp_acl_tcam_region *region,
                              u16 src_offset, u16 dst_offset, u16 size)
{
        char prcr_pl[MLXSW_REG_PRCR_LEN];

        mlxsw_reg_prcr_pack(prcr_pl, MLXSW_REG_PRCR_OP_MOVE,
                            region->tcam_region_info, src_offset,
                            region->tcam_region_info, dst_offset, size);
        mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(prcr), prcr_pl);
}

static int mlxsw_sp_acl_tcam_region_parman_resize(void *priv,
                                                  unsigned long new_count)
{
        struct mlxsw_sp_acl_tcam_region *region = priv;
        struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
        u64 max_tcam_rules;

        max_tcam_rules = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_TCAM_RULES);
        if (new_count > max_tcam_rules)
                return -EINVAL;
        return mlxsw_sp_acl_tcam_region_resize(mlxsw_sp, region, new_count);
}

static void mlxsw_sp_acl_tcam_region_parman_move(void *priv,
                                                 unsigned long from_index,
                                                 unsigned long to_index,
                                                 unsigned long count)
{
        struct mlxsw_sp_acl_tcam_region *region = priv;
        struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;

        mlxsw_sp_acl_tcam_region_move(mlxsw_sp, region,
                                      from_index, to_index, count);
}

static const struct parman_ops mlxsw_sp_acl_tcam_region_parman_ops = {
        .base_count     = MLXSW_SP_ACL_TCAM_REGION_BASE_COUNT,
        .resize_step    = MLXSW_SP_ACL_TCAM_REGION_RESIZE_STEP,
        .resize         = mlxsw_sp_acl_tcam_region_parman_resize,
        .move           = mlxsw_sp_acl_tcam_region_parman_move,
        .algo           = PARMAN_ALGO_TYPE_LSORT,
};

static struct mlxsw_sp_acl_tcam_region *
mlxsw_sp_acl_tcam_region_create(struct mlxsw_sp *mlxsw_sp,
                                struct mlxsw_sp_acl_tcam *tcam,
                                struct mlxsw_afk_element_usage *elusage)
{
        struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
        struct mlxsw_sp_acl_tcam_region *region;
        int err;

        region = kzalloc(sizeof(*region), GFP_KERNEL);
        if (!region)
                return ERR_PTR(-ENOMEM);
        INIT_LIST_HEAD(&region->chunk_list);
        region->mlxsw_sp = mlxsw_sp;

        region->parman = parman_create(&mlxsw_sp_acl_tcam_region_parman_ops,
                                       region);
        if (!region->parman) {
                err = -ENOMEM;
                goto err_parman_create;
        }

        region->key_info = mlxsw_afk_key_info_get(afk, elusage);
        if (IS_ERR(region->key_info)) {
                err = PTR_ERR(region->key_info);
                goto err_key_info_get;
        }

        err = mlxsw_sp_acl_tcam_region_id_get(tcam, &region->id);
        if (err)
                goto err_region_id_get;

        err = mlxsw_sp_acl_tcam_region_alloc(mlxsw_sp, region);
        if (err)
                goto err_tcam_region_alloc;

        err = mlxsw_sp_acl_tcam_region_enable(mlxsw_sp, region);
        if (err)
                goto err_tcam_region_enable;

        err = mlxsw_sp_acl_tcam_region_catchall_add(mlxsw_sp, region);
        if (err)
                goto err_tcam_region_catchall_add;

        return region;

err_tcam_region_catchall_add:
        mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
err_tcam_region_enable:
        mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
err_tcam_region_alloc:
        mlxsw_sp_acl_tcam_region_id_put(tcam, region->id);
err_region_id_get:
        mlxsw_afk_key_info_put(region->key_info);
err_key_info_get:
        parman_destroy(region->parman);
err_parman_create:
        kfree(region);
        return ERR_PTR(err);
}

static void
mlxsw_sp_acl_tcam_region_destroy(struct mlxsw_sp *mlxsw_sp,
                                 struct mlxsw_sp_acl_tcam_region *region)
{
        mlxsw_sp_acl_tcam_region_catchall_del(mlxsw_sp, region);
        mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
        mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
        mlxsw_sp_acl_tcam_region_id_put(region->group->tcam, region->id);
        mlxsw_afk_key_info_put(region->key_info);
        parman_destroy(region->parman);
        kfree(region);
}

static int
mlxsw_sp_acl_tcam_chunk_assoc(struct mlxsw_sp *mlxsw_sp,
                              struct mlxsw_sp_acl_tcam_group *group,
                              unsigned int priority,
                              struct mlxsw_afk_element_usage *elusage,
                              struct mlxsw_sp_acl_tcam_chunk *chunk)
{
        struct mlxsw_sp_acl_tcam_region *region;
        bool region_created = false;
        bool need_split;
        int err;

        region = mlxsw_sp_acl_tcam_group_region_find(group, priority, elusage,
                                                     &need_split);
        if (region && need_split) {
                /* According to priority, the chunk should belong to an
                 * existing region. However, this chunk needs elements
                 * that region does not contain. We need to split the existing
                 * region into two and create a new region for this chunk
                 * in between. This is not supported now.
                 */
                return -EOPNOTSUPP;
        }
        if (!region) {
                struct mlxsw_afk_element_usage region_elusage;

                mlxsw_sp_acl_tcam_group_use_patterns(group, elusage,
                                                     &region_elusage);
                region = mlxsw_sp_acl_tcam_region_create(mlxsw_sp, group->tcam,
                                                         &region_elusage);
                if (IS_ERR(region))
                        return PTR_ERR(region);
                region_created = true;
        }

        chunk->region = region;
        list_add_tail(&chunk->list, &region->chunk_list);

        if (!region_created)
                return 0;

        err = mlxsw_sp_acl_tcam_group_region_attach(mlxsw_sp, group, region);
        if (err)
                goto err_group_region_attach;

        return 0;

err_group_region_attach:
        mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, region);
        return err;
}

static void
mlxsw_sp_acl_tcam_chunk_deassoc(struct mlxsw_sp *mlxsw_sp,
                                struct mlxsw_sp_acl_tcam_chunk *chunk)
{
        struct mlxsw_sp_acl_tcam_region *region = chunk->region;

        list_del(&chunk->list);
        if (list_empty(&region->chunk_list)) {
                mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, region);
                mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, region);
        }
}

static struct mlxsw_sp_acl_tcam_chunk *
mlxsw_sp_acl_tcam_chunk_create(struct mlxsw_sp *mlxsw_sp,
                               struct mlxsw_sp_acl_tcam_group *group,
                               unsigned int priority,
                               struct mlxsw_afk_element_usage *elusage)
{
        struct mlxsw_sp_acl_tcam_chunk *chunk;
        int err;

        if (priority == MLXSW_SP_ACL_TCAM_CATCHALL_PRIO)
                return ERR_PTR(-EINVAL);

        chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
        if (!chunk)
                return ERR_PTR(-ENOMEM);
        chunk->priority = priority;
        chunk->group = group;
        chunk->ref_count = 1;

        err = mlxsw_sp_acl_tcam_chunk_assoc(mlxsw_sp, group, priority,
                                            elusage, chunk);
        if (err)
                goto err_chunk_assoc;

        parman_prio_init(chunk->region->parman, &chunk->parman_prio, priority);

        err = rhashtable_insert_fast(&group->chunk_ht, &chunk->ht_node,
                                     mlxsw_sp_acl_tcam_chunk_ht_params);
        if (err)
                goto err_rhashtable_insert;

        return chunk;

err_rhashtable_insert:
        parman_prio_fini(&chunk->parman_prio);
        mlxsw_sp_acl_tcam_chunk_deassoc(mlxsw_sp, chunk);
err_chunk_assoc:
        kfree(chunk);
        return ERR_PTR(err);
}

static void
mlxsw_sp_acl_tcam_chunk_destroy(struct mlxsw_sp *mlxsw_sp,
                                struct mlxsw_sp_acl_tcam_chunk *chunk)
{
        struct mlxsw_sp_acl_tcam_group *group = chunk->group;

        rhashtable_remove_fast(&group->chunk_ht, &chunk->ht_node,
                               mlxsw_sp_acl_tcam_chunk_ht_params);
        parman_prio_fini(&chunk->parman_prio);
        mlxsw_sp_acl_tcam_chunk_deassoc(mlxsw_sp, chunk);
        kfree(chunk);
}

static struct mlxsw_sp_acl_tcam_chunk *
mlxsw_sp_acl_tcam_chunk_get(struct mlxsw_sp *mlxsw_sp,
                            struct mlxsw_sp_acl_tcam_group *group,
                            unsigned int priority,
                            struct mlxsw_afk_element_usage *elusage)
{
        struct mlxsw_sp_acl_tcam_chunk *chunk;

        chunk = rhashtable_lookup_fast(&group->chunk_ht, &priority,
                                       mlxsw_sp_acl_tcam_chunk_ht_params);
        if (chunk) {
                if (WARN_ON(!mlxsw_afk_key_info_subset(chunk->region->key_info,
                                                       elusage)))
                        return ERR_PTR(-EINVAL);
                chunk->ref_count++;
                return chunk;
        }
        return mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, group,
                                              priority, elusage);
}

static void mlxsw_sp_acl_tcam_chunk_put(struct mlxsw_sp *mlxsw_sp,
                                        struct mlxsw_sp_acl_tcam_chunk *chunk)
{
        if (--chunk->ref_count)
                return;
        mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, chunk);
}

static int mlxsw_sp_acl_tcam_entry_add(struct mlxsw_sp *mlxsw_sp,
                                       struct mlxsw_sp_acl_tcam_group *group,
                                       struct mlxsw_sp_acl_tcam_entry *entry,
                                       struct mlxsw_sp_acl_rule_info *rulei)
{
        struct mlxsw_sp_acl_tcam_chunk *chunk;
        struct mlxsw_sp_acl_tcam_region *region;
        int err;

        chunk = mlxsw_sp_acl_tcam_chunk_get(mlxsw_sp, group, rulei->priority,
                                            &rulei->values.elusage);
        if (IS_ERR(chunk))
                return PTR_ERR(chunk);

        region = chunk->region;
        err = parman_item_add(region->parman, &chunk->parman_prio,
                              &entry->parman_item);
        if (err)
                goto err_parman_item_add;

        err = mlxsw_sp_acl_tcam_region_entry_insert(mlxsw_sp, region,
                                                    entry->parman_item.index,
                                                    rulei);
        if (err)
                goto err_rule_insert;
        entry->chunk = chunk;

        return 0;

err_rule_insert:
        parman_item_remove(region->parman, &chunk->parman_prio,
                           &entry->parman_item);
err_parman_item_add:
        mlxsw_sp_acl_tcam_chunk_put(mlxsw_sp, chunk);
        return err;
}

static void mlxsw_sp_acl_tcam_entry_del(struct mlxsw_sp *mlxsw_sp,
                                        struct mlxsw_sp_acl_tcam_entry *entry)
{
        struct mlxsw_sp_acl_tcam_chunk *chunk = entry->chunk;
        struct mlxsw_sp_acl_tcam_region *region = chunk->region;

        mlxsw_sp_acl_tcam_region_entry_remove(mlxsw_sp, region,
                                              entry->parman_item.index);
        parman_item_remove(region->parman, &chunk->parman_prio,
                           &entry->parman_item);
        mlxsw_sp_acl_tcam_chunk_put(mlxsw_sp, chunk);
}

static int
mlxsw_sp_acl_tcam_entry_activity_get(struct mlxsw_sp *mlxsw_sp,
                                     struct mlxsw_sp_acl_tcam_entry *entry,
                                     bool *activity)
{
        struct mlxsw_sp_acl_tcam_chunk *chunk = entry->chunk;
        struct mlxsw_sp_acl_tcam_region *region = chunk->region;

        return mlxsw_sp_acl_tcam_region_entry_activity_get(mlxsw_sp, region,
                                                           entry->parman_item.index,
                                                           activity);
}

static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv4[] = {
        MLXSW_AFK_ELEMENT_SRC_SYS_PORT,
        MLXSW_AFK_ELEMENT_DMAC,
        MLXSW_AFK_ELEMENT_SMAC,
        MLXSW_AFK_ELEMENT_ETHERTYPE,
        MLXSW_AFK_ELEMENT_IP_PROTO,
        MLXSW_AFK_ELEMENT_SRC_IP4,
        MLXSW_AFK_ELEMENT_DST_IP4,
        MLXSW_AFK_ELEMENT_DST_L4_PORT,
        MLXSW_AFK_ELEMENT_SRC_L4_PORT,
        MLXSW_AFK_ELEMENT_VID,
        MLXSW_AFK_ELEMENT_PCP,
        MLXSW_AFK_ELEMENT_TCP_FLAGS,
        MLXSW_AFK_ELEMENT_IP_TTL_,
        MLXSW_AFK_ELEMENT_IP_ECN,
        MLXSW_AFK_ELEMENT_IP_DSCP,
};

static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv6[] = {
        MLXSW_AFK_ELEMENT_ETHERTYPE,
        MLXSW_AFK_ELEMENT_IP_PROTO,

        MLXSW_AFK_ELEMENT_SRC_IP6_HI,
        MLXSW_AFK_ELEMENT_SRC_IP6_LO,
        MLXSW_AFK_ELEMENT_DST_IP6_HI,
        MLXSW_AFK_ELEMENT_DST_IP6_LO,
        MLXSW_AFK_ELEMENT_DST_L4_PORT,
        MLXSW_AFK_ELEMENT_SRC_L4_PORT,
};

static const struct mlxsw_sp_acl_tcam_pattern mlxsw_sp_acl_tcam_patterns[] = {
        {
                .elements = mlxsw_sp_acl_tcam_pattern_ipv4,
                .elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv4),
        },
        {
                .elements = mlxsw_sp_acl_tcam_pattern_ipv6,
                .elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv6),
        },
};

#define MLXSW_SP_ACL_TCAM_PATTERNS_COUNT \
        ARRAY_SIZE(mlxsw_sp_acl_tcam_patterns)

struct mlxsw_sp_acl_tcam_flower_ruleset {
        struct mlxsw_sp_acl_tcam_group group;
};

struct mlxsw_sp_acl_tcam_flower_rule {
        struct mlxsw_sp_acl_tcam_entry entry;
};

static int
mlxsw_sp_acl_tcam_flower_ruleset_add(struct mlxsw_sp *mlxsw_sp,
                                     void *priv, void *ruleset_priv)
{
        struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
        struct mlxsw_sp_acl_tcam *tcam = priv;

        return mlxsw_sp_acl_tcam_group_add(mlxsw_sp, tcam, &ruleset->group,
                                           mlxsw_sp_acl_tcam_patterns,
                                           MLXSW_SP_ACL_TCAM_PATTERNS_COUNT);
}

static void
mlxsw_sp_acl_tcam_flower_ruleset_del(struct mlxsw_sp *mlxsw_sp,
                                     void *ruleset_priv)
{
        struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;

        mlxsw_sp_acl_tcam_group_del(mlxsw_sp, &ruleset->group);
}

static int
mlxsw_sp_acl_tcam_flower_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
                                      void *ruleset_priv,
                                      struct net_device *dev, bool ingress)
{
        struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;

        return mlxsw_sp_acl_tcam_group_bind(mlxsw_sp, &ruleset->group,
                                            dev, ingress);
}

static void
mlxsw_sp_acl_tcam_flower_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
                                        void *ruleset_priv)
{
        struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;

        mlxsw_sp_acl_tcam_group_unbind(mlxsw_sp, &ruleset->group);
}

static u16
mlxsw_sp_acl_tcam_flower_ruleset_group_id(void *ruleset_priv)
{
        struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;

        return mlxsw_sp_acl_tcam_group_id(&ruleset->group);
}

static int
mlxsw_sp_acl_tcam_flower_rule_add(struct mlxsw_sp *mlxsw_sp,
                                  void *ruleset_priv, void *rule_priv,
                                  struct mlxsw_sp_acl_rule_info *rulei)
{
        struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
        struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;

        return mlxsw_sp_acl_tcam_entry_add(mlxsw_sp, &ruleset->group,
                                           &rule->entry, rulei);
}

static void
mlxsw_sp_acl_tcam_flower_rule_del(struct mlxsw_sp *mlxsw_sp, void *rule_priv)
{
        struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;

        mlxsw_sp_acl_tcam_entry_del(mlxsw_sp, &rule->entry);
}

static int
mlxsw_sp_acl_tcam_flower_rule_activity_get(struct mlxsw_sp *mlxsw_sp,
                                           void *rule_priv, bool *activity)
{
        struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;

        return mlxsw_sp_acl_tcam_entry_activity_get(mlxsw_sp, &rule->entry,
                                                    activity);
}

static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_flower_ops = {
        .ruleset_priv_size      = sizeof(struct mlxsw_sp_acl_tcam_flower_ruleset),
        .ruleset_add            = mlxsw_sp_acl_tcam_flower_ruleset_add,
        .ruleset_del            = mlxsw_sp_acl_tcam_flower_ruleset_del,
        .ruleset_bind           = mlxsw_sp_acl_tcam_flower_ruleset_bind,
        .ruleset_unbind         = mlxsw_sp_acl_tcam_flower_ruleset_unbind,
        .ruleset_group_id       = mlxsw_sp_acl_tcam_flower_ruleset_group_id,
        .rule_priv_size         = sizeof(struct mlxsw_sp_acl_tcam_flower_rule),
        .rule_add               = mlxsw_sp_acl_tcam_flower_rule_add,
        .rule_del               = mlxsw_sp_acl_tcam_flower_rule_del,
        .rule_activity_get      = mlxsw_sp_acl_tcam_flower_rule_activity_get,
};

static const struct mlxsw_sp_acl_profile_ops *
mlxsw_sp_acl_tcam_profile_ops_arr[] = {
        [MLXSW_SP_ACL_PROFILE_FLOWER] = &mlxsw_sp_acl_tcam_flower_ops,
};

static const struct mlxsw_sp_acl_profile_ops *
mlxsw_sp_acl_tcam_profile_ops(struct mlxsw_sp *mlxsw_sp,
                              enum mlxsw_sp_acl_profile profile)
{
        const struct mlxsw_sp_acl_profile_ops *ops;

        if (WARN_ON(profile >= ARRAY_SIZE(mlxsw_sp_acl_tcam_profile_ops_arr)))
                return NULL;
        ops = mlxsw_sp_acl_tcam_profile_ops_arr[profile];
        if (WARN_ON(!ops))
                return NULL;
        return ops;
}

const struct mlxsw_sp_acl_ops mlxsw_sp_acl_tcam_ops = {
        .priv_size              = sizeof(struct mlxsw_sp_acl_tcam),
        .init                   = mlxsw_sp_acl_tcam_init,
        .fini                   = mlxsw_sp_acl_tcam_fini,
        .profile_ops            = mlxsw_sp_acl_tcam_profile_ops,
};