// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019 Mellanox Technologies. */

#include <net/xdp_sock.h>
#include "umem.h"
#include "setup.h"
#include "en/params.h"

static int mlx5e_xsk_map_umem(struct mlx5e_priv *priv,
                              struct xdp_umem *umem)
{
        struct device *dev = priv->mdev->device;
        u32 i;

        for (i = 0; i < umem->npgs; i++) {
                dma_addr_t dma = dma_map_page(dev, umem->pgs[i], 0, PAGE_SIZE,
                                              DMA_BIDIRECTIONAL);

                if (unlikely(dma_mapping_error(dev, dma)))
                        goto err_unmap;
                umem->pages[i].dma = dma;
        }

        return 0;

err_unmap:
        while (i--) {
                dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
                               DMA_BIDIRECTIONAL);
                umem->pages[i].dma = 0;
        }

        return -ENOMEM;
}

static void mlx5e_xsk_unmap_umem(struct mlx5e_priv *priv,
                                 struct xdp_umem *umem)
{
        struct device *dev = priv->mdev->device;
        u32 i;

        for (i = 0; i < umem->npgs; i++) {
                dma_unmap_page(dev, umem->pages[i].dma, PAGE_SIZE,
                               DMA_BIDIRECTIONAL);
                umem->pages[i].dma = 0;
        }
}

static int mlx5e_xsk_get_umems(struct mlx5e_xsk *xsk)
{
        if (!xsk->umems) {
                xsk->umems = kcalloc(MLX5E_MAX_NUM_CHANNELS,
                                     sizeof(*xsk->umems), GFP_KERNEL);
                if (unlikely(!xsk->umems))
                        return -ENOMEM;
        }

        xsk->refcnt++;
        xsk->ever_used = true;

        return 0;
}

static void mlx5e_xsk_put_umems(struct mlx5e_xsk *xsk)
{
        if (!--xsk->refcnt) {
                kfree(xsk->umems);
                xsk->umems = NULL;
        }
}

static int mlx5e_xsk_add_umem(struct mlx5e_xsk *xsk, struct xdp_umem *umem, u16 ix)
{
        int err;

        err = mlx5e_xsk_get_umems(xsk);
        if (unlikely(err))
                return err;

        xsk->umems[ix] = umem;
        return 0;
}

static void mlx5e_xsk_remove_umem(struct mlx5e_xsk *xsk, u16 ix)
{
        xsk->umems[ix] = NULL;

        mlx5e_xsk_put_umems(xsk);
}

static bool mlx5e_xsk_is_umem_sane(struct xdp_umem *umem)
{
        return umem->headroom <= 0xffff && umem->chunk_size_nohr <= 0xffff;
}

void mlx5e_build_xsk_param(struct xdp_umem *umem, struct mlx5e_xsk_param *xsk)
{
        xsk->headroom = umem->headroom;
        xsk->chunk_size = umem->chunk_size_nohr + umem->headroom;
}

static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
                                   struct xdp_umem *umem, u16 ix)
{
        struct mlx5e_params *params = &priv->channels.params;
        struct mlx5e_xsk_param xsk;
        struct mlx5e_channel *c;
        int err;

        if (unlikely(mlx5e_xsk_get_umem(&priv->channels.params, &priv->xsk, ix)))
                return -EBUSY;

        if (unlikely(!mlx5e_xsk_is_umem_sane(umem)))
                return -EINVAL;

        err = mlx5e_xsk_map_umem(priv, umem);
        if (unlikely(err))
                return err;

        err = mlx5e_xsk_add_umem(&priv->xsk, umem, ix);
        if (unlikely(err))
                goto err_unmap_umem;

        mlx5e_build_xsk_param(umem, &xsk);

        if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
                /* XSK objects will be created on open. */
                goto validate_closed;
        }

        if (!params->xdp_prog) {
                /* XSK objects will be created when an XDP program is set,
                 * and the channels are reopened.
                 */
                goto validate_closed;
        }

        c = priv->channels.c[ix];

        err = mlx5e_open_xsk(priv, params, &xsk, umem, c);
        if (unlikely(err))
                goto err_remove_umem;

        mlx5e_activate_xsk(c);

        /* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
         * any Fill Ring entries at the setup stage.
         */

        err = mlx5e_xsk_redirect_rqt_to_channel(priv, priv->channels.c[ix]);
        if (unlikely(err))
                goto err_deactivate;

        return 0;

err_deactivate:
        mlx5e_deactivate_xsk(c);
        mlx5e_close_xsk(c);

err_remove_umem:
        mlx5e_xsk_remove_umem(&priv->xsk, ix);

err_unmap_umem:
        mlx5e_xsk_unmap_umem(priv, umem);

        return err;

validate_closed:
        /* Check the configuration in advance, rather than fail at a later stage
         * (in mlx5e_xdp_set or on open) and end up with no channels.
         */
        if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
                err = -EINVAL;
                goto err_remove_umem;
        }

        return 0;
}

static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
{
        struct xdp_umem *umem = mlx5e_xsk_get_umem(&priv->channels.params,
                                                   &priv->xsk, ix);
        struct mlx5e_channel *c;

        if (unlikely(!umem))
                return -EINVAL;

        if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
                goto remove_umem;

        /* XSK RQ and SQ are only created if XDP program is set. */
        if (!priv->channels.params.xdp_prog)
                goto remove_umem;

        c = priv->channels.c[ix];
        mlx5e_xsk_redirect_rqt_to_drop(priv, ix);
        mlx5e_deactivate_xsk(c);
        mlx5e_close_xsk(c);

remove_umem:
        mlx5e_xsk_remove_umem(&priv->xsk, ix);
        mlx5e_xsk_unmap_umem(priv, umem);

        return 0;
}

static int mlx5e_xsk_enable_umem(struct mlx5e_priv *priv, struct xdp_umem *umem,
                                 u16 ix)
{
        int err;

        mutex_lock(&priv->state_lock);
        err = mlx5e_xsk_enable_locked(priv, umem, ix);
        mutex_unlock(&priv->state_lock);

        return err;
}

static int mlx5e_xsk_disable_umem(struct mlx5e_priv *priv, u16 ix)
{
        int err;

        mutex_lock(&priv->state_lock);
        err = mlx5e_xsk_disable_locked(priv, ix);
        mutex_unlock(&priv->state_lock);

        return err;
}

int mlx5e_xsk_setup_umem(struct net_device *dev, struct xdp_umem *umem, u16 qid)
{
        struct mlx5e_priv *priv = netdev_priv(dev);
        struct mlx5e_params *params = &priv->channels.params;
        u16 ix;

        if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))
                return -EINVAL;

        return umem ? mlx5e_xsk_enable_umem(priv, umem, ix) :
                      mlx5e_xsk_disable_umem(priv, ix);
}

int mlx5e_xsk_resize_reuseq(struct xdp_umem *umem, u32 nentries)
{
        struct xdp_umem_fq_reuse *reuseq;

        reuseq = xsk_reuseq_prepare(nentries);
        if (unlikely(!reuseq))
                return -ENOMEM;
        xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));

        return 0;
}

u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params *params, struct mlx5e_xsk *xsk)
{
        u16 res = xsk->refcnt ? params->num_channels : 0;

        while (res) {
                if (mlx5e_xsk_get_umem(params, xsk, res - 1))
                        break;
                --res;
        }

        return res;
}