// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2015 HGST, a Western Digital Company.
 */
#include <linux/module.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <rdma/ib_verbs.h>

#include <trace/events/rdma_core.h>

/* # of WCs to poll for with a single call to ib_poll_cq */
#define IB_POLL_BATCH                   16
#define IB_POLL_BATCH_DIRECT            8

/* # of WCs to iterate over before yielding */
#define IB_POLL_BUDGET_IRQ              256
#define IB_POLL_BUDGET_WORKQUEUE        65536

#define IB_POLL_FLAGS \
        (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)

static const struct dim_cq_moder
rdma_dim_prof[RDMA_DIM_PARAMS_NUM_PROFILES] = {
        {1,   0, 1,  0},
        {1,   0, 4,  0},
        {2,   0, 4,  0},
        {2,   0, 8,  0},
        {4,   0, 8,  0},
        {16,  0, 8,  0},
        {16,  0, 16, 0},
        {32,  0, 16, 0},
        {32,  0, 32, 0},
};

static void ib_cq_rdma_dim_work(struct work_struct *w)
{
        struct dim *dim = container_of(w, struct dim, work);
        struct ib_cq *cq = dim->priv;

        u16 usec = rdma_dim_prof[dim->profile_ix].usec;
        u16 comps = rdma_dim_prof[dim->profile_ix].comps;

        dim->state = DIM_START_MEASURE;

        trace_cq_modify(cq, comps, usec);
        cq->device->ops.modify_cq(cq, comps, usec);
}

static void rdma_dim_init(struct ib_cq *cq)
{
        struct dim *dim;

        if (!cq->device->ops.modify_cq || !cq->device->use_cq_dim ||
            cq->poll_ctx == IB_POLL_DIRECT)
                return;

        dim = kzalloc(sizeof(struct dim), GFP_KERNEL);
        if (!dim)
                return;

        dim->state = DIM_START_MEASURE;
        dim->tune_state = DIM_GOING_RIGHT;
        dim->profile_ix = RDMA_DIM_START_PROFILE;
        dim->priv = cq;
        cq->dim = dim;

        INIT_WORK(&dim->work, ib_cq_rdma_dim_work);
}

static int __poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
{
        int rc;

        rc = ib_poll_cq(cq, num_entries, wc);
        trace_cq_poll(cq, num_entries, rc);
        return rc;
}

static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,
                           int batch)
{
        int i, n, completed = 0;

        trace_cq_process(cq);

        /*
         * budget might be (-1) if the caller does not
         * want to bound this call, thus we need unsigned
         * minimum here.
         */
        while ((n = __poll_cq(cq, min_t(u32, batch,
                                        budget - completed), wcs)) > 0) {
                for (i = 0; i < n; i++) {
                        struct ib_wc *wc = &wcs[i];

                        if (wc->wr_cqe)
                                wc->wr_cqe->done(cq, wc);
                        else
                                WARN_ON_ONCE(wc->status == IB_WC_SUCCESS);
                }

                completed += n;

                if (n != batch || (budget != -1 && completed >= budget))
                        break;
        }

        return completed;
}

/**
 * ib_process_direct_cq - process a CQ in caller context
 * @cq:         CQ to process
 * @budget:     number of CQEs to poll for
 *
 * This function is used to process all outstanding CQ entries.
 * It does not offload CQ processing to a different context and does
 * not ask for completion interrupts from the HCA.
 * Using direct processing on CQ with non IB_POLL_DIRECT type may trigger
 * concurrent processing.
 *
 * Note: do not pass -1 as %budget unless it is guaranteed that the number
 * of completions that will be processed is small.
 */
int ib_process_cq_direct(struct ib_cq *cq, int budget)
{
        struct ib_wc wcs[IB_POLL_BATCH_DIRECT];

        return __ib_process_cq(cq, budget, wcs, IB_POLL_BATCH_DIRECT);
}
EXPORT_SYMBOL(ib_process_cq_direct);

static void ib_cq_completion_direct(struct ib_cq *cq, void *private)
{
        WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq);
}

static int ib_poll_handler(struct irq_poll *iop, int budget)
{
        struct ib_cq *cq = container_of(iop, struct ib_cq, iop);
        struct dim *dim = cq->dim;
        int completed;

        completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
        if (completed < budget) {
                irq_poll_complete(&cq->iop);
                if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) {
                        trace_cq_reschedule(cq);
                        irq_poll_sched(&cq->iop);
                }
        }

        if (dim)
                rdma_dim(dim, completed);

        return completed;
}

static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)
{
        trace_cq_schedule(cq);
        irq_poll_sched(&cq->iop);
}

static void ib_cq_poll_work(struct work_struct *work)
{
        struct ib_cq *cq = container_of(work, struct ib_cq, work);
        int completed;

        completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, cq->wc,
                                    IB_POLL_BATCH);
        if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
            ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
                queue_work(cq->comp_wq, &cq->work);
        else if (cq->dim)
                rdma_dim(cq->dim, completed);
}

static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
{
        trace_cq_schedule(cq);
        queue_work(cq->comp_wq, &cq->work);
}

/**
 * __ib_alloc_cq_user - allocate a completion queue
 * @dev:                device to allocate the CQ for
 * @private:            driver private data, accessible from cq->cq_context
 * @nr_cqe:             number of CQEs to allocate
 * @comp_vector:        HCA completion vectors for this CQ
 * @poll_ctx:           context to poll the CQ from.
 * @caller:             module owner name.
 * @udata:              Valid user data or NULL for kernel object
 *
 * This is the proper interface to allocate a CQ for in-kernel users. A
 * CQ allocated with this interface will automatically be polled from the
 * specified context. The ULP must use wr->wr_cqe instead of wr->wr_id
 * to use this CQ abstraction.
 */
struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,
                                 int nr_cqe, int comp_vector,
                                 enum ib_poll_context poll_ctx,
                                 const char *caller, struct ib_udata *udata)
{
        struct ib_cq_init_attr cq_attr = {
                .cqe            = nr_cqe,
                .comp_vector    = comp_vector,
        };
        struct ib_cq *cq;
        int ret = -ENOMEM;

        cq = rdma_zalloc_drv_obj(dev, ib_cq);
        if (!cq)
                return ERR_PTR(ret);

        cq->device = dev;
        cq->cq_context = private;
        cq->poll_ctx = poll_ctx;
        atomic_set(&cq->usecnt, 0);

        cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);
        if (!cq->wc)
                goto out_free_cq;

        cq->res.type = RDMA_RESTRACK_CQ;
        rdma_restrack_set_task(&cq->res, caller);

        ret = dev->ops.create_cq(cq, &cq_attr, NULL);
        if (ret)
                goto out_free_wc;

        rdma_restrack_kadd(&cq->res);

        rdma_dim_init(cq);

        switch (cq->poll_ctx) {
        case IB_POLL_DIRECT:
                cq->comp_handler = ib_cq_completion_direct;
                break;
        case IB_POLL_SOFTIRQ:
                cq->comp_handler = ib_cq_completion_softirq;

                irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler);
                ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
                break;
        case IB_POLL_WORKQUEUE:
        case IB_POLL_UNBOUND_WORKQUEUE:
                cq->comp_handler = ib_cq_completion_workqueue;
                INIT_WORK(&cq->work, ib_cq_poll_work);
                ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
                cq->comp_wq = (cq->poll_ctx == IB_POLL_WORKQUEUE) ?
                                ib_comp_wq : ib_comp_unbound_wq;
                break;
        default:
                ret = -EINVAL;
                goto out_destroy_cq;
        }

        trace_cq_alloc(cq, nr_cqe, comp_vector, poll_ctx);
        return cq;

out_destroy_cq:
        rdma_restrack_del(&cq->res);
        cq->device->ops.destroy_cq(cq, udata);
out_free_wc:
        kfree(cq->wc);
out_free_cq:
        kfree(cq);
        trace_cq_alloc_error(nr_cqe, comp_vector, poll_ctx, ret);
        return ERR_PTR(ret);
}
EXPORT_SYMBOL(__ib_alloc_cq_user);

/**
 * __ib_alloc_cq_any - allocate a completion queue
 * @dev:                device to allocate the CQ for
 * @private:            driver private data, accessible from cq->cq_context
 * @nr_cqe:             number of CQEs to allocate
 * @poll_ctx:           context to poll the CQ from
 * @caller:             module owner name
 *
 * Attempt to spread ULP Completion Queues over each device's interrupt
 * vectors. A simple best-effort mechanism is used.
 */
struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
                                int nr_cqe, enum ib_poll_context poll_ctx,
                                const char *caller)
{
        static atomic_t counter;
        int comp_vector = 0;

        if (dev->num_comp_vectors > 1)
                comp_vector =
                        atomic_inc_return(&counter) %
                        min_t(int, dev->num_comp_vectors, num_online_cpus());

        return __ib_alloc_cq_user(dev, private, nr_cqe, comp_vector, poll_ctx,
                                  caller, NULL);
}
EXPORT_SYMBOL(__ib_alloc_cq_any);

/**
 * ib_free_cq_user - free a completion queue
 * @cq:         completion queue to free.
 * @udata:      User data or NULL for kernel object
 */
void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
{
        if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
                return;

        switch (cq->poll_ctx) {
        case IB_POLL_DIRECT:
                break;
        case IB_POLL_SOFTIRQ:
                irq_poll_disable(&cq->iop);
                break;
        case IB_POLL_WORKQUEUE:
        case IB_POLL_UNBOUND_WORKQUEUE:
                cancel_work_sync(&cq->work);
                break;
        default:
                WARN_ON_ONCE(1);
        }

        trace_cq_free(cq);
        rdma_restrack_del(&cq->res);
        cq->device->ops.destroy_cq(cq, udata);
        if (cq->dim)
                cancel_work_sync(&cq->dim->work);
        kfree(cq->dim);
        kfree(cq->wc);
        kfree(cq);
}
EXPORT_SYMBOL(ib_free_cq_user);