// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2016-2019 HabanaLabs, Ltd.
 * All Rights Reserved.
 */

#include "habanalabs.h"

#include <linux/slab.h>

/*
 * hl_queue_add_ptr - add to pi or ci and checks if it wraps around
 *
 * @ptr: the current pi/ci value
 * @val: the amount to add
 *
 * Add val to ptr. It can go until twice the queue length.
 */
inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)
{
        ptr += val;
        ptr &= ((HL_QUEUE_LENGTH << 1) - 1);
        return ptr;
}
static inline int queue_ci_get(atomic_t *ci, u32 queue_len)
{
        return atomic_read(ci) & ((queue_len << 1) - 1);
}

static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)
{
        int delta = (q->pi - queue_ci_get(&q->ci, queue_len));

        if (delta >= 0)
                return (queue_len - delta);
        else
                return (abs(delta) - queue_len);
}

void hl_hw_queue_update_ci(struct hl_cs *cs)
{
        struct hl_device *hdev = cs->ctx->hdev;
        struct hl_hw_queue *q;
        int i;

        if (hdev->disabled)
                return;

        q = &hdev->kernel_queues[0];

        /* There are no internal queues if H/W queues are being used */
        if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)
                return;

        /* We must increment CI for every queue that will never get a
         * completion, there are 2 scenarios this can happen:
         * 1. All queues of a non completion CS will never get a completion.
         * 2. Internal queues never gets completion.
         */
        for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {
                if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT)
                        atomic_add(cs->jobs_in_queue_cnt[i], &q->ci);
        }
}

/*
 * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a
 *                                H/W queue.
 * @hdev: pointer to habanalabs device structure
 * @q: pointer to habanalabs queue structure
 * @ctl: BD's control word
 * @len: BD's length
 * @ptr: BD's pointer
 *
 * This function assumes there is enough space on the queue to submit a new
 * BD to it. It initializes the next BD and calls the device specific
 * function to set the pi (and doorbell)
 *
 * This function must be called when the scheduler mutex is taken
 *
 */
void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
                u32 ctl, u32 len, u64 ptr)
{
        struct hl_bd *bd;

        bd = q->kernel_address;
        bd += hl_pi_2_offset(q->pi);
        bd->ctl = cpu_to_le32(ctl);
        bd->len = cpu_to_le32(len);
        bd->ptr = cpu_to_le64(ptr);

        q->pi = hl_queue_inc_ptr(q->pi);
        hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
}

/*
 * ext_queue_sanity_checks - perform some sanity checks on external queue
 *
 * @hdev              : pointer to hl_device structure
 * @q                 : pointer to hl_hw_queue structure
 * @num_of_entries    : how many entries to check for space
 * @reserve_cq_entry  : whether to reserve an entry in the cq
 *
 * H/W queues spinlock should be taken before calling this function
 *
 * Perform the following:
 * - Make sure we have enough space in the h/w queue
 * - Make sure we have enough space in the completion queue
 * - Reserve space in the completion queue (needs to be reversed if there
 *   is a failure down the road before the actual submission of work). Only
 *   do this action if reserve_cq_entry is true
 *
 */
static int ext_queue_sanity_checks(struct hl_device *hdev,
                                struct hl_hw_queue *q, int num_of_entries,
                                bool reserve_cq_entry)
{
        atomic_t *free_slots =
                        &hdev->completion_queue[q->cq_id].free_slots_cnt;
        int free_slots_cnt;

        /* Check we have enough space in the queue */
        free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);

        if (free_slots_cnt < num_of_entries) {
                dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
                        q->hw_queue_id, num_of_entries);
                return -EAGAIN;
        }

        if (reserve_cq_entry) {
                /*
                 * Check we have enough space in the completion queue
                 * Add -1 to counter (decrement) unless counter was already 0
                 * In that case, CQ is full so we can't submit a new CB because
                 * we won't get ack on its completion
                 * atomic_add_unless will return 0 if counter was already 0
                 */
                if (atomic_add_negative(num_of_entries * -1, free_slots)) {
                        dev_dbg(hdev->dev, "No space for %d on CQ %d\n",
                                num_of_entries, q->hw_queue_id);
                        atomic_add(num_of_entries, free_slots);
                        return -EAGAIN;
                }
        }

        return 0;
}

/*
 * int_queue_sanity_checks - perform some sanity checks on internal queue
 *
 * @hdev              : pointer to hl_device structure
 * @q                 : pointer to hl_hw_queue structure
 * @num_of_entries    : how many entries to check for space
 *
 * H/W queues spinlock should be taken before calling this function
 *
 * Perform the following:
 * - Make sure we have enough space in the h/w queue
 *
 */
static int int_queue_sanity_checks(struct hl_device *hdev,
                                        struct hl_hw_queue *q,
                                        int num_of_entries)
{
        int free_slots_cnt;

        if (num_of_entries > q->int_queue_len) {
                dev_err(hdev->dev,
                        "Cannot populate queue %u with %u jobs\n",
                        q->hw_queue_id, num_of_entries);
                return -ENOMEM;
        }

        /* Check we have enough space in the queue */
        free_slots_cnt = queue_free_slots(q, q->int_queue_len);

        if (free_slots_cnt < num_of_entries) {
                dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
                        q->hw_queue_id, num_of_entries);
                return -EAGAIN;
        }

        return 0;
}

/*
 * hw_queue_sanity_checks() - Make sure we have enough space in the h/w queue
 * @hdev: Pointer to hl_device structure.
 * @q: Pointer to hl_hw_queue structure.
 * @num_of_entries: How many entries to check for space.
 *
 * Notice: We do not reserve queue entries so this function mustn't be called
 *         more than once per CS for the same queue
 *
 */
static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,
                                        int num_of_entries)
{
        int free_slots_cnt;

        /* Check we have enough space in the queue */
        free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);

        if (free_slots_cnt < num_of_entries) {
                dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
                        q->hw_queue_id, num_of_entries);
                return -EAGAIN;
        }

        return 0;
}

/*
 * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion
 *
 * @hdev: pointer to hl_device structure
 * @hw_queue_id: Queue's type
 * @cb_size: size of CB
 * @cb_ptr: pointer to CB location
 *
 * This function sends a single CB, that must NOT generate a completion entry.
 * Sending CPU messages can be done instead via 'hl_hw_queue_submit_bd()'
 */
int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
                                u32 cb_size, u64 cb_ptr)
{
        struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
        int rc = 0;

        hdev->asic_funcs->hw_queues_lock(hdev);

        if (hdev->disabled) {
                rc = -EPERM;
                goto out;
        }

        /*
         * hl_hw_queue_send_cb_no_cmpl() is called for queues of a H/W queue
         * type only on init phase, when the queues are empty and being tested,
         * so there is no need for sanity checks.
         */
        if (q->queue_type != QUEUE_TYPE_HW) {
                rc = ext_queue_sanity_checks(hdev, q, 1, false);
                if (rc)
                        goto out;
        }

        hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);

out:
        hdev->asic_funcs->hw_queues_unlock(hdev);

        return rc;
}

/*
 * ext_queue_schedule_job - submit a JOB to an external queue
 *
 * @job: pointer to the job that needs to be submitted to the queue
 *
 * This function must be called when the scheduler mutex is taken
 *
 */
static void ext_queue_schedule_job(struct hl_cs_job *job)
{
        struct hl_device *hdev = job->cs->ctx->hdev;
        struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
        struct hl_cq_entry cq_pkt;
        struct hl_cq *cq;
        u64 cq_addr;
        struct hl_cb *cb;
        u32 ctl;
        u32 len;
        u64 ptr;

        /*
         * Update the JOB ID inside the BD CTL so the device would know what
         * to write in the completion queue
         */
        ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK);

        cb = job->patched_cb;
        len = job->job_cb_size;
        ptr = cb->bus_address;

        /* Skip completion flow in case this is a non completion CS */
        if (!cs_needs_completion(job->cs))
                goto submit_bd;

        cq_pkt.data = cpu_to_le32(
                        ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)
                                & CQ_ENTRY_SHADOW_INDEX_MASK) |
                        FIELD_PREP(CQ_ENTRY_SHADOW_INDEX_VALID_MASK, 1) |
                        FIELD_PREP(CQ_ENTRY_READY_MASK, 1));

        /*
         * No need to protect pi_offset because scheduling to the
         * H/W queues is done under the scheduler mutex
         *
         * No need to check if CQ is full because it was already
         * checked in ext_queue_sanity_checks
         */
        cq = &hdev->completion_queue[q->cq_id];
        cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry);

        hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,
                                                cq_addr,
                                                le32_to_cpu(cq_pkt.data),
                                                q->msi_vec,
                                                job->contains_dma_pkt);

        q->shadow_queue[hl_pi_2_offset(q->pi)] = job;

        cq->pi = hl_cq_inc_ptr(cq->pi);

submit_bd:
        hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
}

/*
 * int_queue_schedule_job - submit a JOB to an internal queue
 *
 * @job: pointer to the job that needs to be submitted to the queue
 *
 * This function must be called when the scheduler mutex is taken
 *
 */
static void int_queue_schedule_job(struct hl_cs_job *job)
{
        struct hl_device *hdev = job->cs->ctx->hdev;
        struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
        struct hl_bd bd;
        __le64 *pi;

        bd.ctl = 0;
        bd.len = cpu_to_le32(job->job_cb_size);

        if (job->is_kernel_allocated_cb)
                /* bus_address is actually a mmu mapped address
                 * allocated from an internal pool
                 */
                bd.ptr = cpu_to_le64(job->user_cb->bus_address);
        else
                bd.ptr = cpu_to_le64((u64) (uintptr_t) job->user_cb);

        pi = q->kernel_address + (q->pi & (q->int_queue_len - 1)) * sizeof(bd);

        q->pi++;
        q->pi &= ((q->int_queue_len << 1) - 1);

        hdev->asic_funcs->pqe_write(hdev, pi, &bd);

        hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
}

/*
 * hw_queue_schedule_job - submit a JOB to a H/W queue
 *
 * @job: pointer to the job that needs to be submitted to the queue
 *
 * This function must be called when the scheduler mutex is taken
 *
 */
static void hw_queue_schedule_job(struct hl_cs_job *job)
{
        struct hl_device *hdev = job->cs->ctx->hdev;
        struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
        u64 ptr;
        u32 offset, ctl, len;

        /*
         * Upon PQE completion, COMP_DATA is used as the write data to the
         * completion queue (QMAN HBW message), and COMP_OFFSET is used as the
         * write address offset in the SM block (QMAN LBW message).
         * The write address offset is calculated as "COMP_OFFSET << 2".
         */
        offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1);
        ctl = ((offset << BD_CTL_COMP_OFFSET_SHIFT) & BD_CTL_COMP_OFFSET_MASK) |
                ((q->pi << BD_CTL_COMP_DATA_SHIFT) & BD_CTL_COMP_DATA_MASK);

        len = job->job_cb_size;

        /*
         * A patched CB is created only if a user CB was allocated by driver and
         * MMU is disabled. If MMU is enabled, the user CB should be used
         * instead. If the user CB wasn't allocated by driver, assume that it
         * holds an address.
         */
        if (job->patched_cb)
                ptr = job->patched_cb->bus_address;
        else if (job->is_kernel_allocated_cb)
                ptr = job->user_cb->bus_address;
        else
                ptr = (u64) (uintptr_t) job->user_cb;

        hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
}

static int init_signal_cs(struct hl_device *hdev,
                struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
{
        struct hl_sync_stream_properties *prop;
        struct hl_hw_sob *hw_sob;
        u32 q_idx;
        int rc = 0;

        q_idx = job->hw_queue_id;
        prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
        hw_sob = &prop->hw_sob[prop->curr_sob_offset];

        cs_cmpl->hw_sob = hw_sob;
        cs_cmpl->sob_val = prop->next_sob_val;

        dev_dbg(hdev->dev,
                "generate signal CB, sob_id: %d, sob val: %u, q_idx: %d, seq: %llu\n",
                cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx,
                cs_cmpl->cs_seq);

        /* we set an EB since we must make sure all oeprations are done
         * when sending the signal
         */
        hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,
                                cs_cmpl->hw_sob->sob_id, 0, true);

        rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1,
                                                                false);

        return rc;
}

void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
                        struct hl_cs *cs, struct hl_cs_job *job,
                        struct hl_cs_compl *cs_cmpl)
{
        struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl;
        u32 offset = 0;

        cs_cmpl->hw_sob = handle->hw_sob;

        /* Note that encaps_sig_wait_offset was validated earlier in the flow
         * for offset value which exceeds the max reserved signal count.
         * always decrement 1 of the offset since when the user
         * set offset 1 for example he mean to wait only for the first
         * signal only, which will be pre_sob_val, and if he set offset 2
         * then the value required is (pre_sob_val + 1) and so on...
         * if user set wait offset to 0, then treat it as legacy wait cs,
         * wait for the next signal.
         */
        if (job->encaps_sig_wait_offset)
                offset = job->encaps_sig_wait_offset - 1;

        cs_cmpl->sob_val = handle->pre_sob_val + offset;
}

static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,
                struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
{
        struct hl_gen_wait_properties wait_prop;
        struct hl_sync_stream_properties *prop;
        struct hl_cs_compl *signal_cs_cmpl;
        u32 q_idx;

        q_idx = job->hw_queue_id;
        prop = &hdev->kernel_queues[q_idx].sync_stream_prop;

        signal_cs_cmpl = container_of(cs->signal_fence,
                                        struct hl_cs_compl,
                                        base_fence);

        if (cs->encaps_signals) {
                /* use the encaps signal handle stored earlier in the flow
                 * and set the SOB information from the encaps
                 * signals handle
                 */
                hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl);

                dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, offset: %u\n",
                                cs->encaps_sig_hdl->q_idx,
                                cs->encaps_sig_hdl->cs_seq,
                                cs_cmpl->sob_val,
                                job->encaps_sig_wait_offset);
        } else {
                /* Copy the SOB id and value of the signal CS */
                cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
                cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
        }

        /* check again if the signal cs already completed.
         * if yes then don't send any wait cs since the hw_sob
         * could be in reset already. if signal is not completed
         * then get refcount to hw_sob to prevent resetting the sob
         * while wait cs is not submitted.
         * note that this check is protected by two locks,
         * hw queue lock and completion object lock,
         * and the same completion object lock also protects
         * the hw_sob reset handler function.
         * The hw_queue lock prevent out of sync of hw_sob
         * refcount value, changed by signal/wait flows.
         */
        spin_lock(&signal_cs_cmpl->lock);

        if (completion_done(&cs->signal_fence->completion)) {
                spin_unlock(&signal_cs_cmpl->lock);
                return -EINVAL;
        }

        kref_get(&cs_cmpl->hw_sob->kref);

        spin_unlock(&signal_cs_cmpl->lock);

        dev_dbg(hdev->dev,
                "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d, seq: %llu\n",
                cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
                prop->base_mon_id, q_idx, cs->sequence);

        wait_prop.data = (void *) job->patched_cb;
        wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
        wait_prop.sob_mask = 0x1;
        wait_prop.sob_val = cs_cmpl->sob_val;
        wait_prop.mon_id = prop->base_mon_id;
        wait_prop.q_idx = q_idx;
        wait_prop.size = 0;

        hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);

        mb();
        hl_fence_put(cs->signal_fence);
        cs->signal_fence = NULL;

        return 0;
}

/*
 * init_signal_wait_cs - initialize a signal/wait CS
 * @cs: pointer to the signal/wait CS
 *
 * H/W queues spinlock should be taken before calling this function
 */
static int init_signal_wait_cs(struct hl_cs *cs)
{
        struct hl_ctx *ctx = cs->ctx;
        struct hl_device *hdev = ctx->hdev;
        struct hl_cs_job *job;
        struct hl_cs_compl *cs_cmpl =
                        container_of(cs->fence, struct hl_cs_compl, base_fence);
        int rc = 0;

        /* There is only one job in a signal/wait CS */
        job = list_first_entry(&cs->job_list, struct hl_cs_job,
                                cs_node);

        if (cs->type & CS_TYPE_SIGNAL)
                rc = init_signal_cs(hdev, job, cs_cmpl);
        else if (cs->type & CS_TYPE_WAIT)
                rc = init_wait_cs(hdev, cs, job, cs_cmpl);

        return rc;
}

static int encaps_sig_first_staged_cs_handler
                        (struct hl_device *hdev, struct hl_cs *cs)
{
        struct hl_cs_compl *cs_cmpl =
                        container_of(cs->fence,
                                        struct hl_cs_compl, base_fence);
        struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
        struct hl_encaps_signals_mgr *mgr;
        int rc = 0;

        mgr = &hdev->compute_ctx->sig_mgr;

        spin_lock(&mgr->lock);
        encaps_sig_hdl = idr_find(&mgr->handles, cs->encaps_sig_hdl_id);
        if (encaps_sig_hdl) {
                /*
                 * Set handler CS sequence,
                 * the CS which contains the encapsulated signals.
                 */
                encaps_sig_hdl->cs_seq = cs->sequence;
                /* store the handle and set encaps signal indication,
                 * to be used later in cs_do_release to put the last
                 * reference to encaps signals handlers.
                 */
                cs_cmpl->encaps_signals = true;
                cs_cmpl->encaps_sig_hdl = encaps_sig_hdl;

                /* set hw_sob pointer in completion object
                 * since it's used in cs_do_release flow to put
                 * refcount to sob
                 */
                cs_cmpl->hw_sob = encaps_sig_hdl->hw_sob;
                cs_cmpl->sob_val = encaps_sig_hdl->pre_sob_val +
                                                encaps_sig_hdl->count;

                dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob(%u), val(%u)\n",
                                cs->sequence, encaps_sig_hdl->id,
                                encaps_sig_hdl->count,
                                encaps_sig_hdl->q_idx,
                                cs_cmpl->hw_sob->sob_id,
                                cs_cmpl->sob_val);

        } else {
                dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n",
                                cs->encaps_sig_hdl_id);
                rc = -EINVAL;
        }

        spin_unlock(&mgr->lock);

        return rc;
}

/*
 * hl_hw_queue_schedule_cs - schedule a command submission
 * @cs: pointer to the CS
 */
int hl_hw_queue_schedule_cs(struct hl_cs *cs)
{
        enum hl_device_status status;
        struct hl_cs_counters_atomic *cntr;
        struct hl_ctx *ctx = cs->ctx;
        struct hl_device *hdev = ctx->hdev;
        struct hl_cs_job *job, *tmp;
        struct hl_hw_queue *q;
        int rc = 0, i, cq_cnt;
        bool first_entry;
        u32 max_queues;

        cntr = &hdev->aggregated_cs_counters;

        hdev->asic_funcs->hw_queues_lock(hdev);

        if (!hl_device_operational(hdev, &status)) {
                atomic64_inc(&cntr->device_in_reset_drop_cnt);
                atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt);
                dev_err(hdev->dev,
                        "device is %s, CS rejected!\n", hdev->status[status]);
                rc = -EPERM;
                goto out;
        }

        max_queues = hdev->asic_prop.max_queues;

        q = &hdev->kernel_queues[0];
        for (i = 0, cq_cnt = 0 ; i < max_queues ; i++, q++) {
                if (cs->jobs_in_queue_cnt[i]) {
                        switch (q->queue_type) {
                        case QUEUE_TYPE_EXT:
                                rc = ext_queue_sanity_checks(hdev, q,
                                                cs->jobs_in_queue_cnt[i],
                                                cs_needs_completion(cs) ?
                                                                true : false);
                                break;
                        case QUEUE_TYPE_INT:
                                rc = int_queue_sanity_checks(hdev, q,
                                                cs->jobs_in_queue_cnt[i]);
                                break;
                        case QUEUE_TYPE_HW:
                                rc = hw_queue_sanity_checks(hdev, q,
                                                cs->jobs_in_queue_cnt[i]);
                                break;
                        default:
                                dev_err(hdev->dev, "Queue type %d is invalid\n",
                                        q->queue_type);
                                rc = -EINVAL;
                                break;
                        }

                        if (rc) {
                                atomic64_inc(
                                        &ctx->cs_counters.queue_full_drop_cnt);
                                atomic64_inc(&cntr->queue_full_drop_cnt);
                                goto unroll_cq_resv;
                        }

                        if (q->queue_type == QUEUE_TYPE_EXT)
                                cq_cnt++;
                }
        }

        if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) {
                rc = init_signal_wait_cs(cs);
                if (rc)
                        goto unroll_cq_resv;
        } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) {
                rc = hdev->asic_funcs->collective_wait_init_cs(cs);
                if (rc)
                        goto unroll_cq_resv;
        }


        if (cs->encaps_signals && cs->staged_first) {
                rc = encaps_sig_first_staged_cs_handler(hdev, cs);
                if (rc)
                        goto unroll_cq_resv;
        }

        spin_lock(&hdev->cs_mirror_lock);

        /* Verify staged CS exists and add to the staged list */
        if (cs->staged_cs && !cs->staged_first) {
                struct hl_cs *staged_cs;

                staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);
                if (!staged_cs) {
                        dev_err(hdev->dev,
                                "Cannot find staged submission sequence %llu",
                                cs->staged_sequence);
                        rc = -EINVAL;
                        goto unlock_cs_mirror;
                }

                if (is_staged_cs_last_exists(hdev, staged_cs)) {
                        dev_err(hdev->dev,
                                "Staged submission sequence %llu already submitted",
                                cs->staged_sequence);
                        rc = -EINVAL;
                        goto unlock_cs_mirror;
                }

                list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);

                /* update stream map of the first CS */
                if (hdev->supports_wait_for_multi_cs)
                        staged_cs->fence->stream_master_qid_map |=
                                        cs->fence->stream_master_qid_map;
        }

        list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);

        /* Queue TDR if the CS is the first entry and if timeout is wanted */
        first_entry = list_first_entry(&hdev->cs_mirror_list,
                                        struct hl_cs, mirror_node) == cs;
        if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&
                                first_entry && cs_needs_timeout(cs)) {
                cs->tdr_active = true;
                schedule_delayed_work(&cs->work_tdr, cs->timeout_jiffies);

        }

        spin_unlock(&hdev->cs_mirror_lock);

        list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
                switch (job->queue_type) {
                case QUEUE_TYPE_EXT:
                        ext_queue_schedule_job(job);
                        break;
                case QUEUE_TYPE_INT:
                        int_queue_schedule_job(job);
                        break;
                case QUEUE_TYPE_HW:
                        hw_queue_schedule_job(job);
                        break;
                default:
                        break;
                }

        cs->submitted = true;

        goto out;

unlock_cs_mirror:
        spin_unlock(&hdev->cs_mirror_lock);
unroll_cq_resv:
        q = &hdev->kernel_queues[0];
        for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) {
                if ((q->queue_type == QUEUE_TYPE_EXT) &&
                                                (cs->jobs_in_queue_cnt[i])) {
                        atomic_t *free_slots =
                                &hdev->completion_queue[i].free_slots_cnt;
                        atomic_add(cs->jobs_in_queue_cnt[i], free_slots);
                        cq_cnt--;
                }
        }

out:
        hdev->asic_funcs->hw_queues_unlock(hdev);

        return rc;
}

/*
 * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue
 *
 * @hdev: pointer to hl_device structure
 * @hw_queue_id: which queue to increment its ci
 */
void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)
{
        struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];

        atomic_inc(&q->ci);
}

static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
                                        bool is_cpu_queue)
{
        void *p;
        int rc;

        if (is_cpu_queue)
                p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
                                                        HL_QUEUE_SIZE_IN_BYTES,
                                                        &q->bus_address);
        else
                p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
                                                HL_QUEUE_SIZE_IN_BYTES,
                                                &q->bus_address,
                                                GFP_KERNEL | __GFP_ZERO);
        if (!p)
                return -ENOMEM;

        q->kernel_address = p;

        q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH,
                                        sizeof(*q->shadow_queue),
                                        GFP_KERNEL);
        if (!q->shadow_queue) {
                dev_err(hdev->dev,
                        "Failed to allocate shadow queue for H/W queue %d\n",
                        q->hw_queue_id);
                rc = -ENOMEM;
                goto free_queue;
        }

        /* Make sure read/write pointers are initialized to start of queue */
        atomic_set(&q->ci, 0);
        q->pi = 0;

        return 0;

free_queue:
        if (is_cpu_queue)
                hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
                                        HL_QUEUE_SIZE_IN_BYTES,
                                        q->kernel_address);
        else
                hdev->asic_funcs->asic_dma_free_coherent(hdev,
                                        HL_QUEUE_SIZE_IN_BYTES,
                                        q->kernel_address,
                                        q->bus_address);

        return rc;
}

static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
{
        void *p;

        p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,
                                        &q->bus_address, &q->int_queue_len);
        if (!p) {
                dev_err(hdev->dev,
                        "Failed to get base address for internal queue %d\n",
                        q->hw_queue_id);
                return -EFAULT;
        }

        q->kernel_address = p;
        q->pi = 0;
        atomic_set(&q->ci, 0);

        return 0;
}

static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
{
        return ext_and_cpu_queue_init(hdev, q, true);
}

static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
{
        return ext_and_cpu_queue_init(hdev, q, false);
}

static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
{
        void *p;

        p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
                                                HL_QUEUE_SIZE_IN_BYTES,
                                                &q->bus_address,
                                                GFP_KERNEL | __GFP_ZERO);
        if (!p)
                return -ENOMEM;

        q->kernel_address = p;

        /* Make sure read/write pointers are initialized to start of queue */
        atomic_set(&q->ci, 0);
        q->pi = 0;

        return 0;
}

static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx)
{
        struct hl_sync_stream_properties *sync_stream_prop;
        struct asic_fixed_properties *prop = &hdev->asic_prop;
        struct hl_hw_sob *hw_sob;
        int sob, reserved_mon_idx, queue_idx;

        sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop;

        /* We use 'collective_mon_idx' as a running index in order to reserve
         * monitors for collective master/slave queues.
         * collective master queue gets 2 reserved monitors
         * collective slave queue gets 1 reserved monitor
         */
        if (hdev->kernel_queues[q_idx].collective_mode ==
                        HL_COLLECTIVE_MASTER) {
                reserved_mon_idx = hdev->collective_mon_idx;

                /* reserve the first monitor for collective master queue */
                sync_stream_prop->collective_mstr_mon_id[0] =
                        prop->collective_first_mon + reserved_mon_idx;

                /* reserve the second monitor for collective master queue */
                sync_stream_prop->collective_mstr_mon_id[1] =
                        prop->collective_first_mon + reserved_mon_idx + 1;

                hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS;
        } else if (hdev->kernel_queues[q_idx].collective_mode ==
                        HL_COLLECTIVE_SLAVE) {
                reserved_mon_idx = hdev->collective_mon_idx++;

                /* reserve a monitor for collective slave queue */
                sync_stream_prop->collective_slave_mon_id =
                        prop->collective_first_mon + reserved_mon_idx;
        }

        if (!hdev->kernel_queues[q_idx].supports_sync_stream)
                return;

        queue_idx = hdev->sync_stream_queue_idx++;

        sync_stream_prop->base_sob_id = prop->sync_stream_first_sob +
                        (queue_idx * HL_RSVD_SOBS);
        sync_stream_prop->base_mon_id = prop->sync_stream_first_mon +
                        (queue_idx * HL_RSVD_MONS);
        sync_stream_prop->next_sob_val = 1;
        sync_stream_prop->curr_sob_offset = 0;

        for (sob = 0 ; sob < HL_RSVD_SOBS ; sob++) {
                hw_sob = &sync_stream_prop->hw_sob[sob];
                hw_sob->hdev = hdev;
                hw_sob->sob_id = sync_stream_prop->base_sob_id + sob;
                hw_sob->sob_addr =
                        hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);
                hw_sob->q_idx = q_idx;
                kref_init(&hw_sob->kref);
        }
}

static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx)
{
        struct hl_sync_stream_properties *prop =
                        &hdev->kernel_queues[q_idx].sync_stream_prop;

        /*
         * In case we got here due to a stuck CS, the refcnt might be bigger
         * than 1 and therefore we reset it.
         */
        kref_init(&prop->hw_sob[prop->curr_sob_offset].kref);
        prop->curr_sob_offset = 0;
        prop->next_sob_val = 1;
}

/*
 * queue_init - main initialization function for H/W queue object
 *
 * @hdev: pointer to hl_device device structure
 * @q: pointer to hl_hw_queue queue structure
 * @hw_queue_id: The id of the H/W queue
 *
 * Allocate dma-able memory for the queue and initialize fields
 * Returns 0 on success
 */
static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
                        u32 hw_queue_id)
{
        int rc;

        q->hw_queue_id = hw_queue_id;

        switch (q->queue_type) {
        case QUEUE_TYPE_EXT:
                rc = ext_queue_init(hdev, q);
                break;
        case QUEUE_TYPE_INT:
                rc = int_queue_init(hdev, q);
                break;
        case QUEUE_TYPE_CPU:
                rc = cpu_queue_init(hdev, q);
                break;
        case QUEUE_TYPE_HW:

                rc = hw_queue_init(hdev, q);
                break;
        case QUEUE_TYPE_NA:
                q->valid = 0;
                return 0;
        default:
                dev_crit(hdev->dev, "wrong queue type %d during init\n",
                        q->queue_type);
                rc = -EINVAL;
                break;
        }

        sync_stream_queue_init(hdev, q->hw_queue_id);

        if (rc)
                return rc;

        q->valid = 1;

        return 0;
}

/*
 * hw_queue_fini - destroy queue
 *
 * @hdev: pointer to hl_device device structure
 * @q: pointer to hl_hw_queue queue structure
 *
 * Free the queue memory
 */
static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)
{
        if (!q->valid)
                return;

        /*
         * If we arrived here, there are no jobs waiting on this queue
         * so we can safely remove it.
         * This is because this function can only called when:
         * 1. Either a context is deleted, which only can occur if all its
         *    jobs were finished
         * 2. A context wasn't able to be created due to failure or timeout,
         *    which means there are no jobs on the queue yet
         *
         * The only exception are the queues of the kernel context, but
         * if they are being destroyed, it means that the entire module is
         * being removed. If the module is removed, it means there is no open
         * user context. It also means that if a job was submitted by
         * the kernel driver (e.g. context creation), the job itself was
         * released by the kernel driver when a timeout occurred on its
         * Completion. Thus, we don't need to release it again.
         */

        if (q->queue_type == QUEUE_TYPE_INT)
                return;

        kfree(q->shadow_queue);

        if (q->queue_type == QUEUE_TYPE_CPU)
                hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
                                        HL_QUEUE_SIZE_IN_BYTES,
                                        q->kernel_address);
        else
                hdev->asic_funcs->asic_dma_free_coherent(hdev,
                                        HL_QUEUE_SIZE_IN_BYTES,
                                        q->kernel_address,
                                        q->bus_address);
}

int hl_hw_queues_create(struct hl_device *hdev)
{
        struct asic_fixed_properties *asic = &hdev->asic_prop;
        struct hl_hw_queue *q;
        int i, rc, q_ready_cnt;

        hdev->kernel_queues = kcalloc(asic->max_queues,
                                sizeof(*hdev->kernel_queues), GFP_KERNEL);

        if (!hdev->kernel_queues) {
                dev_err(hdev->dev, "Not enough memory for H/W queues\n");
                return -ENOMEM;
        }

        /* Initialize the H/W queues */
        for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;
                        i < asic->max_queues ; i++, q_ready_cnt++, q++) {

                q->queue_type = asic->hw_queues_props[i].type;
                q->supports_sync_stream =
                                asic->hw_queues_props[i].supports_sync_stream;
                q->collective_mode = asic->hw_queues_props[i].collective_mode;
                rc = queue_init(hdev, q, i);
                if (rc) {
                        dev_err(hdev->dev,
                                "failed to initialize queue %d\n", i);
                        goto release_queues;
                }
        }

        return 0;

release_queues:
        for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)
                queue_fini(hdev, q);

        kfree(hdev->kernel_queues);

        return rc;
}

void hl_hw_queues_destroy(struct hl_device *hdev)
{
        struct hl_hw_queue *q;
        u32 max_queues = hdev->asic_prop.max_queues;
        int i;

        for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)
                queue_fini(hdev, q);

        kfree(hdev->kernel_queues);
}

void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)
{
        struct hl_hw_queue *q;
        u32 max_queues = hdev->asic_prop.max_queues;
        int i;

        for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {
                if ((!q->valid) ||
                        ((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))
                        continue;
                q->pi = 0;
                atomic_set(&q->ci, 0);

                if (q->supports_sync_stream)
                        sync_stream_queue_reset(hdev, q->hw_queue_id);
        }
}