/*
 * Copyright (c) 2005-2011 Atheros Communications Inc.
 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <linux/pci.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/bitops.h>

#include "core.h"
#include "debug.h"

#include "targaddrs.h"
#include "bmi.h"

#include "hif.h"
#include "htc.h"

#include "ce.h"
#include "pci.h"

enum ath10k_pci_irq_mode {
        ATH10K_PCI_IRQ_AUTO = 0,
        ATH10K_PCI_IRQ_LEGACY = 1,
        ATH10K_PCI_IRQ_MSI = 2,
};

enum ath10k_pci_reset_mode {
        ATH10K_PCI_RESET_AUTO = 0,
        ATH10K_PCI_RESET_WARM_ONLY = 1,
};

static unsigned int ath10k_pci_target_ps;
static unsigned int ath10k_pci_irq_mode = ATH10K_PCI_IRQ_AUTO;
static unsigned int ath10k_pci_reset_mode = ATH10K_PCI_RESET_AUTO;

module_param_named(target_ps, ath10k_pci_target_ps, uint, 0644);
MODULE_PARM_DESC(target_ps, "Enable ath10k Target (SoC) PS option");

module_param_named(irq_mode, ath10k_pci_irq_mode, uint, 0644);
MODULE_PARM_DESC(irq_mode, "0: auto, 1: legacy, 2: msi (default: 0)");

module_param_named(reset_mode, ath10k_pci_reset_mode, uint, 0644);
MODULE_PARM_DESC(reset_mode, "0: auto, 1: warm only (default: 0)");

/* how long wait to wait for target to initialise, in ms */
#define ATH10K_PCI_TARGET_WAIT 3000
#define ATH10K_PCI_NUM_WARM_RESET_ATTEMPTS 3

#define QCA988X_2_0_DEVICE_ID   (0x003c)

static DEFINE_PCI_DEVICE_TABLE(ath10k_pci_id_table) = {
        { PCI_VDEVICE(ATHEROS, QCA988X_2_0_DEVICE_ID) }, /* PCI-E QCA988X V2 */
        {0}
};

static int ath10k_pci_diag_read_access(struct ath10k *ar, u32 address,
                                       u32 *data);

static int ath10k_pci_post_rx(struct ath10k *ar);
static int ath10k_pci_post_rx_pipe(struct ath10k_pci_pipe *pipe_info,
                                             int num);
static void ath10k_pci_rx_pipe_cleanup(struct ath10k_pci_pipe *pipe_info);
static int ath10k_pci_cold_reset(struct ath10k *ar);
static int ath10k_pci_warm_reset(struct ath10k *ar);
static int ath10k_pci_wait_for_target_init(struct ath10k *ar);
static int ath10k_pci_init_irq(struct ath10k *ar);
static int ath10k_pci_deinit_irq(struct ath10k *ar);
static int ath10k_pci_request_irq(struct ath10k *ar);
static void ath10k_pci_free_irq(struct ath10k *ar);
static int ath10k_pci_bmi_wait(struct ath10k_ce_pipe *tx_pipe,
                               struct ath10k_ce_pipe *rx_pipe,
                               struct bmi_xfer *xfer);

static const struct ce_attr host_ce_config_wlan[] = {
        /* CE0: host->target HTC control and raw streams */
        {
                .flags = CE_ATTR_FLAGS,
                .src_nentries = 16,
                .src_sz_max = 256,
                .dest_nentries = 0,
        },

        /* CE1: target->host HTT + HTC control */
        {
                .flags = CE_ATTR_FLAGS,
                .src_nentries = 0,
                .src_sz_max = 512,
                .dest_nentries = 512,
        },

        /* CE2: target->host WMI */
        {
                .flags = CE_ATTR_FLAGS,
                .src_nentries = 0,
                .src_sz_max = 2048,
                .dest_nentries = 32,
        },

        /* CE3: host->target WMI */
        {
                .flags = CE_ATTR_FLAGS,
                .src_nentries = 32,
                .src_sz_max = 2048,
                .dest_nentries = 0,
        },

        /* CE4: host->target HTT */
        {
                .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
                .src_nentries = CE_HTT_H2T_MSG_SRC_NENTRIES,
                .src_sz_max = 256,
                .dest_nentries = 0,
        },

        /* CE5: unused */
        {
                .flags = CE_ATTR_FLAGS,
                .src_nentries = 0,
                .src_sz_max = 0,
                .dest_nentries = 0,
        },

        /* CE6: target autonomous hif_memcpy */
        {
                .flags = CE_ATTR_FLAGS,
                .src_nentries = 0,
                .src_sz_max = 0,
                .dest_nentries = 0,
        },

        /* CE7: ce_diag, the Diagnostic Window */
        {
                .flags = CE_ATTR_FLAGS,
                .src_nentries = 2,
                .src_sz_max = DIAG_TRANSFER_LIMIT,
                .dest_nentries = 2,
        },
};

/* Target firmware's Copy Engine configuration. */
static const struct ce_pipe_config target_ce_config_wlan[] = {
        /* CE0: host->target HTC control and raw streams */
        {
                .pipenum = 0,
                .pipedir = PIPEDIR_OUT,
                .nentries = 32,
                .nbytes_max = 256,
                .flags = CE_ATTR_FLAGS,
                .reserved = 0,
        },

        /* CE1: target->host HTT + HTC control */
        {
                .pipenum = 1,
                .pipedir = PIPEDIR_IN,
                .nentries = 32,
                .nbytes_max = 512,
                .flags = CE_ATTR_FLAGS,
                .reserved = 0,
        },

        /* CE2: target->host WMI */
        {
                .pipenum = 2,
                .pipedir = PIPEDIR_IN,
                .nentries = 32,
                .nbytes_max = 2048,
                .flags = CE_ATTR_FLAGS,
                .reserved = 0,
        },

        /* CE3: host->target WMI */
        {
                .pipenum = 3,
                .pipedir = PIPEDIR_OUT,
                .nentries = 32,
                .nbytes_max = 2048,
                .flags = CE_ATTR_FLAGS,
                .reserved = 0,
        },

        /* CE4: host->target HTT */
        {
                .pipenum = 4,
                .pipedir = PIPEDIR_OUT,
                .nentries = 256,
                .nbytes_max = 256,
                .flags = CE_ATTR_FLAGS,
                .reserved = 0,
        },

        /* NB: 50% of src nentries, since tx has 2 frags */

        /* CE5: unused */
        {
                .pipenum = 5,
                .pipedir = PIPEDIR_OUT,
                .nentries = 32,
                .nbytes_max = 2048,
                .flags = CE_ATTR_FLAGS,
                .reserved = 0,
        },

        /* CE6: Reserved for target autonomous hif_memcpy */
        {
                .pipenum = 6,
                .pipedir = PIPEDIR_INOUT,
                .nentries = 32,
                .nbytes_max = 4096,
                .flags = CE_ATTR_FLAGS,
                .reserved = 0,
        },

        /* CE7 used only by Host */
};

static bool ath10k_pci_irq_pending(struct ath10k *ar)
{
        u32 cause;

        /* Check if the shared legacy irq is for us */
        cause = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
                                  PCIE_INTR_CAUSE_ADDRESS);
        if (cause & (PCIE_INTR_FIRMWARE_MASK | PCIE_INTR_CE_MASK_ALL))
                return true;

        return false;
}

static void ath10k_pci_disable_and_clear_legacy_irq(struct ath10k *ar)
{
        /* IMPORTANT: INTR_CLR register has to be set after
         * INTR_ENABLE is set to 0, otherwise interrupt can not be
         * really cleared. */
        ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS + PCIE_INTR_ENABLE_ADDRESS,
                           0);
        ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS + PCIE_INTR_CLR_ADDRESS,
                           PCIE_INTR_FIRMWARE_MASK | PCIE_INTR_CE_MASK_ALL);

        /* IMPORTANT: this extra read transaction is required to
         * flush the posted write buffer. */
        (void) ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
                                 PCIE_INTR_ENABLE_ADDRESS);
}

static void ath10k_pci_enable_legacy_irq(struct ath10k *ar)
{
        ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS +
                           PCIE_INTR_ENABLE_ADDRESS,
                           PCIE_INTR_FIRMWARE_MASK | PCIE_INTR_CE_MASK_ALL);

        /* IMPORTANT: this extra read transaction is required to
         * flush the posted write buffer. */
        (void) ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
                                 PCIE_INTR_ENABLE_ADDRESS);
}

static irqreturn_t ath10k_pci_early_irq_handler(int irq, void *arg)
{
        struct ath10k *ar = arg;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        if (ar_pci->num_msi_intrs == 0) {
                if (!ath10k_pci_irq_pending(ar))
                        return IRQ_NONE;

                ath10k_pci_disable_and_clear_legacy_irq(ar);
        }

        tasklet_schedule(&ar_pci->early_irq_tasklet);

        return IRQ_HANDLED;
}

static int ath10k_pci_request_early_irq(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret;

        /* Regardless whether MSI-X/MSI/legacy irqs have been set up the first
         * interrupt from irq vector is triggered in all cases for FW
         * indication/errors */
        ret = request_irq(ar_pci->pdev->irq, ath10k_pci_early_irq_handler,
                          IRQF_SHARED, "ath10k_pci (early)", ar);
        if (ret) {
                ath10k_warn("failed to request early irq: %d\n", ret);
                return ret;
        }

        return 0;
}

static void ath10k_pci_free_early_irq(struct ath10k *ar)
{
        free_irq(ath10k_pci_priv(ar)->pdev->irq, ar);
}

/*
 * Diagnostic read/write access is provided for startup/config/debug usage.
 * Caller must guarantee proper alignment, when applicable, and single user
 * at any moment.
 */
static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
                                    int nbytes)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret = 0;
        u32 buf;
        unsigned int completed_nbytes, orig_nbytes, remaining_bytes;
        unsigned int id;
        unsigned int flags;
        struct ath10k_ce_pipe *ce_diag;
        /* Host buffer address in CE space */
        u32 ce_data;
        dma_addr_t ce_data_base = 0;
        void *data_buf = NULL;
        int i;

        /*
         * This code cannot handle reads to non-memory space. Redirect to the
         * register read fn but preserve the multi word read capability of
         * this fn
         */
        if (address < DRAM_BASE_ADDRESS) {
                if (!IS_ALIGNED(address, 4) ||
                    !IS_ALIGNED((unsigned long)data, 4))
                        return -EIO;

                while ((nbytes >= 4) &&  ((ret = ath10k_pci_diag_read_access(
                                           ar, address, (u32 *)data)) == 0)) {
                        nbytes -= sizeof(u32);
                        address += sizeof(u32);
                        data += sizeof(u32);
                }
                return ret;
        }

        ce_diag = ar_pci->ce_diag;

        /*
         * Allocate a temporary bounce buffer to hold caller's data
         * to be DMA'ed from Target. This guarantees
         *   1) 4-byte alignment
         *   2) Buffer in DMA-able space
         */
        orig_nbytes = nbytes;
        data_buf = (unsigned char *)dma_alloc_coherent(ar->dev,
                                                       orig_nbytes,
                                                       &ce_data_base,
                                                       GFP_ATOMIC);

        if (!data_buf) {
                ret = -ENOMEM;
                goto done;
        }
        memset(data_buf, 0, orig_nbytes);

        remaining_bytes = orig_nbytes;
        ce_data = ce_data_base;
        while (remaining_bytes) {
                nbytes = min_t(unsigned int, remaining_bytes,
                               DIAG_TRANSFER_LIMIT);

                ret = ath10k_ce_recv_buf_enqueue(ce_diag, NULL, ce_data);
                if (ret != 0)
                        goto done;

                /* Request CE to send from Target(!) address to Host buffer */
                /*
                 * The address supplied by the caller is in the
                 * Target CPU virtual address space.
                 *
                 * In order to use this address with the diagnostic CE,
                 * convert it from Target CPU virtual address space
                 * to CE address space
                 */
                ath10k_pci_wake(ar);
                address = TARG_CPU_SPACE_TO_CE_SPACE(ar, ar_pci->mem,
                                                     address);
                ath10k_pci_sleep(ar);

                ret = ath10k_ce_send(ce_diag, NULL, (u32)address, nbytes, 0,
                                 0);
                if (ret)
                        goto done;

                i = 0;
                while (ath10k_ce_completed_send_next(ce_diag, NULL, &buf,
                                                     &completed_nbytes,
                                                     &id) != 0) {
                        mdelay(1);
                        if (i++ > DIAG_ACCESS_CE_TIMEOUT_MS) {
                                ret = -EBUSY;
                                goto done;
                        }
                }

                if (nbytes != completed_nbytes) {
                        ret = -EIO;
                        goto done;
                }

                if (buf != (u32) address) {
                        ret = -EIO;
                        goto done;
                }

                i = 0;
                while (ath10k_ce_completed_recv_next(ce_diag, NULL, &buf,
                                                     &completed_nbytes,
                                                     &id, &flags) != 0) {
                        mdelay(1);

                        if (i++ > DIAG_ACCESS_CE_TIMEOUT_MS) {
                                ret = -EBUSY;
                                goto done;
                        }
                }

                if (nbytes != completed_nbytes) {
                        ret = -EIO;
                        goto done;
                }

                if (buf != ce_data) {
                        ret = -EIO;
                        goto done;
                }

                remaining_bytes -= nbytes;
                address += nbytes;
                ce_data += nbytes;
        }

done:
        if (ret == 0) {
                /* Copy data from allocated DMA buf to caller's buf */
                WARN_ON_ONCE(orig_nbytes & 3);
                for (i = 0; i < orig_nbytes / sizeof(__le32); i++) {
                        ((u32 *)data)[i] =
                                __le32_to_cpu(((__le32 *)data_buf)[i]);
                }
        } else
                ath10k_warn("failed to read diag value at 0x%x: %d\n",
                            address, ret);

        if (data_buf)
                dma_free_coherent(ar->dev, orig_nbytes, data_buf,
                                  ce_data_base);

        return ret;
}

/* Read 4-byte aligned data from Target memory or register */
static int ath10k_pci_diag_read_access(struct ath10k *ar, u32 address,
                                       u32 *data)
{
        /* Assume range doesn't cross this boundary */
        if (address >= DRAM_BASE_ADDRESS)
                return ath10k_pci_diag_read_mem(ar, address, data, sizeof(u32));

        ath10k_pci_wake(ar);
        *data = ath10k_pci_read32(ar, address);
        ath10k_pci_sleep(ar);
        return 0;
}

static int ath10k_pci_diag_write_mem(struct ath10k *ar, u32 address,
                                     const void *data, int nbytes)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret = 0;
        u32 buf;
        unsigned int completed_nbytes, orig_nbytes, remaining_bytes;
        unsigned int id;
        unsigned int flags;
        struct ath10k_ce_pipe *ce_diag;
        void *data_buf = NULL;
        u32 ce_data;    /* Host buffer address in CE space */
        dma_addr_t ce_data_base = 0;
        int i;

        ce_diag = ar_pci->ce_diag;

        /*
         * Allocate a temporary bounce buffer to hold caller's data
         * to be DMA'ed to Target. This guarantees
         *   1) 4-byte alignment
         *   2) Buffer in DMA-able space
         */
        orig_nbytes = nbytes;
        data_buf = (unsigned char *)dma_alloc_coherent(ar->dev,
                                                       orig_nbytes,
                                                       &ce_data_base,
                                                       GFP_ATOMIC);
        if (!data_buf) {
                ret = -ENOMEM;
                goto done;
        }

        /* Copy caller's data to allocated DMA buf */
        WARN_ON_ONCE(orig_nbytes & 3);
        for (i = 0; i < orig_nbytes / sizeof(__le32); i++)
                ((__le32 *)data_buf)[i] = __cpu_to_le32(((u32 *)data)[i]);

        /*
         * The address supplied by the caller is in the
         * Target CPU virtual address space.
         *
         * In order to use this address with the diagnostic CE,
         * convert it from
         *    Target CPU virtual address space
         * to
         *    CE address space
         */
        ath10k_pci_wake(ar);
        address = TARG_CPU_SPACE_TO_CE_SPACE(ar, ar_pci->mem, address);
        ath10k_pci_sleep(ar);

        remaining_bytes = orig_nbytes;
        ce_data = ce_data_base;
        while (remaining_bytes) {
                /* FIXME: check cast */
                nbytes = min_t(int, remaining_bytes, DIAG_TRANSFER_LIMIT);

                /* Set up to receive directly into Target(!) address */
                ret = ath10k_ce_recv_buf_enqueue(ce_diag, NULL, address);
                if (ret != 0)
                        goto done;

                /*
                 * Request CE to send caller-supplied data that
                 * was copied to bounce buffer to Target(!) address.
                 */
                ret = ath10k_ce_send(ce_diag, NULL, (u32) ce_data,
                                     nbytes, 0, 0);
                if (ret != 0)
                        goto done;

                i = 0;
                while (ath10k_ce_completed_send_next(ce_diag, NULL, &buf,
                                                     &completed_nbytes,
                                                     &id) != 0) {
                        mdelay(1);

                        if (i++ > DIAG_ACCESS_CE_TIMEOUT_MS) {
                                ret = -EBUSY;
                                goto done;
                        }
                }

                if (nbytes != completed_nbytes) {
                        ret = -EIO;
                        goto done;
                }

                if (buf != ce_data) {
                        ret = -EIO;
                        goto done;
                }

                i = 0;
                while (ath10k_ce_completed_recv_next(ce_diag, NULL, &buf,
                                                     &completed_nbytes,
                                                     &id, &flags) != 0) {
                        mdelay(1);

                        if (i++ > DIAG_ACCESS_CE_TIMEOUT_MS) {
                                ret = -EBUSY;
                                goto done;
                        }
                }

                if (nbytes != completed_nbytes) {
                        ret = -EIO;
                        goto done;
                }

                if (buf != address) {
                        ret = -EIO;
                        goto done;
                }

                remaining_bytes -= nbytes;
                address += nbytes;
                ce_data += nbytes;
        }

done:
        if (data_buf) {
                dma_free_coherent(ar->dev, orig_nbytes, data_buf,
                                  ce_data_base);
        }

        if (ret != 0)
                ath10k_warn("failed to write diag value at 0x%x: %d\n",
                            address, ret);

        return ret;
}

/* Write 4B data to Target memory or register */
static int ath10k_pci_diag_write_access(struct ath10k *ar, u32 address,
                                        u32 data)
{
        /* Assume range doesn't cross this boundary */
        if (address >= DRAM_BASE_ADDRESS)
                return ath10k_pci_diag_write_mem(ar, address, &data,
                                                 sizeof(u32));

        ath10k_pci_wake(ar);
        ath10k_pci_write32(ar, address, data);
        ath10k_pci_sleep(ar);
        return 0;
}

static bool ath10k_pci_target_is_awake(struct ath10k *ar)
{
        void __iomem *mem = ath10k_pci_priv(ar)->mem;
        u32 val;
        val = ioread32(mem + PCIE_LOCAL_BASE_ADDRESS +
                       RTC_STATE_ADDRESS);
        return (RTC_STATE_V_GET(val) == RTC_STATE_V_ON);
}

int ath10k_do_pci_wake(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        void __iomem *pci_addr = ar_pci->mem;
        int tot_delay = 0;
        int curr_delay = 5;

        if (atomic_read(&ar_pci->keep_awake_count) == 0) {
                /* Force AWAKE */
                iowrite32(PCIE_SOC_WAKE_V_MASK,
                          pci_addr + PCIE_LOCAL_BASE_ADDRESS +
                          PCIE_SOC_WAKE_ADDRESS);
        }
        atomic_inc(&ar_pci->keep_awake_count);

        if (ar_pci->verified_awake)
                return 0;

        for (;;) {
                if (ath10k_pci_target_is_awake(ar)) {
                        ar_pci->verified_awake = true;
                        return 0;
                }

                if (tot_delay > PCIE_WAKE_TIMEOUT) {
                        ath10k_warn("target took longer %d us to wake up (awake count %d)\n",
                                    PCIE_WAKE_TIMEOUT,
                                    atomic_read(&ar_pci->keep_awake_count));
                        return -ETIMEDOUT;
                }

                udelay(curr_delay);
                tot_delay += curr_delay;

                if (curr_delay < 50)
                        curr_delay += 5;
        }
}

void ath10k_do_pci_sleep(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        void __iomem *pci_addr = ar_pci->mem;

        if (atomic_dec_and_test(&ar_pci->keep_awake_count)) {
                /* Allow sleep */
                ar_pci->verified_awake = false;
                iowrite32(PCIE_SOC_WAKE_RESET,
                          pci_addr + PCIE_LOCAL_BASE_ADDRESS +
                          PCIE_SOC_WAKE_ADDRESS);
        }
}

/* Called by lower (CE) layer when a send to Target completes. */
static void ath10k_pci_ce_send_done(struct ath10k_ce_pipe *ce_state)
{
        struct ath10k *ar = ce_state->ar;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct ath10k_hif_cb *cb = &ar_pci->msg_callbacks_current;
        void *transfer_context;
        u32 ce_data;
        unsigned int nbytes;
        unsigned int transfer_id;

        while (ath10k_ce_completed_send_next(ce_state, &transfer_context,
                                             &ce_data, &nbytes,
                                             &transfer_id) == 0) {
                /* no need to call tx completion for NULL pointers */
                if (transfer_context == NULL)
                        continue;

                cb->tx_completion(ar, transfer_context, transfer_id);
        }
}

/* Called by lower (CE) layer when data is received from the Target. */
static void ath10k_pci_ce_recv_data(struct ath10k_ce_pipe *ce_state)
{
        struct ath10k *ar = ce_state->ar;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct ath10k_pci_pipe *pipe_info =  &ar_pci->pipe_info[ce_state->id];
        struct ath10k_hif_cb *cb = &ar_pci->msg_callbacks_current;
        struct sk_buff *skb;
        void *transfer_context;
        u32 ce_data;
        unsigned int nbytes, max_nbytes;
        unsigned int transfer_id;
        unsigned int flags;
        int err;

        while (ath10k_ce_completed_recv_next(ce_state, &transfer_context,
                                             &ce_data, &nbytes, &transfer_id,
                                             &flags) == 0) {
                err = ath10k_pci_post_rx_pipe(pipe_info, 1);
                if (unlikely(err)) {
                        /* FIXME: retry */
                        ath10k_warn("failed to replenish CE rx ring %d: %d\n",
                                    pipe_info->pipe_num, err);
                }

                skb = transfer_context;
                max_nbytes = skb->len + skb_tailroom(skb);
                dma_unmap_single(ar->dev, ATH10K_SKB_CB(skb)->paddr,
                                 max_nbytes, DMA_FROM_DEVICE);

                if (unlikely(max_nbytes < nbytes)) {
                        ath10k_warn("rxed more than expected (nbytes %d, max %d)",
                                    nbytes, max_nbytes);
                        dev_kfree_skb_any(skb);
                        continue;
                }

                skb_put(skb, nbytes);
                cb->rx_completion(ar, skb, pipe_info->pipe_num);
        }
}

static int ath10k_pci_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
                                struct ath10k_hif_sg_item *items, int n_items)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct ath10k_pci_pipe *pci_pipe = &ar_pci->pipe_info[pipe_id];
        struct ath10k_ce_pipe *ce_pipe = pci_pipe->ce_hdl;
        struct ath10k_ce_ring *src_ring = ce_pipe->src_ring;
        unsigned int nentries_mask;
        unsigned int sw_index;
        unsigned int write_index;
        int err, i = 0;

        spin_lock_bh(&ar_pci->ce_lock);

        nentries_mask = src_ring->nentries_mask;
        sw_index = src_ring->sw_index;
        write_index = src_ring->write_index;

        if (unlikely(CE_RING_DELTA(nentries_mask,
                                   write_index, sw_index - 1) < n_items)) {
                err = -ENOBUFS;
                goto err;
        }

        for (i = 0; i < n_items - 1; i++) {
                ath10k_dbg(ATH10K_DBG_PCI,
                           "pci tx item %d paddr 0x%08x len %d n_items %d\n",
                           i, items[i].paddr, items[i].len, n_items);
                ath10k_dbg_dump(ATH10K_DBG_PCI_DUMP, NULL, "item data: ",
                                items[i].vaddr, items[i].len);

                err = ath10k_ce_send_nolock(ce_pipe,
                                            items[i].transfer_context,
                                            items[i].paddr,
                                            items[i].len,
                                            items[i].transfer_id,
                                            CE_SEND_FLAG_GATHER);
                if (err)
                        goto err;
        }

        /* `i` is equal to `n_items -1` after for() */

        ath10k_dbg(ATH10K_DBG_PCI,
                   "pci tx item %d paddr 0x%08x len %d n_items %d\n",
                   i, items[i].paddr, items[i].len, n_items);
        ath10k_dbg_dump(ATH10K_DBG_PCI_DUMP, NULL, "item data: ",
                        items[i].vaddr, items[i].len);

        err = ath10k_ce_send_nolock(ce_pipe,
                                    items[i].transfer_context,
                                    items[i].paddr,
                                    items[i].len,
                                    items[i].transfer_id,
                                    0);
        if (err)
                goto err;

        spin_unlock_bh(&ar_pci->ce_lock);
        return 0;

err:
        for (; i > 0; i--)
                __ath10k_ce_send_revert(ce_pipe);

        spin_unlock_bh(&ar_pci->ce_lock);
        return err;
}

static u16 ath10k_pci_hif_get_free_queue_number(struct ath10k *ar, u8 pipe)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        ath10k_dbg(ATH10K_DBG_PCI, "pci hif get free queue number\n");

        return ath10k_ce_num_free_src_entries(ar_pci->pipe_info[pipe].ce_hdl);
}

static void ath10k_pci_hif_dump_area(struct ath10k *ar)
{
        u32 reg_dump_area = 0;
        u32 reg_dump_values[REG_DUMP_COUNT_QCA988X] = {};
        u32 host_addr;
        int ret;
        u32 i;

        ath10k_err("firmware crashed!\n");
        ath10k_err("hardware name %s version 0x%x\n",
                   ar->hw_params.name, ar->target_version);
        ath10k_err("firmware version: %s\n", ar->hw->wiphy->fw_version);

        host_addr = host_interest_item_address(HI_ITEM(hi_failure_state));
        ret = ath10k_pci_diag_read_mem(ar, host_addr,
                                       &reg_dump_area, sizeof(u32));
        if (ret) {
                ath10k_err("failed to read FW dump area address: %d\n", ret);
                return;
        }

        ath10k_err("target register Dump Location: 0x%08X\n", reg_dump_area);

        ret = ath10k_pci_diag_read_mem(ar, reg_dump_area,
                                       &reg_dump_values[0],
                                       REG_DUMP_COUNT_QCA988X * sizeof(u32));
        if (ret != 0) {
                ath10k_err("failed to read FW dump area: %d\n", ret);
                return;
        }

        BUILD_BUG_ON(REG_DUMP_COUNT_QCA988X % 4);

        ath10k_err("target Register Dump\n");
        for (i = 0; i < REG_DUMP_COUNT_QCA988X; i += 4)
                ath10k_err("[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X\n",
                           i,
                           reg_dump_values[i],
                           reg_dump_values[i + 1],
                           reg_dump_values[i + 2],
                           reg_dump_values[i + 3]);

        queue_work(ar->workqueue, &ar->restart_work);
}

static void ath10k_pci_hif_send_complete_check(struct ath10k *ar, u8 pipe,
                                               int force)
{
        ath10k_dbg(ATH10K_DBG_PCI, "pci hif send complete check\n");

        if (!force) {
                int resources;
                /*
                 * Decide whether to actually poll for completions, or just
                 * wait for a later chance.
                 * If there seem to be plenty of resources left, then just wait
                 * since checking involves reading a CE register, which is a
                 * relatively expensive operation.
                 */
                resources = ath10k_pci_hif_get_free_queue_number(ar, pipe);

                /*
                 * If at least 50% of the total resources are still available,
                 * don't bother checking again yet.
                 */
                if (resources > (host_ce_config_wlan[pipe].src_nentries >> 1))
                        return;
        }
        ath10k_ce_per_engine_service(ar, pipe);
}

static void ath10k_pci_hif_set_callbacks(struct ath10k *ar,
                                         struct ath10k_hif_cb *callbacks)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        ath10k_dbg(ATH10K_DBG_PCI, "pci hif set callbacks\n");

        memcpy(&ar_pci->msg_callbacks_current, callbacks,
               sizeof(ar_pci->msg_callbacks_current));
}

static int ath10k_pci_setup_ce_irq(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        const struct ce_attr *attr;
        struct ath10k_pci_pipe *pipe_info;
        int pipe_num, disable_interrupts;

        for (pipe_num = 0; pipe_num < CE_COUNT; pipe_num++) {
                pipe_info = &ar_pci->pipe_info[pipe_num];

                /* Handle Diagnostic CE specially */
                if (pipe_info->ce_hdl == ar_pci->ce_diag)
                        continue;

                attr = &host_ce_config_wlan[pipe_num];

                if (attr->src_nentries) {
                        disable_interrupts = attr->flags & CE_ATTR_DIS_INTR;
                        ath10k_ce_send_cb_register(pipe_info->ce_hdl,
                                                   ath10k_pci_ce_send_done,
                                                   disable_interrupts);
                }

                if (attr->dest_nentries)
                        ath10k_ce_recv_cb_register(pipe_info->ce_hdl,
                                                   ath10k_pci_ce_recv_data);
        }

        return 0;
}

static void ath10k_pci_kill_tasklet(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int i;

        tasklet_kill(&ar_pci->intr_tq);
        tasklet_kill(&ar_pci->msi_fw_err);
        tasklet_kill(&ar_pci->early_irq_tasklet);

        for (i = 0; i < CE_COUNT; i++)
                tasklet_kill(&ar_pci->pipe_info[i].intr);
}

/* TODO - temporary mapping while we have too few CE's */
static int ath10k_pci_hif_map_service_to_pipe(struct ath10k *ar,
                                              u16 service_id, u8 *ul_pipe,
                                              u8 *dl_pipe, int *ul_is_polled,
                                              int *dl_is_polled)
{
        int ret = 0;

        ath10k_dbg(ATH10K_DBG_PCI, "pci hif map service\n");

        /* polling for received messages not supported */
        *dl_is_polled = 0;

        switch (service_id) {
        case ATH10K_HTC_SVC_ID_HTT_DATA_MSG:
                /*
                 * Host->target HTT gets its own pipe, so it can be polled
                 * while other pipes are interrupt driven.
                 */
                *ul_pipe = 4;
                /*
                 * Use the same target->host pipe for HTC ctrl, HTC raw
                 * streams, and HTT.
                 */
                *dl_pipe = 1;
                break;

        case ATH10K_HTC_SVC_ID_RSVD_CTRL:
        case ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS:
                /*
                 * Note: HTC_RAW_STREAMS_SVC is currently unused, and
                 * HTC_CTRL_RSVD_SVC could share the same pipe as the
                 * WMI services.  So, if another CE is needed, change
                 * this to *ul_pipe = 3, which frees up CE 0.
                 */
                /* *ul_pipe = 3; */
                *ul_pipe = 0;
                *dl_pipe = 1;
                break;

        case ATH10K_HTC_SVC_ID_WMI_DATA_BK:
        case ATH10K_HTC_SVC_ID_WMI_DATA_BE:
        case ATH10K_HTC_SVC_ID_WMI_DATA_VI:
        case ATH10K_HTC_SVC_ID_WMI_DATA_VO:

        case ATH10K_HTC_SVC_ID_WMI_CONTROL:
                *ul_pipe = 3;
                *dl_pipe = 2;
                break;

                /* pipe 5 unused   */
                /* pipe 6 reserved */
                /* pipe 7 reserved */

        default:
                ret = -1;
                break;
        }
        *ul_is_polled =
                (host_ce_config_wlan[*ul_pipe].flags & CE_ATTR_DIS_INTR) != 0;

        return ret;
}

static void ath10k_pci_hif_get_default_pipe(struct ath10k *ar,
                                                u8 *ul_pipe, u8 *dl_pipe)
{
        int ul_is_polled, dl_is_polled;

        ath10k_dbg(ATH10K_DBG_PCI, "pci hif get default pipe\n");

        (void)ath10k_pci_hif_map_service_to_pipe(ar,
                                                 ATH10K_HTC_SVC_ID_RSVD_CTRL,
                                                 ul_pipe,
                                                 dl_pipe,
                                                 &ul_is_polled,
                                                 &dl_is_polled);
}

static int ath10k_pci_post_rx_pipe(struct ath10k_pci_pipe *pipe_info,
                                   int num)
{
        struct ath10k *ar = pipe_info->hif_ce_state;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct ath10k_ce_pipe *ce_state = pipe_info->ce_hdl;
        struct sk_buff *skb;
        dma_addr_t ce_data;
        int i, ret = 0;

        if (pipe_info->buf_sz == 0)
                return 0;

        for (i = 0; i < num; i++) {
                skb = dev_alloc_skb(pipe_info->buf_sz);
                if (!skb) {
                        ath10k_warn("failed to allocate skbuff for pipe %d\n",
                                    num);
                        ret = -ENOMEM;
                        goto err;
                }

                WARN_ONCE((unsigned long)skb->data & 3, "unaligned skb");

                ce_data = dma_map_single(ar->dev, skb->data,
                                         skb->len + skb_tailroom(skb),
                                         DMA_FROM_DEVICE);

                if (unlikely(dma_mapping_error(ar->dev, ce_data))) {
                        ath10k_warn("failed to DMA map sk_buff\n");
                        dev_kfree_skb_any(skb);
                        ret = -EIO;
                        goto err;
                }

                ATH10K_SKB_CB(skb)->paddr = ce_data;

                pci_dma_sync_single_for_device(ar_pci->pdev, ce_data,
                                               pipe_info->buf_sz,
                                               PCI_DMA_FROMDEVICE);

                ret = ath10k_ce_recv_buf_enqueue(ce_state, (void *)skb,
                                                 ce_data);
                if (ret) {
                        ath10k_warn("failed to enqueue to pipe %d: %d\n",
                                    num, ret);
                        goto err;
                }
        }

        return ret;

err:
        ath10k_pci_rx_pipe_cleanup(pipe_info);
        return ret;
}

static int ath10k_pci_post_rx(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct ath10k_pci_pipe *pipe_info;
        const struct ce_attr *attr;
        int pipe_num, ret = 0;

        for (pipe_num = 0; pipe_num < CE_COUNT; pipe_num++) {
                pipe_info = &ar_pci->pipe_info[pipe_num];
                attr = &host_ce_config_wlan[pipe_num];

                if (attr->dest_nentries == 0)
                        continue;

                ret = ath10k_pci_post_rx_pipe(pipe_info,
                                              attr->dest_nentries - 1);
                if (ret) {
                        ath10k_warn("failed to post RX buffer for pipe %d: %d\n",
                                    pipe_num, ret);

                        for (; pipe_num >= 0; pipe_num--) {
                                pipe_info = &ar_pci->pipe_info[pipe_num];
                                ath10k_pci_rx_pipe_cleanup(pipe_info);
                        }
                        return ret;
                }
        }

        return 0;
}

static int ath10k_pci_hif_start(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret, ret_early;

        ath10k_dbg(ATH10K_DBG_BOOT, "boot hif start\n");

        ath10k_pci_free_early_irq(ar);
        ath10k_pci_kill_tasklet(ar);

        ret = ath10k_pci_request_irq(ar);
        if (ret) {
                ath10k_warn("failed to post RX buffers for all pipes: %d\n",
                            ret);
                goto err_early_irq;
        }

        ret = ath10k_pci_setup_ce_irq(ar);
        if (ret) {
                ath10k_warn("failed to setup CE interrupts: %d\n", ret);
                goto err_stop;
        }

        /* Post buffers once to start things off. */
        ret = ath10k_pci_post_rx(ar);
        if (ret) {
                ath10k_warn("failed to post RX buffers for all pipes: %d\n",
                            ret);
                goto err_stop;
        }

        ar_pci->started = 1;
        return 0;

err_stop:
        ath10k_ce_disable_interrupts(ar);
        ath10k_pci_free_irq(ar);
        ath10k_pci_kill_tasklet(ar);
err_early_irq:
        /* Though there should be no interrupts (device was reset)
         * power_down() expects the early IRQ to be installed as per the
         * driver lifecycle. */
        ret_early = ath10k_pci_request_early_irq(ar);
        if (ret_early)
                ath10k_warn("failed to re-enable early irq: %d\n", ret_early);

        return ret;
}

static void ath10k_pci_rx_pipe_cleanup(struct ath10k_pci_pipe *pipe_info)
{
        struct ath10k *ar;
        struct ath10k_pci *ar_pci;
        struct ath10k_ce_pipe *ce_hdl;
        u32 buf_sz;
        struct sk_buff *netbuf;
        u32 ce_data;

        buf_sz = pipe_info->buf_sz;

        /* Unused Copy Engine */
        if (buf_sz == 0)
                return;

        ar = pipe_info->hif_ce_state;
        ar_pci = ath10k_pci_priv(ar);

        if (!ar_pci->started)
                return;

        ce_hdl = pipe_info->ce_hdl;

        while (ath10k_ce_revoke_recv_next(ce_hdl, (void **)&netbuf,
                                          &ce_data) == 0) {
                dma_unmap_single(ar->dev, ATH10K_SKB_CB(netbuf)->paddr,
                                 netbuf->len + skb_tailroom(netbuf),
                                 DMA_FROM_DEVICE);
                dev_kfree_skb_any(netbuf);
        }
}

static void ath10k_pci_tx_pipe_cleanup(struct ath10k_pci_pipe *pipe_info)
{
        struct ath10k *ar;
        struct ath10k_pci *ar_pci;
        struct ath10k_ce_pipe *ce_hdl;
        struct sk_buff *netbuf;
        u32 ce_data;
        unsigned int nbytes;
        unsigned int id;
        u32 buf_sz;

        buf_sz = pipe_info->buf_sz;

        /* Unused Copy Engine */
        if (buf_sz == 0)
                return;

        ar = pipe_info->hif_ce_state;
        ar_pci = ath10k_pci_priv(ar);

        if (!ar_pci->started)
                return;

        ce_hdl = pipe_info->ce_hdl;

        while (ath10k_ce_cancel_send_next(ce_hdl, (void **)&netbuf,
                                          &ce_data, &nbytes, &id) == 0) {
                /* no need to call tx completion for NULL pointers */
                if (!netbuf)
                        continue;

                ar_pci->msg_callbacks_current.tx_completion(ar,
                                                            netbuf,
                                                            id);
        }
}

/*
 * Cleanup residual buffers for device shutdown:
 *    buffers that were enqueued for receive
 *    buffers that were to be sent
 * Note: Buffers that had completed but which were
 * not yet processed are on a completion queue. They
 * are handled when the completion thread shuts down.
 */
static void ath10k_pci_buffer_cleanup(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int pipe_num;

        for (pipe_num = 0; pipe_num < CE_COUNT; pipe_num++) {
                struct ath10k_pci_pipe *pipe_info;

                pipe_info = &ar_pci->pipe_info[pipe_num];
                ath10k_pci_rx_pipe_cleanup(pipe_info);
                ath10k_pci_tx_pipe_cleanup(pipe_info);
        }
}

static void ath10k_pci_ce_deinit(struct ath10k *ar)
{
        int i;

        for (i = 0; i < CE_COUNT; i++)
                ath10k_ce_deinit_pipe(ar, i);
}

static void ath10k_pci_hif_stop(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret;

        ath10k_dbg(ATH10K_DBG_BOOT, "boot hif stop\n");

        if (WARN_ON(!ar_pci->started))
                return;

        ret = ath10k_ce_disable_interrupts(ar);
        if (ret)
                ath10k_warn("failed to disable CE interrupts: %d\n", ret);

        ath10k_pci_free_irq(ar);
        ath10k_pci_kill_tasklet(ar);

        ret = ath10k_pci_request_early_irq(ar);
        if (ret)
                ath10k_warn("failed to re-enable early irq: %d\n", ret);

        /* At this point, asynchronous threads are stopped, the target should
         * not DMA nor interrupt. We process the leftovers and then free
         * everything else up. */

        ath10k_pci_buffer_cleanup(ar);

        /* Make the sure the device won't access any structures on the host by
         * resetting it. The device was fed with PCI CE ringbuffer
         * configuration during init. If ringbuffers are freed and the device
         * were to access them this could lead to memory corruption on the
         * host. */
        ath10k_pci_warm_reset(ar);

        ar_pci->started = 0;
}

static int ath10k_pci_hif_exchange_bmi_msg(struct ath10k *ar,
                                           void *req, u32 req_len,
                                           void *resp, u32 *resp_len)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct ath10k_pci_pipe *pci_tx = &ar_pci->pipe_info[BMI_CE_NUM_TO_TARG];
        struct ath10k_pci_pipe *pci_rx = &ar_pci->pipe_info[BMI_CE_NUM_TO_HOST];
        struct ath10k_ce_pipe *ce_tx = pci_tx->ce_hdl;
        struct ath10k_ce_pipe *ce_rx = pci_rx->ce_hdl;
        dma_addr_t req_paddr = 0;
        dma_addr_t resp_paddr = 0;
        struct bmi_xfer xfer = {};
        void *treq, *tresp = NULL;
        int ret = 0;

        might_sleep();

        if (resp && !resp_len)
                return -EINVAL;

        if (resp && resp_len && *resp_len == 0)
                return -EINVAL;

        treq = kmemdup(req, req_len, GFP_KERNEL);
        if (!treq)
                return -ENOMEM;

        req_paddr = dma_map_single(ar->dev, treq, req_len, DMA_TO_DEVICE);
        ret = dma_mapping_error(ar->dev, req_paddr);
        if (ret)
                goto err_dma;

        if (resp && resp_len) {
                tresp = kzalloc(*resp_len, GFP_KERNEL);
                if (!tresp) {
                        ret = -ENOMEM;
                        goto err_req;
                }

                resp_paddr = dma_map_single(ar->dev, tresp, *resp_len,
                                            DMA_FROM_DEVICE);
                ret = dma_mapping_error(ar->dev, resp_paddr);
                if (ret)
                        goto err_req;

                xfer.wait_for_resp = true;
                xfer.resp_len = 0;

                ath10k_ce_recv_buf_enqueue(ce_rx, &xfer, resp_paddr);
        }

        init_completion(&xfer.done);

        ret = ath10k_ce_send(ce_tx, &xfer, req_paddr, req_len, -1, 0);
        if (ret)
                goto err_resp;

        ret = ath10k_pci_bmi_wait(ce_tx, ce_rx, &xfer);
        if (ret) {
                u32 unused_buffer;
                unsigned int unused_nbytes;
                unsigned int unused_id;

                ath10k_ce_cancel_send_next(ce_tx, NULL, &unused_buffer,
                                           &unused_nbytes, &unused_id);
        } else {
                /* non-zero means we did not time out */
                ret = 0;
        }

err_resp:
        if (resp) {
                u32 unused_buffer;

                ath10k_ce_revoke_recv_next(ce_rx, NULL, &unused_buffer);
                dma_unmap_single(ar->dev, resp_paddr,
                                 *resp_len, DMA_FROM_DEVICE);
        }
err_req:
        dma_unmap_single(ar->dev, req_paddr, req_len, DMA_TO_DEVICE);

        if (ret == 0 && resp_len) {
                *resp_len = min(*resp_len, xfer.resp_len);
                memcpy(resp, tresp, xfer.resp_len);
        }
err_dma:
        kfree(treq);
        kfree(tresp);

        return ret;
}

static void ath10k_pci_bmi_send_done(struct ath10k_ce_pipe *ce_state)
{
        struct bmi_xfer *xfer;
        u32 ce_data;
        unsigned int nbytes;
        unsigned int transfer_id;

        if (ath10k_ce_completed_send_next(ce_state, (void **)&xfer, &ce_data,
                                          &nbytes, &transfer_id))
                return;

        if (xfer->wait_for_resp)
                return;

        complete(&xfer->done);
}

static void ath10k_pci_bmi_recv_data(struct ath10k_ce_pipe *ce_state)
{
        struct bmi_xfer *xfer;
        u32 ce_data;
        unsigned int nbytes;
        unsigned int transfer_id;
        unsigned int flags;

        if (ath10k_ce_completed_recv_next(ce_state, (void **)&xfer, &ce_data,
                                          &nbytes, &transfer_id, &flags))
                return;

        if (!xfer->wait_for_resp) {
                ath10k_warn("unexpected: BMI data received; ignoring\n");
                return;
        }

        xfer->resp_len = nbytes;
        complete(&xfer->done);
}

static int ath10k_pci_bmi_wait(struct ath10k_ce_pipe *tx_pipe,
                               struct ath10k_ce_pipe *rx_pipe,
                               struct bmi_xfer *xfer)
{
        unsigned long timeout = jiffies + BMI_COMMUNICATION_TIMEOUT_HZ;

        while (time_before_eq(jiffies, timeout)) {
                ath10k_pci_bmi_send_done(tx_pipe);
                ath10k_pci_bmi_recv_data(rx_pipe);

                if (completion_done(&xfer->done))
                        return 0;

                schedule();
        }

        return -ETIMEDOUT;
}

/*
 * Map from service/endpoint to Copy Engine.
 * This table is derived from the CE_PCI TABLE, above.
 * It is passed to the Target at startup for use by firmware.
 */
static const struct service_to_pipe target_service_to_ce_map_wlan[] = {
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_VO,
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 3,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_VO,
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 2,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_BK,
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 3,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_BK,
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 2,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_BE,
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 3,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_BE,
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 2,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_VI,
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 3,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_DATA_VI,
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 2,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_CONTROL,
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 3,
        },
        {
                 ATH10K_HTC_SVC_ID_WMI_CONTROL,
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 2,
        },
        {
                 ATH10K_HTC_SVC_ID_RSVD_CTRL,
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 0,             /* could be moved to 3 (share with WMI) */
        },
        {
                 ATH10K_HTC_SVC_ID_RSVD_CTRL,
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 1,
        },
        {
                 ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS,    /* not currently used */
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 0,
        },
        {
                 ATH10K_HTC_SVC_ID_TEST_RAW_STREAMS,    /* not currently used */
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 1,
        },
        {
                 ATH10K_HTC_SVC_ID_HTT_DATA_MSG,
                 PIPEDIR_OUT,           /* out = UL = host -> target */
                 4,
        },
        {
                 ATH10K_HTC_SVC_ID_HTT_DATA_MSG,
                 PIPEDIR_IN,            /* in = DL = target -> host */
                 1,
        },

        /* (Additions here) */

        {                               /* Must be last */
                 0,
                 0,
                 0,
        },
};

/*
 * Send an interrupt to the device to wake up the Target CPU
 * so it has an opportunity to notice any changed state.
 */
static int ath10k_pci_wake_target_cpu(struct ath10k *ar)
{
        int ret;
        u32 core_ctrl;

        ret = ath10k_pci_diag_read_access(ar, SOC_CORE_BASE_ADDRESS |
                                              CORE_CTRL_ADDRESS,
                                          &core_ctrl);
        if (ret) {
                ath10k_warn("failed to read core_ctrl: %d\n", ret);
                return ret;
        }

        /* A_INUM_FIRMWARE interrupt to Target CPU */
        core_ctrl |= CORE_CTRL_CPU_INTR_MASK;

        ret = ath10k_pci_diag_write_access(ar, SOC_CORE_BASE_ADDRESS |
                                               CORE_CTRL_ADDRESS,
                                           core_ctrl);
        if (ret) {
                ath10k_warn("failed to set target CPU interrupt mask: %d\n",
                            ret);
                return ret;
        }

        return 0;
}

static int ath10k_pci_init_config(struct ath10k *ar)
{
        u32 interconnect_targ_addr;
        u32 pcie_state_targ_addr = 0;
        u32 pipe_cfg_targ_addr = 0;
        u32 svc_to_pipe_map = 0;
        u32 pcie_config_flags = 0;
        u32 ealloc_value;
        u32 ealloc_targ_addr;
        u32 flag2_value;
        u32 flag2_targ_addr;
        int ret = 0;

        /* Download to Target the CE Config and the service-to-CE map */
        interconnect_targ_addr =
                host_interest_item_address(HI_ITEM(hi_interconnect_state));

        /* Supply Target-side CE configuration */
        ret = ath10k_pci_diag_read_access(ar, interconnect_targ_addr,
                                          &pcie_state_targ_addr);
        if (ret != 0) {
                ath10k_err("Failed to get pcie state addr: %d\n", ret);
                return ret;
        }

        if (pcie_state_targ_addr == 0) {
                ret = -EIO;
                ath10k_err("Invalid pcie state addr\n");
                return ret;
        }

        ret = ath10k_pci_diag_read_access(ar, pcie_state_targ_addr +
                                          offsetof(struct pcie_state,
                                                   pipe_cfg_addr),
                                          &pipe_cfg_targ_addr);
        if (ret != 0) {
                ath10k_err("Failed to get pipe cfg addr: %d\n", ret);
                return ret;
        }

        if (pipe_cfg_targ_addr == 0) {
                ret = -EIO;
                ath10k_err("Invalid pipe cfg addr\n");
                return ret;
        }

        ret = ath10k_pci_diag_write_mem(ar, pipe_cfg_targ_addr,
                                 target_ce_config_wlan,
                                 sizeof(target_ce_config_wlan));

        if (ret != 0) {
                ath10k_err("Failed to write pipe cfg: %d\n", ret);
                return ret;
        }

        ret = ath10k_pci_diag_read_access(ar, pcie_state_targ_addr +
                                          offsetof(struct pcie_state,
                                                   svc_to_pipe_map),
                                          &svc_to_pipe_map);
        if (ret != 0) {
                ath10k_err("Failed to get svc/pipe map: %d\n", ret);
                return ret;
        }

        if (svc_to_pipe_map == 0) {
                ret = -EIO;
                ath10k_err("Invalid svc_to_pipe map\n");
                return ret;
        }

        ret = ath10k_pci_diag_write_mem(ar, svc_to_pipe_map,
                                 target_service_to_ce_map_wlan,
                                 sizeof(target_service_to_ce_map_wlan));
        if (ret != 0) {
                ath10k_err("Failed to write svc/pipe map: %d\n", ret);
                return ret;
        }

        ret = ath10k_pci_diag_read_access(ar, pcie_state_targ_addr +
                                          offsetof(struct pcie_state,
                                                   config_flags),
                                          &pcie_config_flags);
        if (ret != 0) {
                ath10k_err("Failed to get pcie config_flags: %d\n", ret);
                return ret;
        }

        pcie_config_flags &= ~PCIE_CONFIG_FLAG_ENABLE_L1;

        ret = ath10k_pci_diag_write_mem(ar, pcie_state_targ_addr +
                                 offsetof(struct pcie_state, config_flags),
                                 &pcie_config_flags,
                                 sizeof(pcie_config_flags));
        if (ret != 0) {
                ath10k_err("Failed to write pcie config_flags: %d\n", ret);
                return ret;
        }

        /* configure early allocation */
        ealloc_targ_addr = host_interest_item_address(HI_ITEM(hi_early_alloc));

        ret = ath10k_pci_diag_read_access(ar, ealloc_targ_addr, &ealloc_value);
        if (ret != 0) {
                ath10k_err("Faile to get early alloc val: %d\n", ret);
                return ret;
        }

        /* first bank is switched to IRAM */
        ealloc_value |= ((HI_EARLY_ALLOC_MAGIC << HI_EARLY_ALLOC_MAGIC_SHIFT) &
                         HI_EARLY_ALLOC_MAGIC_MASK);
        ealloc_value |= ((1 << HI_EARLY_ALLOC_IRAM_BANKS_SHIFT) &
                         HI_EARLY_ALLOC_IRAM_BANKS_MASK);

        ret = ath10k_pci_diag_write_access(ar, ealloc_targ_addr, ealloc_value);
        if (ret != 0) {
                ath10k_err("Failed to set early alloc val: %d\n", ret);
                return ret;
        }

        /* Tell Target to proceed with initialization */
        flag2_targ_addr = host_interest_item_address(HI_ITEM(hi_option_flag2));

        ret = ath10k_pci_diag_read_access(ar, flag2_targ_addr, &flag2_value);
        if (ret != 0) {
                ath10k_err("Failed to get option val: %d\n", ret);
                return ret;
        }

        flag2_value |= HI_OPTION_EARLY_CFG_DONE;

        ret = ath10k_pci_diag_write_access(ar, flag2_targ_addr, flag2_value);
        if (ret != 0) {
                ath10k_err("Failed to set option val: %d\n", ret);
                return ret;
        }

        return 0;
}

static int ath10k_pci_alloc_ce(struct ath10k *ar)
{
        int i, ret;

        for (i = 0; i < CE_COUNT; i++) {
                ret = ath10k_ce_alloc_pipe(ar, i, &host_ce_config_wlan[i]);
                if (ret) {
                        ath10k_err("failed to allocate copy engine pipe %d: %d\n",
                                   i, ret);
                        return ret;
                }
        }

        return 0;
}

static void ath10k_pci_free_ce(struct ath10k *ar)
{
        int i;

        for (i = 0; i < CE_COUNT; i++)
                ath10k_ce_free_pipe(ar, i);
}

static int ath10k_pci_ce_init(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct ath10k_pci_pipe *pipe_info;
        const struct ce_attr *attr;
        int pipe_num, ret;

        for (pipe_num = 0; pipe_num < CE_COUNT; pipe_num++) {
                pipe_info = &ar_pci->pipe_info[pipe_num];
                pipe_info->ce_hdl = &ar_pci->ce_states[pipe_num];
                pipe_info->pipe_num = pipe_num;
                pipe_info->hif_ce_state = ar;
                attr = &host_ce_config_wlan[pipe_num];

                ret = ath10k_ce_init_pipe(ar, pipe_num, attr);
                if (ret) {
                        ath10k_err("failed to initialize copy engine pipe %d: %d\n",
                                   pipe_num, ret);
                        return ret;
                }

                if (pipe_num == CE_COUNT - 1) {
                        /*
                         * Reserve the ultimate CE for
                         * diagnostic Window support
                         */
                        ar_pci->ce_diag = pipe_info->ce_hdl;
                        continue;
                }

                pipe_info->buf_sz = (size_t) (attr->src_sz_max);
        }

        return 0;
}

static void ath10k_pci_fw_interrupt_handler(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        u32 fw_indicator;

        ath10k_pci_wake(ar);

        fw_indicator = ath10k_pci_read32(ar, FW_INDICATOR_ADDRESS);

        if (fw_indicator & FW_IND_EVENT_PENDING) {
                /* ACK: clear Target-side pending event */
                ath10k_pci_write32(ar, FW_INDICATOR_ADDRESS,
                                   fw_indicator & ~FW_IND_EVENT_PENDING);

                if (ar_pci->started) {
                        ath10k_pci_hif_dump_area(ar);
                } else {
                        /*
                         * Probable Target failure before we're prepared
                         * to handle it.  Generally unexpected.
                         */
                        ath10k_warn("early firmware event indicated\n");
                }
        }

        ath10k_pci_sleep(ar);
}

/* this function effectively clears target memory controller assert line */
static void ath10k_pci_warm_reset_si0(struct ath10k *ar)
{
        u32 val;

        val = ath10k_pci_soc_read32(ar, SOC_RESET_CONTROL_ADDRESS);
        ath10k_pci_soc_write32(ar, SOC_RESET_CONTROL_ADDRESS,
                               val | SOC_RESET_CONTROL_SI0_RST_MASK);
        val = ath10k_pci_soc_read32(ar, SOC_RESET_CONTROL_ADDRESS);

        msleep(10);

        val = ath10k_pci_soc_read32(ar, SOC_RESET_CONTROL_ADDRESS);
        ath10k_pci_soc_write32(ar, SOC_RESET_CONTROL_ADDRESS,
                               val & ~SOC_RESET_CONTROL_SI0_RST_MASK);
        val = ath10k_pci_soc_read32(ar, SOC_RESET_CONTROL_ADDRESS);

        msleep(10);
}

static int ath10k_pci_warm_reset(struct ath10k *ar)
{
        int ret = 0;
        u32 val;

        ath10k_dbg(ATH10K_DBG_BOOT, "boot warm reset\n");

        ret = ath10k_do_pci_wake(ar);
        if (ret) {
                ath10k_err("failed to wake up target: %d\n", ret);
                return ret;
        }

        /* debug */
        val = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
                                PCIE_INTR_CAUSE_ADDRESS);
        ath10k_dbg(ATH10K_DBG_BOOT, "boot host cpu intr cause: 0x%08x\n", val);

        val = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
                                CPU_INTR_ADDRESS);
        ath10k_dbg(ATH10K_DBG_BOOT, "boot target cpu intr cause: 0x%08x\n",
                   val);

        /* disable pending irqs */
        ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS +
                           PCIE_INTR_ENABLE_ADDRESS, 0);

        ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS +
                           PCIE_INTR_CLR_ADDRESS, ~0);

        msleep(100);

        /* clear fw indicator */
        ath10k_pci_write32(ar, FW_INDICATOR_ADDRESS, 0);

        /* clear target LF timer interrupts */
        val = ath10k_pci_read32(ar, RTC_SOC_BASE_ADDRESS +
                                SOC_LF_TIMER_CONTROL0_ADDRESS);
        ath10k_pci_write32(ar, RTC_SOC_BASE_ADDRESS +
                           SOC_LF_TIMER_CONTROL0_ADDRESS,
                           val & ~SOC_LF_TIMER_CONTROL0_ENABLE_MASK);

        /* reset CE */
        val = ath10k_pci_read32(ar, RTC_SOC_BASE_ADDRESS +
                                SOC_RESET_CONTROL_ADDRESS);
        ath10k_pci_write32(ar, RTC_SOC_BASE_ADDRESS + SOC_RESET_CONTROL_ADDRESS,
                           val | SOC_RESET_CONTROL_CE_RST_MASK);
        val = ath10k_pci_read32(ar, RTC_SOC_BASE_ADDRESS +
                                SOC_RESET_CONTROL_ADDRESS);
        msleep(10);

        /* unreset CE */
        ath10k_pci_write32(ar, RTC_SOC_BASE_ADDRESS + SOC_RESET_CONTROL_ADDRESS,
                           val & ~SOC_RESET_CONTROL_CE_RST_MASK);
        val = ath10k_pci_read32(ar, RTC_SOC_BASE_ADDRESS +
                                SOC_RESET_CONTROL_ADDRESS);
        msleep(10);

        ath10k_pci_warm_reset_si0(ar);

        /* debug */
        val = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
                                PCIE_INTR_CAUSE_ADDRESS);
        ath10k_dbg(ATH10K_DBG_BOOT, "boot host cpu intr cause: 0x%08x\n", val);

        val = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
                                CPU_INTR_ADDRESS);
        ath10k_dbg(ATH10K_DBG_BOOT, "boot target cpu intr cause: 0x%08x\n",
                   val);

        /* CPU warm reset */
        val = ath10k_pci_read32(ar, RTC_SOC_BASE_ADDRESS +
                                SOC_RESET_CONTROL_ADDRESS);
        ath10k_pci_write32(ar, RTC_SOC_BASE_ADDRESS + SOC_RESET_CONTROL_ADDRESS,
                           val | SOC_RESET_CONTROL_CPU_WARM_RST_MASK);

        val = ath10k_pci_read32(ar, RTC_SOC_BASE_ADDRESS +
                                SOC_RESET_CONTROL_ADDRESS);
        ath10k_dbg(ATH10K_DBG_BOOT, "boot target reset state: 0x%08x\n", val);

        msleep(100);

        ath10k_dbg(ATH10K_DBG_BOOT, "boot warm reset complete\n");

        ath10k_do_pci_sleep(ar);
        return ret;
}

static int __ath10k_pci_hif_power_up(struct ath10k *ar, bool cold_reset)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        const char *irq_mode;
        int ret;

        /*
         * Bring the target up cleanly.
         *
         * The target may be in an undefined state with an AUX-powered Target
         * and a Host in WoW mode. If the Host crashes, loses power, or is
         * restarted (without unloading the driver) then the Target is left
         * (aux) powered and running. On a subsequent driver load, the Target
         * is in an unexpected state. We try to catch that here in order to
         * reset the Target and retry the probe.
         */
        if (cold_reset)
                ret = ath10k_pci_cold_reset(ar);
        else
                ret = ath10k_pci_warm_reset(ar);

        if (ret) {
                ath10k_err("failed to reset target: %d\n", ret);
                goto err;
        }

        if (!test_bit(ATH10K_PCI_FEATURE_SOC_POWER_SAVE, ar_pci->features))
                /* Force AWAKE forever */
                ath10k_do_pci_wake(ar);

        ret = ath10k_pci_ce_init(ar);
        if (ret) {
                ath10k_err("failed to initialize CE: %d\n", ret);
                goto err_ps;
        }

        ret = ath10k_ce_disable_interrupts(ar);
        if (ret) {
                ath10k_err("failed to disable CE interrupts: %d\n", ret);
                goto err_ce;
        }

        ret = ath10k_pci_init_irq(ar);
        if (ret) {
                ath10k_err("failed to init irqs: %d\n", ret);
                goto err_ce;
        }

        ret = ath10k_pci_request_early_irq(ar);
        if (ret) {
                ath10k_err("failed to request early irq: %d\n", ret);
                goto err_deinit_irq;
        }

        ret = ath10k_pci_wait_for_target_init(ar);
        if (ret) {
                ath10k_err("failed to wait for target to init: %d\n", ret);
                goto err_free_early_irq;
        }

        ret = ath10k_pci_init_config(ar);
        if (ret) {
                ath10k_err("failed to setup init config: %d\n", ret);
                goto err_free_early_irq;
        }

        ret = ath10k_pci_wake_target_cpu(ar);
        if (ret) {
                ath10k_err("could not wake up target CPU: %d\n", ret);
                goto err_free_early_irq;
        }

        if (ar_pci->num_msi_intrs > 1)
                irq_mode = "MSI-X";
        else if (ar_pci->num_msi_intrs == 1)

                irq_mode = "MSI";
        else
                irq_mode = "legacy";

        if (!test_bit(ATH10K_FLAG_FIRST_BOOT_DONE, &ar->dev_flags))
                ath10k_info("pci irq %s irq_mode %d reset_mode %d\n",
                            irq_mode, ath10k_pci_irq_mode,
                            ath10k_pci_reset_mode);

        return 0;

err_free_early_irq:
        ath10k_pci_free_early_irq(ar);
err_deinit_irq:
        ath10k_pci_deinit_irq(ar);
err_ce:
        ath10k_pci_ce_deinit(ar);
        ath10k_pci_warm_reset(ar);
err_ps:
        if (!test_bit(ATH10K_PCI_FEATURE_SOC_POWER_SAVE, ar_pci->features))
                ath10k_do_pci_sleep(ar);
err:
        return ret;
}

static int ath10k_pci_hif_power_up_warm(struct ath10k *ar)
{
        int i, ret;

        /*
         * Sometime warm reset succeeds after retries.
         *
         * FIXME: It might be possible to tune ath10k_pci_warm_reset() to work
         * at first try.
         */
        for (i = 0; i < ATH10K_PCI_NUM_WARM_RESET_ATTEMPTS; i++) {
                ret = __ath10k_pci_hif_power_up(ar, false);
                if (ret == 0)
                        break;

                ath10k_warn("failed to warm reset (attempt %d out of %d): %d\n",
                            i + 1, ATH10K_PCI_NUM_WARM_RESET_ATTEMPTS, ret);
        }

        return ret;
}

static int ath10k_pci_hif_power_up(struct ath10k *ar)
{
        int ret;

        ath10k_dbg(ATH10K_DBG_BOOT, "boot hif power up\n");

        /*
         * Hardware CUS232 version 2 has some issues with cold reset and the
         * preferred (and safer) way to perform a device reset is through a
         * warm reset.
         *
         * Warm reset doesn't always work though so fall back to cold reset may
         * be necessary.
         */
        ret = ath10k_pci_hif_power_up_warm(ar);
        if (ret) {
                ath10k_warn("failed to power up target using warm reset: %d\n",
                            ret);

                if (ath10k_pci_reset_mode == ATH10K_PCI_RESET_WARM_ONLY)
                        return ret;

                ath10k_warn("trying cold reset\n");

                ret = __ath10k_pci_hif_power_up(ar, true);
                if (ret) {
                        ath10k_err("failed to power up target using cold reset too (%d)\n",
                                   ret);
                        return ret;
                }
        }

        return 0;
}

static void ath10k_pci_hif_power_down(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        ath10k_dbg(ATH10K_DBG_BOOT, "boot hif power down\n");

        ath10k_pci_free_early_irq(ar);
        ath10k_pci_kill_tasklet(ar);
        ath10k_pci_deinit_irq(ar);
        ath10k_pci_ce_deinit(ar);
        ath10k_pci_warm_reset(ar);

        if (!test_bit(ATH10K_PCI_FEATURE_SOC_POWER_SAVE, ar_pci->features))
                ath10k_do_pci_sleep(ar);
}

#ifdef CONFIG_PM

#define ATH10K_PCI_PM_CONTROL 0x44

static int ath10k_pci_hif_suspend(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct pci_dev *pdev = ar_pci->pdev;
        u32 val;

        pci_read_config_dword(pdev, ATH10K_PCI_PM_CONTROL, &val);

        if ((val & 0x000000ff) != 0x3) {
                pci_save_state(pdev);
                pci_disable_device(pdev);
                pci_write_config_dword(pdev, ATH10K_PCI_PM_CONTROL,
                                       (val & 0xffffff00) | 0x03);
        }

        return 0;
}

static int ath10k_pci_hif_resume(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        struct pci_dev *pdev = ar_pci->pdev;
        u32 val;

        pci_read_config_dword(pdev, ATH10K_PCI_PM_CONTROL, &val);

        if ((val & 0x000000ff) != 0) {
                pci_restore_state(pdev);
                pci_write_config_dword(pdev, ATH10K_PCI_PM_CONTROL,
                                       val & 0xffffff00);
                /*
                 * Suspend/Resume resets the PCI configuration space,
                 * so we have to re-disable the RETRY_TIMEOUT register (0x41)
                 * to keep PCI Tx retries from interfering with C3 CPU state
                 */
                pci_read_config_dword(pdev, 0x40, &val);

                if ((val & 0x0000ff00) != 0)
                        pci_write_config_dword(pdev, 0x40, val & 0xffff00ff);
        }

        return 0;
}
#endif

static const struct ath10k_hif_ops ath10k_pci_hif_ops = {
        .tx_sg                  = ath10k_pci_hif_tx_sg,
        .exchange_bmi_msg       = ath10k_pci_hif_exchange_bmi_msg,
        .start                  = ath10k_pci_hif_start,
        .stop                   = ath10k_pci_hif_stop,
        .map_service_to_pipe    = ath10k_pci_hif_map_service_to_pipe,
        .get_default_pipe       = ath10k_pci_hif_get_default_pipe,
        .send_complete_check    = ath10k_pci_hif_send_complete_check,
        .set_callbacks          = ath10k_pci_hif_set_callbacks,
        .get_free_queue_number  = ath10k_pci_hif_get_free_queue_number,
        .power_up               = ath10k_pci_hif_power_up,
        .power_down             = ath10k_pci_hif_power_down,
#ifdef CONFIG_PM
        .suspend                = ath10k_pci_hif_suspend,
        .resume                 = ath10k_pci_hif_resume,
#endif
};

static void ath10k_pci_ce_tasklet(unsigned long ptr)
{
        struct ath10k_pci_pipe *pipe = (struct ath10k_pci_pipe *)ptr;
        struct ath10k_pci *ar_pci = pipe->ar_pci;

        ath10k_ce_per_engine_service(ar_pci->ar, pipe->pipe_num);
}

static void ath10k_msi_err_tasklet(unsigned long data)
{
        struct ath10k *ar = (struct ath10k *)data;

        ath10k_pci_fw_interrupt_handler(ar);
}

/*
 * Handler for a per-engine interrupt on a PARTICULAR CE.
 * This is used in cases where each CE has a private MSI interrupt.
 */
static irqreturn_t ath10k_pci_per_engine_handler(int irq, void *arg)
{
        struct ath10k *ar = arg;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ce_id = irq - ar_pci->pdev->irq - MSI_ASSIGN_CE_INITIAL;

        if (ce_id < 0 || ce_id >= ARRAY_SIZE(ar_pci->pipe_info)) {
                ath10k_warn("unexpected/invalid irq %d ce_id %d\n", irq, ce_id);
                return IRQ_HANDLED;
        }

        /*
         * NOTE: We are able to derive ce_id from irq because we
         * use a one-to-one mapping for CE's 0..5.
         * CE's 6 & 7 do not use interrupts at all.
         *
         * This mapping must be kept in sync with the mapping
         * used by firmware.
         */
        tasklet_schedule(&ar_pci->pipe_info[ce_id].intr);
        return IRQ_HANDLED;
}

static irqreturn_t ath10k_pci_msi_fw_handler(int irq, void *arg)
{
        struct ath10k *ar = arg;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        tasklet_schedule(&ar_pci->msi_fw_err);
        return IRQ_HANDLED;
}

/*
 * Top-level interrupt handler for all PCI interrupts from a Target.
 * When a block of MSI interrupts is allocated, this top-level handler
 * is not used; instead, we directly call the correct sub-handler.
 */
static irqreturn_t ath10k_pci_interrupt_handler(int irq, void *arg)
{
        struct ath10k *ar = arg;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        if (ar_pci->num_msi_intrs == 0) {
                if (!ath10k_pci_irq_pending(ar))
                        return IRQ_NONE;

                ath10k_pci_disable_and_clear_legacy_irq(ar);
        }

        tasklet_schedule(&ar_pci->intr_tq);

        return IRQ_HANDLED;
}

static void ath10k_pci_early_irq_tasklet(unsigned long data)
{
        struct ath10k *ar = (struct ath10k *)data;
        u32 fw_ind;
        int ret;

        ret = ath10k_pci_wake(ar);
        if (ret) {
                ath10k_warn("failed to wake target in early irq tasklet: %d\n",
                            ret);
                return;
        }

        fw_ind = ath10k_pci_read32(ar, FW_INDICATOR_ADDRESS);
        if (fw_ind & FW_IND_EVENT_PENDING) {
                ath10k_pci_write32(ar, FW_INDICATOR_ADDRESS,
                                   fw_ind & ~FW_IND_EVENT_PENDING);
                ath10k_pci_hif_dump_area(ar);
        }

        ath10k_pci_sleep(ar);
        ath10k_pci_enable_legacy_irq(ar);
}

static void ath10k_pci_tasklet(unsigned long data)
{
        struct ath10k *ar = (struct ath10k *)data;
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        ath10k_pci_fw_interrupt_handler(ar); /* FIXME: Handle FW error */
        ath10k_ce_per_engine_service_any(ar);

        /* Re-enable legacy irq that was disabled in the irq handler */
        if (ar_pci->num_msi_intrs == 0)
                ath10k_pci_enable_legacy_irq(ar);
}

static int ath10k_pci_request_irq_msix(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret, i;

        ret = request_irq(ar_pci->pdev->irq + MSI_ASSIGN_FW,
                          ath10k_pci_msi_fw_handler,
                          IRQF_SHARED, "ath10k_pci", ar);
        if (ret) {
                ath10k_warn("failed to request MSI-X fw irq %d: %d\n",
                            ar_pci->pdev->irq + MSI_ASSIGN_FW, ret);
                return ret;
        }

        for (i = MSI_ASSIGN_CE_INITIAL; i <= MSI_ASSIGN_CE_MAX; i++) {
                ret = request_irq(ar_pci->pdev->irq + i,
                                  ath10k_pci_per_engine_handler,
                                  IRQF_SHARED, "ath10k_pci", ar);
                if (ret) {
                        ath10k_warn("failed to request MSI-X ce irq %d: %d\n",
                                    ar_pci->pdev->irq + i, ret);

                        for (i--; i >= MSI_ASSIGN_CE_INITIAL; i--)
                                free_irq(ar_pci->pdev->irq + i, ar);

                        free_irq(ar_pci->pdev->irq + MSI_ASSIGN_FW, ar);
                        return ret;
                }
        }

        return 0;
}

static int ath10k_pci_request_irq_msi(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret;

        ret = request_irq(ar_pci->pdev->irq,
                          ath10k_pci_interrupt_handler,
                          IRQF_SHARED, "ath10k_pci", ar);
        if (ret) {
                ath10k_warn("failed to request MSI irq %d: %d\n",
                            ar_pci->pdev->irq, ret);
                return ret;
        }

        return 0;
}

static int ath10k_pci_request_irq_legacy(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int ret;

        ret = request_irq(ar_pci->pdev->irq,
                          ath10k_pci_interrupt_handler,
                          IRQF_SHARED, "ath10k_pci", ar);
        if (ret) {
                ath10k_warn("failed to request legacy irq %d: %d\n",
                            ar_pci->pdev->irq, ret);
                return ret;
        }

        return 0;
}

static int ath10k_pci_request_irq(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        switch (ar_pci->num_msi_intrs) {
        case 0:
                return ath10k_pci_request_irq_legacy(ar);
        case 1:
                return ath10k_pci_request_irq_msi(ar);
        case MSI_NUM_REQUEST:
                return ath10k_pci_request_irq_msix(ar);
        }

        ath10k_warn("unknown irq configuration upon request\n");
        return -EINVAL;
}

static void ath10k_pci_free_irq(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int i;

        /* There's at least one interrupt irregardless whether its legacy INTR
         * or MSI or MSI-X */
        for (i = 0; i < max(1, ar_pci->num_msi_intrs); i++)
                free_irq(ar_pci->pdev->irq + i, ar);
}

static void ath10k_pci_init_irq_tasklets(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        int i;

        tasklet_init(&ar_pci->intr_tq, ath10k_pci_tasklet, (unsigned long)ar);
        tasklet_init(&ar_pci->msi_fw_err, ath10k_msi_err_tasklet,
                     (unsigned long)ar);
        tasklet_init(&ar_pci->early_irq_tasklet, ath10k_pci_early_irq_tasklet,
                     (unsigned long)ar);

        for (i = 0; i < CE_COUNT; i++) {
                ar_pci->pipe_info[i].ar_pci = ar_pci;
                tasklet_init(&ar_pci->pipe_info[i].intr, ath10k_pci_ce_tasklet,
                             (unsigned long)&ar_pci->pipe_info[i]);
        }
}

static int ath10k_pci_init_irq(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        bool msix_supported = test_bit(ATH10K_PCI_FEATURE_MSI_X,
                                       ar_pci->features);
        int ret;

        ath10k_pci_init_irq_tasklets(ar);

        if (ath10k_pci_irq_mode != ATH10K_PCI_IRQ_AUTO &&
            !test_bit(ATH10K_FLAG_FIRST_BOOT_DONE, &ar->dev_flags))
                ath10k_info("limiting irq mode to: %d\n", ath10k_pci_irq_mode);

        /* Try MSI-X */
        if (ath10k_pci_irq_mode == ATH10K_PCI_IRQ_AUTO && msix_supported) {
                ar_pci->num_msi_intrs = MSI_NUM_REQUEST;
                ret = pci_enable_msi_range(ar_pci->pdev, ar_pci->num_msi_intrs,
                                                         ar_pci->num_msi_intrs);
                if (ret > 0)
                        return 0;

                /* fall-through */
        }

        /* Try MSI */
        if (ath10k_pci_irq_mode != ATH10K_PCI_IRQ_LEGACY) {
                ar_pci->num_msi_intrs = 1;
                ret = pci_enable_msi(ar_pci->pdev);
                if (ret == 0)
                        return 0;

                /* fall-through */
        }

        /* Try legacy irq
         *
         * A potential race occurs here: The CORE_BASE write
         * depends on target correctly decoding AXI address but
         * host won't know when target writes BAR to CORE_CTRL.
         * This write might get lost if target has NOT written BAR.
         * For now, fix the race by repeating the write in below
         * synchronization checking. */
        ar_pci->num_msi_intrs = 0;

        ret = ath10k_pci_wake(ar);
        if (ret) {
                ath10k_warn("failed to wake target: %d\n", ret);
                return ret;
        }

        ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS + PCIE_INTR_ENABLE_ADDRESS,
                           PCIE_INTR_FIRMWARE_MASK | PCIE_INTR_CE_MASK_ALL);
        ath10k_pci_sleep(ar);

        return 0;
}

static int ath10k_pci_deinit_irq_legacy(struct ath10k *ar)
{
        int ret;

        ret = ath10k_pci_wake(ar);
        if (ret) {
                ath10k_warn("failed to wake target: %d\n", ret);
                return ret;
        }

        ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS + PCIE_INTR_ENABLE_ADDRESS,
                           0);
        ath10k_pci_sleep(ar);

        return 0;
}

static int ath10k_pci_deinit_irq(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);

        switch (ar_pci->num_msi_intrs) {
        case 0:
                return ath10k_pci_deinit_irq_legacy(ar);
        case 1:
                /* fall-through */
        case MSI_NUM_REQUEST:
                pci_disable_msi(ar_pci->pdev);
                return 0;
        default:
                pci_disable_msi(ar_pci->pdev);
        }

        ath10k_warn("unknown irq configuration upon deinit\n");
        return -EINVAL;
}

static int ath10k_pci_wait_for_target_init(struct ath10k *ar)
{
        struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
        unsigned long timeout;
        int ret;
        u32 val;

        ath10k_dbg(ATH10K_DBG_BOOT, "boot waiting target to initialise\n");

        ret = ath10k_pci_wake(ar);
        if (ret) {
                ath10k_err("failed to wake up target for init: %d\n", ret);
                return ret;
        }

        timeout = jiffies + msecs_to_jiffies(ATH10K_PCI_TARGET_WAIT);

        do {
                val = ath10k_pci_read32(ar, FW_INDICATOR_ADDRESS);

                ath10k_dbg(ATH10K_DBG_BOOT, "boot target indicator %x\n", val);

                /* target should never return this */
                if (val == 0xffffffff)
                        continue;

                /* the device has crashed so don't bother trying anymore */
                if (val & FW_IND_EVENT_PENDING)
                        break;

                if (val & FW_IND_INITIALIZED)
                        break;

                if (ar_pci->num_msi_intrs == 0)
                        /* Fix potential race by repeating CORE_BASE writes */
                        ath10k_pci_soc_write32(ar, PCIE_INTR_ENABLE_ADDRESS,
                                               PCIE_INTR_FIRMWARE_MASK |
                                               PCIE_INTR_CE_MASK_ALL);

                mdelay(10);
        } while (time_before(jiffies, timeout));

        if (val == 0xffffffff) {
                ath10k_err("failed to read device register, device is gone\n");
                ret = -EIO;
                goto out;
        }

        if (val & FW_IND_EVENT_PENDING) {
                ath10k_warn("device has crashed during init\n");
                ath10k_pci_write32(ar, FW_INDICATOR_ADDRESS,
                                   val & ~FW_IND_EVENT_PENDING);
                ath10k_pci_hif_dump_area(ar);
                ret = -ECOMM;
                goto out;
        }

        if (!(val & FW_IND_INITIALIZED)) {
                ath10k_err("failed to receive initialized event from target: %08x\n",
                           val);
                ret = -ETIMEDOUT;
                goto out;
        }

        ath10k_dbg(ATH10K_DBG_BOOT, "boot target initialised\n");

out:
        ath10k_pci_sleep(ar);
        return ret;
}

static int ath10k_pci_cold_reset(struct ath10k *ar)
{
        int i, ret;
        u32 val;

        ath10k_dbg(ATH10K_DBG_BOOT, "boot cold reset\n");

        ret = ath10k_do_pci_wake(ar);
        if (ret) {
                ath10k_err("failed to wake up target: %d\n",
                           ret);
                return ret;
        }

        /* Put Target, including PCIe, into RESET. */
        val = ath10k_pci_reg_read32(ar, SOC_GLOBAL_RESET_ADDRESS);
        val |= 1;
        ath10k_pci_reg_write32(ar, SOC_GLOBAL_RESET_ADDRESS, val);

        for (i = 0; i < ATH_PCI_RESET_WAIT_MAX; i++) {
                if (ath10k_pci_reg_read32(ar, RTC_STATE_ADDRESS) &
                                          RTC_STATE_COLD_RESET_MASK)
                        break;
                msleep(1);
        }

        /* Pull Target, including PCIe, out of RESET. */
        val &= ~1;
        ath10k_pci_reg_write32(ar, SOC_GLOBAL_RESET_ADDRESS, val);

        for (i = 0; i < ATH_PCI_RESET_WAIT_MAX; i++) {
                if (!(ath10k_pci_reg_read32(ar, RTC_STATE_ADDRESS) &
                                            RTC_STATE_COLD_RESET_MASK))
                        break;
                msleep(1);
        }

        ath10k_do_pci_sleep(ar);

        ath10k_dbg(ATH10K_DBG_BOOT, "boot cold reset complete\n");

        return 0;
}

static void ath10k_pci_dump_features(struct ath10k_pci *ar_pci)
{
        int i;

        for (i = 0; i < ATH10K_PCI_FEATURE_COUNT; i++) {
                if (!test_bit(i, ar_pci->features))
                        continue;

                switch (i) {
                case ATH10K_PCI_FEATURE_MSI_X:
                        ath10k_dbg(ATH10K_DBG_BOOT, "device supports MSI-X\n");
                        break;
                case ATH10K_PCI_FEATURE_SOC_POWER_SAVE:
                        ath10k_dbg(ATH10K_DBG_BOOT, "QCA98XX SoC power save enabled\n");
                        break;
                }
        }
}

static int ath10k_pci_probe(struct pci_dev *pdev,
                            const struct pci_device_id *pci_dev)
{
        void __iomem *mem;
        int ret = 0;
        struct ath10k *ar;
        struct ath10k_pci *ar_pci;
        u32 lcr_val, chip_id;

        ath10k_dbg(ATH10K_DBG_PCI, "pci probe\n");

        ar_pci = kzalloc(sizeof(*ar_pci), GFP_KERNEL);
        if (ar_pci == NULL)
                return -ENOMEM;

        ar_pci->pdev = pdev;
        ar_pci->dev = &pdev->dev;

        switch (pci_dev->device) {
        case QCA988X_2_0_DEVICE_ID:
                set_bit(ATH10K_PCI_FEATURE_MSI_X, ar_pci->features);
                break;
        default:
                ret = -ENODEV;
                ath10k_err("Unknown device ID: %d\n", pci_dev->device);
                goto err_ar_pci;
        }

        if (ath10k_pci_target_ps)
                set_bit(ATH10K_PCI_FEATURE_SOC_POWER_SAVE, ar_pci->features);

        ath10k_pci_dump_features(ar_pci);

        ar = ath10k_core_create(ar_pci, ar_pci->dev, &ath10k_pci_hif_ops);
        if (!ar) {
                ath10k_err("failed to create driver core\n");
                ret = -EINVAL;
                goto err_ar_pci;
        }

        ar_pci->ar = ar;
        atomic_set(&ar_pci->keep_awake_count, 0);

        pci_set_drvdata(pdev, ar);

        ret = pci_enable_device(pdev);
        if (ret) {
                ath10k_err("failed to enable PCI device: %d\n", ret);
                goto err_ar;
        }

        /* Request MMIO resources */
        ret = pci_request_region(pdev, BAR_NUM, "ath");
        if (ret) {
                ath10k_err("failed to request MMIO region: %d\n", ret);
                goto err_device;
        }

        /*
         * Target structures have a limit of 32 bit DMA pointers.
         * DMA pointers can be wider than 32 bits by default on some systems.
         */
        ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
        if (ret) {
                ath10k_err("failed to set DMA mask to 32-bit: %d\n", ret);
                goto err_region;
        }

        ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
        if (ret) {
                ath10k_err("failed to set consistent DMA mask to 32-bit\n");
                goto err_region;
        }

        /* Set bus master bit in PCI_COMMAND to enable DMA */
        pci_set_master(pdev);

        /*
         * Temporary FIX: disable ASPM
         * Will be removed after the OTP is programmed
         */
        pci_read_config_dword(pdev, 0x80, &lcr_val);
        pci_write_config_dword(pdev, 0x80, (lcr_val & 0xffffff00));

        /* Arrange for access to Target SoC registers. */
        mem = pci_iomap(pdev, BAR_NUM, 0);
        if (!mem) {
                ath10k_err("failed to perform IOMAP for BAR%d\n", BAR_NUM);
                ret = -EIO;
                goto err_master;
        }

        ar_pci->mem = mem;

        spin_lock_init(&ar_pci->ce_lock);

        ret = ath10k_do_pci_wake(ar);
        if (ret) {
                ath10k_err("Failed to get chip id: %d\n", ret);
                goto err_iomap;
        }

        chip_id = ath10k_pci_soc_read32(ar, SOC_CHIP_ID_ADDRESS);

        ath10k_do_pci_sleep(ar);

        ret = ath10k_pci_alloc_ce(ar);
        if (ret) {
                ath10k_err("failed to allocate copy engine pipes: %d\n", ret);
                goto err_iomap;
        }

        ath10k_dbg(ATH10K_DBG_BOOT, "boot pci_mem 0x%p\n", ar_pci->mem);

        ret = ath10k_core_register(ar, chip_id);
        if (ret) {
                ath10k_err("failed to register driver core: %d\n", ret);
                goto err_free_ce;
        }

        return 0;

err_free_ce:
        ath10k_pci_free_ce(ar);
err_iomap:
        pci_iounmap(pdev, mem);
err_master:
        pci_clear_master(pdev);
err_region:
        pci_release_region(pdev, BAR_NUM);
err_device:
        pci_disable_device(pdev);
err_ar:
        ath10k_core_destroy(ar);
err_ar_pci:
        /* call HIF PCI free here */
        kfree(ar_pci);

        return ret;
}

static void ath10k_pci_remove(struct pci_dev *pdev)
{
        struct ath10k *ar = pci_get_drvdata(pdev);
        struct ath10k_pci *ar_pci;

        ath10k_dbg(ATH10K_DBG_PCI, "pci remove\n");

        if (!ar)
                return;

        ar_pci = ath10k_pci_priv(ar);

        if (!ar_pci)
                return;

        ath10k_core_unregister(ar);
        ath10k_pci_free_ce(ar);

        pci_iounmap(pdev, ar_pci->mem);
        pci_release_region(pdev, BAR_NUM);
        pci_clear_master(pdev);
        pci_disable_device(pdev);

        ath10k_core_destroy(ar);
        kfree(ar_pci);
}

MODULE_DEVICE_TABLE(pci, ath10k_pci_id_table);

static struct pci_driver ath10k_pci_driver = {
        .name = "ath10k_pci",
        .id_table = ath10k_pci_id_table,
        .probe = ath10k_pci_probe,
        .remove = ath10k_pci_remove,
};

static int __init ath10k_pci_init(void)
{
        int ret;

        ret = pci_register_driver(&ath10k_pci_driver);
        if (ret)
                ath10k_err("failed to register PCI driver: %d\n", ret);

        return ret;
}
module_init(ath10k_pci_init);

static void __exit ath10k_pci_exit(void)
{
        pci_unregister_driver(&ath10k_pci_driver);
}

module_exit(ath10k_pci_exit);

MODULE_AUTHOR("Qualcomm Atheros");
MODULE_DESCRIPTION("Driver support for Atheros QCA988X PCIe devices");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_FIRMWARE(QCA988X_HW_2_0_FW_DIR "/" QCA988X_HW_2_0_FW_2_FILE);
MODULE_FIRMWARE(QCA988X_HW_2_0_FW_DIR "/" QCA988X_HW_2_0_BOARD_DATA_FILE);