/*
 * Intel MIC Platform Software Stack (MPSS)
 *
 * Copyright(c) 2013 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 * Intel MIC Host driver.
 *
 */
#include <linux/pci.h>
#include <linux/sched.h>
#include <linux/uaccess.h>

#include <linux/mic_common.h>
#include "../common/mic_dev.h"
#include "mic_device.h"
#include "mic_smpt.h"
#include "mic_virtio.h"

/*
 * Initiates the copies across the PCIe bus from card memory to
 * a user space buffer.
 */
static int mic_virtio_copy_to_user(struct mic_vdev *mvdev,
                void __user *ubuf, size_t len, u64 addr)
{
        int err;
        void __iomem *dbuf = mvdev->mdev->aper.va + addr;
        /*
         * We are copying from IO below an should ideally use something
         * like copy_to_user_fromio(..) if it existed.
         */
        if (copy_to_user(ubuf, (void __force *)dbuf, len)) {
                err = -EFAULT;
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, err);
                goto err;
        }
        mvdev->in_bytes += len;
        err = 0;
err:
        return err;
}

/*
 * Initiates copies across the PCIe bus from a user space
 * buffer to card memory.
 */
static int mic_virtio_copy_from_user(struct mic_vdev *mvdev,
                void __user *ubuf, size_t len, u64 addr)
{
        int err;
        void __iomem *dbuf = mvdev->mdev->aper.va + addr;
        /*
         * We are copying to IO below and should ideally use something
         * like copy_from_user_toio(..) if it existed.
         */
        if (copy_from_user((void __force *)dbuf, ubuf, len)) {
                err = -EFAULT;
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, err);
                goto err;
        }
        mvdev->out_bytes += len;
        err = 0;
err:
        return err;
}

#define MIC_VRINGH_READ true

/* The function to call to notify the card about added buffers */
static void mic_notify(struct vringh *vrh)
{
        struct mic_vringh *mvrh = container_of(vrh, struct mic_vringh, vrh);
        struct mic_vdev *mvdev = mvrh->mvdev;
        s8 db = mvdev->dc->h2c_vdev_db;

        if (db != -1)
                mvdev->mdev->ops->send_intr(mvdev->mdev, db);
}

/* Determine the total number of bytes consumed in a VRINGH KIOV */
static inline u32 mic_vringh_iov_consumed(struct vringh_kiov *iov)
{
        int i;
        u32 total = iov->consumed;

        for (i = 0; i < iov->i; i++)
                total += iov->iov[i].iov_len;
        return total;
}

/*
 * Traverse the VRINGH KIOV and issue the APIs to trigger the copies.
 * This API is heavily based on the vringh_iov_xfer(..) implementation
 * in vringh.c. The reason we cannot reuse vringh_iov_pull_kern(..)
 * and vringh_iov_push_kern(..) directly is because there is no
 * way to override the VRINGH xfer(..) routines as of v3.10.
 */
static int mic_vringh_copy(struct mic_vdev *mvdev, struct vringh_kiov *iov,
        void __user *ubuf, size_t len, bool read, size_t *out_len)
{
        int ret = 0;
        size_t partlen, tot_len = 0;

        while (len && iov->i < iov->used) {
                partlen = min(iov->iov[iov->i].iov_len, len);
                if (read)
                        ret = mic_virtio_copy_to_user(mvdev,
                                ubuf, partlen,
                                (u64)iov->iov[iov->i].iov_base);
                else
                        ret = mic_virtio_copy_from_user(mvdev,
                                ubuf, partlen,
                                (u64)iov->iov[iov->i].iov_base);
                if (ret) {
                        dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                __func__, __LINE__, ret);
                        break;
                }
                len -= partlen;
                ubuf += partlen;
                tot_len += partlen;
                iov->consumed += partlen;
                iov->iov[iov->i].iov_len -= partlen;
                iov->iov[iov->i].iov_base += partlen;
                if (!iov->iov[iov->i].iov_len) {
                        /* Fix up old iov element then increment. */
                        iov->iov[iov->i].iov_len = iov->consumed;
                        iov->iov[iov->i].iov_base -= iov->consumed;

                        iov->consumed = 0;
                        iov->i++;
                }
        }
        *out_len = tot_len;
        return ret;
}

/*
 * Use the standard VRINGH infrastructure in the kernel to fetch new
 * descriptors, initiate the copies and update the used ring.
 */
static int _mic_virtio_copy(struct mic_vdev *mvdev,
        struct mic_copy_desc *copy)
{
        int ret = 0;
        u32 iovcnt = copy->iovcnt;
        struct iovec iov;
        struct iovec __user *u_iov = copy->iov;
        void __user *ubuf = NULL;
        struct mic_vringh *mvr = &mvdev->mvr[copy->vr_idx];
        struct vringh_kiov *riov = &mvr->riov;
        struct vringh_kiov *wiov = &mvr->wiov;
        struct vringh *vrh = &mvr->vrh;
        u16 *head = &mvr->head;
        struct mic_vring *vr = &mvr->vring;
        size_t len = 0, out_len;

        copy->out_len = 0;
        /* Fetch a new IOVEC if all previous elements have been processed */
        if (riov->i == riov->used && wiov->i == wiov->used) {
                ret = vringh_getdesc_kern(vrh, riov, wiov,
                                head, GFP_KERNEL);
                /* Check if there are available descriptors */
                if (ret <= 0)
                        return ret;
        }
        while (iovcnt) {
                if (!len) {
                        /* Copy over a new iovec from user space. */
                        ret = copy_from_user(&iov, u_iov, sizeof(*u_iov));
                        if (ret) {
                                ret = -EINVAL;
                                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                        __func__, __LINE__, ret);
                                break;
                        }
                        len = iov.iov_len;
                        ubuf = iov.iov_base;
                }
                /* Issue all the read descriptors first */
                ret = mic_vringh_copy(mvdev, riov, ubuf, len,
                        MIC_VRINGH_READ, &out_len);
                if (ret) {
                        dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                __func__, __LINE__, ret);
                        break;
                }
                len -= out_len;
                ubuf += out_len;
                copy->out_len += out_len;
                /* Issue the write descriptors next */
                ret = mic_vringh_copy(mvdev, wiov, ubuf, len,
                        !MIC_VRINGH_READ, &out_len);
                if (ret) {
                        dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                __func__, __LINE__, ret);
                        break;
                }
                len -= out_len;
                ubuf += out_len;
                copy->out_len += out_len;
                if (!len) {
                        /* One user space iovec is now completed */
                        iovcnt--;
                        u_iov++;
                }
                /* Exit loop if all elements in KIOVs have been processed. */
                if (riov->i == riov->used && wiov->i == wiov->used)
                        break;
        }
        /*
         * Update the used ring if a descriptor was available and some data was
         * copied in/out and the user asked for a used ring update.
         */
        if (*head != USHRT_MAX && copy->out_len && copy->update_used) {
                u32 total = 0;

                /* Determine the total data consumed */
                total += mic_vringh_iov_consumed(riov);
                total += mic_vringh_iov_consumed(wiov);
                vringh_complete_kern(vrh, *head, total);
                *head = USHRT_MAX;
                if (vringh_need_notify_kern(vrh) > 0)
                        vringh_notify(vrh);
                vringh_kiov_cleanup(riov);
                vringh_kiov_cleanup(wiov);
                /* Update avail idx for user space */
                vr->info->avail_idx = vrh->last_avail_idx;
        }
        return ret;
}

static inline int mic_verify_copy_args(struct mic_vdev *mvdev,
                struct mic_copy_desc *copy)
{
        if (copy->vr_idx >= mvdev->dd->num_vq) {
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, -EINVAL);
                return -EINVAL;
        }
        return 0;
}

/* Copy a specified number of virtio descriptors in a chain */
int mic_virtio_copy_desc(struct mic_vdev *mvdev,
                struct mic_copy_desc *copy)
{
        int err;
        struct mic_vringh *mvr = &mvdev->mvr[copy->vr_idx];

        err = mic_verify_copy_args(mvdev, copy);
        if (err)
                return err;

        mutex_lock(&mvr->vr_mutex);
        if (!mic_vdevup(mvdev)) {
                err = -ENODEV;
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, err);
                goto err;
        }
        err = _mic_virtio_copy(mvdev, copy);
        if (err) {
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, err);
        }
err:
        mutex_unlock(&mvr->vr_mutex);
        return err;
}

static void mic_virtio_init_post(struct mic_vdev *mvdev)
{
        struct mic_vqconfig *vqconfig = mic_vq_config(mvdev->dd);
        int i;

        for (i = 0; i < mvdev->dd->num_vq; i++) {
                if (!le64_to_cpu(vqconfig[i].used_address)) {
                        dev_warn(mic_dev(mvdev), "used_address zero??\n");
                        continue;
                }
                mvdev->mvr[i].vrh.vring.used =
                        (void __force *)mvdev->mdev->aper.va +
                        le64_to_cpu(vqconfig[i].used_address);
        }

        mvdev->dc->used_address_updated = 0;

        dev_dbg(mic_dev(mvdev), "%s: device type %d LINKUP\n",
                __func__, mvdev->virtio_id);
}

static inline void mic_virtio_device_reset(struct mic_vdev *mvdev)
{
        int i;

        dev_dbg(mic_dev(mvdev), "%s: status %d device type %d RESET\n",
                __func__, mvdev->dd->status, mvdev->virtio_id);

        for (i = 0; i < mvdev->dd->num_vq; i++)
                /*
                 * Avoid lockdep false positive. The + 1 is for the mic
                 * mutex which is held in the reset devices code path.
                 */
                mutex_lock_nested(&mvdev->mvr[i].vr_mutex, i + 1);

        /* 0 status means "reset" */
        mvdev->dd->status = 0;
        mvdev->dc->vdev_reset = 0;
        mvdev->dc->host_ack = 1;

        for (i = 0; i < mvdev->dd->num_vq; i++) {
                struct vringh *vrh = &mvdev->mvr[i].vrh;
                mvdev->mvr[i].vring.info->avail_idx = 0;
                vrh->completed = 0;
                vrh->last_avail_idx = 0;
                vrh->last_used_idx = 0;
        }

        for (i = 0; i < mvdev->dd->num_vq; i++)
                mutex_unlock(&mvdev->mvr[i].vr_mutex);
}

void mic_virtio_reset_devices(struct mic_device *mdev)
{
        struct list_head *pos, *tmp;
        struct mic_vdev *mvdev;

        dev_dbg(mdev->sdev->parent, "%s\n",  __func__);

        list_for_each_safe(pos, tmp, &mdev->vdev_list) {
                mvdev = list_entry(pos, struct mic_vdev, list);
                mic_virtio_device_reset(mvdev);
                mvdev->poll_wake = 1;
                wake_up(&mvdev->waitq);
        }
}

void mic_bh_handler(struct work_struct *work)
{
        struct mic_vdev *mvdev = container_of(work, struct mic_vdev,
                        virtio_bh_work);

        if (mvdev->dc->used_address_updated)
                mic_virtio_init_post(mvdev);

        if (mvdev->dc->vdev_reset)
                mic_virtio_device_reset(mvdev);

        mvdev->poll_wake = 1;
        wake_up(&mvdev->waitq);
}

static irqreturn_t mic_virtio_intr_handler(int irq, void *data)
{
        struct mic_vdev *mvdev = data;
        struct mic_device *mdev = mvdev->mdev;

        mdev->ops->intr_workarounds(mdev);
        schedule_work(&mvdev->virtio_bh_work);
        return IRQ_HANDLED;
}

int mic_virtio_config_change(struct mic_vdev *mvdev,
                        void __user *argp)
{
        DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake);
        int ret = 0, retry, i;
        struct mic_bootparam *bootparam = mvdev->mdev->dp;
        s8 db = bootparam->h2c_config_db;

        mutex_lock(&mvdev->mdev->mic_mutex);
        for (i = 0; i < mvdev->dd->num_vq; i++)
                mutex_lock_nested(&mvdev->mvr[i].vr_mutex, i + 1);

        if (db == -1 || mvdev->dd->type == -1) {
                ret = -EIO;
                goto exit;
        }

        if (copy_from_user(mic_vq_configspace(mvdev->dd),
                           argp, mvdev->dd->config_len)) {
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, -EFAULT);
                ret = -EFAULT;
                goto exit;
        }
        mvdev->dc->config_change = MIC_VIRTIO_PARAM_CONFIG_CHANGED;
        mvdev->mdev->ops->send_intr(mvdev->mdev, db);

        for (retry = 100; retry--;) {
                ret = wait_event_timeout(wake,
                        mvdev->dc->guest_ack, msecs_to_jiffies(100));
                if (ret)
                        break;
        }

        dev_dbg(mic_dev(mvdev),
                "%s %d retry: %d\n", __func__, __LINE__, retry);
        mvdev->dc->config_change = 0;
        mvdev->dc->guest_ack = 0;
exit:
        for (i = 0; i < mvdev->dd->num_vq; i++)
                mutex_unlock(&mvdev->mvr[i].vr_mutex);
        mutex_unlock(&mvdev->mdev->mic_mutex);
        return ret;
}

static int mic_copy_dp_entry(struct mic_vdev *mvdev,
                                        void __user *argp,
                                        __u8 *type,
                                        struct mic_device_desc **devpage)
{
        struct mic_device *mdev = mvdev->mdev;
        struct mic_device_desc dd, *dd_config, *devp;
        struct mic_vqconfig *vqconfig;
        int ret = 0, i;
        bool slot_found = false;

        if (copy_from_user(&dd, argp, sizeof(dd))) {
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, -EFAULT);
                return -EFAULT;
        }

        if (mic_aligned_desc_size(&dd) > MIC_MAX_DESC_BLK_SIZE ||
            dd.num_vq > MIC_MAX_VRINGS) {
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, -EINVAL);
                return -EINVAL;
        }

        dd_config = kmalloc(mic_desc_size(&dd), GFP_KERNEL);
        if (dd_config == NULL) {
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, -ENOMEM);
                return -ENOMEM;
        }
        if (copy_from_user(dd_config, argp, mic_desc_size(&dd))) {
                ret = -EFAULT;
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, ret);
                goto exit;
        }

        vqconfig = mic_vq_config(dd_config);
        for (i = 0; i < dd.num_vq; i++) {
                if (le16_to_cpu(vqconfig[i].num) > MIC_MAX_VRING_ENTRIES) {
                        ret =  -EINVAL;
                        dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                __func__, __LINE__, ret);
                        goto exit;
                }
        }

        /* Find the first free device page entry */
        for (i = sizeof(struct mic_bootparam);
                i < MIC_DP_SIZE - mic_total_desc_size(dd_config);
                i += mic_total_desc_size(devp)) {
                devp = mdev->dp + i;
                if (devp->type == 0 || devp->type == -1) {
                        slot_found = true;
                        break;
                }
        }
        if (!slot_found) {
                ret =  -EINVAL;
                dev_err(mic_dev(mvdev), "%s %d err %d\n",
                        __func__, __LINE__, ret);
                goto exit;
        }
        /*
         * Save off the type before doing the memcpy. Type will be set in the
         * end after completing all initialization for the new device.
         */
        *type = dd_config->type;
        dd_config->type = 0;
        memcpy(devp, dd_config, mic_desc_size(dd_config));

        *devpage = devp;
exit:
        kfree(dd_config);
        return ret;
}

static void mic_init_device_ctrl(struct mic_vdev *mvdev,
                                struct mic_device_desc *devpage)
{
        struct mic_device_ctrl *dc;

        dc = (void *)devpage + mic_aligned_desc_size(devpage);

        dc->config_change = 0;
        dc->guest_ack = 0;
        dc->vdev_reset = 0;
        dc->host_ack = 0;
        dc->used_address_updated = 0;
        dc->c2h_vdev_db = -1;
        dc->h2c_vdev_db = -1;
        mvdev->dc = dc;
}

int mic_virtio_add_device(struct mic_vdev *mvdev,
                        void __user *argp)
{
        struct mic_device *mdev = mvdev->mdev;
        struct mic_device_desc *dd = NULL;
        struct mic_vqconfig *vqconfig;
        int vr_size, i, j, ret;
        u8 type = 0;
        s8 db;
        char irqname[10];
        struct mic_bootparam *bootparam = mdev->dp;
        u16 num;
        dma_addr_t vr_addr;

        mutex_lock(&mdev->mic_mutex);

        ret = mic_copy_dp_entry(mvdev, argp, &type, &dd);
        if (ret) {
                mutex_unlock(&mdev->mic_mutex);
                return ret;
        }

        mic_init_device_ctrl(mvdev, dd);

        mvdev->dd = dd;
        mvdev->virtio_id = type;
        vqconfig = mic_vq_config(dd);
        INIT_WORK(&mvdev->virtio_bh_work, mic_bh_handler);

        for (i = 0; i < dd->num_vq; i++) {
                struct mic_vringh *mvr = &mvdev->mvr[i];
                struct mic_vring *vr = &mvdev->mvr[i].vring;
                num = le16_to_cpu(vqconfig[i].num);
                mutex_init(&mvr->vr_mutex);
                vr_size = PAGE_ALIGN(vring_size(num, MIC_VIRTIO_RING_ALIGN) +
                        sizeof(struct _mic_vring_info));
                vr->va = (void *)
                        __get_free_pages(GFP_KERNEL | __GFP_ZERO,
                                         get_order(vr_size));
                if (!vr->va) {
                        ret = -ENOMEM;
                        dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                __func__, __LINE__, ret);
                        goto err;
                }
                vr->len = vr_size;
                vr->info = vr->va + vring_size(num, MIC_VIRTIO_RING_ALIGN);
                vr->info->magic = cpu_to_le32(MIC_MAGIC + mvdev->virtio_id + i);
                vr_addr = mic_map_single(mdev, vr->va, vr_size);
                if (mic_map_error(vr_addr)) {
                        free_pages((unsigned long)vr->va, get_order(vr_size));
                        ret = -ENOMEM;
                        dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                __func__, __LINE__, ret);
                        goto err;
                }
                vqconfig[i].address = cpu_to_le64(vr_addr);

                vring_init(&vr->vr, num, vr->va, MIC_VIRTIO_RING_ALIGN);
                ret = vringh_init_kern(&mvr->vrh,
                        *(u32 *)mic_vq_features(mvdev->dd), num, false,
                        vr->vr.desc, vr->vr.avail, vr->vr.used);
                if (ret) {
                        dev_err(mic_dev(mvdev), "%s %d err %d\n",
                                __func__, __LINE__, ret);
                        goto err;
                }
                vringh_kiov_init(&mvr->riov, NULL, 0);
                vringh_kiov_init(&mvr->wiov, NULL, 0);
                mvr->head = USHRT_MAX;
                mvr->mvdev = mvdev;
                mvr->vrh.notify = mic_notify;
                dev_dbg(mdev->sdev->parent,
                        "%s %d index %d va %p info %p vr_size 0x%x\n",
                        __func__, __LINE__, i, vr->va, vr->info, vr_size);
        }

        snprintf(irqname, sizeof(irqname), "mic%dvirtio%d", mdev->id,
                 mvdev->virtio_id);
        mvdev->virtio_db = mic_next_db(mdev);
        mvdev->virtio_cookie = mic_request_irq(mdev, mic_virtio_intr_handler,
                        irqname, mvdev, mvdev->virtio_db, MIC_INTR_DB);
        if (IS_ERR(mvdev->virtio_cookie)) {
                ret = PTR_ERR(mvdev->virtio_cookie);
                dev_dbg(mdev->sdev->parent, "request irq failed\n");
                goto err;
        }

        mvdev->dc->c2h_vdev_db = mvdev->virtio_db;

        list_add_tail(&mvdev->list, &mdev->vdev_list);
        /*
         * Order the type update with previous stores. This write barrier
         * is paired with the corresponding read barrier before the uncached
         * system memory read of the type, on the card while scanning the
         * device page.
         */
        smp_wmb();
        dd->type = type;

        dev_dbg(mdev->sdev->parent, "Added virtio device id %d\n", dd->type);

        db = bootparam->h2c_config_db;
        if (db != -1)
                mdev->ops->send_intr(mdev, db);
        mutex_unlock(&mdev->mic_mutex);
        return 0;
err:
        vqconfig = mic_vq_config(dd);
        for (j = 0; j < i; j++) {
                struct mic_vringh *mvr = &mvdev->mvr[j];
                mic_unmap_single(mdev, le64_to_cpu(vqconfig[j].address),
                                 mvr->vring.len);
                free_pages((unsigned long)mvr->vring.va,
                           get_order(mvr->vring.len));
        }
        mutex_unlock(&mdev->mic_mutex);
        return ret;
}

void mic_virtio_del_device(struct mic_vdev *mvdev)
{
        struct list_head *pos, *tmp;
        struct mic_vdev *tmp_mvdev;
        struct mic_device *mdev = mvdev->mdev;
        DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake);
        int i, ret, retry;
        struct mic_vqconfig *vqconfig;
        struct mic_bootparam *bootparam = mdev->dp;
        s8 db;

        mutex_lock(&mdev->mic_mutex);
        db = bootparam->h2c_config_db;
        if (db == -1)
                goto skip_hot_remove;
        dev_dbg(mdev->sdev->parent,
                "Requesting hot remove id %d\n", mvdev->virtio_id);
        mvdev->dc->config_change = MIC_VIRTIO_PARAM_DEV_REMOVE;
        mdev->ops->send_intr(mdev, db);
        for (retry = 100; retry--;) {
                ret = wait_event_timeout(wake,
                        mvdev->dc->guest_ack, msecs_to_jiffies(100));
                if (ret)
                        break;
        }
        dev_dbg(mdev->sdev->parent,
                "Device id %d config_change %d guest_ack %d retry %d\n",
                mvdev->virtio_id, mvdev->dc->config_change,
                mvdev->dc->guest_ack, retry);
        mvdev->dc->config_change = 0;
        mvdev->dc->guest_ack = 0;
skip_hot_remove:
        mic_free_irq(mdev, mvdev->virtio_cookie, mvdev);
        flush_work(&mvdev->virtio_bh_work);
        vqconfig = mic_vq_config(mvdev->dd);
        for (i = 0; i < mvdev->dd->num_vq; i++) {
                struct mic_vringh *mvr = &mvdev->mvr[i];
                vringh_kiov_cleanup(&mvr->riov);
                vringh_kiov_cleanup(&mvr->wiov);
                mic_unmap_single(mdev, le64_to_cpu(vqconfig[i].address),
                                 mvr->vring.len);
                free_pages((unsigned long)mvr->vring.va,
                           get_order(mvr->vring.len));
        }

        list_for_each_safe(pos, tmp, &mdev->vdev_list) {
                tmp_mvdev = list_entry(pos, struct mic_vdev, list);
                if (tmp_mvdev == mvdev) {
                        list_del(pos);
                        dev_dbg(mdev->sdev->parent,
                                "Removing virtio device id %d\n",
                                mvdev->virtio_id);
                        break;
                }
        }
        /*
         * Order the type update with previous stores. This write barrier
         * is paired with the corresponding read barrier before the uncached
         * system memory read of the type, on the card while scanning the
         * device page.
         */
        smp_wmb();
        mvdev->dd->type = -1;
        mutex_unlock(&mdev->mic_mutex);
}