linux/drivers/scsi/vmw_pvscsi.c
<<
>>
Prefs
   1/*
   2 * Linux driver for VMware's para-virtualized SCSI HBA.
   3 *
   4 * Copyright (C) 2008-2014, VMware, Inc. All Rights Reserved.
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License as published by the
   8 * Free Software Foundation; version 2 of the License and no later version.
   9 *
  10 * This program is distributed in the hope that it will be useful, but
  11 * WITHOUT ANY WARRANTY; without even the implied warranty of
  12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  13 * NON INFRINGEMENT.  See the GNU General Public License for more
  14 * details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with this program; if not, write to the Free Software
  18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 *
  20 * Maintained by: Jim Gill <jgill@vmware.com>
  21 *
  22 */
  23
  24#include <linux/kernel.h>
  25#include <linux/module.h>
  26#include <linux/interrupt.h>
  27#include <linux/slab.h>
  28#include <linux/workqueue.h>
  29#include <linux/pci.h>
  30
  31#include <scsi/scsi.h>
  32#include <scsi/scsi_host.h>
  33#include <scsi/scsi_cmnd.h>
  34#include <scsi/scsi_device.h>
  35#include <scsi/scsi_tcq.h>
  36
  37#include "vmw_pvscsi.h"
  38
  39#define PVSCSI_LINUX_DRIVER_DESC "VMware PVSCSI driver"
  40
  41MODULE_DESCRIPTION(PVSCSI_LINUX_DRIVER_DESC);
  42MODULE_AUTHOR("VMware, Inc.");
  43MODULE_LICENSE("GPL");
  44MODULE_VERSION(PVSCSI_DRIVER_VERSION_STRING);
  45
  46#define PVSCSI_DEFAULT_NUM_PAGES_PER_RING       8
  47#define PVSCSI_DEFAULT_NUM_PAGES_MSG_RING       1
  48#define PVSCSI_DEFAULT_QUEUE_DEPTH              254
  49#define SGL_SIZE                                PAGE_SIZE
  50
  51struct pvscsi_sg_list {
  52        struct PVSCSISGElement sge[PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT];
  53};
  54
  55struct pvscsi_ctx {
  56        /*
  57         * The index of the context in cmd_map serves as the context ID for a
  58         * 1-to-1 mapping completions back to requests.
  59         */
  60        struct scsi_cmnd        *cmd;
  61        struct pvscsi_sg_list   *sgl;
  62        struct list_head        list;
  63        dma_addr_t              dataPA;
  64        dma_addr_t              sensePA;
  65        dma_addr_t              sglPA;
  66        struct completion       *abort_cmp;
  67};
  68
  69struct pvscsi_adapter {
  70        char                            *mmioBase;
  71        u8                              rev;
  72        bool                            use_msg;
  73        bool                            use_req_threshold;
  74
  75        spinlock_t                      hw_lock;
  76
  77        struct workqueue_struct         *workqueue;
  78        struct work_struct              work;
  79
  80        struct PVSCSIRingReqDesc        *req_ring;
  81        unsigned                        req_pages;
  82        unsigned                        req_depth;
  83        dma_addr_t                      reqRingPA;
  84
  85        struct PVSCSIRingCmpDesc        *cmp_ring;
  86        unsigned                        cmp_pages;
  87        dma_addr_t                      cmpRingPA;
  88
  89        struct PVSCSIRingMsgDesc        *msg_ring;
  90        unsigned                        msg_pages;
  91        dma_addr_t                      msgRingPA;
  92
  93        struct PVSCSIRingsState         *rings_state;
  94        dma_addr_t                      ringStatePA;
  95
  96        struct pci_dev                  *dev;
  97        struct Scsi_Host                *host;
  98
  99        struct list_head                cmd_pool;
 100        struct pvscsi_ctx               *cmd_map;
 101};
 102
 103
 104/* Command line parameters */
 105static int pvscsi_ring_pages;
 106static int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
 107static int pvscsi_cmd_per_lun    = PVSCSI_DEFAULT_QUEUE_DEPTH;
 108static bool pvscsi_disable_msi;
 109static bool pvscsi_disable_msix;
 110static bool pvscsi_use_msg       = true;
 111static bool pvscsi_use_req_threshold = true;
 112
 113#define PVSCSI_RW (S_IRUSR | S_IWUSR)
 114
 115module_param_named(ring_pages, pvscsi_ring_pages, int, PVSCSI_RW);
 116MODULE_PARM_DESC(ring_pages, "Number of pages per req/cmp ring - (default="
 117                 __stringify(PVSCSI_DEFAULT_NUM_PAGES_PER_RING)
 118                 "[up to 16 targets],"
 119                 __stringify(PVSCSI_SETUP_RINGS_MAX_NUM_PAGES)
 120                 "[for 16+ targets])");
 121
 122module_param_named(msg_ring_pages, pvscsi_msg_ring_pages, int, PVSCSI_RW);
 123MODULE_PARM_DESC(msg_ring_pages, "Number of pages for the msg ring - (default="
 124                 __stringify(PVSCSI_DEFAULT_NUM_PAGES_MSG_RING) ")");
 125
 126module_param_named(cmd_per_lun, pvscsi_cmd_per_lun, int, PVSCSI_RW);
 127MODULE_PARM_DESC(cmd_per_lun, "Maximum commands per lun - (default="
 128                 __stringify(PVSCSI_DEFAULT_QUEUE_DEPTH) ")");
 129
 130module_param_named(disable_msi, pvscsi_disable_msi, bool, PVSCSI_RW);
 131MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)");
 132
 133module_param_named(disable_msix, pvscsi_disable_msix, bool, PVSCSI_RW);
 134MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)");
 135
 136module_param_named(use_msg, pvscsi_use_msg, bool, PVSCSI_RW);
 137MODULE_PARM_DESC(use_msg, "Use msg ring when available - (default=1)");
 138
 139module_param_named(use_req_threshold, pvscsi_use_req_threshold,
 140                   bool, PVSCSI_RW);
 141MODULE_PARM_DESC(use_req_threshold, "Use driver-based request coalescing if configured - (default=1)");
 142
 143static const struct pci_device_id pvscsi_pci_tbl[] = {
 144        { PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_PVSCSI) },
 145        { 0 }
 146};
 147
 148MODULE_DEVICE_TABLE(pci, pvscsi_pci_tbl);
 149
 150static struct device *
 151pvscsi_dev(const struct pvscsi_adapter *adapter)
 152{
 153        return &(adapter->dev->dev);
 154}
 155
 156static struct pvscsi_ctx *
 157pvscsi_find_context(const struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
 158{
 159        struct pvscsi_ctx *ctx, *end;
 160
 161        end = &adapter->cmd_map[adapter->req_depth];
 162        for (ctx = adapter->cmd_map; ctx < end; ctx++)
 163                if (ctx->cmd == cmd)
 164                        return ctx;
 165
 166        return NULL;
 167}
 168
 169static struct pvscsi_ctx *
 170pvscsi_acquire_context(struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
 171{
 172        struct pvscsi_ctx *ctx;
 173
 174        if (list_empty(&adapter->cmd_pool))
 175                return NULL;
 176
 177        ctx = list_first_entry(&adapter->cmd_pool, struct pvscsi_ctx, list);
 178        ctx->cmd = cmd;
 179        list_del(&ctx->list);
 180
 181        return ctx;
 182}
 183
 184static void pvscsi_release_context(struct pvscsi_adapter *adapter,
 185                                   struct pvscsi_ctx *ctx)
 186{
 187        ctx->cmd = NULL;
 188        ctx->abort_cmp = NULL;
 189        list_add(&ctx->list, &adapter->cmd_pool);
 190}
 191
 192/*
 193 * Map a pvscsi_ctx struct to a context ID field value; we map to a simple
 194 * non-zero integer. ctx always points to an entry in cmd_map array, hence
 195 * the return value is always >=1.
 196 */
 197static u64 pvscsi_map_context(const struct pvscsi_adapter *adapter,
 198                              const struct pvscsi_ctx *ctx)
 199{
 200        return ctx - adapter->cmd_map + 1;
 201}
 202
 203static struct pvscsi_ctx *
 204pvscsi_get_context(const struct pvscsi_adapter *adapter, u64 context)
 205{
 206        return &adapter->cmd_map[context - 1];
 207}
 208
 209static void pvscsi_reg_write(const struct pvscsi_adapter *adapter,
 210                             u32 offset, u32 val)
 211{
 212        writel(val, adapter->mmioBase + offset);
 213}
 214
 215static u32 pvscsi_reg_read(const struct pvscsi_adapter *adapter, u32 offset)
 216{
 217        return readl(adapter->mmioBase + offset);
 218}
 219
 220static u32 pvscsi_read_intr_status(const struct pvscsi_adapter *adapter)
 221{
 222        return pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_INTR_STATUS);
 223}
 224
 225static void pvscsi_write_intr_status(const struct pvscsi_adapter *adapter,
 226                                     u32 val)
 227{
 228        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_STATUS, val);
 229}
 230
 231static void pvscsi_unmask_intr(const struct pvscsi_adapter *adapter)
 232{
 233        u32 intr_bits;
 234
 235        intr_bits = PVSCSI_INTR_CMPL_MASK;
 236        if (adapter->use_msg)
 237                intr_bits |= PVSCSI_INTR_MSG_MASK;
 238
 239        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, intr_bits);
 240}
 241
 242static void pvscsi_mask_intr(const struct pvscsi_adapter *adapter)
 243{
 244        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, 0);
 245}
 246
 247static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter,
 248                                  u32 cmd, const void *desc, size_t len)
 249{
 250        const u32 *ptr = desc;
 251        size_t i;
 252
 253        len /= sizeof(*ptr);
 254        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd);
 255        for (i = 0; i < len; i++)
 256                pvscsi_reg_write(adapter,
 257                                 PVSCSI_REG_OFFSET_COMMAND_DATA, ptr[i]);
 258}
 259
 260static void pvscsi_abort_cmd(const struct pvscsi_adapter *adapter,
 261                             const struct pvscsi_ctx *ctx)
 262{
 263        struct PVSCSICmdDescAbortCmd cmd = { 0 };
 264
 265        cmd.target = ctx->cmd->device->id;
 266        cmd.context = pvscsi_map_context(adapter, ctx);
 267
 268        pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd));
 269}
 270
 271static void pvscsi_kick_rw_io(const struct pvscsi_adapter *adapter)
 272{
 273        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
 274}
 275
 276static void pvscsi_process_request_ring(const struct pvscsi_adapter *adapter)
 277{
 278        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
 279}
 280
 281static int scsi_is_rw(unsigned char op)
 282{
 283        return op == READ_6  || op == WRITE_6 ||
 284               op == READ_10 || op == WRITE_10 ||
 285               op == READ_12 || op == WRITE_12 ||
 286               op == READ_16 || op == WRITE_16;
 287}
 288
 289static void pvscsi_kick_io(const struct pvscsi_adapter *adapter,
 290                           unsigned char op)
 291{
 292        if (scsi_is_rw(op)) {
 293                struct PVSCSIRingsState *s = adapter->rings_state;
 294
 295                if (!adapter->use_req_threshold ||
 296                    s->reqProdIdx - s->reqConsIdx >= s->reqCallThreshold)
 297                        pvscsi_kick_rw_io(adapter);
 298        } else {
 299                pvscsi_process_request_ring(adapter);
 300        }
 301}
 302
 303static void ll_adapter_reset(const struct pvscsi_adapter *adapter)
 304{
 305        dev_dbg(pvscsi_dev(adapter), "Adapter Reset on %p\n", adapter);
 306
 307        pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
 308}
 309
 310static void ll_bus_reset(const struct pvscsi_adapter *adapter)
 311{
 312        dev_dbg(pvscsi_dev(adapter), "Resetting bus on %p\n", adapter);
 313
 314        pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_BUS, NULL, 0);
 315}
 316
 317static void ll_device_reset(const struct pvscsi_adapter *adapter, u32 target)
 318{
 319        struct PVSCSICmdDescResetDevice cmd = { 0 };
 320
 321        dev_dbg(pvscsi_dev(adapter), "Resetting device: target=%u\n", target);
 322
 323        cmd.target = target;
 324
 325        pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_DEVICE,
 326                              &cmd, sizeof(cmd));
 327}
 328
 329static void pvscsi_create_sg(struct pvscsi_ctx *ctx,
 330                             struct scatterlist *sg, unsigned count)
 331{
 332        unsigned i;
 333        struct PVSCSISGElement *sge;
 334
 335        BUG_ON(count > PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT);
 336
 337        sge = &ctx->sgl->sge[0];
 338        for (i = 0; i < count; i++, sg++) {
 339                sge[i].addr   = sg_dma_address(sg);
 340                sge[i].length = sg_dma_len(sg);
 341                sge[i].flags  = 0;
 342        }
 343}
 344
 345/*
 346 * Map all data buffers for a command into PCI space and
 347 * setup the scatter/gather list if needed.
 348 */
 349static int pvscsi_map_buffers(struct pvscsi_adapter *adapter,
 350                              struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd,
 351                              struct PVSCSIRingReqDesc *e)
 352{
 353        unsigned count;
 354        unsigned bufflen = scsi_bufflen(cmd);
 355        struct scatterlist *sg;
 356
 357        e->dataLen = bufflen;
 358        e->dataAddr = 0;
 359        if (bufflen == 0)
 360                return 0;
 361
 362        sg = scsi_sglist(cmd);
 363        count = scsi_sg_count(cmd);
 364        if (count != 0) {
 365                int segs = scsi_dma_map(cmd);
 366
 367                if (segs == -ENOMEM) {
 368                        scmd_printk(KERN_ERR, cmd,
 369                                    "vmw_pvscsi: Failed to map cmd sglist for DMA.\n");
 370                        return -ENOMEM;
 371                } else if (segs > 1) {
 372                        pvscsi_create_sg(ctx, sg, segs);
 373
 374                        e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
 375                        ctx->sglPA = pci_map_single(adapter->dev, ctx->sgl,
 376                                                    SGL_SIZE, PCI_DMA_TODEVICE);
 377                        if (pci_dma_mapping_error(adapter->dev, ctx->sglPA)) {
 378                                scmd_printk(KERN_ERR, cmd,
 379                                            "vmw_pvscsi: Failed to map ctx sglist for DMA.\n");
 380                                scsi_dma_unmap(cmd);
 381                                ctx->sglPA = 0;
 382                                return -ENOMEM;
 383                        }
 384                        e->dataAddr = ctx->sglPA;
 385                } else
 386                        e->dataAddr = sg_dma_address(sg);
 387        } else {
 388                /*
 389                 * In case there is no S/G list, scsi_sglist points
 390                 * directly to the buffer.
 391                 */
 392                ctx->dataPA = pci_map_single(adapter->dev, sg, bufflen,
 393                                             cmd->sc_data_direction);
 394                if (pci_dma_mapping_error(adapter->dev, ctx->dataPA)) {
 395                        scmd_printk(KERN_ERR, cmd,
 396                                    "vmw_pvscsi: Failed to map direct data buffer for DMA.\n");
 397                        return -ENOMEM;
 398                }
 399                e->dataAddr = ctx->dataPA;
 400        }
 401
 402        return 0;
 403}
 404
 405static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter,
 406                                 struct pvscsi_ctx *ctx)
 407{
 408        struct scsi_cmnd *cmd;
 409        unsigned bufflen;
 410
 411        cmd = ctx->cmd;
 412        bufflen = scsi_bufflen(cmd);
 413
 414        if (bufflen != 0) {
 415                unsigned count = scsi_sg_count(cmd);
 416
 417                if (count != 0) {
 418                        scsi_dma_unmap(cmd);
 419                        if (ctx->sglPA) {
 420                                pci_unmap_single(adapter->dev, ctx->sglPA,
 421                                                 SGL_SIZE, PCI_DMA_TODEVICE);
 422                                ctx->sglPA = 0;
 423                        }
 424                } else
 425                        pci_unmap_single(adapter->dev, ctx->dataPA, bufflen,
 426                                         cmd->sc_data_direction);
 427        }
 428        if (cmd->sense_buffer)
 429                pci_unmap_single(adapter->dev, ctx->sensePA,
 430                                 SCSI_SENSE_BUFFERSIZE, PCI_DMA_FROMDEVICE);
 431}
 432
 433static int pvscsi_allocate_rings(struct pvscsi_adapter *adapter)
 434{
 435        adapter->rings_state = pci_alloc_consistent(adapter->dev, PAGE_SIZE,
 436                                                    &adapter->ringStatePA);
 437        if (!adapter->rings_state)
 438                return -ENOMEM;
 439
 440        adapter->req_pages = min(PVSCSI_MAX_NUM_PAGES_REQ_RING,
 441                                 pvscsi_ring_pages);
 442        adapter->req_depth = adapter->req_pages
 443                                        * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
 444        adapter->req_ring = pci_alloc_consistent(adapter->dev,
 445                                                 adapter->req_pages * PAGE_SIZE,
 446                                                 &adapter->reqRingPA);
 447        if (!adapter->req_ring)
 448                return -ENOMEM;
 449
 450        adapter->cmp_pages = min(PVSCSI_MAX_NUM_PAGES_CMP_RING,
 451                                 pvscsi_ring_pages);
 452        adapter->cmp_ring = pci_alloc_consistent(adapter->dev,
 453                                                 adapter->cmp_pages * PAGE_SIZE,
 454                                                 &adapter->cmpRingPA);
 455        if (!adapter->cmp_ring)
 456                return -ENOMEM;
 457
 458        BUG_ON(!IS_ALIGNED(adapter->ringStatePA, PAGE_SIZE));
 459        BUG_ON(!IS_ALIGNED(adapter->reqRingPA, PAGE_SIZE));
 460        BUG_ON(!IS_ALIGNED(adapter->cmpRingPA, PAGE_SIZE));
 461
 462        if (!adapter->use_msg)
 463                return 0;
 464
 465        adapter->msg_pages = min(PVSCSI_MAX_NUM_PAGES_MSG_RING,
 466                                 pvscsi_msg_ring_pages);
 467        adapter->msg_ring = pci_alloc_consistent(adapter->dev,
 468                                                 adapter->msg_pages * PAGE_SIZE,
 469                                                 &adapter->msgRingPA);
 470        if (!adapter->msg_ring)
 471                return -ENOMEM;
 472        BUG_ON(!IS_ALIGNED(adapter->msgRingPA, PAGE_SIZE));
 473
 474        return 0;
 475}
 476
 477static void pvscsi_setup_all_rings(const struct pvscsi_adapter *adapter)
 478{
 479        struct PVSCSICmdDescSetupRings cmd = { 0 };
 480        dma_addr_t base;
 481        unsigned i;
 482
 483        cmd.ringsStatePPN   = adapter->ringStatePA >> PAGE_SHIFT;
 484        cmd.reqRingNumPages = adapter->req_pages;
 485        cmd.cmpRingNumPages = adapter->cmp_pages;
 486
 487        base = adapter->reqRingPA;
 488        for (i = 0; i < adapter->req_pages; i++) {
 489                cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
 490                base += PAGE_SIZE;
 491        }
 492
 493        base = adapter->cmpRingPA;
 494        for (i = 0; i < adapter->cmp_pages; i++) {
 495                cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
 496                base += PAGE_SIZE;
 497        }
 498
 499        memset(adapter->rings_state, 0, PAGE_SIZE);
 500        memset(adapter->req_ring, 0, adapter->req_pages * PAGE_SIZE);
 501        memset(adapter->cmp_ring, 0, adapter->cmp_pages * PAGE_SIZE);
 502
 503        pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_RINGS,
 504                              &cmd, sizeof(cmd));
 505
 506        if (adapter->use_msg) {
 507                struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
 508
 509                cmd_msg.numPages = adapter->msg_pages;
 510
 511                base = adapter->msgRingPA;
 512                for (i = 0; i < adapter->msg_pages; i++) {
 513                        cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
 514                        base += PAGE_SIZE;
 515                }
 516                memset(adapter->msg_ring, 0, adapter->msg_pages * PAGE_SIZE);
 517
 518                pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_MSG_RING,
 519                                      &cmd_msg, sizeof(cmd_msg));
 520        }
 521}
 522
 523static int pvscsi_change_queue_depth(struct scsi_device *sdev, int qdepth)
 524{
 525        if (!sdev->tagged_supported)
 526                qdepth = 1;
 527        return scsi_change_queue_depth(sdev, qdepth);
 528}
 529
 530/*
 531 * Pull a completion descriptor off and pass the completion back
 532 * to the SCSI mid layer.
 533 */
 534static void pvscsi_complete_request(struct pvscsi_adapter *adapter,
 535                                    const struct PVSCSIRingCmpDesc *e)
 536{
 537        struct pvscsi_ctx *ctx;
 538        struct scsi_cmnd *cmd;
 539        struct completion *abort_cmp;
 540        u32 btstat = e->hostStatus;
 541        u32 sdstat = e->scsiStatus;
 542
 543        ctx = pvscsi_get_context(adapter, e->context);
 544        cmd = ctx->cmd;
 545        abort_cmp = ctx->abort_cmp;
 546        pvscsi_unmap_buffers(adapter, ctx);
 547        pvscsi_release_context(adapter, ctx);
 548        if (abort_cmp) {
 549                /*
 550                 * The command was requested to be aborted. Just signal that
 551                 * the request completed and swallow the actual cmd completion
 552                 * here. The abort handler will post a completion for this
 553                 * command indicating that it got successfully aborted.
 554                 */
 555                complete(abort_cmp);
 556                return;
 557        }
 558
 559        cmd->result = 0;
 560        if (sdstat != SAM_STAT_GOOD &&
 561            (btstat == BTSTAT_SUCCESS ||
 562             btstat == BTSTAT_LINKED_COMMAND_COMPLETED ||
 563             btstat == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG)) {
 564                if (sdstat == SAM_STAT_COMMAND_TERMINATED) {
 565                        cmd->result = (DID_RESET << 16);
 566                } else {
 567                        cmd->result = (DID_OK << 16) | sdstat;
 568                        if (sdstat == SAM_STAT_CHECK_CONDITION &&
 569                            cmd->sense_buffer)
 570                                cmd->result |= (DRIVER_SENSE << 24);
 571                }
 572        } else
 573                switch (btstat) {
 574                case BTSTAT_SUCCESS:
 575                case BTSTAT_LINKED_COMMAND_COMPLETED:
 576                case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
 577                        /* If everything went fine, let's move on..  */
 578                        cmd->result = (DID_OK << 16);
 579                        break;
 580
 581                case BTSTAT_DATARUN:
 582                case BTSTAT_DATA_UNDERRUN:
 583                        /* Report residual data in underruns */
 584                        scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
 585                        cmd->result = (DID_ERROR << 16);
 586                        break;
 587
 588                case BTSTAT_SELTIMEO:
 589                        /* Our emulation returns this for non-connected devs */
 590                        cmd->result = (DID_BAD_TARGET << 16);
 591                        break;
 592
 593                case BTSTAT_LUNMISMATCH:
 594                case BTSTAT_TAGREJECT:
 595                case BTSTAT_BADMSG:
 596                        cmd->result = (DRIVER_INVALID << 24);
 597                        /* fall through */
 598
 599                case BTSTAT_HAHARDWARE:
 600                case BTSTAT_INVPHASE:
 601                case BTSTAT_HATIMEOUT:
 602                case BTSTAT_NORESPONSE:
 603                case BTSTAT_DISCONNECT:
 604                case BTSTAT_HASOFTWARE:
 605                case BTSTAT_BUSFREE:
 606                case BTSTAT_SENSFAILED:
 607                        cmd->result |= (DID_ERROR << 16);
 608                        break;
 609
 610                case BTSTAT_SENTRST:
 611                case BTSTAT_RECVRST:
 612                case BTSTAT_BUSRESET:
 613                        cmd->result = (DID_RESET << 16);
 614                        break;
 615
 616                case BTSTAT_ABORTQUEUE:
 617                        cmd->result = (DID_BUS_BUSY << 16);
 618                        break;
 619
 620                case BTSTAT_SCSIPARITY:
 621                        cmd->result = (DID_PARITY << 16);
 622                        break;
 623
 624                default:
 625                        cmd->result = (DID_ERROR << 16);
 626                        scmd_printk(KERN_DEBUG, cmd,
 627                                    "Unknown completion status: 0x%x\n",
 628                                    btstat);
 629        }
 630
 631        dev_dbg(&cmd->device->sdev_gendev,
 632                "cmd=%p %x ctx=%p result=0x%x status=0x%x,%x\n",
 633                cmd, cmd->cmnd[0], ctx, cmd->result, btstat, sdstat);
 634
 635        cmd->scsi_done(cmd);
 636}
 637
 638/*
 639 * barrier usage : Since the PVSCSI device is emulated, there could be cases
 640 * where we may want to serialize some accesses between the driver and the
 641 * emulation layer. We use compiler barriers instead of the more expensive
 642 * memory barriers because PVSCSI is only supported on X86 which has strong
 643 * memory access ordering.
 644 */
 645static void pvscsi_process_completion_ring(struct pvscsi_adapter *adapter)
 646{
 647        struct PVSCSIRingsState *s = adapter->rings_state;
 648        struct PVSCSIRingCmpDesc *ring = adapter->cmp_ring;
 649        u32 cmp_entries = s->cmpNumEntriesLog2;
 650
 651        while (s->cmpConsIdx != s->cmpProdIdx) {
 652                struct PVSCSIRingCmpDesc *e = ring + (s->cmpConsIdx &
 653                                                      MASK(cmp_entries));
 654                /*
 655                 * This barrier() ensures that *e is not dereferenced while
 656                 * the device emulation still writes data into the slot.
 657                 * Since the device emulation advances s->cmpProdIdx only after
 658                 * updating the slot we want to check it first.
 659                 */
 660                barrier();
 661                pvscsi_complete_request(adapter, e);
 662                /*
 663                 * This barrier() ensures that compiler doesn't reorder write
 664                 * to s->cmpConsIdx before the read of (*e) inside
 665                 * pvscsi_complete_request. Otherwise, device emulation may
 666                 * overwrite *e before we had a chance to read it.
 667                 */
 668                barrier();
 669                s->cmpConsIdx++;
 670        }
 671}
 672
 673/*
 674 * Translate a Linux SCSI request into a request ring entry.
 675 */
 676static int pvscsi_queue_ring(struct pvscsi_adapter *adapter,
 677                             struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd)
 678{
 679        struct PVSCSIRingsState *s;
 680        struct PVSCSIRingReqDesc *e;
 681        struct scsi_device *sdev;
 682        u32 req_entries;
 683
 684        s = adapter->rings_state;
 685        sdev = cmd->device;
 686        req_entries = s->reqNumEntriesLog2;
 687
 688        /*
 689         * If this condition holds, we might have room on the request ring, but
 690         * we might not have room on the completion ring for the response.
 691         * However, we have already ruled out this possibility - we would not
 692         * have successfully allocated a context if it were true, since we only
 693         * have one context per request entry.  Check for it anyway, since it
 694         * would be a serious bug.
 695         */
 696        if (s->reqProdIdx - s->cmpConsIdx >= 1 << req_entries) {
 697                scmd_printk(KERN_ERR, cmd, "vmw_pvscsi: "
 698                            "ring full: reqProdIdx=%d cmpConsIdx=%d\n",
 699                            s->reqProdIdx, s->cmpConsIdx);
 700                return -1;
 701        }
 702
 703        e = adapter->req_ring + (s->reqProdIdx & MASK(req_entries));
 704
 705        e->bus    = sdev->channel;
 706        e->target = sdev->id;
 707        memset(e->lun, 0, sizeof(e->lun));
 708        e->lun[1] = sdev->lun;
 709
 710        if (cmd->sense_buffer) {
 711                ctx->sensePA = pci_map_single(adapter->dev, cmd->sense_buffer,
 712                                              SCSI_SENSE_BUFFERSIZE,
 713                                              PCI_DMA_FROMDEVICE);
 714                if (pci_dma_mapping_error(adapter->dev, ctx->sensePA)) {
 715                        scmd_printk(KERN_ERR, cmd,
 716                                    "vmw_pvscsi: Failed to map sense buffer for DMA.\n");
 717                        ctx->sensePA = 0;
 718                        return -ENOMEM;
 719                }
 720                e->senseAddr = ctx->sensePA;
 721                e->senseLen = SCSI_SENSE_BUFFERSIZE;
 722        } else {
 723                e->senseLen  = 0;
 724                e->senseAddr = 0;
 725        }
 726        e->cdbLen   = cmd->cmd_len;
 727        e->vcpuHint = smp_processor_id();
 728        memcpy(e->cdb, cmd->cmnd, e->cdbLen);
 729
 730        e->tag = SIMPLE_QUEUE_TAG;
 731
 732        if (cmd->sc_data_direction == DMA_FROM_DEVICE)
 733                e->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
 734        else if (cmd->sc_data_direction == DMA_TO_DEVICE)
 735                e->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
 736        else if (cmd->sc_data_direction == DMA_NONE)
 737                e->flags = PVSCSI_FLAG_CMD_DIR_NONE;
 738        else
 739                e->flags = 0;
 740
 741        if (pvscsi_map_buffers(adapter, ctx, cmd, e) != 0) {
 742                if (cmd->sense_buffer) {
 743                        pci_unmap_single(adapter->dev, ctx->sensePA,
 744                                         SCSI_SENSE_BUFFERSIZE,
 745                                         PCI_DMA_FROMDEVICE);
 746                        ctx->sensePA = 0;
 747                }
 748                return -ENOMEM;
 749        }
 750
 751        e->context = pvscsi_map_context(adapter, ctx);
 752
 753        barrier();
 754
 755        s->reqProdIdx++;
 756
 757        return 0;
 758}
 759
 760static int pvscsi_queue_lck(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
 761{
 762        struct Scsi_Host *host = cmd->device->host;
 763        struct pvscsi_adapter *adapter = shost_priv(host);
 764        struct pvscsi_ctx *ctx;
 765        unsigned long flags;
 766
 767        spin_lock_irqsave(&adapter->hw_lock, flags);
 768
 769        ctx = pvscsi_acquire_context(adapter, cmd);
 770        if (!ctx || pvscsi_queue_ring(adapter, ctx, cmd) != 0) {
 771                if (ctx)
 772                        pvscsi_release_context(adapter, ctx);
 773                spin_unlock_irqrestore(&adapter->hw_lock, flags);
 774                return SCSI_MLQUEUE_HOST_BUSY;
 775        }
 776
 777        cmd->scsi_done = done;
 778
 779        dev_dbg(&cmd->device->sdev_gendev,
 780                "queued cmd %p, ctx %p, op=%x\n", cmd, ctx, cmd->cmnd[0]);
 781
 782        spin_unlock_irqrestore(&adapter->hw_lock, flags);
 783
 784        pvscsi_kick_io(adapter, cmd->cmnd[0]);
 785
 786        return 0;
 787}
 788
 789static DEF_SCSI_QCMD(pvscsi_queue)
 790
 791static int pvscsi_abort(struct scsi_cmnd *cmd)
 792{
 793        struct pvscsi_adapter *adapter = shost_priv(cmd->device->host);
 794        struct pvscsi_ctx *ctx;
 795        unsigned long flags;
 796        int result = SUCCESS;
 797        DECLARE_COMPLETION_ONSTACK(abort_cmp);
 798        int done;
 799
 800        scmd_printk(KERN_DEBUG, cmd, "task abort on host %u, %p\n",
 801                    adapter->host->host_no, cmd);
 802
 803        spin_lock_irqsave(&adapter->hw_lock, flags);
 804
 805        /*
 806         * Poll the completion ring first - we might be trying to abort
 807         * a command that is waiting to be dispatched in the completion ring.
 808         */
 809        pvscsi_process_completion_ring(adapter);
 810
 811        /*
 812         * If there is no context for the command, it either already succeeded
 813         * or else was never properly issued.  Not our problem.
 814         */
 815        ctx = pvscsi_find_context(adapter, cmd);
 816        if (!ctx) {
 817                scmd_printk(KERN_DEBUG, cmd, "Failed to abort cmd %p\n", cmd);
 818                goto out;
 819        }
 820
 821        /*
 822         * Mark that the command has been requested to be aborted and issue
 823         * the abort.
 824         */
 825        ctx->abort_cmp = &abort_cmp;
 826
 827        pvscsi_abort_cmd(adapter, ctx);
 828        spin_unlock_irqrestore(&adapter->hw_lock, flags);
 829        /* Wait for 2 secs for the completion. */
 830        done = wait_for_completion_timeout(&abort_cmp, msecs_to_jiffies(2000));
 831        spin_lock_irqsave(&adapter->hw_lock, flags);
 832
 833        if (!done) {
 834                /*
 835                 * Failed to abort the command, unmark the fact that it
 836                 * was requested to be aborted.
 837                 */
 838                ctx->abort_cmp = NULL;
 839                result = FAILED;
 840                scmd_printk(KERN_DEBUG, cmd,
 841                            "Failed to get completion for aborted cmd %p\n",
 842                            cmd);
 843                goto out;
 844        }
 845
 846        /*
 847         * Successfully aborted the command.
 848         */
 849        cmd->result = (DID_ABORT << 16);
 850        cmd->scsi_done(cmd);
 851
 852out:
 853        spin_unlock_irqrestore(&adapter->hw_lock, flags);
 854        return result;
 855}
 856
 857/*
 858 * Abort all outstanding requests.  This is only safe to use if the completion
 859 * ring will never be walked again or the device has been reset, because it
 860 * destroys the 1-1 mapping between context field passed to emulation and our
 861 * request structure.
 862 */
 863static void pvscsi_reset_all(struct pvscsi_adapter *adapter)
 864{
 865        unsigned i;
 866
 867        for (i = 0; i < adapter->req_depth; i++) {
 868                struct pvscsi_ctx *ctx = &adapter->cmd_map[i];
 869                struct scsi_cmnd *cmd = ctx->cmd;
 870                if (cmd) {
 871                        scmd_printk(KERN_ERR, cmd,
 872                                    "Forced reset on cmd %p\n", cmd);
 873                        pvscsi_unmap_buffers(adapter, ctx);
 874                        pvscsi_release_context(adapter, ctx);
 875                        cmd->result = (DID_RESET << 16);
 876                        cmd->scsi_done(cmd);
 877                }
 878        }
 879}
 880
 881static int pvscsi_host_reset(struct scsi_cmnd *cmd)
 882{
 883        struct Scsi_Host *host = cmd->device->host;
 884        struct pvscsi_adapter *adapter = shost_priv(host);
 885        unsigned long flags;
 886        bool use_msg;
 887
 888        scmd_printk(KERN_INFO, cmd, "SCSI Host reset\n");
 889
 890        spin_lock_irqsave(&adapter->hw_lock, flags);
 891
 892        use_msg = adapter->use_msg;
 893
 894        if (use_msg) {
 895                adapter->use_msg = 0;
 896                spin_unlock_irqrestore(&adapter->hw_lock, flags);
 897
 898                /*
 899                 * Now that we know that the ISR won't add more work on the
 900                 * workqueue we can safely flush any outstanding work.
 901                 */
 902                flush_workqueue(adapter->workqueue);
 903                spin_lock_irqsave(&adapter->hw_lock, flags);
 904        }
 905
 906        /*
 907         * We're going to tear down the entire ring structure and set it back
 908         * up, so stalling new requests until all completions are flushed and
 909         * the rings are back in place.
 910         */
 911
 912        pvscsi_process_request_ring(adapter);
 913
 914        ll_adapter_reset(adapter);
 915
 916        /*
 917         * Now process any completions.  Note we do this AFTER adapter reset,
 918         * which is strange, but stops races where completions get posted
 919         * between processing the ring and issuing the reset.  The backend will
 920         * not touch the ring memory after reset, so the immediately pre-reset
 921         * completion ring state is still valid.
 922         */
 923        pvscsi_process_completion_ring(adapter);
 924
 925        pvscsi_reset_all(adapter);
 926        adapter->use_msg = use_msg;
 927        pvscsi_setup_all_rings(adapter);
 928        pvscsi_unmask_intr(adapter);
 929
 930        spin_unlock_irqrestore(&adapter->hw_lock, flags);
 931
 932        return SUCCESS;
 933}
 934
 935static int pvscsi_bus_reset(struct scsi_cmnd *cmd)
 936{
 937        struct Scsi_Host *host = cmd->device->host;
 938        struct pvscsi_adapter *adapter = shost_priv(host);
 939        unsigned long flags;
 940
 941        scmd_printk(KERN_INFO, cmd, "SCSI Bus reset\n");
 942
 943        /*
 944         * We don't want to queue new requests for this bus after
 945         * flushing all pending requests to emulation, since new
 946         * requests could then sneak in during this bus reset phase,
 947         * so take the lock now.
 948         */
 949        spin_lock_irqsave(&adapter->hw_lock, flags);
 950
 951        pvscsi_process_request_ring(adapter);
 952        ll_bus_reset(adapter);
 953        pvscsi_process_completion_ring(adapter);
 954
 955        spin_unlock_irqrestore(&adapter->hw_lock, flags);
 956
 957        return SUCCESS;
 958}
 959
 960static int pvscsi_device_reset(struct scsi_cmnd *cmd)
 961{
 962        struct Scsi_Host *host = cmd->device->host;
 963        struct pvscsi_adapter *adapter = shost_priv(host);
 964        unsigned long flags;
 965
 966        scmd_printk(KERN_INFO, cmd, "SCSI device reset on scsi%u:%u\n",
 967                    host->host_no, cmd->device->id);
 968
 969        /*
 970         * We don't want to queue new requests for this device after flushing
 971         * all pending requests to emulation, since new requests could then
 972         * sneak in during this device reset phase, so take the lock now.
 973         */
 974        spin_lock_irqsave(&adapter->hw_lock, flags);
 975
 976        pvscsi_process_request_ring(adapter);
 977        ll_device_reset(adapter, cmd->device->id);
 978        pvscsi_process_completion_ring(adapter);
 979
 980        spin_unlock_irqrestore(&adapter->hw_lock, flags);
 981
 982        return SUCCESS;
 983}
 984
 985static struct scsi_host_template pvscsi_template;
 986
 987static const char *pvscsi_info(struct Scsi_Host *host)
 988{
 989        struct pvscsi_adapter *adapter = shost_priv(host);
 990        static char buf[256];
 991
 992        sprintf(buf, "VMware PVSCSI storage adapter rev %d, req/cmp/msg rings: "
 993                "%u/%u/%u pages, cmd_per_lun=%u", adapter->rev,
 994                adapter->req_pages, adapter->cmp_pages, adapter->msg_pages,
 995                pvscsi_template.cmd_per_lun);
 996
 997        return buf;
 998}
 999
1000static struct scsi_host_template pvscsi_template = {
1001        .module                         = THIS_MODULE,
1002        .name                           = "VMware PVSCSI Host Adapter",
1003        .proc_name                      = "vmw_pvscsi",
1004        .info                           = pvscsi_info,
1005        .queuecommand                   = pvscsi_queue,
1006        .this_id                        = -1,
1007        .sg_tablesize                   = PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT,
1008        .dma_boundary                   = UINT_MAX,
1009        .max_sectors                    = 0xffff,
1010        .use_clustering                 = ENABLE_CLUSTERING,
1011        .change_queue_depth             = pvscsi_change_queue_depth,
1012        .eh_abort_handler               = pvscsi_abort,
1013        .eh_device_reset_handler        = pvscsi_device_reset,
1014        .eh_bus_reset_handler           = pvscsi_bus_reset,
1015        .eh_host_reset_handler          = pvscsi_host_reset,
1016};
1017
1018static void pvscsi_process_msg(const struct pvscsi_adapter *adapter,
1019                               const struct PVSCSIRingMsgDesc *e)
1020{
1021        struct PVSCSIRingsState *s = adapter->rings_state;
1022        struct Scsi_Host *host = adapter->host;
1023        struct scsi_device *sdev;
1024
1025        printk(KERN_INFO "vmw_pvscsi: msg type: 0x%x - MSG RING: %u/%u (%u) \n",
1026               e->type, s->msgProdIdx, s->msgConsIdx, s->msgNumEntriesLog2);
1027
1028        BUILD_BUG_ON(PVSCSI_MSG_LAST != 2);
1029
1030        if (e->type == PVSCSI_MSG_DEV_ADDED) {
1031                struct PVSCSIMsgDescDevStatusChanged *desc;
1032                desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
1033
1034                printk(KERN_INFO
1035                       "vmw_pvscsi: msg: device added at scsi%u:%u:%u\n",
1036                       desc->bus, desc->target, desc->lun[1]);
1037
1038                if (!scsi_host_get(host))
1039                        return;
1040
1041                sdev = scsi_device_lookup(host, desc->bus, desc->target,
1042                                          desc->lun[1]);
1043                if (sdev) {
1044                        printk(KERN_INFO "vmw_pvscsi: device already exists\n");
1045                        scsi_device_put(sdev);
1046                } else
1047                        scsi_add_device(adapter->host, desc->bus,
1048                                        desc->target, desc->lun[1]);
1049
1050                scsi_host_put(host);
1051        } else if (e->type == PVSCSI_MSG_DEV_REMOVED) {
1052                struct PVSCSIMsgDescDevStatusChanged *desc;
1053                desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
1054
1055                printk(KERN_INFO
1056                       "vmw_pvscsi: msg: device removed at scsi%u:%u:%u\n",
1057                       desc->bus, desc->target, desc->lun[1]);
1058
1059                if (!scsi_host_get(host))
1060                        return;
1061
1062                sdev = scsi_device_lookup(host, desc->bus, desc->target,
1063                                          desc->lun[1]);
1064                if (sdev) {
1065                        scsi_remove_device(sdev);
1066                        scsi_device_put(sdev);
1067                } else
1068                        printk(KERN_INFO
1069                               "vmw_pvscsi: failed to lookup scsi%u:%u:%u\n",
1070                               desc->bus, desc->target, desc->lun[1]);
1071
1072                scsi_host_put(host);
1073        }
1074}
1075
1076static int pvscsi_msg_pending(const struct pvscsi_adapter *adapter)
1077{
1078        struct PVSCSIRingsState *s = adapter->rings_state;
1079
1080        return s->msgProdIdx != s->msgConsIdx;
1081}
1082
1083static void pvscsi_process_msg_ring(const struct pvscsi_adapter *adapter)
1084{
1085        struct PVSCSIRingsState *s = adapter->rings_state;
1086        struct PVSCSIRingMsgDesc *ring = adapter->msg_ring;
1087        u32 msg_entries = s->msgNumEntriesLog2;
1088
1089        while (pvscsi_msg_pending(adapter)) {
1090                struct PVSCSIRingMsgDesc *e = ring + (s->msgConsIdx &
1091                                                      MASK(msg_entries));
1092
1093                barrier();
1094                pvscsi_process_msg(adapter, e);
1095                barrier();
1096                s->msgConsIdx++;
1097        }
1098}
1099
1100static void pvscsi_msg_workqueue_handler(struct work_struct *data)
1101{
1102        struct pvscsi_adapter *adapter;
1103
1104        adapter = container_of(data, struct pvscsi_adapter, work);
1105
1106        pvscsi_process_msg_ring(adapter);
1107}
1108
1109static int pvscsi_setup_msg_workqueue(struct pvscsi_adapter *adapter)
1110{
1111        char name[32];
1112
1113        if (!pvscsi_use_msg)
1114                return 0;
1115
1116        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
1117                         PVSCSI_CMD_SETUP_MSG_RING);
1118
1119        if (pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS) == -1)
1120                return 0;
1121
1122        snprintf(name, sizeof(name),
1123                 "vmw_pvscsi_wq_%u", adapter->host->host_no);
1124
1125        adapter->workqueue = create_singlethread_workqueue(name);
1126        if (!adapter->workqueue) {
1127                printk(KERN_ERR "vmw_pvscsi: failed to create work queue\n");
1128                return 0;
1129        }
1130        INIT_WORK(&adapter->work, pvscsi_msg_workqueue_handler);
1131
1132        return 1;
1133}
1134
1135static bool pvscsi_setup_req_threshold(struct pvscsi_adapter *adapter,
1136                                      bool enable)
1137{
1138        u32 val;
1139
1140        if (!pvscsi_use_req_threshold)
1141                return false;
1142
1143        pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
1144                         PVSCSI_CMD_SETUP_REQCALLTHRESHOLD);
1145        val = pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS);
1146        if (val == -1) {
1147                printk(KERN_INFO "vmw_pvscsi: device does not support req_threshold\n");
1148                return false;
1149        } else {
1150                struct PVSCSICmdDescSetupReqCall cmd_msg = { 0 };
1151                cmd_msg.enable = enable;
1152                printk(KERN_INFO
1153                       "vmw_pvscsi: %sabling reqCallThreshold\n",
1154                        enable ? "en" : "dis");
1155                pvscsi_write_cmd_desc(adapter,
1156                                      PVSCSI_CMD_SETUP_REQCALLTHRESHOLD,
1157                                      &cmd_msg, sizeof(cmd_msg));
1158                return pvscsi_reg_read(adapter,
1159                                       PVSCSI_REG_OFFSET_COMMAND_STATUS) != 0;
1160        }
1161}
1162
1163static irqreturn_t pvscsi_isr(int irq, void *devp)
1164{
1165        struct pvscsi_adapter *adapter = devp;
1166        unsigned long flags;
1167
1168        spin_lock_irqsave(&adapter->hw_lock, flags);
1169        pvscsi_process_completion_ring(adapter);
1170        if (adapter->use_msg && pvscsi_msg_pending(adapter))
1171                queue_work(adapter->workqueue, &adapter->work);
1172        spin_unlock_irqrestore(&adapter->hw_lock, flags);
1173
1174        return IRQ_HANDLED;
1175}
1176
1177static irqreturn_t pvscsi_shared_isr(int irq, void *devp)
1178{
1179        struct pvscsi_adapter *adapter = devp;
1180        u32 val = pvscsi_read_intr_status(adapter);
1181
1182        if (!(val & PVSCSI_INTR_ALL_SUPPORTED))
1183                return IRQ_NONE;
1184        pvscsi_write_intr_status(devp, val);
1185        return pvscsi_isr(irq, devp);
1186}
1187
1188static void pvscsi_free_sgls(const struct pvscsi_adapter *adapter)
1189{
1190        struct pvscsi_ctx *ctx = adapter->cmd_map;
1191        unsigned i;
1192
1193        for (i = 0; i < adapter->req_depth; ++i, ++ctx)
1194                free_pages((unsigned long)ctx->sgl, get_order(SGL_SIZE));
1195}
1196
1197static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter)
1198{
1199        free_irq(pci_irq_vector(adapter->dev, 0), adapter);
1200        pci_free_irq_vectors(adapter->dev);
1201}
1202
1203static void pvscsi_release_resources(struct pvscsi_adapter *adapter)
1204{
1205        pvscsi_shutdown_intr(adapter);
1206
1207        if (adapter->workqueue)
1208                destroy_workqueue(adapter->workqueue);
1209
1210        if (adapter->mmioBase)
1211                pci_iounmap(adapter->dev, adapter->mmioBase);
1212
1213        pci_release_regions(adapter->dev);
1214
1215        if (adapter->cmd_map) {
1216                pvscsi_free_sgls(adapter);
1217                kfree(adapter->cmd_map);
1218        }
1219
1220        if (adapter->rings_state)
1221                pci_free_consistent(adapter->dev, PAGE_SIZE,
1222                                    adapter->rings_state, adapter->ringStatePA);
1223
1224        if (adapter->req_ring)
1225                pci_free_consistent(adapter->dev,
1226                                    adapter->req_pages * PAGE_SIZE,
1227                                    adapter->req_ring, adapter->reqRingPA);
1228
1229        if (adapter->cmp_ring)
1230                pci_free_consistent(adapter->dev,
1231                                    adapter->cmp_pages * PAGE_SIZE,
1232                                    adapter->cmp_ring, adapter->cmpRingPA);
1233
1234        if (adapter->msg_ring)
1235                pci_free_consistent(adapter->dev,
1236                                    adapter->msg_pages * PAGE_SIZE,
1237                                    adapter->msg_ring, adapter->msgRingPA);
1238}
1239
1240/*
1241 * Allocate scatter gather lists.
1242 *
1243 * These are statically allocated.  Trying to be clever was not worth it.
1244 *
1245 * Dynamic allocation can fail, and we can't go deep into the memory
1246 * allocator, since we're a SCSI driver, and trying too hard to allocate
1247 * memory might generate disk I/O.  We also don't want to fail disk I/O
1248 * in that case because we can't get an allocation - the I/O could be
1249 * trying to swap out data to free memory.  Since that is pathological,
1250 * just use a statically allocated scatter list.
1251 *
1252 */
1253static int pvscsi_allocate_sg(struct pvscsi_adapter *adapter)
1254{
1255        struct pvscsi_ctx *ctx;
1256        int i;
1257
1258        ctx = adapter->cmd_map;
1259        BUILD_BUG_ON(sizeof(struct pvscsi_sg_list) > SGL_SIZE);
1260
1261        for (i = 0; i < adapter->req_depth; ++i, ++ctx) {
1262                ctx->sgl = (void *)__get_free_pages(GFP_KERNEL,
1263                                                    get_order(SGL_SIZE));
1264                ctx->sglPA = 0;
1265                BUG_ON(!IS_ALIGNED(((unsigned long)ctx->sgl), PAGE_SIZE));
1266                if (!ctx->sgl) {
1267                        for (; i >= 0; --i, --ctx) {
1268                                free_pages((unsigned long)ctx->sgl,
1269                                           get_order(SGL_SIZE));
1270                                ctx->sgl = NULL;
1271                        }
1272                        return -ENOMEM;
1273                }
1274        }
1275
1276        return 0;
1277}
1278
1279/*
1280 * Query the device, fetch the config info and return the
1281 * maximum number of targets on the adapter. In case of
1282 * failure due to any reason return default i.e. 16.
1283 */
1284static u32 pvscsi_get_max_targets(struct pvscsi_adapter *adapter)
1285{
1286        struct PVSCSICmdDescConfigCmd cmd;
1287        struct PVSCSIConfigPageHeader *header;
1288        struct device *dev;
1289        dma_addr_t configPagePA;
1290        void *config_page;
1291        u32 numPhys = 16;
1292
1293        dev = pvscsi_dev(adapter);
1294        config_page = pci_alloc_consistent(adapter->dev, PAGE_SIZE,
1295                                           &configPagePA);
1296        if (!config_page) {
1297                dev_warn(dev, "vmw_pvscsi: failed to allocate memory for config page\n");
1298                goto exit;
1299        }
1300        BUG_ON(configPagePA & ~PAGE_MASK);
1301
1302        /* Fetch config info from the device. */
1303        cmd.configPageAddress = ((u64)PVSCSI_CONFIG_CONTROLLER_ADDRESS) << 32;
1304        cmd.configPageNum = PVSCSI_CONFIG_PAGE_CONTROLLER;
1305        cmd.cmpAddr = configPagePA;
1306        cmd._pad = 0;
1307
1308        /*
1309         * Mark the completion page header with error values. If the device
1310         * completes the command successfully, it sets the status values to
1311         * indicate success.
1312         */
1313        header = config_page;
1314        memset(header, 0, sizeof *header);
1315        header->hostStatus = BTSTAT_INVPARAM;
1316        header->scsiStatus = SDSTAT_CHECK;
1317
1318        pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_CONFIG, &cmd, sizeof cmd);
1319
1320        if (header->hostStatus == BTSTAT_SUCCESS &&
1321            header->scsiStatus == SDSTAT_GOOD) {
1322                struct PVSCSIConfigPageController *config;
1323
1324                config = config_page;
1325                numPhys = config->numPhys;
1326        } else
1327                dev_warn(dev, "vmw_pvscsi: PVSCSI_CMD_CONFIG failed. hostStatus = 0x%x, scsiStatus = 0x%x\n",
1328                         header->hostStatus, header->scsiStatus);
1329        pci_free_consistent(adapter->dev, PAGE_SIZE, config_page, configPagePA);
1330exit:
1331        return numPhys;
1332}
1333
1334static int pvscsi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1335{
1336        unsigned int irq_flag = PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_LEGACY;
1337        struct pvscsi_adapter *adapter;
1338        struct pvscsi_adapter adapter_temp;
1339        struct Scsi_Host *host = NULL;
1340        unsigned int i;
1341        int error;
1342        u32 max_id;
1343
1344        error = -ENODEV;
1345
1346        if (pci_enable_device(pdev))
1347                return error;
1348
1349        if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0 &&
1350            pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
1351                printk(KERN_INFO "vmw_pvscsi: using 64bit dma\n");
1352        } else if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) == 0 &&
1353                   pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) == 0) {
1354                printk(KERN_INFO "vmw_pvscsi: using 32bit dma\n");
1355        } else {
1356                printk(KERN_ERR "vmw_pvscsi: failed to set DMA mask\n");
1357                goto out_disable_device;
1358        }
1359
1360        /*
1361         * Let's use a temp pvscsi_adapter struct until we find the number of
1362         * targets on the adapter, after that we will switch to the real
1363         * allocated struct.
1364         */
1365        adapter = &adapter_temp;
1366        memset(adapter, 0, sizeof(*adapter));
1367        adapter->dev  = pdev;
1368        adapter->rev = pdev->revision;
1369
1370        if (pci_request_regions(pdev, "vmw_pvscsi")) {
1371                printk(KERN_ERR "vmw_pvscsi: pci memory selection failed\n");
1372                goto out_disable_device;
1373        }
1374
1375        for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1376                if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE_IO))
1377                        continue;
1378
1379                if (pci_resource_len(pdev, i) < PVSCSI_MEM_SPACE_SIZE)
1380                        continue;
1381
1382                break;
1383        }
1384
1385        if (i == DEVICE_COUNT_RESOURCE) {
1386                printk(KERN_ERR
1387                       "vmw_pvscsi: adapter has no suitable MMIO region\n");
1388                goto out_release_resources_and_disable;
1389        }
1390
1391        adapter->mmioBase = pci_iomap(pdev, i, PVSCSI_MEM_SPACE_SIZE);
1392
1393        if (!adapter->mmioBase) {
1394                printk(KERN_ERR
1395                       "vmw_pvscsi: can't iomap for BAR %d memsize %lu\n",
1396                       i, PVSCSI_MEM_SPACE_SIZE);
1397                goto out_release_resources_and_disable;
1398        }
1399
1400        pci_set_master(pdev);
1401
1402        /*
1403         * Ask the device for max number of targets before deciding the
1404         * default pvscsi_ring_pages value.
1405         */
1406        max_id = pvscsi_get_max_targets(adapter);
1407        printk(KERN_INFO "vmw_pvscsi: max_id: %u\n", max_id);
1408
1409        if (pvscsi_ring_pages == 0)
1410                /*
1411                 * Set the right default value. Up to 16 it is 8, above it is
1412                 * max.
1413                 */
1414                pvscsi_ring_pages = (max_id > 16) ?
1415                        PVSCSI_SETUP_RINGS_MAX_NUM_PAGES :
1416                        PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
1417        printk(KERN_INFO
1418               "vmw_pvscsi: setting ring_pages to %d\n",
1419               pvscsi_ring_pages);
1420
1421        pvscsi_template.can_queue =
1422                min(PVSCSI_MAX_NUM_PAGES_REQ_RING, pvscsi_ring_pages) *
1423                PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
1424        pvscsi_template.cmd_per_lun =
1425                min(pvscsi_template.can_queue, pvscsi_cmd_per_lun);
1426        host = scsi_host_alloc(&pvscsi_template, sizeof(struct pvscsi_adapter));
1427        if (!host) {
1428                printk(KERN_ERR "vmw_pvscsi: failed to allocate host\n");
1429                goto out_release_resources_and_disable;
1430        }
1431
1432        /*
1433         * Let's use the real pvscsi_adapter struct here onwards.
1434         */
1435        adapter = shost_priv(host);
1436        memset(adapter, 0, sizeof(*adapter));
1437        adapter->dev  = pdev;
1438        adapter->host = host;
1439        /*
1440         * Copy back what we already have to the allocated adapter struct.
1441         */
1442        adapter->rev = adapter_temp.rev;
1443        adapter->mmioBase = adapter_temp.mmioBase;
1444
1445        spin_lock_init(&adapter->hw_lock);
1446        host->max_channel = 0;
1447        host->max_lun     = 1;
1448        host->max_cmd_len = 16;
1449        host->max_id      = max_id;
1450
1451        pci_set_drvdata(pdev, host);
1452
1453        ll_adapter_reset(adapter);
1454
1455        adapter->use_msg = pvscsi_setup_msg_workqueue(adapter);
1456
1457        error = pvscsi_allocate_rings(adapter);
1458        if (error) {
1459                printk(KERN_ERR "vmw_pvscsi: unable to allocate ring memory\n");
1460                goto out_release_resources;
1461        }
1462
1463        /*
1464         * From this point on we should reset the adapter if anything goes
1465         * wrong.
1466         */
1467        pvscsi_setup_all_rings(adapter);
1468
1469        adapter->cmd_map = kcalloc(adapter->req_depth,
1470                                   sizeof(struct pvscsi_ctx), GFP_KERNEL);
1471        if (!adapter->cmd_map) {
1472                printk(KERN_ERR "vmw_pvscsi: failed to allocate memory.\n");
1473                error = -ENOMEM;
1474                goto out_reset_adapter;
1475        }
1476
1477        INIT_LIST_HEAD(&adapter->cmd_pool);
1478        for (i = 0; i < adapter->req_depth; i++) {
1479                struct pvscsi_ctx *ctx = adapter->cmd_map + i;
1480                list_add(&ctx->list, &adapter->cmd_pool);
1481        }
1482
1483        error = pvscsi_allocate_sg(adapter);
1484        if (error) {
1485                printk(KERN_ERR "vmw_pvscsi: unable to allocate s/g table\n");
1486                goto out_reset_adapter;
1487        }
1488
1489        if (pvscsi_disable_msix)
1490                irq_flag &= ~PCI_IRQ_MSIX;
1491        if (pvscsi_disable_msi)
1492                irq_flag &= ~PCI_IRQ_MSI;
1493
1494        error = pci_alloc_irq_vectors(adapter->dev, 1, 1, irq_flag);
1495        if (error < 0)
1496                goto out_reset_adapter;
1497
1498        adapter->use_req_threshold = pvscsi_setup_req_threshold(adapter, true);
1499        printk(KERN_DEBUG "vmw_pvscsi: driver-based request coalescing %sabled\n",
1500               adapter->use_req_threshold ? "en" : "dis");
1501
1502        if (adapter->dev->msix_enabled || adapter->dev->msi_enabled) {
1503                printk(KERN_INFO "vmw_pvscsi: using MSI%s\n",
1504                        adapter->dev->msix_enabled ? "-X" : "");
1505                error = request_irq(pci_irq_vector(pdev, 0), pvscsi_isr,
1506                                0, "vmw_pvscsi", adapter);
1507        } else {
1508                printk(KERN_INFO "vmw_pvscsi: using INTx\n");
1509                error = request_irq(pci_irq_vector(pdev, 0), pvscsi_shared_isr,
1510                                IRQF_SHARED, "vmw_pvscsi", adapter);
1511        }
1512
1513        if (error) {
1514                printk(KERN_ERR
1515                       "vmw_pvscsi: unable to request IRQ: %d\n", error);
1516                goto out_reset_adapter;
1517        }
1518
1519        error = scsi_add_host(host, &pdev->dev);
1520        if (error) {
1521                printk(KERN_ERR
1522                       "vmw_pvscsi: scsi_add_host failed: %d\n", error);
1523                goto out_reset_adapter;
1524        }
1525
1526        dev_info(&pdev->dev, "VMware PVSCSI rev %d host #%u\n",
1527                 adapter->rev, host->host_no);
1528
1529        pvscsi_unmask_intr(adapter);
1530
1531        scsi_scan_host(host);
1532
1533        return 0;
1534
1535out_reset_adapter:
1536        ll_adapter_reset(adapter);
1537out_release_resources:
1538        pvscsi_release_resources(adapter);
1539        scsi_host_put(host);
1540out_disable_device:
1541        pci_disable_device(pdev);
1542
1543        return error;
1544
1545out_release_resources_and_disable:
1546        pvscsi_release_resources(adapter);
1547        goto out_disable_device;
1548}
1549
1550static void __pvscsi_shutdown(struct pvscsi_adapter *adapter)
1551{
1552        pvscsi_mask_intr(adapter);
1553
1554        if (adapter->workqueue)
1555                flush_workqueue(adapter->workqueue);
1556
1557        pvscsi_shutdown_intr(adapter);
1558
1559        pvscsi_process_request_ring(adapter);
1560        pvscsi_process_completion_ring(adapter);
1561        ll_adapter_reset(adapter);
1562}
1563
1564static void pvscsi_shutdown(struct pci_dev *dev)
1565{
1566        struct Scsi_Host *host = pci_get_drvdata(dev);
1567        struct pvscsi_adapter *adapter = shost_priv(host);
1568
1569        __pvscsi_shutdown(adapter);
1570}
1571
1572static void pvscsi_remove(struct pci_dev *pdev)
1573{
1574        struct Scsi_Host *host = pci_get_drvdata(pdev);
1575        struct pvscsi_adapter *adapter = shost_priv(host);
1576
1577        scsi_remove_host(host);
1578
1579        __pvscsi_shutdown(adapter);
1580        pvscsi_release_resources(adapter);
1581
1582        scsi_host_put(host);
1583
1584        pci_disable_device(pdev);
1585}
1586
1587static struct pci_driver pvscsi_pci_driver = {
1588        .name           = "vmw_pvscsi",
1589        .id_table       = pvscsi_pci_tbl,
1590        .probe          = pvscsi_probe,
1591        .remove         = pvscsi_remove,
1592        .shutdown       = pvscsi_shutdown,
1593};
1594
1595static int __init pvscsi_init(void)
1596{
1597        pr_info("%s - version %s\n",
1598                PVSCSI_LINUX_DRIVER_DESC, PVSCSI_DRIVER_VERSION_STRING);
1599        return pci_register_driver(&pvscsi_pci_driver);
1600}
1601
1602static void __exit pvscsi_exit(void)
1603{
1604        pci_unregister_driver(&pvscsi_pci_driver);
1605}
1606
1607module_init(pvscsi_init);
1608module_exit(pvscsi_exit);
1609