linux/drivers/block/rsxx/dma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3* Filename: dma.c
   4*
   5* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
   6*       Philip Kelleher <pjk1939@linux.vnet.ibm.com>
   7*
   8* (C) Copyright 2013 IBM Corporation
   9*/
  10
  11#include <linux/slab.h>
  12#include "rsxx_priv.h"
  13
  14struct rsxx_dma {
  15        struct list_head         list;
  16        u8                       cmd;
  17        unsigned int             laddr;     /* Logical address */
  18        struct {
  19                u32              off;
  20                u32              cnt;
  21        } sub_page;
  22        dma_addr_t               dma_addr;
  23        struct page              *page;
  24        unsigned int             pg_off;    /* Page Offset */
  25        rsxx_dma_cb              cb;
  26        void                     *cb_data;
  27};
  28
  29/* This timeout is used to detect a stalled DMA channel */
  30#define DMA_ACTIVITY_TIMEOUT    msecs_to_jiffies(10000)
  31
  32struct hw_status {
  33        u8      status;
  34        u8      tag;
  35        __le16  count;
  36        __le32  _rsvd2;
  37        __le64  _rsvd3;
  38} __packed;
  39
  40enum rsxx_dma_status {
  41        DMA_SW_ERR    = 0x1,
  42        DMA_HW_FAULT  = 0x2,
  43        DMA_CANCELLED = 0x4,
  44};
  45
  46struct hw_cmd {
  47        u8      command;
  48        u8      tag;
  49        u8      _rsvd;
  50        u8      sub_page; /* Bit[0:2]: 512byte offset */
  51                          /* Bit[4:6]: 512byte count */
  52        __le32  device_addr;
  53        __le64  host_addr;
  54} __packed;
  55
  56enum rsxx_hw_cmd {
  57        HW_CMD_BLK_DISCARD      = 0x70,
  58        HW_CMD_BLK_WRITE        = 0x80,
  59        HW_CMD_BLK_READ         = 0xC0,
  60        HW_CMD_BLK_RECON_READ   = 0xE0,
  61};
  62
  63enum rsxx_hw_status {
  64        HW_STATUS_CRC           = 0x01,
  65        HW_STATUS_HARD_ERR      = 0x02,
  66        HW_STATUS_SOFT_ERR      = 0x04,
  67        HW_STATUS_FAULT         = 0x08,
  68};
  69
  70static struct kmem_cache *rsxx_dma_pool;
  71
  72struct dma_tracker {
  73        int                     next_tag;
  74        struct rsxx_dma *dma;
  75};
  76
  77#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
  78                (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
  79
  80struct dma_tracker_list {
  81        spinlock_t              lock;
  82        int                     head;
  83        struct dma_tracker      list[];
  84};
  85
  86
  87/*----------------- Misc Utility Functions -------------------*/
  88static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
  89{
  90        unsigned long long tgt_addr8;
  91
  92        tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
  93                      card->_stripe.upper_mask) |
  94                    ((addr8) & card->_stripe.lower_mask);
  95        do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
  96        return tgt_addr8;
  97}
  98
  99static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
 100{
 101        unsigned int tgt;
 102
 103        tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
 104
 105        return tgt;
 106}
 107
 108void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
 109{
 110        /* Reset all DMA Command/Status Queues */
 111        iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
 112}
 113
 114static unsigned int get_dma_size(struct rsxx_dma *dma)
 115{
 116        if (dma->sub_page.cnt)
 117                return dma->sub_page.cnt << 9;
 118        else
 119                return RSXX_HW_BLK_SIZE;
 120}
 121
 122
 123/*----------------- DMA Tracker -------------------*/
 124static void set_tracker_dma(struct dma_tracker_list *trackers,
 125                            int tag,
 126                            struct rsxx_dma *dma)
 127{
 128        trackers->list[tag].dma = dma;
 129}
 130
 131static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
 132                                            int tag)
 133{
 134        return trackers->list[tag].dma;
 135}
 136
 137static int pop_tracker(struct dma_tracker_list *trackers)
 138{
 139        int tag;
 140
 141        spin_lock(&trackers->lock);
 142        tag = trackers->head;
 143        if (tag != -1) {
 144                trackers->head = trackers->list[tag].next_tag;
 145                trackers->list[tag].next_tag = -1;
 146        }
 147        spin_unlock(&trackers->lock);
 148
 149        return tag;
 150}
 151
 152static void push_tracker(struct dma_tracker_list *trackers, int tag)
 153{
 154        spin_lock(&trackers->lock);
 155        trackers->list[tag].next_tag = trackers->head;
 156        trackers->head = tag;
 157        trackers->list[tag].dma = NULL;
 158        spin_unlock(&trackers->lock);
 159}
 160
 161
 162/*----------------- Interrupt Coalescing -------------*/
 163/*
 164 * Interrupt Coalescing Register Format:
 165 * Interrupt Timer (64ns units) [15:0]
 166 * Interrupt Count [24:16]
 167 * Reserved [31:25]
 168*/
 169#define INTR_COAL_LATENCY_MASK       (0x0000ffff)
 170
 171#define INTR_COAL_COUNT_SHIFT        16
 172#define INTR_COAL_COUNT_BITS         9
 173#define INTR_COAL_COUNT_MASK         (((1 << INTR_COAL_COUNT_BITS) - 1) << \
 174                                        INTR_COAL_COUNT_SHIFT)
 175#define INTR_COAL_LATENCY_UNITS_NS   64
 176
 177
 178static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
 179{
 180        u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
 181
 182        if (mode == RSXX_INTR_COAL_DISABLED)
 183                return 0;
 184
 185        return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
 186                        (latency_units & INTR_COAL_LATENCY_MASK);
 187
 188}
 189
 190static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
 191{
 192        int i;
 193        u32 q_depth = 0;
 194        u32 intr_coal;
 195
 196        if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
 197            unlikely(card->eeh_state))
 198                return;
 199
 200        for (i = 0; i < card->n_targets; i++)
 201                q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
 202
 203        intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
 204                                      q_depth / 2,
 205                                      card->config.data.intr_coal.latency);
 206        iowrite32(intr_coal, card->regmap + INTR_COAL);
 207}
 208
 209/*----------------- RSXX DMA Handling -------------------*/
 210static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma)
 211{
 212        if (dma->cmd != HW_CMD_BLK_DISCARD) {
 213                if (!dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
 214                        dma_unmap_page(&ctrl->card->dev->dev, dma->dma_addr,
 215                                       get_dma_size(dma),
 216                                       dma->cmd == HW_CMD_BLK_WRITE ?
 217                                                   DMA_TO_DEVICE :
 218                                                   DMA_FROM_DEVICE);
 219                }
 220        }
 221
 222        kmem_cache_free(rsxx_dma_pool, dma);
 223}
 224
 225static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
 226                                  struct rsxx_dma *dma,
 227                                  unsigned int status)
 228{
 229        if (status & DMA_SW_ERR)
 230                ctrl->stats.dma_sw_err++;
 231        if (status & DMA_HW_FAULT)
 232                ctrl->stats.dma_hw_fault++;
 233        if (status & DMA_CANCELLED)
 234                ctrl->stats.dma_cancelled++;
 235
 236        if (dma->cb)
 237                dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
 238
 239        rsxx_free_dma(ctrl, dma);
 240}
 241
 242int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
 243                           struct list_head *q, unsigned int done)
 244{
 245        struct rsxx_dma *dma;
 246        struct rsxx_dma *tmp;
 247        int cnt = 0;
 248
 249        list_for_each_entry_safe(dma, tmp, q, list) {
 250                list_del(&dma->list);
 251                if (done & COMPLETE_DMA)
 252                        rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
 253                else
 254                        rsxx_free_dma(ctrl, dma);
 255                cnt++;
 256        }
 257
 258        return cnt;
 259}
 260
 261static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
 262                                 struct rsxx_dma *dma)
 263{
 264        /*
 265         * Requeued DMAs go to the front of the queue so they are issued
 266         * first.
 267         */
 268        spin_lock_bh(&ctrl->queue_lock);
 269        ctrl->stats.sw_q_depth++;
 270        list_add(&dma->list, &ctrl->queue);
 271        spin_unlock_bh(&ctrl->queue_lock);
 272}
 273
 274static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
 275                                      struct rsxx_dma *dma,
 276                                      u8 hw_st)
 277{
 278        unsigned int status = 0;
 279        int requeue_cmd = 0;
 280
 281        dev_dbg(CARD_TO_DEV(ctrl->card),
 282                "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
 283                dma->cmd, dma->laddr, hw_st);
 284
 285        if (hw_st & HW_STATUS_CRC)
 286                ctrl->stats.crc_errors++;
 287        if (hw_st & HW_STATUS_HARD_ERR)
 288                ctrl->stats.hard_errors++;
 289        if (hw_st & HW_STATUS_SOFT_ERR)
 290                ctrl->stats.soft_errors++;
 291
 292        switch (dma->cmd) {
 293        case HW_CMD_BLK_READ:
 294                if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
 295                        if (ctrl->card->scrub_hard) {
 296                                dma->cmd = HW_CMD_BLK_RECON_READ;
 297                                requeue_cmd = 1;
 298                                ctrl->stats.reads_retried++;
 299                        } else {
 300                                status |= DMA_HW_FAULT;
 301                                ctrl->stats.reads_failed++;
 302                        }
 303                } else if (hw_st & HW_STATUS_FAULT) {
 304                        status |= DMA_HW_FAULT;
 305                        ctrl->stats.reads_failed++;
 306                }
 307
 308                break;
 309        case HW_CMD_BLK_RECON_READ:
 310                if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
 311                        /* Data could not be reconstructed. */
 312                        status |= DMA_HW_FAULT;
 313                        ctrl->stats.reads_failed++;
 314                }
 315
 316                break;
 317        case HW_CMD_BLK_WRITE:
 318                status |= DMA_HW_FAULT;
 319                ctrl->stats.writes_failed++;
 320
 321                break;
 322        case HW_CMD_BLK_DISCARD:
 323                status |= DMA_HW_FAULT;
 324                ctrl->stats.discards_failed++;
 325
 326                break;
 327        default:
 328                dev_err(CARD_TO_DEV(ctrl->card),
 329                        "Unknown command in DMA!(cmd: x%02x "
 330                           "laddr x%08x st: x%02x\n",
 331                           dma->cmd, dma->laddr, hw_st);
 332                status |= DMA_SW_ERR;
 333
 334                break;
 335        }
 336
 337        if (requeue_cmd)
 338                rsxx_requeue_dma(ctrl, dma);
 339        else
 340                rsxx_complete_dma(ctrl, dma, status);
 341}
 342
 343static void dma_engine_stalled(struct timer_list *t)
 344{
 345        struct rsxx_dma_ctrl *ctrl = from_timer(ctrl, t, activity_timer);
 346        int cnt;
 347
 348        if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
 349            unlikely(ctrl->card->eeh_state))
 350                return;
 351
 352        if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
 353                /*
 354                 * The dma engine was stalled because the SW_CMD_IDX write
 355                 * was lost. Issue it again to recover.
 356                 */
 357                dev_warn(CARD_TO_DEV(ctrl->card),
 358                        "SW_CMD_IDX write was lost, re-writing...\n");
 359                iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 360                mod_timer(&ctrl->activity_timer,
 361                          jiffies + DMA_ACTIVITY_TIMEOUT);
 362        } else {
 363                dev_warn(CARD_TO_DEV(ctrl->card),
 364                        "DMA channel %d has stalled, faulting interface.\n",
 365                        ctrl->id);
 366                ctrl->card->dma_fault = 1;
 367
 368                /* Clean up the DMA queue */
 369                spin_lock(&ctrl->queue_lock);
 370                cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
 371                spin_unlock(&ctrl->queue_lock);
 372
 373                cnt += rsxx_dma_cancel(ctrl);
 374
 375                if (cnt)
 376                        dev_info(CARD_TO_DEV(ctrl->card),
 377                                "Freed %d queued DMAs on channel %d\n",
 378                                cnt, ctrl->id);
 379        }
 380}
 381
 382static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
 383{
 384        struct rsxx_dma *dma;
 385        int tag;
 386        int cmds_pending = 0;
 387        struct hw_cmd *hw_cmd_buf;
 388        int dir;
 389
 390        hw_cmd_buf = ctrl->cmd.buf;
 391
 392        if (unlikely(ctrl->card->halt) ||
 393            unlikely(ctrl->card->eeh_state))
 394                return;
 395
 396        while (1) {
 397                spin_lock_bh(&ctrl->queue_lock);
 398                if (list_empty(&ctrl->queue)) {
 399                        spin_unlock_bh(&ctrl->queue_lock);
 400                        break;
 401                }
 402                spin_unlock_bh(&ctrl->queue_lock);
 403
 404                tag = pop_tracker(ctrl->trackers);
 405                if (tag == -1)
 406                        break;
 407
 408                spin_lock_bh(&ctrl->queue_lock);
 409                dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
 410                list_del(&dma->list);
 411                ctrl->stats.sw_q_depth--;
 412                spin_unlock_bh(&ctrl->queue_lock);
 413
 414                /*
 415                 * This will catch any DMAs that slipped in right before the
 416                 * fault, but was queued after all the other DMAs were
 417                 * cancelled.
 418                 */
 419                if (unlikely(ctrl->card->dma_fault)) {
 420                        push_tracker(ctrl->trackers, tag);
 421                        rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
 422                        continue;
 423                }
 424
 425                if (dma->cmd != HW_CMD_BLK_DISCARD) {
 426                        if (dma->cmd == HW_CMD_BLK_WRITE)
 427                                dir = DMA_TO_DEVICE;
 428                        else
 429                                dir = DMA_FROM_DEVICE;
 430
 431                        /*
 432                         * The function dma_map_page is placed here because we
 433                         * can only, by design, issue up to 255 commands to the
 434                         * hardware at one time per DMA channel. So the maximum
 435                         * amount of mapped memory would be 255 * 4 channels *
 436                         * 4096 Bytes which is less than 2GB, the limit of a x8
 437                         * Non-HWWD PCIe slot. This way the dma_map_page
 438                         * function should never fail because of a lack of
 439                         * mappable memory.
 440                         */
 441                        dma->dma_addr = dma_map_page(&ctrl->card->dev->dev, dma->page,
 442                                        dma->pg_off, dma->sub_page.cnt << 9, dir);
 443                        if (dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
 444                                push_tracker(ctrl->trackers, tag);
 445                                rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
 446                                continue;
 447                        }
 448                }
 449
 450                set_tracker_dma(ctrl->trackers, tag, dma);
 451                hw_cmd_buf[ctrl->cmd.idx].command  = dma->cmd;
 452                hw_cmd_buf[ctrl->cmd.idx].tag      = tag;
 453                hw_cmd_buf[ctrl->cmd.idx]._rsvd    = 0;
 454                hw_cmd_buf[ctrl->cmd.idx].sub_page =
 455                                        ((dma->sub_page.cnt & 0x7) << 4) |
 456                                         (dma->sub_page.off & 0x7);
 457
 458                hw_cmd_buf[ctrl->cmd.idx].device_addr =
 459                                        cpu_to_le32(dma->laddr);
 460
 461                hw_cmd_buf[ctrl->cmd.idx].host_addr =
 462                                        cpu_to_le64(dma->dma_addr);
 463
 464                dev_dbg(CARD_TO_DEV(ctrl->card),
 465                        "Issue DMA%d(laddr %d tag %d) to idx %d\n",
 466                        ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
 467
 468                ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
 469                cmds_pending++;
 470
 471                if (dma->cmd == HW_CMD_BLK_WRITE)
 472                        ctrl->stats.writes_issued++;
 473                else if (dma->cmd == HW_CMD_BLK_DISCARD)
 474                        ctrl->stats.discards_issued++;
 475                else
 476                        ctrl->stats.reads_issued++;
 477        }
 478
 479        /* Let HW know we've queued commands. */
 480        if (cmds_pending) {
 481                atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
 482                mod_timer(&ctrl->activity_timer,
 483                          jiffies + DMA_ACTIVITY_TIMEOUT);
 484
 485                if (unlikely(ctrl->card->eeh_state)) {
 486                        del_timer_sync(&ctrl->activity_timer);
 487                        return;
 488                }
 489
 490                iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 491        }
 492}
 493
 494static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
 495{
 496        struct rsxx_dma *dma;
 497        unsigned long flags;
 498        u16 count;
 499        u8 status;
 500        u8 tag;
 501        struct hw_status *hw_st_buf;
 502
 503        hw_st_buf = ctrl->status.buf;
 504
 505        if (unlikely(ctrl->card->halt) ||
 506            unlikely(ctrl->card->dma_fault) ||
 507            unlikely(ctrl->card->eeh_state))
 508                return;
 509
 510        count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
 511
 512        while (count == ctrl->e_cnt) {
 513                /*
 514                 * The read memory-barrier is necessary to keep aggressive
 515                 * processors/optimizers (such as the PPC Apple G5) from
 516                 * reordering the following status-buffer tag & status read
 517                 * *before* the count read on subsequent iterations of the
 518                 * loop!
 519                 */
 520                rmb();
 521
 522                status = hw_st_buf[ctrl->status.idx].status;
 523                tag    = hw_st_buf[ctrl->status.idx].tag;
 524
 525                dma = get_tracker_dma(ctrl->trackers, tag);
 526                if (dma == NULL) {
 527                        spin_lock_irqsave(&ctrl->card->irq_lock, flags);
 528                        rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
 529                        spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
 530
 531                        dev_err(CARD_TO_DEV(ctrl->card),
 532                                "No tracker for tag %d "
 533                                "(idx %d id %d)\n",
 534                                tag, ctrl->status.idx, ctrl->id);
 535                        return;
 536                }
 537
 538                dev_dbg(CARD_TO_DEV(ctrl->card),
 539                        "Completing DMA%d"
 540                        "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
 541                        ctrl->id, dma->laddr, tag, status, count,
 542                        ctrl->status.idx);
 543
 544                atomic_dec(&ctrl->stats.hw_q_depth);
 545
 546                mod_timer(&ctrl->activity_timer,
 547                          jiffies + DMA_ACTIVITY_TIMEOUT);
 548
 549                if (status)
 550                        rsxx_handle_dma_error(ctrl, dma, status);
 551                else
 552                        rsxx_complete_dma(ctrl, dma, 0);
 553
 554                push_tracker(ctrl->trackers, tag);
 555
 556                ctrl->status.idx = (ctrl->status.idx + 1) &
 557                                   RSXX_CS_IDX_MASK;
 558                ctrl->e_cnt++;
 559
 560                count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
 561        }
 562
 563        dma_intr_coal_auto_tune(ctrl->card);
 564
 565        if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
 566                del_timer_sync(&ctrl->activity_timer);
 567
 568        spin_lock_irqsave(&ctrl->card->irq_lock, flags);
 569        rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
 570        spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
 571
 572        spin_lock_bh(&ctrl->queue_lock);
 573        if (ctrl->stats.sw_q_depth)
 574                queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
 575        spin_unlock_bh(&ctrl->queue_lock);
 576}
 577
 578static void rsxx_schedule_issue(struct work_struct *work)
 579{
 580        struct rsxx_dma_ctrl *ctrl;
 581
 582        ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
 583
 584        mutex_lock(&ctrl->work_lock);
 585        rsxx_issue_dmas(ctrl);
 586        mutex_unlock(&ctrl->work_lock);
 587}
 588
 589static void rsxx_schedule_done(struct work_struct *work)
 590{
 591        struct rsxx_dma_ctrl *ctrl;
 592
 593        ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
 594
 595        mutex_lock(&ctrl->work_lock);
 596        rsxx_dma_done(ctrl);
 597        mutex_unlock(&ctrl->work_lock);
 598}
 599
 600static blk_status_t rsxx_queue_discard(struct rsxx_cardinfo *card,
 601                                  struct list_head *q,
 602                                  unsigned int laddr,
 603                                  rsxx_dma_cb cb,
 604                                  void *cb_data)
 605{
 606        struct rsxx_dma *dma;
 607
 608        dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
 609        if (!dma)
 610                return BLK_STS_RESOURCE;
 611
 612        dma->cmd          = HW_CMD_BLK_DISCARD;
 613        dma->laddr        = laddr;
 614        dma->dma_addr     = 0;
 615        dma->sub_page.off = 0;
 616        dma->sub_page.cnt = 0;
 617        dma->page         = NULL;
 618        dma->pg_off       = 0;
 619        dma->cb           = cb;
 620        dma->cb_data      = cb_data;
 621
 622        dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
 623
 624        list_add_tail(&dma->list, q);
 625
 626        return 0;
 627}
 628
 629static blk_status_t rsxx_queue_dma(struct rsxx_cardinfo *card,
 630                              struct list_head *q,
 631                              int dir,
 632                              unsigned int dma_off,
 633                              unsigned int dma_len,
 634                              unsigned int laddr,
 635                              struct page *page,
 636                              unsigned int pg_off,
 637                              rsxx_dma_cb cb,
 638                              void *cb_data)
 639{
 640        struct rsxx_dma *dma;
 641
 642        dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
 643        if (!dma)
 644                return BLK_STS_RESOURCE;
 645
 646        dma->cmd          = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
 647        dma->laddr        = laddr;
 648        dma->sub_page.off = (dma_off >> 9);
 649        dma->sub_page.cnt = (dma_len >> 9);
 650        dma->page         = page;
 651        dma->pg_off       = pg_off;
 652        dma->cb           = cb;
 653        dma->cb_data      = cb_data;
 654
 655        dev_dbg(CARD_TO_DEV(card),
 656                "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
 657                dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
 658                dma->sub_page.cnt, dma->page, dma->pg_off);
 659
 660        /* Queue the DMA */
 661        list_add_tail(&dma->list, q);
 662
 663        return 0;
 664}
 665
 666blk_status_t rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
 667                           struct bio *bio,
 668                           atomic_t *n_dmas,
 669                           rsxx_dma_cb cb,
 670                           void *cb_data)
 671{
 672        struct list_head dma_list[RSXX_MAX_TARGETS];
 673        struct bio_vec bvec;
 674        struct bvec_iter iter;
 675        unsigned long long addr8;
 676        unsigned int laddr;
 677        unsigned int bv_len;
 678        unsigned int bv_off;
 679        unsigned int dma_off;
 680        unsigned int dma_len;
 681        int dma_cnt[RSXX_MAX_TARGETS];
 682        int tgt;
 683        blk_status_t st;
 684        int i;
 685
 686        addr8 = bio->bi_iter.bi_sector << 9; /* sectors are 512 bytes */
 687        atomic_set(n_dmas, 0);
 688
 689        for (i = 0; i < card->n_targets; i++) {
 690                INIT_LIST_HEAD(&dma_list[i]);
 691                dma_cnt[i] = 0;
 692        }
 693
 694        if (bio_op(bio) == REQ_OP_DISCARD) {
 695                bv_len = bio->bi_iter.bi_size;
 696
 697                while (bv_len > 0) {
 698                        tgt   = rsxx_get_dma_tgt(card, addr8);
 699                        laddr = rsxx_addr8_to_laddr(addr8, card);
 700
 701                        st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
 702                                                    cb, cb_data);
 703                        if (st)
 704                                goto bvec_err;
 705
 706                        dma_cnt[tgt]++;
 707                        atomic_inc(n_dmas);
 708                        addr8  += RSXX_HW_BLK_SIZE;
 709                        bv_len -= RSXX_HW_BLK_SIZE;
 710                }
 711        } else {
 712                bio_for_each_segment(bvec, bio, iter) {
 713                        bv_len = bvec.bv_len;
 714                        bv_off = bvec.bv_offset;
 715
 716                        while (bv_len > 0) {
 717                                tgt   = rsxx_get_dma_tgt(card, addr8);
 718                                laddr = rsxx_addr8_to_laddr(addr8, card);
 719                                dma_off = addr8 & RSXX_HW_BLK_MASK;
 720                                dma_len = min(bv_len,
 721                                              RSXX_HW_BLK_SIZE - dma_off);
 722
 723                                st = rsxx_queue_dma(card, &dma_list[tgt],
 724                                                        bio_data_dir(bio),
 725                                                        dma_off, dma_len,
 726                                                        laddr, bvec.bv_page,
 727                                                        bv_off, cb, cb_data);
 728                                if (st)
 729                                        goto bvec_err;
 730
 731                                dma_cnt[tgt]++;
 732                                atomic_inc(n_dmas);
 733                                addr8  += dma_len;
 734                                bv_off += dma_len;
 735                                bv_len -= dma_len;
 736                        }
 737                }
 738        }
 739
 740        for (i = 0; i < card->n_targets; i++) {
 741                if (!list_empty(&dma_list[i])) {
 742                        spin_lock_bh(&card->ctrl[i].queue_lock);
 743                        card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
 744                        list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
 745                        spin_unlock_bh(&card->ctrl[i].queue_lock);
 746
 747                        queue_work(card->ctrl[i].issue_wq,
 748                                   &card->ctrl[i].issue_dma_work);
 749                }
 750        }
 751
 752        return 0;
 753
 754bvec_err:
 755        for (i = 0; i < card->n_targets; i++)
 756                rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i],
 757                                        FREE_DMA);
 758        return st;
 759}
 760
 761
 762/*----------------- DMA Engine Initialization & Setup -------------------*/
 763int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
 764{
 765        ctrl->status.buf = dma_alloc_coherent(&dev->dev, STATUS_BUFFER_SIZE8,
 766                                &ctrl->status.dma_addr, GFP_KERNEL);
 767        ctrl->cmd.buf = dma_alloc_coherent(&dev->dev, COMMAND_BUFFER_SIZE8,
 768                                &ctrl->cmd.dma_addr, GFP_KERNEL);
 769        if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
 770                return -ENOMEM;
 771
 772        memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
 773        iowrite32(lower_32_bits(ctrl->status.dma_addr),
 774                ctrl->regmap + SB_ADD_LO);
 775        iowrite32(upper_32_bits(ctrl->status.dma_addr),
 776                ctrl->regmap + SB_ADD_HI);
 777
 778        memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
 779        iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
 780        iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
 781
 782        ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
 783        if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
 784                dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
 785                        ctrl->status.idx);
 786                return -EINVAL;
 787        }
 788        iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
 789        iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
 790
 791        ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
 792        if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
 793                dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
 794                        ctrl->status.idx);
 795                return -EINVAL;
 796        }
 797        iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
 798        iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 799
 800        return 0;
 801}
 802
 803static int rsxx_dma_ctrl_init(struct pci_dev *dev,
 804                                  struct rsxx_dma_ctrl *ctrl)
 805{
 806        int i;
 807        int st;
 808
 809        memset(&ctrl->stats, 0, sizeof(ctrl->stats));
 810
 811        ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
 812        if (!ctrl->trackers)
 813                return -ENOMEM;
 814
 815        ctrl->trackers->head = 0;
 816        for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
 817                ctrl->trackers->list[i].next_tag = i + 1;
 818                ctrl->trackers->list[i].dma = NULL;
 819        }
 820        ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
 821        spin_lock_init(&ctrl->trackers->lock);
 822
 823        spin_lock_init(&ctrl->queue_lock);
 824        mutex_init(&ctrl->work_lock);
 825        INIT_LIST_HEAD(&ctrl->queue);
 826
 827        timer_setup(&ctrl->activity_timer, dma_engine_stalled, 0);
 828
 829        ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
 830        if (!ctrl->issue_wq)
 831                return -ENOMEM;
 832
 833        ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
 834        if (!ctrl->done_wq)
 835                return -ENOMEM;
 836
 837        INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
 838        INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
 839
 840        st = rsxx_hw_buffers_init(dev, ctrl);
 841        if (st)
 842                return st;
 843
 844        return 0;
 845}
 846
 847static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
 848                              unsigned int stripe_size8)
 849{
 850        if (!is_power_of_2(stripe_size8)) {
 851                dev_err(CARD_TO_DEV(card),
 852                        "stripe_size is NOT a power of 2!\n");
 853                return -EINVAL;
 854        }
 855
 856        card->_stripe.lower_mask = stripe_size8 - 1;
 857
 858        card->_stripe.upper_mask  = ~(card->_stripe.lower_mask);
 859        card->_stripe.upper_shift = ffs(card->n_targets) - 1;
 860
 861        card->_stripe.target_mask = card->n_targets - 1;
 862        card->_stripe.target_shift = ffs(stripe_size8) - 1;
 863
 864        dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask   = x%016llx\n",
 865                card->_stripe.lower_mask);
 866        dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift  = x%016llx\n",
 867                card->_stripe.upper_shift);
 868        dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask   = x%016llx\n",
 869                card->_stripe.upper_mask);
 870        dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask  = x%016llx\n",
 871                card->_stripe.target_mask);
 872        dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
 873                card->_stripe.target_shift);
 874
 875        return 0;
 876}
 877
 878int rsxx_dma_configure(struct rsxx_cardinfo *card)
 879{
 880        u32 intr_coal;
 881
 882        intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
 883                                      card->config.data.intr_coal.count,
 884                                      card->config.data.intr_coal.latency);
 885        iowrite32(intr_coal, card->regmap + INTR_COAL);
 886
 887        return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
 888}
 889
 890int rsxx_dma_setup(struct rsxx_cardinfo *card)
 891{
 892        unsigned long flags;
 893        int st;
 894        int i;
 895
 896        dev_info(CARD_TO_DEV(card),
 897                "Initializing %d DMA targets\n",
 898                card->n_targets);
 899
 900        /* Regmap is divided up into 4K chunks. One for each DMA channel */
 901        for (i = 0; i < card->n_targets; i++)
 902                card->ctrl[i].regmap = card->regmap + (i * 4096);
 903
 904        card->dma_fault = 0;
 905
 906        /* Reset the DMA queues */
 907        rsxx_dma_queue_reset(card);
 908
 909        /************* Setup DMA Control *************/
 910        for (i = 0; i < card->n_targets; i++) {
 911                st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
 912                if (st)
 913                        goto failed_dma_setup;
 914
 915                card->ctrl[i].card = card;
 916                card->ctrl[i].id = i;
 917        }
 918
 919        card->scrub_hard = 1;
 920
 921        if (card->config_valid)
 922                rsxx_dma_configure(card);
 923
 924        /* Enable the interrupts after all setup has completed. */
 925        for (i = 0; i < card->n_targets; i++) {
 926                spin_lock_irqsave(&card->irq_lock, flags);
 927                rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
 928                spin_unlock_irqrestore(&card->irq_lock, flags);
 929        }
 930
 931        return 0;
 932
 933failed_dma_setup:
 934        for (i = 0; i < card->n_targets; i++) {
 935                struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
 936
 937                if (ctrl->issue_wq) {
 938                        destroy_workqueue(ctrl->issue_wq);
 939                        ctrl->issue_wq = NULL;
 940                }
 941
 942                if (ctrl->done_wq) {
 943                        destroy_workqueue(ctrl->done_wq);
 944                        ctrl->done_wq = NULL;
 945                }
 946
 947                if (ctrl->trackers)
 948                        vfree(ctrl->trackers);
 949
 950                if (ctrl->status.buf)
 951                        dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
 952                                          ctrl->status.buf,
 953                                          ctrl->status.dma_addr);
 954                if (ctrl->cmd.buf)
 955                        dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
 956                                          ctrl->cmd.buf, ctrl->cmd.dma_addr);
 957        }
 958
 959        return st;
 960}
 961
 962int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
 963{
 964        struct rsxx_dma *dma;
 965        int i;
 966        int cnt = 0;
 967
 968        /* Clean up issued DMAs */
 969        for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
 970                dma = get_tracker_dma(ctrl->trackers, i);
 971                if (dma) {
 972                        atomic_dec(&ctrl->stats.hw_q_depth);
 973                        rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
 974                        push_tracker(ctrl->trackers, i);
 975                        cnt++;
 976                }
 977        }
 978
 979        return cnt;
 980}
 981
 982void rsxx_dma_destroy(struct rsxx_cardinfo *card)
 983{
 984        struct rsxx_dma_ctrl *ctrl;
 985        int i;
 986
 987        for (i = 0; i < card->n_targets; i++) {
 988                ctrl = &card->ctrl[i];
 989
 990                if (ctrl->issue_wq) {
 991                        destroy_workqueue(ctrl->issue_wq);
 992                        ctrl->issue_wq = NULL;
 993                }
 994
 995                if (ctrl->done_wq) {
 996                        destroy_workqueue(ctrl->done_wq);
 997                        ctrl->done_wq = NULL;
 998                }
 999
1000                if (timer_pending(&ctrl->activity_timer))
1001                        del_timer_sync(&ctrl->activity_timer);
1002
1003                /* Clean up the DMA queue */
1004                spin_lock_bh(&ctrl->queue_lock);
1005                rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
1006                spin_unlock_bh(&ctrl->queue_lock);
1007
1008                rsxx_dma_cancel(ctrl);
1009
1010                vfree(ctrl->trackers);
1011
1012                dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
1013                                  ctrl->status.buf, ctrl->status.dma_addr);
1014                dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
1015                                  ctrl->cmd.buf, ctrl->cmd.dma_addr);
1016        }
1017}
1018
1019int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
1020{
1021        int i;
1022        int j;
1023        int cnt;
1024        struct rsxx_dma *dma;
1025        struct list_head *issued_dmas;
1026
1027        issued_dmas = kcalloc(card->n_targets, sizeof(*issued_dmas),
1028                              GFP_KERNEL);
1029        if (!issued_dmas)
1030                return -ENOMEM;
1031
1032        for (i = 0; i < card->n_targets; i++) {
1033                INIT_LIST_HEAD(&issued_dmas[i]);
1034                cnt = 0;
1035                for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1036                        dma = get_tracker_dma(card->ctrl[i].trackers, j);
1037                        if (dma == NULL)
1038                                continue;
1039
1040                        if (dma->cmd == HW_CMD_BLK_WRITE)
1041                                card->ctrl[i].stats.writes_issued--;
1042                        else if (dma->cmd == HW_CMD_BLK_DISCARD)
1043                                card->ctrl[i].stats.discards_issued--;
1044                        else
1045                                card->ctrl[i].stats.reads_issued--;
1046
1047                        if (dma->cmd != HW_CMD_BLK_DISCARD) {
1048                                dma_unmap_page(&card->dev->dev, dma->dma_addr,
1049                                               get_dma_size(dma),
1050                                               dma->cmd == HW_CMD_BLK_WRITE ?
1051                                               DMA_TO_DEVICE :
1052                                               DMA_FROM_DEVICE);
1053                        }
1054
1055                        list_add_tail(&dma->list, &issued_dmas[i]);
1056                        push_tracker(card->ctrl[i].trackers, j);
1057                        cnt++;
1058                }
1059
1060                spin_lock_bh(&card->ctrl[i].queue_lock);
1061                list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1062
1063                atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1064                card->ctrl[i].stats.sw_q_depth += cnt;
1065                card->ctrl[i].e_cnt = 0;
1066                spin_unlock_bh(&card->ctrl[i].queue_lock);
1067        }
1068
1069        kfree(issued_dmas);
1070
1071        return 0;
1072}
1073
1074int rsxx_dma_init(void)
1075{
1076        rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1077        if (!rsxx_dma_pool)
1078                return -ENOMEM;
1079
1080        return 0;
1081}
1082
1083
1084void rsxx_dma_cleanup(void)
1085{
1086        kmem_cache_destroy(rsxx_dma_pool);
1087}
1088
1089