1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/slab.h>
26#include "rsxx_priv.h"
27
28struct rsxx_dma {
29 struct list_head list;
30 u8 cmd;
31 unsigned int laddr;
32 struct {
33 u32 off;
34 u32 cnt;
35 } sub_page;
36 dma_addr_t dma_addr;
37 struct page *page;
38 unsigned int pg_off;
39 rsxx_dma_cb cb;
40 void *cb_data;
41};
42
43
44#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
45
46struct hw_status {
47 u8 status;
48 u8 tag;
49 __le16 count;
50 __le32 _rsvd2;
51 __le64 _rsvd3;
52} __packed;
53
54enum rsxx_dma_status {
55 DMA_SW_ERR = 0x1,
56 DMA_HW_FAULT = 0x2,
57 DMA_CANCELLED = 0x4,
58};
59
60struct hw_cmd {
61 u8 command;
62 u8 tag;
63 u8 _rsvd;
64 u8 sub_page;
65
66 __le32 device_addr;
67 __le64 host_addr;
68} __packed;
69
70enum rsxx_hw_cmd {
71 HW_CMD_BLK_DISCARD = 0x70,
72 HW_CMD_BLK_WRITE = 0x80,
73 HW_CMD_BLK_READ = 0xC0,
74 HW_CMD_BLK_RECON_READ = 0xE0,
75};
76
77enum rsxx_hw_status {
78 HW_STATUS_CRC = 0x01,
79 HW_STATUS_HARD_ERR = 0x02,
80 HW_STATUS_SOFT_ERR = 0x04,
81 HW_STATUS_FAULT = 0x08,
82};
83
84static struct kmem_cache *rsxx_dma_pool;
85
86struct dma_tracker {
87 int next_tag;
88 struct rsxx_dma *dma;
89};
90
91#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
92 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
93
94struct dma_tracker_list {
95 spinlock_t lock;
96 int head;
97 struct dma_tracker list[0];
98};
99
100
101
102static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
103{
104 unsigned long long tgt_addr8;
105
106 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
107 card->_stripe.upper_mask) |
108 ((addr8) & card->_stripe.lower_mask);
109 do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
110 return tgt_addr8;
111}
112
113static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
114{
115 unsigned int tgt;
116
117 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
118
119 return tgt;
120}
121
122void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
123{
124
125 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
126}
127
128static unsigned int get_dma_size(struct rsxx_dma *dma)
129{
130 if (dma->sub_page.cnt)
131 return dma->sub_page.cnt << 9;
132 else
133 return RSXX_HW_BLK_SIZE;
134}
135
136
137
138static void set_tracker_dma(struct dma_tracker_list *trackers,
139 int tag,
140 struct rsxx_dma *dma)
141{
142 trackers->list[tag].dma = dma;
143}
144
145static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
146 int tag)
147{
148 return trackers->list[tag].dma;
149}
150
151static int pop_tracker(struct dma_tracker_list *trackers)
152{
153 int tag;
154
155 spin_lock(&trackers->lock);
156 tag = trackers->head;
157 if (tag != -1) {
158 trackers->head = trackers->list[tag].next_tag;
159 trackers->list[tag].next_tag = -1;
160 }
161 spin_unlock(&trackers->lock);
162
163 return tag;
164}
165
166static void push_tracker(struct dma_tracker_list *trackers, int tag)
167{
168 spin_lock(&trackers->lock);
169 trackers->list[tag].next_tag = trackers->head;
170 trackers->head = tag;
171 trackers->list[tag].dma = NULL;
172 spin_unlock(&trackers->lock);
173}
174
175
176
177
178
179
180
181
182
183#define INTR_COAL_LATENCY_MASK (0x0000ffff)
184
185#define INTR_COAL_COUNT_SHIFT 16
186#define INTR_COAL_COUNT_BITS 9
187#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
188 INTR_COAL_COUNT_SHIFT)
189#define INTR_COAL_LATENCY_UNITS_NS 64
190
191
192static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
193{
194 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
195
196 if (mode == RSXX_INTR_COAL_DISABLED)
197 return 0;
198
199 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
200 (latency_units & INTR_COAL_LATENCY_MASK);
201
202}
203
204static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
205{
206 int i;
207 u32 q_depth = 0;
208 u32 intr_coal;
209
210 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
211 unlikely(card->eeh_state))
212 return;
213
214 for (i = 0; i < card->n_targets; i++)
215 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
216
217 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
218 q_depth / 2,
219 card->config.data.intr_coal.latency);
220 iowrite32(intr_coal, card->regmap + INTR_COAL);
221}
222
223
224static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
225 struct rsxx_dma *dma,
226 unsigned int status)
227{
228 if (status & DMA_SW_ERR)
229 ctrl->stats.dma_sw_err++;
230 if (status & DMA_HW_FAULT)
231 ctrl->stats.dma_hw_fault++;
232 if (status & DMA_CANCELLED)
233 ctrl->stats.dma_cancelled++;
234
235 if (dma->dma_addr)
236 pci_unmap_page(ctrl->card->dev, dma->dma_addr,
237 get_dma_size(dma),
238 dma->cmd == HW_CMD_BLK_WRITE ?
239 PCI_DMA_TODEVICE :
240 PCI_DMA_FROMDEVICE);
241
242 if (dma->cb)
243 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
244
245 kmem_cache_free(rsxx_dma_pool, dma);
246}
247
248static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
249 struct rsxx_dma *dma)
250{
251
252
253
254
255 spin_lock(&ctrl->queue_lock);
256 list_add(&dma->list, &ctrl->queue);
257 spin_unlock(&ctrl->queue_lock);
258}
259
260static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
261 struct rsxx_dma *dma,
262 u8 hw_st)
263{
264 unsigned int status = 0;
265 int requeue_cmd = 0;
266
267 dev_dbg(CARD_TO_DEV(ctrl->card),
268 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
269 dma->cmd, dma->laddr, hw_st);
270
271 if (hw_st & HW_STATUS_CRC)
272 ctrl->stats.crc_errors++;
273 if (hw_st & HW_STATUS_HARD_ERR)
274 ctrl->stats.hard_errors++;
275 if (hw_st & HW_STATUS_SOFT_ERR)
276 ctrl->stats.soft_errors++;
277
278 switch (dma->cmd) {
279 case HW_CMD_BLK_READ:
280 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
281 if (ctrl->card->scrub_hard) {
282 dma->cmd = HW_CMD_BLK_RECON_READ;
283 requeue_cmd = 1;
284 ctrl->stats.reads_retried++;
285 } else {
286 status |= DMA_HW_FAULT;
287 ctrl->stats.reads_failed++;
288 }
289 } else if (hw_st & HW_STATUS_FAULT) {
290 status |= DMA_HW_FAULT;
291 ctrl->stats.reads_failed++;
292 }
293
294 break;
295 case HW_CMD_BLK_RECON_READ:
296 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
297
298 status |= DMA_HW_FAULT;
299 ctrl->stats.reads_failed++;
300 }
301
302 break;
303 case HW_CMD_BLK_WRITE:
304 status |= DMA_HW_FAULT;
305 ctrl->stats.writes_failed++;
306
307 break;
308 case HW_CMD_BLK_DISCARD:
309 status |= DMA_HW_FAULT;
310 ctrl->stats.discards_failed++;
311
312 break;
313 default:
314 dev_err(CARD_TO_DEV(ctrl->card),
315 "Unknown command in DMA!(cmd: x%02x "
316 "laddr x%08x st: x%02x\n",
317 dma->cmd, dma->laddr, hw_st);
318 status |= DMA_SW_ERR;
319
320 break;
321 }
322
323 if (requeue_cmd)
324 rsxx_requeue_dma(ctrl, dma);
325 else
326 rsxx_complete_dma(ctrl, dma, status);
327}
328
329static void dma_engine_stalled(unsigned long data)
330{
331 struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
332
333 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
334 unlikely(ctrl->card->eeh_state))
335 return;
336
337 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
338
339
340
341
342 dev_warn(CARD_TO_DEV(ctrl->card),
343 "SW_CMD_IDX write was lost, re-writing...\n");
344 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
345 mod_timer(&ctrl->activity_timer,
346 jiffies + DMA_ACTIVITY_TIMEOUT);
347 } else {
348 dev_warn(CARD_TO_DEV(ctrl->card),
349 "DMA channel %d has stalled, faulting interface.\n",
350 ctrl->id);
351 ctrl->card->dma_fault = 1;
352 }
353}
354
355static void rsxx_issue_dmas(struct work_struct *work)
356{
357 struct rsxx_dma_ctrl *ctrl;
358 struct rsxx_dma *dma;
359 int tag;
360 int cmds_pending = 0;
361 struct hw_cmd *hw_cmd_buf;
362
363 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
364 hw_cmd_buf = ctrl->cmd.buf;
365
366 if (unlikely(ctrl->card->halt) ||
367 unlikely(ctrl->card->eeh_state))
368 return;
369
370 while (1) {
371 spin_lock(&ctrl->queue_lock);
372 if (list_empty(&ctrl->queue)) {
373 spin_unlock(&ctrl->queue_lock);
374 break;
375 }
376 spin_unlock(&ctrl->queue_lock);
377
378 tag = pop_tracker(ctrl->trackers);
379 if (tag == -1)
380 break;
381
382 spin_lock(&ctrl->queue_lock);
383 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
384 list_del(&dma->list);
385 ctrl->stats.sw_q_depth--;
386 spin_unlock(&ctrl->queue_lock);
387
388
389
390
391
392
393 if (unlikely(ctrl->card->dma_fault)) {
394 push_tracker(ctrl->trackers, tag);
395 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
396 continue;
397 }
398
399 set_tracker_dma(ctrl->trackers, tag, dma);
400 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
401 hw_cmd_buf[ctrl->cmd.idx].tag = tag;
402 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
403 hw_cmd_buf[ctrl->cmd.idx].sub_page =
404 ((dma->sub_page.cnt & 0x7) << 4) |
405 (dma->sub_page.off & 0x7);
406
407 hw_cmd_buf[ctrl->cmd.idx].device_addr =
408 cpu_to_le32(dma->laddr);
409
410 hw_cmd_buf[ctrl->cmd.idx].host_addr =
411 cpu_to_le64(dma->dma_addr);
412
413 dev_dbg(CARD_TO_DEV(ctrl->card),
414 "Issue DMA%d(laddr %d tag %d) to idx %d\n",
415 ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
416
417 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
418 cmds_pending++;
419
420 if (dma->cmd == HW_CMD_BLK_WRITE)
421 ctrl->stats.writes_issued++;
422 else if (dma->cmd == HW_CMD_BLK_DISCARD)
423 ctrl->stats.discards_issued++;
424 else
425 ctrl->stats.reads_issued++;
426 }
427
428
429 if (cmds_pending) {
430 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
431 mod_timer(&ctrl->activity_timer,
432 jiffies + DMA_ACTIVITY_TIMEOUT);
433
434 if (unlikely(ctrl->card->eeh_state)) {
435 del_timer_sync(&ctrl->activity_timer);
436 return;
437 }
438
439 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
440 }
441}
442
443static void rsxx_dma_done(struct work_struct *work)
444{
445 struct rsxx_dma_ctrl *ctrl;
446 struct rsxx_dma *dma;
447 unsigned long flags;
448 u16 count;
449 u8 status;
450 u8 tag;
451 struct hw_status *hw_st_buf;
452
453 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
454 hw_st_buf = ctrl->status.buf;
455
456 if (unlikely(ctrl->card->halt) ||
457 unlikely(ctrl->card->dma_fault) ||
458 unlikely(ctrl->card->eeh_state))
459 return;
460
461 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
462
463 while (count == ctrl->e_cnt) {
464
465
466
467
468
469
470
471 rmb();
472
473 status = hw_st_buf[ctrl->status.idx].status;
474 tag = hw_st_buf[ctrl->status.idx].tag;
475
476 dma = get_tracker_dma(ctrl->trackers, tag);
477 if (dma == NULL) {
478 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
479 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
480 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
481
482 dev_err(CARD_TO_DEV(ctrl->card),
483 "No tracker for tag %d "
484 "(idx %d id %d)\n",
485 tag, ctrl->status.idx, ctrl->id);
486 return;
487 }
488
489 dev_dbg(CARD_TO_DEV(ctrl->card),
490 "Completing DMA%d"
491 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
492 ctrl->id, dma->laddr, tag, status, count,
493 ctrl->status.idx);
494
495 atomic_dec(&ctrl->stats.hw_q_depth);
496
497 mod_timer(&ctrl->activity_timer,
498 jiffies + DMA_ACTIVITY_TIMEOUT);
499
500 if (status)
501 rsxx_handle_dma_error(ctrl, dma, status);
502 else
503 rsxx_complete_dma(ctrl, dma, 0);
504
505 push_tracker(ctrl->trackers, tag);
506
507 ctrl->status.idx = (ctrl->status.idx + 1) &
508 RSXX_CS_IDX_MASK;
509 ctrl->e_cnt++;
510
511 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
512 }
513
514 dma_intr_coal_auto_tune(ctrl->card);
515
516 if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
517 del_timer_sync(&ctrl->activity_timer);
518
519 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
520 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
521 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
522
523 spin_lock(&ctrl->queue_lock);
524 if (ctrl->stats.sw_q_depth)
525 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
526 spin_unlock(&ctrl->queue_lock);
527}
528
529static int rsxx_cleanup_dma_queue(struct rsxx_cardinfo *card,
530 struct list_head *q)
531{
532 struct rsxx_dma *dma;
533 struct rsxx_dma *tmp;
534 int cnt = 0;
535
536 list_for_each_entry_safe(dma, tmp, q, list) {
537 list_del(&dma->list);
538
539 if (dma->dma_addr)
540 pci_unmap_page(card->dev, dma->dma_addr,
541 get_dma_size(dma),
542 (dma->cmd == HW_CMD_BLK_WRITE) ?
543 PCI_DMA_TODEVICE :
544 PCI_DMA_FROMDEVICE);
545 kmem_cache_free(rsxx_dma_pool, dma);
546 cnt++;
547 }
548
549 return cnt;
550}
551
552static int rsxx_queue_discard(struct rsxx_cardinfo *card,
553 struct list_head *q,
554 unsigned int laddr,
555 rsxx_dma_cb cb,
556 void *cb_data)
557{
558 struct rsxx_dma *dma;
559
560 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
561 if (!dma)
562 return -ENOMEM;
563
564 dma->cmd = HW_CMD_BLK_DISCARD;
565 dma->laddr = laddr;
566 dma->dma_addr = 0;
567 dma->sub_page.off = 0;
568 dma->sub_page.cnt = 0;
569 dma->page = NULL;
570 dma->pg_off = 0;
571 dma->cb = cb;
572 dma->cb_data = cb_data;
573
574 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
575
576 list_add_tail(&dma->list, q);
577
578 return 0;
579}
580
581static int rsxx_queue_dma(struct rsxx_cardinfo *card,
582 struct list_head *q,
583 int dir,
584 unsigned int dma_off,
585 unsigned int dma_len,
586 unsigned int laddr,
587 struct page *page,
588 unsigned int pg_off,
589 rsxx_dma_cb cb,
590 void *cb_data)
591{
592 struct rsxx_dma *dma;
593
594 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
595 if (!dma)
596 return -ENOMEM;
597
598 dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len,
599 dir ? PCI_DMA_TODEVICE :
600 PCI_DMA_FROMDEVICE);
601 if (!dma->dma_addr) {
602 kmem_cache_free(rsxx_dma_pool, dma);
603 return -ENOMEM;
604 }
605
606 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
607 dma->laddr = laddr;
608 dma->sub_page.off = (dma_off >> 9);
609 dma->sub_page.cnt = (dma_len >> 9);
610 dma->page = page;
611 dma->pg_off = pg_off;
612 dma->cb = cb;
613 dma->cb_data = cb_data;
614
615 dev_dbg(CARD_TO_DEV(card),
616 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
617 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
618 dma->sub_page.cnt, dma->page, dma->pg_off);
619
620
621 list_add_tail(&dma->list, q);
622
623 return 0;
624}
625
626int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
627 struct bio *bio,
628 atomic_t *n_dmas,
629 rsxx_dma_cb cb,
630 void *cb_data)
631{
632 struct list_head dma_list[RSXX_MAX_TARGETS];
633 struct bio_vec *bvec;
634 unsigned long long addr8;
635 unsigned int laddr;
636 unsigned int bv_len;
637 unsigned int bv_off;
638 unsigned int dma_off;
639 unsigned int dma_len;
640 int dma_cnt[RSXX_MAX_TARGETS];
641 int tgt;
642 int st;
643 int i;
644
645 addr8 = bio->bi_sector << 9;
646 atomic_set(n_dmas, 0);
647
648 for (i = 0; i < card->n_targets; i++) {
649 INIT_LIST_HEAD(&dma_list[i]);
650 dma_cnt[i] = 0;
651 }
652
653 if (bio->bi_rw & REQ_DISCARD) {
654 bv_len = bio->bi_size;
655
656 while (bv_len > 0) {
657 tgt = rsxx_get_dma_tgt(card, addr8);
658 laddr = rsxx_addr8_to_laddr(addr8, card);
659
660 st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
661 cb, cb_data);
662 if (st)
663 goto bvec_err;
664
665 dma_cnt[tgt]++;
666 atomic_inc(n_dmas);
667 addr8 += RSXX_HW_BLK_SIZE;
668 bv_len -= RSXX_HW_BLK_SIZE;
669 }
670 } else {
671 bio_for_each_segment(bvec, bio, i) {
672 bv_len = bvec->bv_len;
673 bv_off = bvec->bv_offset;
674
675 while (bv_len > 0) {
676 tgt = rsxx_get_dma_tgt(card, addr8);
677 laddr = rsxx_addr8_to_laddr(addr8, card);
678 dma_off = addr8 & RSXX_HW_BLK_MASK;
679 dma_len = min(bv_len,
680 RSXX_HW_BLK_SIZE - dma_off);
681
682 st = rsxx_queue_dma(card, &dma_list[tgt],
683 bio_data_dir(bio),
684 dma_off, dma_len,
685 laddr, bvec->bv_page,
686 bv_off, cb, cb_data);
687 if (st)
688 goto bvec_err;
689
690 dma_cnt[tgt]++;
691 atomic_inc(n_dmas);
692 addr8 += dma_len;
693 bv_off += dma_len;
694 bv_len -= dma_len;
695 }
696 }
697 }
698
699 for (i = 0; i < card->n_targets; i++) {
700 if (!list_empty(&dma_list[i])) {
701 spin_lock(&card->ctrl[i].queue_lock);
702 card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
703 list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
704 spin_unlock(&card->ctrl[i].queue_lock);
705
706 queue_work(card->ctrl[i].issue_wq,
707 &card->ctrl[i].issue_dma_work);
708 }
709 }
710
711 return 0;
712
713bvec_err:
714 for (i = 0; i < card->n_targets; i++)
715 rsxx_cleanup_dma_queue(card, &dma_list[i]);
716
717 return st;
718}
719
720
721
722int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
723{
724 ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8,
725 &ctrl->status.dma_addr);
726 ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8,
727 &ctrl->cmd.dma_addr);
728 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
729 return -ENOMEM;
730
731 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
732 iowrite32(lower_32_bits(ctrl->status.dma_addr),
733 ctrl->regmap + SB_ADD_LO);
734 iowrite32(upper_32_bits(ctrl->status.dma_addr),
735 ctrl->regmap + SB_ADD_HI);
736
737 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
738 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
739 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
740
741 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
742 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
743 dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
744 ctrl->status.idx);
745 return -EINVAL;
746 }
747 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
748 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
749
750 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
751 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
752 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
753 ctrl->status.idx);
754 return -EINVAL;
755 }
756 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
757 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
758
759 return 0;
760}
761
762static int rsxx_dma_ctrl_init(struct pci_dev *dev,
763 struct rsxx_dma_ctrl *ctrl)
764{
765 int i;
766 int st;
767
768 memset(&ctrl->stats, 0, sizeof(ctrl->stats));
769
770 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
771 if (!ctrl->trackers)
772 return -ENOMEM;
773
774 ctrl->trackers->head = 0;
775 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
776 ctrl->trackers->list[i].next_tag = i + 1;
777 ctrl->trackers->list[i].dma = NULL;
778 }
779 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
780 spin_lock_init(&ctrl->trackers->lock);
781
782 spin_lock_init(&ctrl->queue_lock);
783 INIT_LIST_HEAD(&ctrl->queue);
784
785 setup_timer(&ctrl->activity_timer, dma_engine_stalled,
786 (unsigned long)ctrl);
787
788 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
789 if (!ctrl->issue_wq)
790 return -ENOMEM;
791
792 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
793 if (!ctrl->done_wq)
794 return -ENOMEM;
795
796 INIT_WORK(&ctrl->issue_dma_work, rsxx_issue_dmas);
797 INIT_WORK(&ctrl->dma_done_work, rsxx_dma_done);
798
799 st = rsxx_hw_buffers_init(dev, ctrl);
800 if (st)
801 return st;
802
803 return 0;
804}
805
806static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
807 unsigned int stripe_size8)
808{
809 if (!is_power_of_2(stripe_size8)) {
810 dev_err(CARD_TO_DEV(card),
811 "stripe_size is NOT a power of 2!\n");
812 return -EINVAL;
813 }
814
815 card->_stripe.lower_mask = stripe_size8 - 1;
816
817 card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
818 card->_stripe.upper_shift = ffs(card->n_targets) - 1;
819
820 card->_stripe.target_mask = card->n_targets - 1;
821 card->_stripe.target_shift = ffs(stripe_size8) - 1;
822
823 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
824 card->_stripe.lower_mask);
825 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
826 card->_stripe.upper_shift);
827 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
828 card->_stripe.upper_mask);
829 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
830 card->_stripe.target_mask);
831 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
832 card->_stripe.target_shift);
833
834 return 0;
835}
836
837int rsxx_dma_configure(struct rsxx_cardinfo *card)
838{
839 u32 intr_coal;
840
841 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
842 card->config.data.intr_coal.count,
843 card->config.data.intr_coal.latency);
844 iowrite32(intr_coal, card->regmap + INTR_COAL);
845
846 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
847}
848
849int rsxx_dma_setup(struct rsxx_cardinfo *card)
850{
851 unsigned long flags;
852 int st;
853 int i;
854
855 dev_info(CARD_TO_DEV(card),
856 "Initializing %d DMA targets\n",
857 card->n_targets);
858
859
860 for (i = 0; i < card->n_targets; i++)
861 card->ctrl[i].regmap = card->regmap + (i * 4096);
862
863 card->dma_fault = 0;
864
865
866 rsxx_dma_queue_reset(card);
867
868
869 for (i = 0; i < card->n_targets; i++) {
870 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
871 if (st)
872 goto failed_dma_setup;
873
874 card->ctrl[i].card = card;
875 card->ctrl[i].id = i;
876 }
877
878 card->scrub_hard = 1;
879
880 if (card->config_valid)
881 rsxx_dma_configure(card);
882
883
884 for (i = 0; i < card->n_targets; i++) {
885 spin_lock_irqsave(&card->irq_lock, flags);
886 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
887 spin_unlock_irqrestore(&card->irq_lock, flags);
888 }
889
890 return 0;
891
892failed_dma_setup:
893 for (i = 0; i < card->n_targets; i++) {
894 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
895
896 if (ctrl->issue_wq) {
897 destroy_workqueue(ctrl->issue_wq);
898 ctrl->issue_wq = NULL;
899 }
900
901 if (ctrl->done_wq) {
902 destroy_workqueue(ctrl->done_wq);
903 ctrl->done_wq = NULL;
904 }
905
906 if (ctrl->trackers)
907 vfree(ctrl->trackers);
908
909 if (ctrl->status.buf)
910 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
911 ctrl->status.buf,
912 ctrl->status.dma_addr);
913 if (ctrl->cmd.buf)
914 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
915 ctrl->cmd.buf, ctrl->cmd.dma_addr);
916 }
917
918 return st;
919}
920
921
922void rsxx_dma_destroy(struct rsxx_cardinfo *card)
923{
924 struct rsxx_dma_ctrl *ctrl;
925 struct rsxx_dma *dma;
926 int i, j;
927 int cnt = 0;
928
929 for (i = 0; i < card->n_targets; i++) {
930 ctrl = &card->ctrl[i];
931
932 if (ctrl->issue_wq) {
933 destroy_workqueue(ctrl->issue_wq);
934 ctrl->issue_wq = NULL;
935 }
936
937 if (ctrl->done_wq) {
938 destroy_workqueue(ctrl->done_wq);
939 ctrl->done_wq = NULL;
940 }
941
942 if (timer_pending(&ctrl->activity_timer))
943 del_timer_sync(&ctrl->activity_timer);
944
945
946 spin_lock(&ctrl->queue_lock);
947 cnt = rsxx_cleanup_dma_queue(card, &ctrl->queue);
948 spin_unlock(&ctrl->queue_lock);
949
950 if (cnt)
951 dev_info(CARD_TO_DEV(card),
952 "Freed %d queued DMAs on channel %d\n",
953 cnt, i);
954
955
956 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
957 dma = get_tracker_dma(ctrl->trackers, j);
958 if (dma) {
959 pci_unmap_page(card->dev, dma->dma_addr,
960 get_dma_size(dma),
961 (dma->cmd == HW_CMD_BLK_WRITE) ?
962 PCI_DMA_TODEVICE :
963 PCI_DMA_FROMDEVICE);
964 kmem_cache_free(rsxx_dma_pool, dma);
965 cnt++;
966 }
967 }
968
969 if (cnt)
970 dev_info(CARD_TO_DEV(card),
971 "Freed %d pending DMAs on channel %d\n",
972 cnt, i);
973
974 vfree(ctrl->trackers);
975
976 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
977 ctrl->status.buf, ctrl->status.dma_addr);
978 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
979 ctrl->cmd.buf, ctrl->cmd.dma_addr);
980 }
981}
982
983int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
984{
985 int i;
986 int j;
987 int cnt;
988 struct rsxx_dma *dma;
989 struct list_head *issued_dmas;
990
991 issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets,
992 GFP_KERNEL);
993 if (!issued_dmas)
994 return -ENOMEM;
995
996 for (i = 0; i < card->n_targets; i++) {
997 INIT_LIST_HEAD(&issued_dmas[i]);
998 cnt = 0;
999 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1000 dma = get_tracker_dma(card->ctrl[i].trackers, j);
1001 if (dma == NULL)
1002 continue;
1003
1004 if (dma->cmd == HW_CMD_BLK_WRITE)
1005 card->ctrl[i].stats.writes_issued--;
1006 else if (dma->cmd == HW_CMD_BLK_DISCARD)
1007 card->ctrl[i].stats.discards_issued--;
1008 else
1009 card->ctrl[i].stats.reads_issued--;
1010
1011 list_add_tail(&dma->list, &issued_dmas[i]);
1012 push_tracker(card->ctrl[i].trackers, j);
1013 cnt++;
1014 }
1015
1016 spin_lock(&card->ctrl[i].queue_lock);
1017 list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1018
1019 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1020 card->ctrl[i].stats.sw_q_depth += cnt;
1021 card->ctrl[i].e_cnt = 0;
1022
1023 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1024 if (dma->dma_addr)
1025 pci_unmap_page(card->dev, dma->dma_addr,
1026 get_dma_size(dma),
1027 dma->cmd == HW_CMD_BLK_WRITE ?
1028 PCI_DMA_TODEVICE :
1029 PCI_DMA_FROMDEVICE);
1030 }
1031 spin_unlock(&card->ctrl[i].queue_lock);
1032 }
1033
1034 kfree(issued_dmas);
1035
1036 return 0;
1037}
1038
1039void rsxx_eeh_cancel_dmas(struct rsxx_cardinfo *card)
1040{
1041 struct rsxx_dma *dma;
1042 struct rsxx_dma *tmp;
1043 int i;
1044
1045 for (i = 0; i < card->n_targets; i++) {
1046 spin_lock(&card->ctrl[i].queue_lock);
1047 list_for_each_entry_safe(dma, tmp, &card->ctrl[i].queue, list) {
1048 list_del(&dma->list);
1049
1050 rsxx_complete_dma(&card->ctrl[i], dma, DMA_CANCELLED);
1051 }
1052 spin_unlock(&card->ctrl[i].queue_lock);
1053 }
1054}
1055
1056int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
1057{
1058 struct rsxx_dma *dma;
1059 int i;
1060
1061 for (i = 0; i < card->n_targets; i++) {
1062 spin_lock(&card->ctrl[i].queue_lock);
1063 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1064 dma->dma_addr = pci_map_page(card->dev, dma->page,
1065 dma->pg_off, get_dma_size(dma),
1066 dma->cmd == HW_CMD_BLK_WRITE ?
1067 PCI_DMA_TODEVICE :
1068 PCI_DMA_FROMDEVICE);
1069 if (!dma->dma_addr) {
1070 spin_unlock(&card->ctrl[i].queue_lock);
1071 kmem_cache_free(rsxx_dma_pool, dma);
1072 return -ENOMEM;
1073 }
1074 }
1075 spin_unlock(&card->ctrl[i].queue_lock);
1076 }
1077
1078 return 0;
1079}
1080
1081int rsxx_dma_init(void)
1082{
1083 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1084 if (!rsxx_dma_pool)
1085 return -ENOMEM;
1086
1087 return 0;
1088}
1089
1090
1091void rsxx_dma_cleanup(void)
1092{
1093 kmem_cache_destroy(rsxx_dma_pool);
1094}
1095
1096
