1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/slab.h>
26#include "rsxx_priv.h"
27
28struct rsxx_dma {
29 struct list_head list;
30 u8 cmd;
31 unsigned int laddr;
32 struct {
33 u32 off;
34 u32 cnt;
35 } sub_page;
36 dma_addr_t dma_addr;
37 struct page *page;
38 unsigned int pg_off;
39 rsxx_dma_cb cb;
40 void *cb_data;
41};
42
43
44#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
45
46struct hw_status {
47 u8 status;
48 u8 tag;
49 __le16 count;
50 __le32 _rsvd2;
51 __le64 _rsvd3;
52} __packed;
53
54enum rsxx_dma_status {
55 DMA_SW_ERR = 0x1,
56 DMA_HW_FAULT = 0x2,
57 DMA_CANCELLED = 0x4,
58};
59
60struct hw_cmd {
61 u8 command;
62 u8 tag;
63 u8 _rsvd;
64 u8 sub_page;
65
66 __le32 device_addr;
67 __le64 host_addr;
68} __packed;
69
70enum rsxx_hw_cmd {
71 HW_CMD_BLK_DISCARD = 0x70,
72 HW_CMD_BLK_WRITE = 0x80,
73 HW_CMD_BLK_READ = 0xC0,
74 HW_CMD_BLK_RECON_READ = 0xE0,
75};
76
77enum rsxx_hw_status {
78 HW_STATUS_CRC = 0x01,
79 HW_STATUS_HARD_ERR = 0x02,
80 HW_STATUS_SOFT_ERR = 0x04,
81 HW_STATUS_FAULT = 0x08,
82};
83
84static struct kmem_cache *rsxx_dma_pool;
85
86struct dma_tracker {
87 int next_tag;
88 struct rsxx_dma *dma;
89};
90
91#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
92 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
93
94struct dma_tracker_list {
95 spinlock_t lock;
96 int head;
97 struct dma_tracker list[0];
98};
99
100
101
102static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
103{
104 unsigned long long tgt_addr8;
105
106 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
107 card->_stripe.upper_mask) |
108 ((addr8) & card->_stripe.lower_mask);
109 do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
110 return tgt_addr8;
111}
112
113static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
114{
115 unsigned int tgt;
116
117 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
118
119 return tgt;
120}
121
122void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
123{
124
125 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
126}
127
128static unsigned int get_dma_size(struct rsxx_dma *dma)
129{
130 if (dma->sub_page.cnt)
131 return dma->sub_page.cnt << 9;
132 else
133 return RSXX_HW_BLK_SIZE;
134}
135
136
137
138static void set_tracker_dma(struct dma_tracker_list *trackers,
139 int tag,
140 struct rsxx_dma *dma)
141{
142 trackers->list[tag].dma = dma;
143}
144
145static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
146 int tag)
147{
148 return trackers->list[tag].dma;
149}
150
151static int pop_tracker(struct dma_tracker_list *trackers)
152{
153 int tag;
154
155 spin_lock(&trackers->lock);
156 tag = trackers->head;
157 if (tag != -1) {
158 trackers->head = trackers->list[tag].next_tag;
159 trackers->list[tag].next_tag = -1;
160 }
161 spin_unlock(&trackers->lock);
162
163 return tag;
164}
165
166static void push_tracker(struct dma_tracker_list *trackers, int tag)
167{
168 spin_lock(&trackers->lock);
169 trackers->list[tag].next_tag = trackers->head;
170 trackers->head = tag;
171 trackers->list[tag].dma = NULL;
172 spin_unlock(&trackers->lock);
173}
174
175
176
177
178
179
180
181
182
183#define INTR_COAL_LATENCY_MASK (0x0000ffff)
184
185#define INTR_COAL_COUNT_SHIFT 16
186#define INTR_COAL_COUNT_BITS 9
187#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
188 INTR_COAL_COUNT_SHIFT)
189#define INTR_COAL_LATENCY_UNITS_NS 64
190
191
192static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
193{
194 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
195
196 if (mode == RSXX_INTR_COAL_DISABLED)
197 return 0;
198
199 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
200 (latency_units & INTR_COAL_LATENCY_MASK);
201
202}
203
204static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
205{
206 int i;
207 u32 q_depth = 0;
208 u32 intr_coal;
209
210 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
211 unlikely(card->eeh_state))
212 return;
213
214 for (i = 0; i < card->n_targets; i++)
215 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
216
217 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
218 q_depth / 2,
219 card->config.data.intr_coal.latency);
220 iowrite32(intr_coal, card->regmap + INTR_COAL);
221}
222
223
224static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
225 struct rsxx_dma *dma,
226 unsigned int status)
227{
228 if (status & DMA_SW_ERR)
229 ctrl->stats.dma_sw_err++;
230 if (status & DMA_HW_FAULT)
231 ctrl->stats.dma_hw_fault++;
232 if (status & DMA_CANCELLED)
233 ctrl->stats.dma_cancelled++;
234
235 if (dma->dma_addr)
236 pci_unmap_page(ctrl->card->dev, dma->dma_addr,
237 get_dma_size(dma),
238 dma->cmd == HW_CMD_BLK_WRITE ?
239 PCI_DMA_TODEVICE :
240 PCI_DMA_FROMDEVICE);
241
242 if (dma->cb)
243 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
244
245 kmem_cache_free(rsxx_dma_pool, dma);
246}
247
248int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
249 struct list_head *q)
250{
251 struct rsxx_dma *dma;
252 struct rsxx_dma *tmp;
253 int cnt = 0;
254
255 list_for_each_entry_safe(dma, tmp, q, list) {
256 list_del(&dma->list);
257 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
258 cnt++;
259 }
260
261 return cnt;
262}
263
264static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
265 struct rsxx_dma *dma)
266{
267
268
269
270
271 spin_lock_bh(&ctrl->queue_lock);
272 ctrl->stats.sw_q_depth++;
273 list_add(&dma->list, &ctrl->queue);
274 spin_unlock_bh(&ctrl->queue_lock);
275}
276
277static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
278 struct rsxx_dma *dma,
279 u8 hw_st)
280{
281 unsigned int status = 0;
282 int requeue_cmd = 0;
283
284 dev_dbg(CARD_TO_DEV(ctrl->card),
285 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
286 dma->cmd, dma->laddr, hw_st);
287
288 if (hw_st & HW_STATUS_CRC)
289 ctrl->stats.crc_errors++;
290 if (hw_st & HW_STATUS_HARD_ERR)
291 ctrl->stats.hard_errors++;
292 if (hw_st & HW_STATUS_SOFT_ERR)
293 ctrl->stats.soft_errors++;
294
295 switch (dma->cmd) {
296 case HW_CMD_BLK_READ:
297 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
298 if (ctrl->card->scrub_hard) {
299 dma->cmd = HW_CMD_BLK_RECON_READ;
300 requeue_cmd = 1;
301 ctrl->stats.reads_retried++;
302 } else {
303 status |= DMA_HW_FAULT;
304 ctrl->stats.reads_failed++;
305 }
306 } else if (hw_st & HW_STATUS_FAULT) {
307 status |= DMA_HW_FAULT;
308 ctrl->stats.reads_failed++;
309 }
310
311 break;
312 case HW_CMD_BLK_RECON_READ:
313 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
314
315 status |= DMA_HW_FAULT;
316 ctrl->stats.reads_failed++;
317 }
318
319 break;
320 case HW_CMD_BLK_WRITE:
321 status |= DMA_HW_FAULT;
322 ctrl->stats.writes_failed++;
323
324 break;
325 case HW_CMD_BLK_DISCARD:
326 status |= DMA_HW_FAULT;
327 ctrl->stats.discards_failed++;
328
329 break;
330 default:
331 dev_err(CARD_TO_DEV(ctrl->card),
332 "Unknown command in DMA!(cmd: x%02x "
333 "laddr x%08x st: x%02x\n",
334 dma->cmd, dma->laddr, hw_st);
335 status |= DMA_SW_ERR;
336
337 break;
338 }
339
340 if (requeue_cmd)
341 rsxx_requeue_dma(ctrl, dma);
342 else
343 rsxx_complete_dma(ctrl, dma, status);
344}
345
346static void dma_engine_stalled(unsigned long data)
347{
348 struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
349 int cnt;
350
351 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
352 unlikely(ctrl->card->eeh_state))
353 return;
354
355 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
356
357
358
359
360 dev_warn(CARD_TO_DEV(ctrl->card),
361 "SW_CMD_IDX write was lost, re-writing...\n");
362 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
363 mod_timer(&ctrl->activity_timer,
364 jiffies + DMA_ACTIVITY_TIMEOUT);
365 } else {
366 dev_warn(CARD_TO_DEV(ctrl->card),
367 "DMA channel %d has stalled, faulting interface.\n",
368 ctrl->id);
369 ctrl->card->dma_fault = 1;
370
371
372 spin_lock(&ctrl->queue_lock);
373 cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
374 spin_unlock(&ctrl->queue_lock);
375
376 cnt += rsxx_dma_cancel(ctrl);
377
378 if (cnt)
379 dev_info(CARD_TO_DEV(ctrl->card),
380 "Freed %d queued DMAs on channel %d\n",
381 cnt, ctrl->id);
382 }
383}
384
385static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
386{
387 struct rsxx_dma *dma;
388 int tag;
389 int cmds_pending = 0;
390 struct hw_cmd *hw_cmd_buf;
391
392 hw_cmd_buf = ctrl->cmd.buf;
393
394 if (unlikely(ctrl->card->halt) ||
395 unlikely(ctrl->card->eeh_state))
396 return;
397
398 while (1) {
399 spin_lock_bh(&ctrl->queue_lock);
400 if (list_empty(&ctrl->queue)) {
401 spin_unlock_bh(&ctrl->queue_lock);
402 break;
403 }
404 spin_unlock_bh(&ctrl->queue_lock);
405
406 tag = pop_tracker(ctrl->trackers);
407 if (tag == -1)
408 break;
409
410 spin_lock_bh(&ctrl->queue_lock);
411 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
412 list_del(&dma->list);
413 ctrl->stats.sw_q_depth--;
414 spin_unlock_bh(&ctrl->queue_lock);
415
416
417
418
419
420
421 if (unlikely(ctrl->card->dma_fault)) {
422 push_tracker(ctrl->trackers, tag);
423 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
424 continue;
425 }
426
427 set_tracker_dma(ctrl->trackers, tag, dma);
428 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
429 hw_cmd_buf[ctrl->cmd.idx].tag = tag;
430 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
431 hw_cmd_buf[ctrl->cmd.idx].sub_page =
432 ((dma->sub_page.cnt & 0x7) << 4) |
433 (dma->sub_page.off & 0x7);
434
435 hw_cmd_buf[ctrl->cmd.idx].device_addr =
436 cpu_to_le32(dma->laddr);
437
438 hw_cmd_buf[ctrl->cmd.idx].host_addr =
439 cpu_to_le64(dma->dma_addr);
440
441 dev_dbg(CARD_TO_DEV(ctrl->card),
442 "Issue DMA%d(laddr %d tag %d) to idx %d\n",
443 ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
444
445 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
446 cmds_pending++;
447
448 if (dma->cmd == HW_CMD_BLK_WRITE)
449 ctrl->stats.writes_issued++;
450 else if (dma->cmd == HW_CMD_BLK_DISCARD)
451 ctrl->stats.discards_issued++;
452 else
453 ctrl->stats.reads_issued++;
454 }
455
456
457 if (cmds_pending) {
458 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
459 mod_timer(&ctrl->activity_timer,
460 jiffies + DMA_ACTIVITY_TIMEOUT);
461
462 if (unlikely(ctrl->card->eeh_state)) {
463 del_timer_sync(&ctrl->activity_timer);
464 return;
465 }
466
467 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
468 }
469}
470
471static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
472{
473 struct rsxx_dma *dma;
474 unsigned long flags;
475 u16 count;
476 u8 status;
477 u8 tag;
478 struct hw_status *hw_st_buf;
479
480 hw_st_buf = ctrl->status.buf;
481
482 if (unlikely(ctrl->card->halt) ||
483 unlikely(ctrl->card->dma_fault) ||
484 unlikely(ctrl->card->eeh_state))
485 return;
486
487 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
488
489 while (count == ctrl->e_cnt) {
490
491
492
493
494
495
496
497 rmb();
498
499 status = hw_st_buf[ctrl->status.idx].status;
500 tag = hw_st_buf[ctrl->status.idx].tag;
501
502 dma = get_tracker_dma(ctrl->trackers, tag);
503 if (dma == NULL) {
504 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
505 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
506 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
507
508 dev_err(CARD_TO_DEV(ctrl->card),
509 "No tracker for tag %d "
510 "(idx %d id %d)\n",
511 tag, ctrl->status.idx, ctrl->id);
512 return;
513 }
514
515 dev_dbg(CARD_TO_DEV(ctrl->card),
516 "Completing DMA%d"
517 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
518 ctrl->id, dma->laddr, tag, status, count,
519 ctrl->status.idx);
520
521 atomic_dec(&ctrl->stats.hw_q_depth);
522
523 mod_timer(&ctrl->activity_timer,
524 jiffies + DMA_ACTIVITY_TIMEOUT);
525
526 if (status)
527 rsxx_handle_dma_error(ctrl, dma, status);
528 else
529 rsxx_complete_dma(ctrl, dma, 0);
530
531 push_tracker(ctrl->trackers, tag);
532
533 ctrl->status.idx = (ctrl->status.idx + 1) &
534 RSXX_CS_IDX_MASK;
535 ctrl->e_cnt++;
536
537 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
538 }
539
540 dma_intr_coal_auto_tune(ctrl->card);
541
542 if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
543 del_timer_sync(&ctrl->activity_timer);
544
545 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
546 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
547 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
548
549 spin_lock_bh(&ctrl->queue_lock);
550 if (ctrl->stats.sw_q_depth)
551 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
552 spin_unlock_bh(&ctrl->queue_lock);
553}
554
555static void rsxx_schedule_issue(struct work_struct *work)
556{
557 struct rsxx_dma_ctrl *ctrl;
558
559 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
560
561 mutex_lock(&ctrl->work_lock);
562 rsxx_issue_dmas(ctrl);
563 mutex_unlock(&ctrl->work_lock);
564}
565
566static void rsxx_schedule_done(struct work_struct *work)
567{
568 struct rsxx_dma_ctrl *ctrl;
569
570 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
571
572 mutex_lock(&ctrl->work_lock);
573 rsxx_dma_done(ctrl);
574 mutex_unlock(&ctrl->work_lock);
575}
576
577static int rsxx_queue_discard(struct rsxx_cardinfo *card,
578 struct list_head *q,
579 unsigned int laddr,
580 rsxx_dma_cb cb,
581 void *cb_data)
582{
583 struct rsxx_dma *dma;
584
585 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
586 if (!dma)
587 return -ENOMEM;
588
589 dma->cmd = HW_CMD_BLK_DISCARD;
590 dma->laddr = laddr;
591 dma->dma_addr = 0;
592 dma->sub_page.off = 0;
593 dma->sub_page.cnt = 0;
594 dma->page = NULL;
595 dma->pg_off = 0;
596 dma->cb = cb;
597 dma->cb_data = cb_data;
598
599 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
600
601 list_add_tail(&dma->list, q);
602
603 return 0;
604}
605
606static int rsxx_queue_dma(struct rsxx_cardinfo *card,
607 struct list_head *q,
608 int dir,
609 unsigned int dma_off,
610 unsigned int dma_len,
611 unsigned int laddr,
612 struct page *page,
613 unsigned int pg_off,
614 rsxx_dma_cb cb,
615 void *cb_data)
616{
617 struct rsxx_dma *dma;
618
619 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
620 if (!dma)
621 return -ENOMEM;
622
623 dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len,
624 dir ? PCI_DMA_TODEVICE :
625 PCI_DMA_FROMDEVICE);
626 if (!dma->dma_addr) {
627 kmem_cache_free(rsxx_dma_pool, dma);
628 return -ENOMEM;
629 }
630
631 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
632 dma->laddr = laddr;
633 dma->sub_page.off = (dma_off >> 9);
634 dma->sub_page.cnt = (dma_len >> 9);
635 dma->page = page;
636 dma->pg_off = pg_off;
637 dma->cb = cb;
638 dma->cb_data = cb_data;
639
640 dev_dbg(CARD_TO_DEV(card),
641 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
642 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
643 dma->sub_page.cnt, dma->page, dma->pg_off);
644
645
646 list_add_tail(&dma->list, q);
647
648 return 0;
649}
650
651int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
652 struct bio *bio,
653 atomic_t *n_dmas,
654 rsxx_dma_cb cb,
655 void *cb_data)
656{
657 struct list_head dma_list[RSXX_MAX_TARGETS];
658 struct bio_vec *bvec;
659 unsigned long long addr8;
660 unsigned int laddr;
661 unsigned int bv_len;
662 unsigned int bv_off;
663 unsigned int dma_off;
664 unsigned int dma_len;
665 int dma_cnt[RSXX_MAX_TARGETS];
666 int tgt;
667 int st;
668 int i;
669
670 addr8 = bio->bi_sector << 9;
671 atomic_set(n_dmas, 0);
672
673 for (i = 0; i < card->n_targets; i++) {
674 INIT_LIST_HEAD(&dma_list[i]);
675 dma_cnt[i] = 0;
676 }
677
678 if (bio->bi_rw & REQ_DISCARD) {
679 bv_len = bio->bi_size;
680
681 while (bv_len > 0) {
682 tgt = rsxx_get_dma_tgt(card, addr8);
683 laddr = rsxx_addr8_to_laddr(addr8, card);
684
685 st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
686 cb, cb_data);
687 if (st)
688 goto bvec_err;
689
690 dma_cnt[tgt]++;
691 atomic_inc(n_dmas);
692 addr8 += RSXX_HW_BLK_SIZE;
693 bv_len -= RSXX_HW_BLK_SIZE;
694 }
695 } else {
696 bio_for_each_segment(bvec, bio, i) {
697 bv_len = bvec->bv_len;
698 bv_off = bvec->bv_offset;
699
700 while (bv_len > 0) {
701 tgt = rsxx_get_dma_tgt(card, addr8);
702 laddr = rsxx_addr8_to_laddr(addr8, card);
703 dma_off = addr8 & RSXX_HW_BLK_MASK;
704 dma_len = min(bv_len,
705 RSXX_HW_BLK_SIZE - dma_off);
706
707 st = rsxx_queue_dma(card, &dma_list[tgt],
708 bio_data_dir(bio),
709 dma_off, dma_len,
710 laddr, bvec->bv_page,
711 bv_off, cb, cb_data);
712 if (st)
713 goto bvec_err;
714
715 dma_cnt[tgt]++;
716 atomic_inc(n_dmas);
717 addr8 += dma_len;
718 bv_off += dma_len;
719 bv_len -= dma_len;
720 }
721 }
722 }
723
724 for (i = 0; i < card->n_targets; i++) {
725 if (!list_empty(&dma_list[i])) {
726 spin_lock_bh(&card->ctrl[i].queue_lock);
727 card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
728 list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
729 spin_unlock_bh(&card->ctrl[i].queue_lock);
730
731 queue_work(card->ctrl[i].issue_wq,
732 &card->ctrl[i].issue_dma_work);
733 }
734 }
735
736 return 0;
737
738bvec_err:
739 for (i = 0; i < card->n_targets; i++) {
740 spin_lock_bh(&card->ctrl[i].queue_lock);
741 rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]);
742 spin_unlock_bh(&card->ctrl[i].queue_lock);
743 }
744
745 return st;
746}
747
748
749
750int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
751{
752 ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8,
753 &ctrl->status.dma_addr);
754 ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8,
755 &ctrl->cmd.dma_addr);
756 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
757 return -ENOMEM;
758
759 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
760 iowrite32(lower_32_bits(ctrl->status.dma_addr),
761 ctrl->regmap + SB_ADD_LO);
762 iowrite32(upper_32_bits(ctrl->status.dma_addr),
763 ctrl->regmap + SB_ADD_HI);
764
765 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
766 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
767 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
768
769 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
770 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
771 dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
772 ctrl->status.idx);
773 return -EINVAL;
774 }
775 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
776 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
777
778 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
779 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
780 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
781 ctrl->status.idx);
782 return -EINVAL;
783 }
784 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
785 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
786
787 return 0;
788}
789
790static int rsxx_dma_ctrl_init(struct pci_dev *dev,
791 struct rsxx_dma_ctrl *ctrl)
792{
793 int i;
794 int st;
795
796 memset(&ctrl->stats, 0, sizeof(ctrl->stats));
797
798 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
799 if (!ctrl->trackers)
800 return -ENOMEM;
801
802 ctrl->trackers->head = 0;
803 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
804 ctrl->trackers->list[i].next_tag = i + 1;
805 ctrl->trackers->list[i].dma = NULL;
806 }
807 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
808 spin_lock_init(&ctrl->trackers->lock);
809
810 spin_lock_init(&ctrl->queue_lock);
811 mutex_init(&ctrl->work_lock);
812 INIT_LIST_HEAD(&ctrl->queue);
813
814 setup_timer(&ctrl->activity_timer, dma_engine_stalled,
815 (unsigned long)ctrl);
816
817 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
818 if (!ctrl->issue_wq)
819 return -ENOMEM;
820
821 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
822 if (!ctrl->done_wq)
823 return -ENOMEM;
824
825 INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
826 INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
827
828 st = rsxx_hw_buffers_init(dev, ctrl);
829 if (st)
830 return st;
831
832 return 0;
833}
834
835static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
836 unsigned int stripe_size8)
837{
838 if (!is_power_of_2(stripe_size8)) {
839 dev_err(CARD_TO_DEV(card),
840 "stripe_size is NOT a power of 2!\n");
841 return -EINVAL;
842 }
843
844 card->_stripe.lower_mask = stripe_size8 - 1;
845
846 card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
847 card->_stripe.upper_shift = ffs(card->n_targets) - 1;
848
849 card->_stripe.target_mask = card->n_targets - 1;
850 card->_stripe.target_shift = ffs(stripe_size8) - 1;
851
852 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
853 card->_stripe.lower_mask);
854 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
855 card->_stripe.upper_shift);
856 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
857 card->_stripe.upper_mask);
858 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
859 card->_stripe.target_mask);
860 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
861 card->_stripe.target_shift);
862
863 return 0;
864}
865
866int rsxx_dma_configure(struct rsxx_cardinfo *card)
867{
868 u32 intr_coal;
869
870 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
871 card->config.data.intr_coal.count,
872 card->config.data.intr_coal.latency);
873 iowrite32(intr_coal, card->regmap + INTR_COAL);
874
875 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
876}
877
878int rsxx_dma_setup(struct rsxx_cardinfo *card)
879{
880 unsigned long flags;
881 int st;
882 int i;
883
884 dev_info(CARD_TO_DEV(card),
885 "Initializing %d DMA targets\n",
886 card->n_targets);
887
888
889 for (i = 0; i < card->n_targets; i++)
890 card->ctrl[i].regmap = card->regmap + (i * 4096);
891
892 card->dma_fault = 0;
893
894
895 rsxx_dma_queue_reset(card);
896
897
898 for (i = 0; i < card->n_targets; i++) {
899 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
900 if (st)
901 goto failed_dma_setup;
902
903 card->ctrl[i].card = card;
904 card->ctrl[i].id = i;
905 }
906
907 card->scrub_hard = 1;
908
909 if (card->config_valid)
910 rsxx_dma_configure(card);
911
912
913 for (i = 0; i < card->n_targets; i++) {
914 spin_lock_irqsave(&card->irq_lock, flags);
915 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
916 spin_unlock_irqrestore(&card->irq_lock, flags);
917 }
918
919 return 0;
920
921failed_dma_setup:
922 for (i = 0; i < card->n_targets; i++) {
923 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
924
925 if (ctrl->issue_wq) {
926 destroy_workqueue(ctrl->issue_wq);
927 ctrl->issue_wq = NULL;
928 }
929
930 if (ctrl->done_wq) {
931 destroy_workqueue(ctrl->done_wq);
932 ctrl->done_wq = NULL;
933 }
934
935 if (ctrl->trackers)
936 vfree(ctrl->trackers);
937
938 if (ctrl->status.buf)
939 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
940 ctrl->status.buf,
941 ctrl->status.dma_addr);
942 if (ctrl->cmd.buf)
943 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
944 ctrl->cmd.buf, ctrl->cmd.dma_addr);
945 }
946
947 return st;
948}
949
950int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
951{
952 struct rsxx_dma *dma;
953 int i;
954 int cnt = 0;
955
956
957 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
958 dma = get_tracker_dma(ctrl->trackers, i);
959 if (dma) {
960 atomic_dec(&ctrl->stats.hw_q_depth);
961 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
962 push_tracker(ctrl->trackers, i);
963 cnt++;
964 }
965 }
966
967 return cnt;
968}
969
970void rsxx_dma_destroy(struct rsxx_cardinfo *card)
971{
972 struct rsxx_dma_ctrl *ctrl;
973 int i;
974
975 for (i = 0; i < card->n_targets; i++) {
976 ctrl = &card->ctrl[i];
977
978 if (ctrl->issue_wq) {
979 destroy_workqueue(ctrl->issue_wq);
980 ctrl->issue_wq = NULL;
981 }
982
983 if (ctrl->done_wq) {
984 destroy_workqueue(ctrl->done_wq);
985 ctrl->done_wq = NULL;
986 }
987
988 if (timer_pending(&ctrl->activity_timer))
989 del_timer_sync(&ctrl->activity_timer);
990
991
992 spin_lock_bh(&ctrl->queue_lock);
993 rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
994 spin_unlock_bh(&ctrl->queue_lock);
995
996 rsxx_dma_cancel(ctrl);
997
998 vfree(ctrl->trackers);
999
1000 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
1001 ctrl->status.buf, ctrl->status.dma_addr);
1002 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
1003 ctrl->cmd.buf, ctrl->cmd.dma_addr);
1004 }
1005}
1006
1007int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
1008{
1009 int i;
1010 int j;
1011 int cnt;
1012 struct rsxx_dma *dma;
1013 struct list_head *issued_dmas;
1014
1015 issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets,
1016 GFP_KERNEL);
1017 if (!issued_dmas)
1018 return -ENOMEM;
1019
1020 for (i = 0; i < card->n_targets; i++) {
1021 INIT_LIST_HEAD(&issued_dmas[i]);
1022 cnt = 0;
1023 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1024 dma = get_tracker_dma(card->ctrl[i].trackers, j);
1025 if (dma == NULL)
1026 continue;
1027
1028 if (dma->cmd == HW_CMD_BLK_WRITE)
1029 card->ctrl[i].stats.writes_issued--;
1030 else if (dma->cmd == HW_CMD_BLK_DISCARD)
1031 card->ctrl[i].stats.discards_issued--;
1032 else
1033 card->ctrl[i].stats.reads_issued--;
1034
1035 list_add_tail(&dma->list, &issued_dmas[i]);
1036 push_tracker(card->ctrl[i].trackers, j);
1037 cnt++;
1038 }
1039
1040 spin_lock_bh(&card->ctrl[i].queue_lock);
1041 list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1042
1043 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1044 card->ctrl[i].stats.sw_q_depth += cnt;
1045 card->ctrl[i].e_cnt = 0;
1046
1047 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1048 if (dma->dma_addr)
1049 pci_unmap_page(card->dev, dma->dma_addr,
1050 get_dma_size(dma),
1051 dma->cmd == HW_CMD_BLK_WRITE ?
1052 PCI_DMA_TODEVICE :
1053 PCI_DMA_FROMDEVICE);
1054 }
1055 spin_unlock_bh(&card->ctrl[i].queue_lock);
1056 }
1057
1058 kfree(issued_dmas);
1059
1060 return 0;
1061}
1062
1063int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
1064{
1065 struct rsxx_dma *dma;
1066 int i;
1067
1068 for (i = 0; i < card->n_targets; i++) {
1069 spin_lock_bh(&card->ctrl[i].queue_lock);
1070 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1071 dma->dma_addr = pci_map_page(card->dev, dma->page,
1072 dma->pg_off, get_dma_size(dma),
1073 dma->cmd == HW_CMD_BLK_WRITE ?
1074 PCI_DMA_TODEVICE :
1075 PCI_DMA_FROMDEVICE);
1076 if (!dma->dma_addr) {
1077 spin_unlock_bh(&card->ctrl[i].queue_lock);
1078 kmem_cache_free(rsxx_dma_pool, dma);
1079 return -ENOMEM;
1080 }
1081 }
1082 spin_unlock_bh(&card->ctrl[i].queue_lock);
1083 }
1084
1085 return 0;
1086}
1087
1088int rsxx_dma_init(void)
1089{
1090 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1091 if (!rsxx_dma_pool)
1092 return -ENOMEM;
1093
1094 return 0;
1095}
1096
1097
1098void rsxx_dma_cleanup(void)
1099{
1100 kmem_cache_destroy(rsxx_dma_pool);
1101}
1102
1103
