1
2
3#include <linux/bitfield.h>
4#include <linux/dmaengine.h>
5#include <linux/init.h>
6#include <linux/iopoll.h>
7#include <linux/module.h>
8#include <linux/pci.h>
9#include <linux/spinlock.h>
10#include "virt-dma.h"
11
12#define HISI_DMA_SQ_BASE_L 0x0
13#define HISI_DMA_SQ_BASE_H 0x4
14#define HISI_DMA_SQ_DEPTH 0x8
15#define HISI_DMA_SQ_TAIL_PTR 0xc
16#define HISI_DMA_CQ_BASE_L 0x10
17#define HISI_DMA_CQ_BASE_H 0x14
18#define HISI_DMA_CQ_DEPTH 0x18
19#define HISI_DMA_CQ_HEAD_PTR 0x1c
20#define HISI_DMA_CTRL0 0x20
21#define HISI_DMA_CTRL0_QUEUE_EN_S 0
22#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4
23#define HISI_DMA_CTRL1 0x24
24#define HISI_DMA_CTRL1_QUEUE_RESET_S 0
25#define HISI_DMA_Q_FSM_STS 0x30
26#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0)
27#define HISI_DMA_INT_STS 0x40
28#define HISI_DMA_INT_STS_MASK GENMASK(12, 0)
29#define HISI_DMA_INT_MSK 0x44
30#define HISI_DMA_MODE 0x217c
31#define HISI_DMA_OFFSET 0x100
32
33#define HISI_DMA_MSI_NUM 30
34#define HISI_DMA_CHAN_NUM 30
35#define HISI_DMA_Q_DEPTH_VAL 1024
36
37#define PCI_BAR_2 2
38
39enum hisi_dma_mode {
40 EP = 0,
41 RC,
42};
43
44enum hisi_dma_chan_status {
45 DISABLE = -1,
46 IDLE = 0,
47 RUN,
48 CPL,
49 PAUSE,
50 HALT,
51 ABORT,
52 WAIT,
53 BUFFCLR,
54};
55
56struct hisi_dma_sqe {
57 __le32 dw0;
58#define OPCODE_MASK GENMASK(3, 0)
59#define OPCODE_SMALL_PACKAGE 0x1
60#define OPCODE_M2M 0x4
61#define LOCAL_IRQ_EN BIT(8)
62#define ATTR_SRC_MASK GENMASK(14, 12)
63 __le32 dw1;
64 __le32 dw2;
65#define ATTR_DST_MASK GENMASK(26, 24)
66 __le32 length;
67 __le64 src_addr;
68 __le64 dst_addr;
69};
70
71struct hisi_dma_cqe {
72 __le32 rsv0;
73 __le32 rsv1;
74 __le16 sq_head;
75 __le16 rsv2;
76 __le16 rsv3;
77 __le16 w0;
78#define STATUS_MASK GENMASK(15, 1)
79#define STATUS_SUCC 0x0
80#define VALID_BIT BIT(0)
81};
82
83struct hisi_dma_desc {
84 struct virt_dma_desc vd;
85 struct hisi_dma_sqe sqe;
86};
87
88struct hisi_dma_chan {
89 struct virt_dma_chan vc;
90 struct hisi_dma_dev *hdma_dev;
91 struct hisi_dma_sqe *sq;
92 struct hisi_dma_cqe *cq;
93 dma_addr_t sq_dma;
94 dma_addr_t cq_dma;
95 u32 sq_tail;
96 u32 cq_head;
97 u32 qp_num;
98 enum hisi_dma_chan_status status;
99 struct hisi_dma_desc *desc;
100};
101
102struct hisi_dma_dev {
103 struct pci_dev *pdev;
104 void __iomem *base;
105 struct dma_device dma_dev;
106 u32 chan_num;
107 u32 chan_depth;
108 struct hisi_dma_chan chan[];
109};
110
111static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
112{
113 return container_of(c, struct hisi_dma_chan, vc.chan);
114}
115
116static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
117{
118 return container_of(vd, struct hisi_dma_desc, vd);
119}
120
121static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
122 u32 val)
123{
124 writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
125}
126
127static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
128{
129 u32 tmp;
130
131 tmp = readl_relaxed(addr);
132 tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
133 writel_relaxed(tmp, addr);
134}
135
136static void hisi_dma_free_irq_vectors(void *data)
137{
138 pci_free_irq_vectors(data);
139}
140
141static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
142 bool pause)
143{
144 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
145 HISI_DMA_OFFSET;
146
147 hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
148}
149
150static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
151 bool enable)
152{
153 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
154 HISI_DMA_OFFSET;
155
156 hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
157}
158
159static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
160{
161 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
162 HISI_DMA_INT_STS_MASK);
163}
164
165static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
166{
167 void __iomem *base = hdma_dev->base;
168
169 hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
170 HISI_DMA_INT_STS_MASK);
171 hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
172}
173
174static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
175{
176 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
177 HISI_DMA_OFFSET;
178
179 hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
180}
181
182static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
183{
184 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
185 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
186}
187
188static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan)
189{
190 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
191 u32 index = chan->qp_num, tmp;
192 int ret;
193
194 hisi_dma_pause_dma(hdma_dev, index, true);
195 hisi_dma_enable_dma(hdma_dev, index, false);
196 hisi_dma_mask_irq(hdma_dev, index);
197
198 ret = readl_relaxed_poll_timeout(hdma_dev->base +
199 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
200 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
201 if (ret) {
202 dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
203 WARN_ON(1);
204 }
205
206 hisi_dma_do_reset(hdma_dev, index);
207 hisi_dma_reset_qp_point(hdma_dev, index);
208 hisi_dma_pause_dma(hdma_dev, index, false);
209 hisi_dma_enable_dma(hdma_dev, index, true);
210 hisi_dma_unmask_irq(hdma_dev, index);
211
212 ret = readl_relaxed_poll_timeout(hdma_dev->base +
213 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
214 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
215 if (ret) {
216 dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
217 WARN_ON(1);
218 }
219}
220
221static void hisi_dma_free_chan_resources(struct dma_chan *c)
222{
223 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
224 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
225
226 hisi_dma_reset_hw_chan(chan);
227 vchan_free_chan_resources(&chan->vc);
228
229 memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
230 memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
231 chan->sq_tail = 0;
232 chan->cq_head = 0;
233 chan->status = DISABLE;
234}
235
236static void hisi_dma_desc_free(struct virt_dma_desc *vd)
237{
238 kfree(to_hisi_dma_desc(vd));
239}
240
241static struct dma_async_tx_descriptor *
242hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
243 size_t len, unsigned long flags)
244{
245 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
246 struct hisi_dma_desc *desc;
247
248 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
249 if (!desc)
250 return NULL;
251
252 desc->sqe.length = cpu_to_le32(len);
253 desc->sqe.src_addr = cpu_to_le64(src);
254 desc->sqe.dst_addr = cpu_to_le64(dst);
255
256 return vchan_tx_prep(&chan->vc, &desc->vd, flags);
257}
258
259static enum dma_status
260hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
261 struct dma_tx_state *txstate)
262{
263 return dma_cookie_status(c, cookie, txstate);
264}
265
266static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
267{
268 struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
269 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
270 struct hisi_dma_desc *desc;
271 struct virt_dma_desc *vd;
272
273 vd = vchan_next_desc(&chan->vc);
274 if (!vd) {
275 dev_err(&hdma_dev->pdev->dev, "no issued task!\n");
276 chan->desc = NULL;
277 return;
278 }
279 list_del(&vd->node);
280 desc = to_hisi_dma_desc(vd);
281 chan->desc = desc;
282
283 memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
284
285
286 sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
287 sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);
288
289
290 wmb();
291
292
293 chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
294
295
296 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
297 chan->sq_tail);
298}
299
300static void hisi_dma_issue_pending(struct dma_chan *c)
301{
302 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
303 unsigned long flags;
304
305 spin_lock_irqsave(&chan->vc.lock, flags);
306
307 if (vchan_issue_pending(&chan->vc))
308 hisi_dma_start_transfer(chan);
309
310 spin_unlock_irqrestore(&chan->vc.lock, flags);
311}
312
313static int hisi_dma_terminate_all(struct dma_chan *c)
314{
315 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
316 unsigned long flags;
317 LIST_HEAD(head);
318
319 spin_lock_irqsave(&chan->vc.lock, flags);
320
321 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
322 if (chan->desc) {
323 vchan_terminate_vdesc(&chan->desc->vd);
324 chan->desc = NULL;
325 }
326
327 vchan_get_all_descriptors(&chan->vc, &head);
328
329 spin_unlock_irqrestore(&chan->vc.lock, flags);
330
331 vchan_dma_desc_free_list(&chan->vc, &head);
332 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
333
334 return 0;
335}
336
337static void hisi_dma_synchronize(struct dma_chan *c)
338{
339 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
340
341 vchan_synchronize(&chan->vc);
342}
343
344static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
345{
346 size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
347 size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
348 struct device *dev = &hdma_dev->pdev->dev;
349 struct hisi_dma_chan *chan;
350 int i;
351
352 for (i = 0; i < hdma_dev->chan_num; i++) {
353 chan = &hdma_dev->chan[i];
354 chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
355 GFP_KERNEL);
356 if (!chan->sq)
357 return -ENOMEM;
358
359 chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
360 GFP_KERNEL);
361 if (!chan->cq)
362 return -ENOMEM;
363 }
364
365 return 0;
366}
367
368static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
369{
370 struct hisi_dma_chan *chan = &hdma_dev->chan[index];
371 u32 hw_depth = hdma_dev->chan_depth - 1;
372 void __iomem *base = hdma_dev->base;
373
374
375 hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
376 lower_32_bits(chan->sq_dma));
377 hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
378 upper_32_bits(chan->sq_dma));
379 hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
380 lower_32_bits(chan->cq_dma));
381 hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
382 upper_32_bits(chan->cq_dma));
383
384
385 hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
386 hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
387
388
389 hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
390 hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
391}
392
393static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
394{
395 hisi_dma_init_hw_qp(hdma_dev, qp_index);
396 hisi_dma_unmask_irq(hdma_dev, qp_index);
397 hisi_dma_enable_dma(hdma_dev, qp_index, true);
398}
399
400static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
401{
402 hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]);
403}
404
405static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
406{
407 int i;
408
409 for (i = 0; i < hdma_dev->chan_num; i++) {
410 hdma_dev->chan[i].qp_num = i;
411 hdma_dev->chan[i].hdma_dev = hdma_dev;
412 hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
413 vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
414 hisi_dma_enable_qp(hdma_dev, i);
415 }
416}
417
418static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
419{
420 int i;
421
422 for (i = 0; i < hdma_dev->chan_num; i++) {
423 hisi_dma_disable_qp(hdma_dev, i);
424 tasklet_kill(&hdma_dev->chan[i].vc.task);
425 }
426}
427
428static irqreturn_t hisi_dma_irq(int irq, void *data)
429{
430 struct hisi_dma_chan *chan = data;
431 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
432 struct hisi_dma_desc *desc;
433 struct hisi_dma_cqe *cqe;
434 unsigned long flags;
435
436 spin_lock_irqsave(&chan->vc.lock, flags);
437
438 desc = chan->desc;
439 cqe = chan->cq + chan->cq_head;
440 if (desc) {
441 if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
442 chan->cq_head = (chan->cq_head + 1) %
443 hdma_dev->chan_depth;
444 hisi_dma_chan_write(hdma_dev->base,
445 HISI_DMA_CQ_HEAD_PTR, chan->qp_num,
446 chan->cq_head);
447 vchan_cookie_complete(&desc->vd);
448 } else {
449 dev_err(&hdma_dev->pdev->dev, "task error!\n");
450 }
451
452 chan->desc = NULL;
453 }
454
455 spin_unlock_irqrestore(&chan->vc.lock, flags);
456
457 return IRQ_HANDLED;
458}
459
460static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
461{
462 struct pci_dev *pdev = hdma_dev->pdev;
463 int i, ret;
464
465 for (i = 0; i < hdma_dev->chan_num; i++) {
466 ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
467 hisi_dma_irq, IRQF_SHARED, "hisi_dma",
468 &hdma_dev->chan[i]);
469 if (ret)
470 return ret;
471 }
472
473 return 0;
474}
475
476
477static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
478{
479 int ret;
480
481 ret = hisi_dma_alloc_qps_mem(hdma_dev);
482 if (ret) {
483 dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
484 return ret;
485 }
486
487 ret = hisi_dma_request_qps_irq(hdma_dev);
488 if (ret) {
489 dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
490 return ret;
491 }
492
493 hisi_dma_enable_qps(hdma_dev);
494
495 return 0;
496}
497
498static void hisi_dma_disable_hw_channels(void *data)
499{
500 hisi_dma_disable_qps(data);
501}
502
503static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
504 enum hisi_dma_mode mode)
505{
506 writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
507}
508
509static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
510{
511 struct device *dev = &pdev->dev;
512 struct hisi_dma_dev *hdma_dev;
513 struct dma_device *dma_dev;
514 int ret;
515
516 ret = pcim_enable_device(pdev);
517 if (ret) {
518 dev_err(dev, "failed to enable device mem!\n");
519 return ret;
520 }
521
522 ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
523 if (ret) {
524 dev_err(dev, "failed to remap I/O region!\n");
525 return ret;
526 }
527
528 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
529 if (ret)
530 return ret;
531
532 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
533 if (ret)
534 return ret;
535
536 hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
537 if (!hdma_dev)
538 return -EINVAL;
539
540 hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
541 hdma_dev->pdev = pdev;
542 hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
543 hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
544
545 pci_set_drvdata(pdev, hdma_dev);
546 pci_set_master(pdev);
547
548 ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
549 PCI_IRQ_MSI);
550 if (ret < 0) {
551 dev_err(dev, "Failed to allocate MSI vectors!\n");
552 return ret;
553 }
554
555 ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev);
556 if (ret)
557 return ret;
558
559 dma_dev = &hdma_dev->dma_dev;
560 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
561 dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
562 dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
563 dma_dev->device_tx_status = hisi_dma_tx_status;
564 dma_dev->device_issue_pending = hisi_dma_issue_pending;
565 dma_dev->device_terminate_all = hisi_dma_terminate_all;
566 dma_dev->device_synchronize = hisi_dma_synchronize;
567 dma_dev->directions = BIT(DMA_MEM_TO_MEM);
568 dma_dev->dev = dev;
569 INIT_LIST_HEAD(&dma_dev->channels);
570
571 hisi_dma_set_mode(hdma_dev, RC);
572
573 ret = hisi_dma_enable_hw_channels(hdma_dev);
574 if (ret < 0) {
575 dev_err(dev, "failed to enable hw channel!\n");
576 return ret;
577 }
578
579 ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
580 hdma_dev);
581 if (ret)
582 return ret;
583
584 ret = dmaenginem_async_device_register(dma_dev);
585 if (ret < 0)
586 dev_err(dev, "failed to register device!\n");
587
588 return ret;
589}
590
591static const struct pci_device_id hisi_dma_pci_tbl[] = {
592 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
593 { 0, }
594};
595
596static struct pci_driver hisi_dma_pci_driver = {
597 .name = "hisi_dma",
598 .id_table = hisi_dma_pci_tbl,
599 .probe = hisi_dma_probe,
600};
601
602module_pci_driver(hisi_dma_pci_driver);
603
604MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
605MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
606MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
607MODULE_LICENSE("GPL v2");
608MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
609
