1
2
3
4
5
6
7
8
9
10
11
12#include <linux/bitfield.h>
13#include <linux/dma-mapping.h>
14#include <linux/debugfs.h>
15#include <linux/interrupt.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/pci.h>
19
20#include "ptdma.h"
21
22
23static char *pt_error_codes[] = {
24 "",
25 "ERR 01: ILLEGAL_ENGINE",
26 "ERR 03: ILLEGAL_FUNCTION_TYPE",
27 "ERR 04: ILLEGAL_FUNCTION_MODE",
28 "ERR 06: ILLEGAL_FUNCTION_SIZE",
29 "ERR 08: ILLEGAL_FUNCTION_RSVD",
30 "ERR 09: ILLEGAL_BUFFER_LENGTH",
31 "ERR 10: VLSB_FAULT",
32 "ERR 11: ILLEGAL_MEM_ADDR",
33 "ERR 12: ILLEGAL_MEM_SEL",
34 "ERR 13: ILLEGAL_CONTEXT_ID",
35 "ERR 15: 0xF Reserved",
36 "ERR 18: CMD_TIMEOUT",
37 "ERR 19: IDMA0_AXI_SLVERR",
38 "ERR 20: IDMA0_AXI_DECERR",
39 "ERR 21: 0x15 Reserved",
40 "ERR 22: IDMA1_AXI_SLAVE_FAULT",
41 "ERR 23: IDMA1_AIXI_DECERR",
42 "ERR 24: 0x18 Reserved",
43 "ERR 27: 0x1B Reserved",
44 "ERR 38: ODMA0_AXI_SLVERR",
45 "ERR 39: ODMA0_AXI_DECERR",
46 "ERR 40: 0x28 Reserved",
47 "ERR 41: ODMA1_AXI_SLVERR",
48 "ERR 42: ODMA1_AXI_DECERR",
49 "ERR 43: LSB_PARITY_ERR",
50};
51
52static void pt_log_error(struct pt_device *d, int e)
53{
54 dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e);
55}
56
57void pt_start_queue(struct pt_cmd_queue *cmd_q)
58{
59
60 iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control);
61}
62
63void pt_stop_queue(struct pt_cmd_queue *cmd_q)
64{
65
66 iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control);
67}
68
69static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q)
70{
71 bool soc = FIELD_GET(DWORD0_SOC, desc->dw0);
72 u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx];
73 u32 tail;
74
75 if (soc) {
76 desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0);
77 desc->dw0 &= ~DWORD0_SOC;
78 }
79 mutex_lock(&cmd_q->q_mutex);
80
81
82 memcpy(q_desc, desc, 32);
83 cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN;
84
85
86 wmb();
87
88
89 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
90 iowrite32(tail, cmd_q->reg_control + 0x0004);
91
92
93 pt_start_queue(cmd_q);
94 mutex_unlock(&cmd_q->q_mutex);
95
96 return 0;
97}
98
99int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
100 struct pt_passthru_engine *pt_engine)
101{
102 struct ptdma_desc desc;
103
104 cmd_q->cmd_error = 0;
105 cmd_q->total_pt_ops++;
106 memset(&desc, 0, sizeof(desc));
107 desc.dw0 = CMD_DESC_DW0_VAL;
108 desc.length = pt_engine->src_len;
109 desc.src_lo = lower_32_bits(pt_engine->src_dma);
110 desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma);
111 desc.dst_lo = lower_32_bits(pt_engine->dst_dma);
112 desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);
113
114 return pt_core_execute_cmd(&desc, cmd_q);
115}
116
117static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)
118{
119 iowrite32(0, pt->cmd_q.reg_control + 0x000C);
120}
121
122static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)
123{
124 iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);
125}
126
127static void pt_do_cmd_complete(unsigned long data)
128{
129 struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data;
130 struct pt_cmd *cmd = tdata->cmd;
131 struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q;
132 u32 tail;
133
134 if (cmd_q->cmd_error) {
135
136
137
138
139 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
140 pt_log_error(cmd_q->pt, cmd_q->cmd_error);
141 iowrite32(tail, cmd_q->reg_control + 0x0008);
142 }
143
144 cmd->pt_cmd_callback(cmd->data, cmd->ret);
145}
146
147static irqreturn_t pt_core_irq_handler(int irq, void *data)
148{
149 struct pt_device *pt = data;
150 struct pt_cmd_queue *cmd_q = &pt->cmd_q;
151 u32 status;
152
153 pt_core_disable_queue_interrupts(pt);
154 pt->total_interrupts++;
155 status = ioread32(cmd_q->reg_control + 0x0010);
156 if (status) {
157 cmd_q->int_status = status;
158 cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100);
159 cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104);
160
161
162 if ((status & INT_ERROR) && !cmd_q->cmd_error)
163 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
164
165
166 iowrite32(status, cmd_q->reg_control + 0x0010);
167 pt_core_enable_queue_interrupts(pt);
168 pt_do_cmd_complete((ulong)&pt->tdata);
169 }
170 return IRQ_HANDLED;
171}
172
173int pt_core_init(struct pt_device *pt)
174{
175 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
176 struct pt_cmd_queue *cmd_q = &pt->cmd_q;
177 u32 dma_addr_lo, dma_addr_hi;
178 struct device *dev = pt->dev;
179 struct dma_pool *dma_pool;
180 int ret;
181
182
183 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev));
184
185 dma_pool = dma_pool_create(dma_pool_name, dev,
186 PT_DMAPOOL_MAX_SIZE,
187 PT_DMAPOOL_ALIGN, 0);
188 if (!dma_pool)
189 return -ENOMEM;
190
191
192 iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET);
193 iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET);
194 iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET);
195 iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET);
196 iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET);
197
198 cmd_q->pt = pt;
199 cmd_q->dma_pool = dma_pool;
200 mutex_init(&cmd_q->q_mutex);
201
202
203 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
204 cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
205 &cmd_q->qbase_dma,
206 GFP_KERNEL);
207 if (!cmd_q->qbase) {
208 dev_err(dev, "unable to allocate command queue\n");
209 ret = -ENOMEM;
210 goto e_dma_alloc;
211 }
212
213 cmd_q->qidx = 0;
214
215
216 cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR;
217
218
219 pt_core_disable_queue_interrupts(pt);
220
221 cmd_q->qcontrol = 0;
222 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
223
224 ioread32(cmd_q->reg_control + 0x0104);
225 ioread32(cmd_q->reg_control + 0x0100);
226
227
228 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
229
230
231 ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);
232 if (ret)
233 goto e_pool;
234
235
236 cmd_q->qcontrol &= ~CMD_Q_SIZE;
237 cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL);
238
239 cmd_q->qdma_tail = cmd_q->qbase_dma;
240 dma_addr_lo = lower_32_bits(cmd_q->qdma_tail);
241 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004);
242 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008);
243
244 dma_addr_hi = upper_32_bits(cmd_q->qdma_tail);
245 cmd_q->qcontrol |= (dma_addr_hi << 16);
246 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
247
248 pt_core_enable_queue_interrupts(pt);
249
250
251 ret = pt_dmaengine_register(pt);
252 if (ret)
253 goto e_dmaengine;
254
255
256 ptdma_debugfs_setup(pt);
257
258 return 0;
259
260e_dmaengine:
261 free_irq(pt->pt_irq, pt);
262
263e_dma_alloc:
264 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
265
266e_pool:
267 dev_err(dev, "unable to allocate an IRQ\n");
268 dma_pool_destroy(pt->cmd_q.dma_pool);
269
270 return ret;
271}
272
273void pt_core_destroy(struct pt_device *pt)
274{
275 struct device *dev = pt->dev;
276 struct pt_cmd_queue *cmd_q = &pt->cmd_q;
277 struct pt_cmd *cmd;
278
279
280 pt_dmaengine_unregister(pt);
281
282
283 pt_core_disable_queue_interrupts(pt);
284
285
286 pt_stop_queue(cmd_q);
287
288
289 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
290 ioread32(cmd_q->reg_control + 0x0104);
291 ioread32(cmd_q->reg_control + 0x0100);
292
293 free_irq(pt->pt_irq, pt);
294
295 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
296 cmd_q->qbase_dma);
297
298
299 while (!list_empty(&pt->cmd)) {
300
301 cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry);
302 list_del(&cmd->entry);
303 cmd->pt_cmd_callback(cmd->data, -ENODEV);
304 }
305}
306