1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/dma-mapping.h>
20#include <linux/io.h>
21#include <linux/interrupt.h>
22#include <linux/module.h>
23#include <linux/of_address.h>
24#include <linux/soc/ti/knav_qmss.h>
25
26#include "knav_qmss.h"
27
28#define knav_range_offset_to_inst(kdev, range, q) \
29 (range->queue_base_inst + (q << kdev->inst_shift))
30
31static void __knav_acc_notify(struct knav_range_info *range,
32 struct knav_acc_channel *acc)
33{
34 struct knav_device *kdev = range->kdev;
35 struct knav_queue_inst *inst;
36 int range_base, queue;
37
38 range_base = kdev->base_id + range->queue_base;
39
40 if (range->flags & RANGE_MULTI_QUEUE) {
41 for (queue = 0; queue < range->num_queues; queue++) {
42 inst = knav_range_offset_to_inst(kdev, range,
43 queue);
44 if (inst->notify_needed) {
45 inst->notify_needed = 0;
46 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
47 range_base + queue);
48 knav_queue_notify(inst);
49 }
50 }
51 } else {
52 queue = acc->channel - range->acc_info.start_channel;
53 inst = knav_range_offset_to_inst(kdev, range, queue);
54 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
55 range_base + queue);
56 knav_queue_notify(inst);
57 }
58}
59
60static int knav_acc_set_notify(struct knav_range_info *range,
61 struct knav_queue_inst *kq,
62 bool enabled)
63{
64 struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
65 struct knav_device *kdev = range->kdev;
66 u32 mask, offset;
67
68
69
70
71
72 if (!enabled || atomic_read(&kq->desc_count) <= 0)
73 return 0;
74
75 kq->notify_needed = 1;
76 atomic_inc(&kq->acc->retrigger_count);
77 mask = BIT(kq->acc->channel % 32);
78 offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
79 dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
80 kq->acc->name);
81 writel_relaxed(mask, pdsp->intd + offset);
82 return 0;
83}
84
85static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
86{
87 struct knav_acc_channel *acc;
88 struct knav_queue_inst *kq = NULL;
89 struct knav_range_info *range;
90 struct knav_pdsp_info *pdsp;
91 struct knav_acc_info *info;
92 struct knav_device *kdev;
93
94 u32 *list, *list_cpu, val, idx, notifies;
95 int range_base, channel, queue = 0;
96 dma_addr_t list_dma;
97
98 range = _instdata;
99 info = &range->acc_info;
100 kdev = range->kdev;
101 pdsp = range->acc_info.pdsp;
102 acc = range->acc;
103
104 range_base = kdev->base_id + range->queue_base;
105 if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
106 for (queue = 0; queue < range->num_irqs; queue++)
107 if (range->irqs[queue].irq == irq)
108 break;
109 kq = knav_range_offset_to_inst(kdev, range, queue);
110 acc += queue;
111 }
112
113 channel = acc->channel;
114 list_dma = acc->list_dma[acc->list_index];
115 list_cpu = acc->list_cpu[acc->list_index];
116 dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",
117 channel, acc->list_index, list_cpu, &list_dma);
118 if (atomic_read(&acc->retrigger_count)) {
119 atomic_dec(&acc->retrigger_count);
120 __knav_acc_notify(range, acc);
121 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
122
123 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
124 pdsp->intd + ACC_INTD_OFFSET_EOI);
125
126 return IRQ_HANDLED;
127 }
128
129 notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
130 WARN_ON(!notifies);
131 dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
132 DMA_FROM_DEVICE);
133
134 for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
135 list += ACC_LIST_ENTRY_WORDS) {
136 if (ACC_LIST_ENTRY_WORDS == 1) {
137 dev_dbg(kdev->dev,
138 "acc-irq: list %d, entry @%p, %08x\n",
139 acc->list_index, list, list[0]);
140 } else if (ACC_LIST_ENTRY_WORDS == 2) {
141 dev_dbg(kdev->dev,
142 "acc-irq: list %d, entry @%p, %08x %08x\n",
143 acc->list_index, list, list[0], list[1]);
144 } else if (ACC_LIST_ENTRY_WORDS == 4) {
145 dev_dbg(kdev->dev,
146 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
147 acc->list_index, list, list[0], list[1],
148 list[2], list[3]);
149 }
150
151 val = list[ACC_LIST_ENTRY_DESC_IDX];
152 if (!val)
153 break;
154
155 if (range->flags & RANGE_MULTI_QUEUE) {
156 queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
157 if (queue < range_base ||
158 queue >= range_base + range->num_queues) {
159 dev_err(kdev->dev,
160 "bad queue %d, expecting %d-%d\n",
161 queue, range_base,
162 range_base + range->num_queues);
163 break;
164 }
165 queue -= range_base;
166 kq = knav_range_offset_to_inst(kdev, range,
167 queue);
168 }
169
170 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
171 atomic_dec(&kq->desc_count);
172 dev_err(kdev->dev,
173 "acc-irq: queue %d full, entry dropped\n",
174 queue + range_base);
175 continue;
176 }
177
178 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
179 kq->descs[idx] = val;
180 kq->notify_needed = 1;
181 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
182 val, idx, queue + range_base);
183 }
184
185 __knav_acc_notify(range, acc);
186 memset(list_cpu, 0, info->list_size);
187 dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
188 DMA_TO_DEVICE);
189
190
191 acc->list_index ^= 1;
192
193
194 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
195
196
197 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
198 pdsp->intd + ACC_INTD_OFFSET_EOI);
199
200 return IRQ_HANDLED;
201}
202
203static int knav_range_setup_acc_irq(struct knav_range_info *range,
204 int queue, bool enabled)
205{
206 struct knav_device *kdev = range->kdev;
207 struct knav_acc_channel *acc;
208 unsigned long cpu_map;
209 int ret = 0, irq;
210 u32 old, new;
211
212 if (range->flags & RANGE_MULTI_QUEUE) {
213 acc = range->acc;
214 irq = range->irqs[0].irq;
215 cpu_map = range->irqs[0].cpu_map;
216 } else {
217 acc = range->acc + queue;
218 irq = range->irqs[queue].irq;
219 cpu_map = range->irqs[queue].cpu_map;
220 }
221
222 old = acc->open_mask;
223 if (enabled)
224 new = old | BIT(queue);
225 else
226 new = old & ~BIT(queue);
227 acc->open_mask = new;
228
229 dev_dbg(kdev->dev,
230 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
231 old, new, acc->name);
232
233 if (likely(new == old))
234 return 0;
235
236 if (new && !old) {
237 dev_dbg(kdev->dev,
238 "setup-acc-irq: requesting %s for channel %s\n",
239 acc->name, acc->name);
240 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
241 range);
242 if (!ret && cpu_map) {
243 ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map));
244 if (ret) {
245 dev_warn(range->kdev->dev,
246 "Failed to set IRQ affinity\n");
247 return ret;
248 }
249 }
250 }
251
252 if (old && !new) {
253 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
254 acc->name, acc->name);
255 ret = irq_set_affinity_hint(irq, NULL);
256 if (ret)
257 dev_warn(range->kdev->dev,
258 "Failed to set IRQ affinity\n");
259 free_irq(irq, range);
260 }
261
262 return ret;
263}
264
265static const char *knav_acc_result_str(enum knav_acc_result result)
266{
267 static const char * const result_str[] = {
268 [ACC_RET_IDLE] = "idle",
269 [ACC_RET_SUCCESS] = "success",
270 [ACC_RET_INVALID_COMMAND] = "invalid command",
271 [ACC_RET_INVALID_CHANNEL] = "invalid channel",
272 [ACC_RET_INACTIVE_CHANNEL] = "inactive channel",
273 [ACC_RET_ACTIVE_CHANNEL] = "active channel",
274 [ACC_RET_INVALID_QUEUE] = "invalid queue",
275 [ACC_RET_INVALID_RET] = "invalid return code",
276 };
277
278 if (result >= ARRAY_SIZE(result_str))
279 return result_str[ACC_RET_INVALID_RET];
280 else
281 return result_str[result];
282}
283
284static enum knav_acc_result
285knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
286 struct knav_reg_acc_command *cmd)
287{
288 u32 result;
289
290 dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
291 cmd->command, cmd->queue_mask, cmd->list_dma,
292 cmd->queue_num, cmd->timer_config);
293
294 writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
295 writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
296 writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);
297 writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
298 writel_relaxed(cmd->command, &pdsp->acc_command->command);
299
300
301 do {
302 result = readl_relaxed(&pdsp->acc_command->command);
303 } while ((result >> 8) & 0xff);
304
305 return (result >> 24) & 0xff;
306}
307
308static void knav_acc_setup_cmd(struct knav_device *kdev,
309 struct knav_range_info *range,
310 struct knav_reg_acc_command *cmd,
311 int queue)
312{
313 struct knav_acc_info *info = &range->acc_info;
314 struct knav_acc_channel *acc;
315 int queue_base;
316 u32 queue_mask;
317
318 if (range->flags & RANGE_MULTI_QUEUE) {
319 acc = range->acc;
320 queue_base = range->queue_base;
321 queue_mask = BIT(range->num_queues) - 1;
322 } else {
323 acc = range->acc + queue;
324 queue_base = range->queue_base + queue;
325 queue_mask = 0;
326 }
327
328 memset(cmd, 0, sizeof(*cmd));
329 cmd->command = acc->channel;
330 cmd->queue_mask = queue_mask;
331 cmd->list_dma = (u32)acc->list_dma[0];
332 cmd->queue_num = info->list_entries << 16;
333 cmd->queue_num |= queue_base;
334
335 cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
336 if (range->flags & RANGE_MULTI_QUEUE)
337 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
338 cmd->timer_config |= info->pacing_mode << 16;
339 cmd->timer_config |= info->timer_count;
340}
341
342static void knav_acc_stop(struct knav_device *kdev,
343 struct knav_range_info *range,
344 int queue)
345{
346 struct knav_reg_acc_command cmd;
347 struct knav_acc_channel *acc;
348 enum knav_acc_result result;
349
350 acc = range->acc + queue;
351
352 knav_acc_setup_cmd(kdev, range, &cmd, queue);
353 cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
354 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
355
356 dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
357 acc->name, knav_acc_result_str(result));
358}
359
360static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
361 struct knav_range_info *range,
362 int queue)
363{
364 struct knav_reg_acc_command cmd;
365 struct knav_acc_channel *acc;
366 enum knav_acc_result result;
367
368 acc = range->acc + queue;
369
370 knav_acc_setup_cmd(kdev, range, &cmd, queue);
371 cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
372 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
373
374 dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
375 acc->name, knav_acc_result_str(result));
376
377 return result;
378}
379
380static int knav_acc_init_range(struct knav_range_info *range)
381{
382 struct knav_device *kdev = range->kdev;
383 struct knav_acc_channel *acc;
384 enum knav_acc_result result;
385 int queue;
386
387 for (queue = 0; queue < range->num_queues; queue++) {
388 acc = range->acc + queue;
389
390 knav_acc_stop(kdev, range, queue);
391 acc->list_index = 0;
392 result = knav_acc_start(kdev, range, queue);
393
394 if (result != ACC_RET_SUCCESS)
395 return -EIO;
396
397 if (range->flags & RANGE_MULTI_QUEUE)
398 return 0;
399 }
400 return 0;
401}
402
403static int knav_acc_init_queue(struct knav_range_info *range,
404 struct knav_queue_inst *kq)
405{
406 unsigned id = kq->id - range->queue_base;
407
408 kq->descs = devm_kcalloc(range->kdev->dev,
409 ACC_DESCS_MAX, sizeof(u32), GFP_KERNEL);
410 if (!kq->descs)
411 return -ENOMEM;
412
413 kq->acc = range->acc;
414 if ((range->flags & RANGE_MULTI_QUEUE) == 0)
415 kq->acc += id;
416 return 0;
417}
418
419static int knav_acc_open_queue(struct knav_range_info *range,
420 struct knav_queue_inst *inst, unsigned flags)
421{
422 unsigned id = inst->id - range->queue_base;
423
424 return knav_range_setup_acc_irq(range, id, true);
425}
426
427static int knav_acc_close_queue(struct knav_range_info *range,
428 struct knav_queue_inst *inst)
429{
430 unsigned id = inst->id - range->queue_base;
431
432 return knav_range_setup_acc_irq(range, id, false);
433}
434
435static int knav_acc_free_range(struct knav_range_info *range)
436{
437 struct knav_device *kdev = range->kdev;
438 struct knav_acc_channel *acc;
439 struct knav_acc_info *info;
440 int channel, channels;
441
442 info = &range->acc_info;
443
444 if (range->flags & RANGE_MULTI_QUEUE)
445 channels = 1;
446 else
447 channels = range->num_queues;
448
449 for (channel = 0; channel < channels; channel++) {
450 acc = range->acc + channel;
451 if (!acc->list_cpu[0])
452 continue;
453 dma_unmap_single(kdev->dev, acc->list_dma[0],
454 info->mem_size, DMA_BIDIRECTIONAL);
455 free_pages_exact(acc->list_cpu[0], info->mem_size);
456 }
457 devm_kfree(range->kdev->dev, range->acc);
458 return 0;
459}
460
461struct knav_range_ops knav_acc_range_ops = {
462 .set_notify = knav_acc_set_notify,
463 .init_queue = knav_acc_init_queue,
464 .open_queue = knav_acc_open_queue,
465 .close_queue = knav_acc_close_queue,
466 .init_range = knav_acc_init_range,
467 .free_range = knav_acc_free_range,
468};
469
470
471
472
473
474
475
476
477
478
479int knav_init_acc_range(struct knav_device *kdev,
480 struct device_node *node,
481 struct knav_range_info *range)
482{
483 struct knav_acc_channel *acc;
484 struct knav_pdsp_info *pdsp;
485 struct knav_acc_info *info;
486 int ret, channel, channels;
487 int list_size, mem_size;
488 dma_addr_t list_dma;
489 void *list_mem;
490 u32 config[5];
491
492 range->flags |= RANGE_HAS_ACCUMULATOR;
493 info = &range->acc_info;
494
495 ret = of_property_read_u32_array(node, "accumulator", config, 5);
496 if (ret)
497 return ret;
498
499 info->pdsp_id = config[0];
500 info->start_channel = config[1];
501 info->list_entries = config[2];
502 info->pacing_mode = config[3];
503 info->timer_count = config[4] / ACC_DEFAULT_PERIOD;
504
505 if (info->start_channel > ACC_MAX_CHANNEL) {
506 dev_err(kdev->dev, "channel %d invalid for range %s\n",
507 info->start_channel, range->name);
508 return -EINVAL;
509 }
510
511 if (info->pacing_mode > 3) {
512 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
513 info->pacing_mode, range->name);
514 return -EINVAL;
515 }
516
517 pdsp = knav_find_pdsp(kdev, info->pdsp_id);
518 if (!pdsp) {
519 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
520 info->pdsp_id, range->name);
521 return -EINVAL;
522 }
523
524 if (!pdsp->started) {
525 dev_err(kdev->dev, "pdsp id %d not started for range %s\n",
526 info->pdsp_id, range->name);
527 return -ENODEV;
528 }
529
530 info->pdsp = pdsp;
531 channels = range->num_queues;
532 if (of_get_property(node, "multi-queue", NULL)) {
533 range->flags |= RANGE_MULTI_QUEUE;
534 channels = 1;
535 if (range->queue_base & (32 - 1)) {
536 dev_err(kdev->dev,
537 "misaligned multi-queue accumulator range %s\n",
538 range->name);
539 return -EINVAL;
540 }
541 if (range->num_queues > 32) {
542 dev_err(kdev->dev,
543 "too many queues in accumulator range %s\n",
544 range->name);
545 return -EINVAL;
546 }
547 }
548
549
550 list_size = info->list_entries;
551 list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
552 info->list_size = list_size;
553 mem_size = PAGE_ALIGN(list_size * 2);
554 info->mem_size = mem_size;
555 range->acc = devm_kcalloc(kdev->dev, channels, sizeof(*range->acc),
556 GFP_KERNEL);
557 if (!range->acc)
558 return -ENOMEM;
559
560 for (channel = 0; channel < channels; channel++) {
561 acc = range->acc + channel;
562 acc->channel = info->start_channel + channel;
563
564
565 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
566 if (!list_mem)
567 return -ENOMEM;
568
569 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
570 DMA_BIDIRECTIONAL);
571 if (dma_mapping_error(kdev->dev, list_dma)) {
572 free_pages_exact(list_mem, mem_size);
573 return -ENOMEM;
574 }
575
576 memset(list_mem, 0, mem_size);
577 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
578 DMA_TO_DEVICE);
579 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
580 acc->channel);
581 acc->list_cpu[0] = list_mem;
582 acc->list_cpu[1] = list_mem + list_size;
583 acc->list_dma[0] = list_dma;
584 acc->list_dma[1] = list_dma + list_size;
585 dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",
586 acc->name, acc->channel, &list_dma, list_mem);
587 }
588
589 range->ops = &knav_acc_range_ops;
590 return 0;
591}
592EXPORT_SYMBOL_GPL(knav_init_acc_range);
593
