1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/device.h>
22#include <linux/io.h>
23#include <linux/interrupt.h>
24#include <linux/bitops.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
27#include <linux/soc/ti/knav_qmss.h>
28#include <linux/platform_device.h>
29#include <linux/dma-mapping.h>
30#include <linux/of.h>
31#include <linux/of_device.h>
32#include <linux/of_address.h>
33#include <linux/firmware.h>
34
35#include "knav_qmss.h"
36
37#define knav_range_offset_to_inst(kdev, range, q) \
38 (range->queue_base_inst + (q << kdev->inst_shift))
39
40static void __knav_acc_notify(struct knav_range_info *range,
41 struct knav_acc_channel *acc)
42{
43 struct knav_device *kdev = range->kdev;
44 struct knav_queue_inst *inst;
45 int range_base, queue;
46
47 range_base = kdev->base_id + range->queue_base;
48
49 if (range->flags & RANGE_MULTI_QUEUE) {
50 for (queue = 0; queue < range->num_queues; queue++) {
51 inst = knav_range_offset_to_inst(kdev, range,
52 queue);
53 if (inst->notify_needed) {
54 inst->notify_needed = 0;
55 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
56 range_base + queue);
57 knav_queue_notify(inst);
58 }
59 }
60 } else {
61 queue = acc->channel - range->acc_info.start_channel;
62 inst = knav_range_offset_to_inst(kdev, range, queue);
63 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
64 range_base + queue);
65 knav_queue_notify(inst);
66 }
67}
68
69static int knav_acc_set_notify(struct knav_range_info *range,
70 struct knav_queue_inst *kq,
71 bool enabled)
72{
73 struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
74 struct knav_device *kdev = range->kdev;
75 u32 mask, offset;
76
77
78
79
80
81 if (!enabled || atomic_read(&kq->desc_count) <= 0)
82 return 0;
83
84 kq->notify_needed = 1;
85 atomic_inc(&kq->acc->retrigger_count);
86 mask = BIT(kq->acc->channel % 32);
87 offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
88 dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
89 kq->acc->name);
90 writel_relaxed(mask, pdsp->intd + offset);
91 return 0;
92}
93
94static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
95{
96 struct knav_acc_channel *acc;
97 struct knav_queue_inst *kq = NULL;
98 struct knav_range_info *range;
99 struct knav_pdsp_info *pdsp;
100 struct knav_acc_info *info;
101 struct knav_device *kdev;
102
103 u32 *list, *list_cpu, val, idx, notifies;
104 int range_base, channel, queue = 0;
105 dma_addr_t list_dma;
106
107 range = _instdata;
108 info = &range->acc_info;
109 kdev = range->kdev;
110 pdsp = range->acc_info.pdsp;
111 acc = range->acc;
112
113 range_base = kdev->base_id + range->queue_base;
114 if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
115 for (queue = 0; queue < range->num_irqs; queue++)
116 if (range->irqs[queue].irq == irq)
117 break;
118 kq = knav_range_offset_to_inst(kdev, range, queue);
119 acc += queue;
120 }
121
122 channel = acc->channel;
123 list_dma = acc->list_dma[acc->list_index];
124 list_cpu = acc->list_cpu[acc->list_index];
125 dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",
126 channel, acc->list_index, list_cpu, &list_dma);
127 if (atomic_read(&acc->retrigger_count)) {
128 atomic_dec(&acc->retrigger_count);
129 __knav_acc_notify(range, acc);
130 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
131
132 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
133 pdsp->intd + ACC_INTD_OFFSET_EOI);
134
135 return IRQ_HANDLED;
136 }
137
138 notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
139 WARN_ON(!notifies);
140 dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
141 DMA_FROM_DEVICE);
142
143 for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
144 list += ACC_LIST_ENTRY_WORDS) {
145 if (ACC_LIST_ENTRY_WORDS == 1) {
146 dev_dbg(kdev->dev,
147 "acc-irq: list %d, entry @%p, %08x\n",
148 acc->list_index, list, list[0]);
149 } else if (ACC_LIST_ENTRY_WORDS == 2) {
150 dev_dbg(kdev->dev,
151 "acc-irq: list %d, entry @%p, %08x %08x\n",
152 acc->list_index, list, list[0], list[1]);
153 } else if (ACC_LIST_ENTRY_WORDS == 4) {
154 dev_dbg(kdev->dev,
155 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
156 acc->list_index, list, list[0], list[1],
157 list[2], list[3]);
158 }
159
160 val = list[ACC_LIST_ENTRY_DESC_IDX];
161 if (!val)
162 break;
163
164 if (range->flags & RANGE_MULTI_QUEUE) {
165 queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
166 if (queue < range_base ||
167 queue >= range_base + range->num_queues) {
168 dev_err(kdev->dev,
169 "bad queue %d, expecting %d-%d\n",
170 queue, range_base,
171 range_base + range->num_queues);
172 break;
173 }
174 queue -= range_base;
175 kq = knav_range_offset_to_inst(kdev, range,
176 queue);
177 }
178
179 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
180 atomic_dec(&kq->desc_count);
181 dev_err(kdev->dev,
182 "acc-irq: queue %d full, entry dropped\n",
183 queue + range_base);
184 continue;
185 }
186
187 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
188 kq->descs[idx] = val;
189 kq->notify_needed = 1;
190 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
191 val, idx, queue + range_base);
192 }
193
194 __knav_acc_notify(range, acc);
195 memset(list_cpu, 0, info->list_size);
196 dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
197 DMA_TO_DEVICE);
198
199
200 acc->list_index ^= 1;
201
202
203 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
204
205
206 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
207 pdsp->intd + ACC_INTD_OFFSET_EOI);
208
209 return IRQ_HANDLED;
210}
211
212static int knav_range_setup_acc_irq(struct knav_range_info *range,
213 int queue, bool enabled)
214{
215 struct knav_device *kdev = range->kdev;
216 struct knav_acc_channel *acc;
217 unsigned long cpu_map;
218 int ret = 0, irq;
219 u32 old, new;
220
221 if (range->flags & RANGE_MULTI_QUEUE) {
222 acc = range->acc;
223 irq = range->irqs[0].irq;
224 cpu_map = range->irqs[0].cpu_map;
225 } else {
226 acc = range->acc + queue;
227 irq = range->irqs[queue].irq;
228 cpu_map = range->irqs[queue].cpu_map;
229 }
230
231 old = acc->open_mask;
232 if (enabled)
233 new = old | BIT(queue);
234 else
235 new = old & ~BIT(queue);
236 acc->open_mask = new;
237
238 dev_dbg(kdev->dev,
239 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
240 old, new, acc->name);
241
242 if (likely(new == old))
243 return 0;
244
245 if (new && !old) {
246 dev_dbg(kdev->dev,
247 "setup-acc-irq: requesting %s for channel %s\n",
248 acc->name, acc->name);
249 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
250 range);
251 if (!ret && cpu_map) {
252 ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map));
253 if (ret) {
254 dev_warn(range->kdev->dev,
255 "Failed to set IRQ affinity\n");
256 return ret;
257 }
258 }
259 }
260
261 if (old && !new) {
262 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
263 acc->name, acc->name);
264 ret = irq_set_affinity_hint(irq, NULL);
265 if (ret)
266 dev_warn(range->kdev->dev,
267 "Failed to set IRQ affinity\n");
268 free_irq(irq, range);
269 }
270
271 return ret;
272}
273
274static const char *knav_acc_result_str(enum knav_acc_result result)
275{
276 static const char * const result_str[] = {
277 [ACC_RET_IDLE] = "idle",
278 [ACC_RET_SUCCESS] = "success",
279 [ACC_RET_INVALID_COMMAND] = "invalid command",
280 [ACC_RET_INVALID_CHANNEL] = "invalid channel",
281 [ACC_RET_INACTIVE_CHANNEL] = "inactive channel",
282 [ACC_RET_ACTIVE_CHANNEL] = "active channel",
283 [ACC_RET_INVALID_QUEUE] = "invalid queue",
284 [ACC_RET_INVALID_RET] = "invalid return code",
285 };
286
287 if (result >= ARRAY_SIZE(result_str))
288 return result_str[ACC_RET_INVALID_RET];
289 else
290 return result_str[result];
291}
292
293static enum knav_acc_result
294knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
295 struct knav_reg_acc_command *cmd)
296{
297 u32 result;
298
299 dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
300 cmd->command, cmd->queue_mask, cmd->list_dma,
301 cmd->queue_num, cmd->timer_config);
302
303 writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
304 writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
305 writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);
306 writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
307 writel_relaxed(cmd->command, &pdsp->acc_command->command);
308
309
310 do {
311 result = readl_relaxed(&pdsp->acc_command->command);
312 } while ((result >> 8) & 0xff);
313
314 return (result >> 24) & 0xff;
315}
316
317static void knav_acc_setup_cmd(struct knav_device *kdev,
318 struct knav_range_info *range,
319 struct knav_reg_acc_command *cmd,
320 int queue)
321{
322 struct knav_acc_info *info = &range->acc_info;
323 struct knav_acc_channel *acc;
324 int queue_base;
325 u32 queue_mask;
326
327 if (range->flags & RANGE_MULTI_QUEUE) {
328 acc = range->acc;
329 queue_base = range->queue_base;
330 queue_mask = BIT(range->num_queues) - 1;
331 } else {
332 acc = range->acc + queue;
333 queue_base = range->queue_base + queue;
334 queue_mask = 0;
335 }
336
337 memset(cmd, 0, sizeof(*cmd));
338 cmd->command = acc->channel;
339 cmd->queue_mask = queue_mask;
340 cmd->list_dma = (u32)acc->list_dma[0];
341 cmd->queue_num = info->list_entries << 16;
342 cmd->queue_num |= queue_base;
343
344 cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
345 if (range->flags & RANGE_MULTI_QUEUE)
346 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
347 cmd->timer_config |= info->pacing_mode << 16;
348 cmd->timer_config |= info->timer_count;
349}
350
351static void knav_acc_stop(struct knav_device *kdev,
352 struct knav_range_info *range,
353 int queue)
354{
355 struct knav_reg_acc_command cmd;
356 struct knav_acc_channel *acc;
357 enum knav_acc_result result;
358
359 acc = range->acc + queue;
360
361 knav_acc_setup_cmd(kdev, range, &cmd, queue);
362 cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
363 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
364
365 dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
366 acc->name, knav_acc_result_str(result));
367}
368
369static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
370 struct knav_range_info *range,
371 int queue)
372{
373 struct knav_reg_acc_command cmd;
374 struct knav_acc_channel *acc;
375 enum knav_acc_result result;
376
377 acc = range->acc + queue;
378
379 knav_acc_setup_cmd(kdev, range, &cmd, queue);
380 cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
381 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
382
383 dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
384 acc->name, knav_acc_result_str(result));
385
386 return result;
387}
388
389static int knav_acc_init_range(struct knav_range_info *range)
390{
391 struct knav_device *kdev = range->kdev;
392 struct knav_acc_channel *acc;
393 enum knav_acc_result result;
394 int queue;
395
396 for (queue = 0; queue < range->num_queues; queue++) {
397 acc = range->acc + queue;
398
399 knav_acc_stop(kdev, range, queue);
400 acc->list_index = 0;
401 result = knav_acc_start(kdev, range, queue);
402
403 if (result != ACC_RET_SUCCESS)
404 return -EIO;
405
406 if (range->flags & RANGE_MULTI_QUEUE)
407 return 0;
408 }
409 return 0;
410}
411
412static int knav_acc_init_queue(struct knav_range_info *range,
413 struct knav_queue_inst *kq)
414{
415 unsigned id = kq->id - range->queue_base;
416
417 kq->descs = devm_kzalloc(range->kdev->dev,
418 ACC_DESCS_MAX * sizeof(u32), GFP_KERNEL);
419 if (!kq->descs)
420 return -ENOMEM;
421
422 kq->acc = range->acc;
423 if ((range->flags & RANGE_MULTI_QUEUE) == 0)
424 kq->acc += id;
425 return 0;
426}
427
428static int knav_acc_open_queue(struct knav_range_info *range,
429 struct knav_queue_inst *inst, unsigned flags)
430{
431 unsigned id = inst->id - range->queue_base;
432
433 return knav_range_setup_acc_irq(range, id, true);
434}
435
436static int knav_acc_close_queue(struct knav_range_info *range,
437 struct knav_queue_inst *inst)
438{
439 unsigned id = inst->id - range->queue_base;
440
441 return knav_range_setup_acc_irq(range, id, false);
442}
443
444static int knav_acc_free_range(struct knav_range_info *range)
445{
446 struct knav_device *kdev = range->kdev;
447 struct knav_acc_channel *acc;
448 struct knav_acc_info *info;
449 int channel, channels;
450
451 info = &range->acc_info;
452
453 if (range->flags & RANGE_MULTI_QUEUE)
454 channels = 1;
455 else
456 channels = range->num_queues;
457
458 for (channel = 0; channel < channels; channel++) {
459 acc = range->acc + channel;
460 if (!acc->list_cpu[0])
461 continue;
462 dma_unmap_single(kdev->dev, acc->list_dma[0],
463 info->mem_size, DMA_BIDIRECTIONAL);
464 free_pages_exact(acc->list_cpu[0], info->mem_size);
465 }
466 devm_kfree(range->kdev->dev, range->acc);
467 return 0;
468}
469
470struct knav_range_ops knav_acc_range_ops = {
471 .set_notify = knav_acc_set_notify,
472 .init_queue = knav_acc_init_queue,
473 .open_queue = knav_acc_open_queue,
474 .close_queue = knav_acc_close_queue,
475 .init_range = knav_acc_init_range,
476 .free_range = knav_acc_free_range,
477};
478
479
480
481
482
483
484
485
486
487
488int knav_init_acc_range(struct knav_device *kdev,
489 struct device_node *node,
490 struct knav_range_info *range)
491{
492 struct knav_acc_channel *acc;
493 struct knav_pdsp_info *pdsp;
494 struct knav_acc_info *info;
495 int ret, channel, channels;
496 int list_size, mem_size;
497 dma_addr_t list_dma;
498 void *list_mem;
499 u32 config[5];
500
501 range->flags |= RANGE_HAS_ACCUMULATOR;
502 info = &range->acc_info;
503
504 ret = of_property_read_u32_array(node, "accumulator", config, 5);
505 if (ret)
506 return ret;
507
508 info->pdsp_id = config[0];
509 info->start_channel = config[1];
510 info->list_entries = config[2];
511 info->pacing_mode = config[3];
512 info->timer_count = config[4] / ACC_DEFAULT_PERIOD;
513
514 if (info->start_channel > ACC_MAX_CHANNEL) {
515 dev_err(kdev->dev, "channel %d invalid for range %s\n",
516 info->start_channel, range->name);
517 return -EINVAL;
518 }
519
520 if (info->pacing_mode > 3) {
521 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
522 info->pacing_mode, range->name);
523 return -EINVAL;
524 }
525
526 pdsp = knav_find_pdsp(kdev, info->pdsp_id);
527 if (!pdsp) {
528 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
529 info->pdsp_id, range->name);
530 return -EINVAL;
531 }
532
533 if (!pdsp->started) {
534 dev_err(kdev->dev, "pdsp id %d not started for range %s\n",
535 info->pdsp_id, range->name);
536 return -ENODEV;
537 }
538
539 info->pdsp = pdsp;
540 channels = range->num_queues;
541 if (of_get_property(node, "multi-queue", NULL)) {
542 range->flags |= RANGE_MULTI_QUEUE;
543 channels = 1;
544 if (range->queue_base & (32 - 1)) {
545 dev_err(kdev->dev,
546 "misaligned multi-queue accumulator range %s\n",
547 range->name);
548 return -EINVAL;
549 }
550 if (range->num_queues > 32) {
551 dev_err(kdev->dev,
552 "too many queues in accumulator range %s\n",
553 range->name);
554 return -EINVAL;
555 }
556 }
557
558
559 list_size = info->list_entries;
560 list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
561 info->list_size = list_size;
562 mem_size = PAGE_ALIGN(list_size * 2);
563 info->mem_size = mem_size;
564 range->acc = devm_kzalloc(kdev->dev, channels * sizeof(*range->acc),
565 GFP_KERNEL);
566 if (!range->acc)
567 return -ENOMEM;
568
569 for (channel = 0; channel < channels; channel++) {
570 acc = range->acc + channel;
571 acc->channel = info->start_channel + channel;
572
573
574 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
575 if (!list_mem)
576 return -ENOMEM;
577
578 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
579 DMA_BIDIRECTIONAL);
580 if (dma_mapping_error(kdev->dev, list_dma)) {
581 free_pages_exact(list_mem, mem_size);
582 return -ENOMEM;
583 }
584
585 memset(list_mem, 0, mem_size);
586 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
587 DMA_TO_DEVICE);
588 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
589 acc->channel);
590 acc->list_cpu[0] = list_mem;
591 acc->list_cpu[1] = list_mem + list_size;
592 acc->list_dma[0] = list_dma;
593 acc->list_dma[1] = list_dma + list_size;
594 dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",
595 acc->name, acc->channel, &list_dma, list_mem);
596 }
597
598 range->ops = &knav_acc_range_ops;
599 return 0;
600}
601EXPORT_SYMBOL_GPL(knav_init_acc_range);
602
