1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/device.h>
22#include <linux/io.h>
23#include <linux/interrupt.h>
24#include <linux/bitops.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
27#include <linux/soc/ti/knav_qmss.h>
28#include <linux/platform_device.h>
29#include <linux/dma-mapping.h>
30#include <linux/of.h>
31#include <linux/of_device.h>
32#include <linux/of_address.h>
33#include <linux/firmware.h>
34
35#include "knav_qmss.h"
36
37#define knav_range_offset_to_inst(kdev, range, q) \
38 (range->queue_base_inst + (q << kdev->inst_shift))
39
40static void __knav_acc_notify(struct knav_range_info *range,
41 struct knav_acc_channel *acc)
42{
43 struct knav_device *kdev = range->kdev;
44 struct knav_queue_inst *inst;
45 int range_base, queue;
46
47 range_base = kdev->base_id + range->queue_base;
48
49 if (range->flags & RANGE_MULTI_QUEUE) {
50 for (queue = 0; queue < range->num_queues; queue++) {
51 inst = knav_range_offset_to_inst(kdev, range,
52 queue);
53 if (inst->notify_needed) {
54 inst->notify_needed = 0;
55 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
56 range_base + queue);
57 knav_queue_notify(inst);
58 }
59 }
60 } else {
61 queue = acc->channel - range->acc_info.start_channel;
62 inst = knav_range_offset_to_inst(kdev, range, queue);
63 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
64 range_base + queue);
65 knav_queue_notify(inst);
66 }
67}
68
69static int knav_acc_set_notify(struct knav_range_info *range,
70 struct knav_queue_inst *kq,
71 bool enabled)
72{
73 struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
74 struct knav_device *kdev = range->kdev;
75 u32 mask, offset;
76
77
78
79
80
81 if (!enabled || atomic_read(&kq->desc_count) <= 0)
82 return 0;
83
84 kq->notify_needed = 1;
85 atomic_inc(&kq->acc->retrigger_count);
86 mask = BIT(kq->acc->channel % 32);
87 offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
88 dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
89 kq->acc->name);
90 writel_relaxed(mask, pdsp->intd + offset);
91 return 0;
92}
93
94static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
95{
96 struct knav_acc_channel *acc;
97 struct knav_queue_inst *kq = NULL;
98 struct knav_range_info *range;
99 struct knav_pdsp_info *pdsp;
100 struct knav_acc_info *info;
101 struct knav_device *kdev;
102
103 u32 *list, *list_cpu, val, idx, notifies;
104 int range_base, channel, queue = 0;
105 dma_addr_t list_dma;
106
107 range = _instdata;
108 info = &range->acc_info;
109 kdev = range->kdev;
110 pdsp = range->acc_info.pdsp;
111 acc = range->acc;
112
113 range_base = kdev->base_id + range->queue_base;
114 if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
115 for (queue = 0; queue < range->num_irqs; queue++)
116 if (range->irqs[queue].irq == irq)
117 break;
118 kq = knav_range_offset_to_inst(kdev, range, queue);
119 acc += queue;
120 }
121
122 channel = acc->channel;
123 list_dma = acc->list_dma[acc->list_index];
124 list_cpu = acc->list_cpu[acc->list_index];
125 dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, phys %x\n",
126 channel, acc->list_index, list_cpu, list_dma);
127 if (atomic_read(&acc->retrigger_count)) {
128 atomic_dec(&acc->retrigger_count);
129 __knav_acc_notify(range, acc);
130 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
131
132 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
133 pdsp->intd + ACC_INTD_OFFSET_EOI);
134
135 return IRQ_HANDLED;
136 }
137
138 notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
139 WARN_ON(!notifies);
140 dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
141 DMA_FROM_DEVICE);
142
143 for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
144 list += ACC_LIST_ENTRY_WORDS) {
145 if (ACC_LIST_ENTRY_WORDS == 1) {
146 dev_dbg(kdev->dev,
147 "acc-irq: list %d, entry @%p, %08x\n",
148 acc->list_index, list, list[0]);
149 } else if (ACC_LIST_ENTRY_WORDS == 2) {
150 dev_dbg(kdev->dev,
151 "acc-irq: list %d, entry @%p, %08x %08x\n",
152 acc->list_index, list, list[0], list[1]);
153 } else if (ACC_LIST_ENTRY_WORDS == 4) {
154 dev_dbg(kdev->dev,
155 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
156 acc->list_index, list, list[0], list[1],
157 list[2], list[3]);
158 }
159
160 val = list[ACC_LIST_ENTRY_DESC_IDX];
161 if (!val)
162 break;
163
164 if (range->flags & RANGE_MULTI_QUEUE) {
165 queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
166 if (queue < range_base ||
167 queue >= range_base + range->num_queues) {
168 dev_err(kdev->dev,
169 "bad queue %d, expecting %d-%d\n",
170 queue, range_base,
171 range_base + range->num_queues);
172 break;
173 }
174 queue -= range_base;
175 kq = knav_range_offset_to_inst(kdev, range,
176 queue);
177 }
178
179 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
180 atomic_dec(&kq->desc_count);
181 dev_err(kdev->dev,
182 "acc-irq: queue %d full, entry dropped\n",
183 queue + range_base);
184 continue;
185 }
186
187 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
188 kq->descs[idx] = val;
189 kq->notify_needed = 1;
190 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
191 val, idx, queue + range_base);
192 }
193
194 __knav_acc_notify(range, acc);
195 memset(list_cpu, 0, info->list_size);
196 dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
197 DMA_TO_DEVICE);
198
199
200 acc->list_index ^= 1;
201
202
203 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
204
205
206 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
207 pdsp->intd + ACC_INTD_OFFSET_EOI);
208
209 return IRQ_HANDLED;
210}
211
212static int knav_range_setup_acc_irq(struct knav_range_info *range,
213 int queue, bool enabled)
214{
215 struct knav_device *kdev = range->kdev;
216 struct knav_acc_channel *acc;
217 unsigned long cpu_map;
218 int ret = 0, irq;
219 u32 old, new;
220
221 if (range->flags & RANGE_MULTI_QUEUE) {
222 acc = range->acc;
223 irq = range->irqs[0].irq;
224 cpu_map = range->irqs[0].cpu_map;
225 } else {
226 acc = range->acc + queue;
227 irq = range->irqs[queue].irq;
228 cpu_map = range->irqs[queue].cpu_map;
229 }
230
231 old = acc->open_mask;
232 if (enabled)
233 new = old | BIT(queue);
234 else
235 new = old & ~BIT(queue);
236 acc->open_mask = new;
237
238 dev_dbg(kdev->dev,
239 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
240 old, new, acc->name);
241
242 if (likely(new == old))
243 return 0;
244
245 if (new && !old) {
246 dev_dbg(kdev->dev,
247 "setup-acc-irq: requesting %s for channel %s\n",
248 acc->name, acc->name);
249 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
250 range);
251 if (!ret && cpu_map) {
252 ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map));
253 if (ret) {
254 dev_warn(range->kdev->dev,
255 "Failed to set IRQ affinity\n");
256 return ret;
257 }
258 }
259 }
260
261 if (old && !new) {
262 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
263 acc->name, acc->name);
264 free_irq(irq, range);
265 }
266
267 return ret;
268}
269
270static const char *knav_acc_result_str(enum knav_acc_result result)
271{
272 static const char * const result_str[] = {
273 [ACC_RET_IDLE] = "idle",
274 [ACC_RET_SUCCESS] = "success",
275 [ACC_RET_INVALID_COMMAND] = "invalid command",
276 [ACC_RET_INVALID_CHANNEL] = "invalid channel",
277 [ACC_RET_INACTIVE_CHANNEL] = "inactive channel",
278 [ACC_RET_ACTIVE_CHANNEL] = "active channel",
279 [ACC_RET_INVALID_QUEUE] = "invalid queue",
280 [ACC_RET_INVALID_RET] = "invalid return code",
281 };
282
283 if (result >= ARRAY_SIZE(result_str))
284 return result_str[ACC_RET_INVALID_RET];
285 else
286 return result_str[result];
287}
288
289static enum knav_acc_result
290knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
291 struct knav_reg_acc_command *cmd)
292{
293 u32 result;
294
295 dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
296 cmd->command, cmd->queue_mask, cmd->list_phys,
297 cmd->queue_num, cmd->timer_config);
298
299 writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
300 writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
301 writel_relaxed(cmd->list_phys, &pdsp->acc_command->list_phys);
302 writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
303 writel_relaxed(cmd->command, &pdsp->acc_command->command);
304
305
306 do {
307 result = readl_relaxed(&pdsp->acc_command->command);
308 } while ((result >> 8) & 0xff);
309
310 return (result >> 24) & 0xff;
311}
312
313static void knav_acc_setup_cmd(struct knav_device *kdev,
314 struct knav_range_info *range,
315 struct knav_reg_acc_command *cmd,
316 int queue)
317{
318 struct knav_acc_info *info = &range->acc_info;
319 struct knav_acc_channel *acc;
320 int queue_base;
321 u32 queue_mask;
322
323 if (range->flags & RANGE_MULTI_QUEUE) {
324 acc = range->acc;
325 queue_base = range->queue_base;
326 queue_mask = BIT(range->num_queues) - 1;
327 } else {
328 acc = range->acc + queue;
329 queue_base = range->queue_base + queue;
330 queue_mask = 0;
331 }
332
333 memset(cmd, 0, sizeof(*cmd));
334 cmd->command = acc->channel;
335 cmd->queue_mask = queue_mask;
336 cmd->list_phys = acc->list_dma[0];
337 cmd->queue_num = info->list_entries << 16;
338 cmd->queue_num |= queue_base;
339
340 cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
341 if (range->flags & RANGE_MULTI_QUEUE)
342 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
343 cmd->timer_config |= info->pacing_mode << 16;
344 cmd->timer_config |= info->timer_count;
345}
346
347static void knav_acc_stop(struct knav_device *kdev,
348 struct knav_range_info *range,
349 int queue)
350{
351 struct knav_reg_acc_command cmd;
352 struct knav_acc_channel *acc;
353 enum knav_acc_result result;
354
355 acc = range->acc + queue;
356
357 knav_acc_setup_cmd(kdev, range, &cmd, queue);
358 cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
359 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
360
361 dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
362 acc->name, knav_acc_result_str(result));
363}
364
365static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
366 struct knav_range_info *range,
367 int queue)
368{
369 struct knav_reg_acc_command cmd;
370 struct knav_acc_channel *acc;
371 enum knav_acc_result result;
372
373 acc = range->acc + queue;
374
375 knav_acc_setup_cmd(kdev, range, &cmd, queue);
376 cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
377 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
378
379 dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
380 acc->name, knav_acc_result_str(result));
381
382 return result;
383}
384
385static int knav_acc_init_range(struct knav_range_info *range)
386{
387 struct knav_device *kdev = range->kdev;
388 struct knav_acc_channel *acc;
389 enum knav_acc_result result;
390 int queue;
391
392 for (queue = 0; queue < range->num_queues; queue++) {
393 acc = range->acc + queue;
394
395 knav_acc_stop(kdev, range, queue);
396 acc->list_index = 0;
397 result = knav_acc_start(kdev, range, queue);
398
399 if (result != ACC_RET_SUCCESS)
400 return -EIO;
401
402 if (range->flags & RANGE_MULTI_QUEUE)
403 return 0;
404 }
405 return 0;
406}
407
408static int knav_acc_init_queue(struct knav_range_info *range,
409 struct knav_queue_inst *kq)
410{
411 unsigned id = kq->id - range->queue_base;
412
413 kq->descs = devm_kzalloc(range->kdev->dev,
414 ACC_DESCS_MAX * sizeof(u32), GFP_KERNEL);
415 if (!kq->descs)
416 return -ENOMEM;
417
418 kq->acc = range->acc;
419 if ((range->flags & RANGE_MULTI_QUEUE) == 0)
420 kq->acc += id;
421 return 0;
422}
423
424static int knav_acc_open_queue(struct knav_range_info *range,
425 struct knav_queue_inst *inst, unsigned flags)
426{
427 unsigned id = inst->id - range->queue_base;
428
429 return knav_range_setup_acc_irq(range, id, true);
430}
431
432static int knav_acc_close_queue(struct knav_range_info *range,
433 struct knav_queue_inst *inst)
434{
435 unsigned id = inst->id - range->queue_base;
436
437 return knav_range_setup_acc_irq(range, id, false);
438}
439
440static int knav_acc_free_range(struct knav_range_info *range)
441{
442 struct knav_device *kdev = range->kdev;
443 struct knav_acc_channel *acc;
444 struct knav_acc_info *info;
445 int channel, channels;
446
447 info = &range->acc_info;
448
449 if (range->flags & RANGE_MULTI_QUEUE)
450 channels = 1;
451 else
452 channels = range->num_queues;
453
454 for (channel = 0; channel < channels; channel++) {
455 acc = range->acc + channel;
456 if (!acc->list_cpu[0])
457 continue;
458 dma_unmap_single(kdev->dev, acc->list_dma[0],
459 info->mem_size, DMA_BIDIRECTIONAL);
460 free_pages_exact(acc->list_cpu[0], info->mem_size);
461 }
462 devm_kfree(range->kdev->dev, range->acc);
463 return 0;
464}
465
466struct knav_range_ops knav_acc_range_ops = {
467 .set_notify = knav_acc_set_notify,
468 .init_queue = knav_acc_init_queue,
469 .open_queue = knav_acc_open_queue,
470 .close_queue = knav_acc_close_queue,
471 .init_range = knav_acc_init_range,
472 .free_range = knav_acc_free_range,
473};
474
475
476
477
478
479
480
481
482
483
484int knav_init_acc_range(struct knav_device *kdev,
485 struct device_node *node,
486 struct knav_range_info *range)
487{
488 struct knav_acc_channel *acc;
489 struct knav_pdsp_info *pdsp;
490 struct knav_acc_info *info;
491 int ret, channel, channels;
492 int list_size, mem_size;
493 dma_addr_t list_dma;
494 void *list_mem;
495 u32 config[5];
496
497 range->flags |= RANGE_HAS_ACCUMULATOR;
498 info = &range->acc_info;
499
500 ret = of_property_read_u32_array(node, "accumulator", config, 5);
501 if (ret)
502 return ret;
503
504 info->pdsp_id = config[0];
505 info->start_channel = config[1];
506 info->list_entries = config[2];
507 info->pacing_mode = config[3];
508 info->timer_count = config[4] / ACC_DEFAULT_PERIOD;
509
510 if (info->start_channel > ACC_MAX_CHANNEL) {
511 dev_err(kdev->dev, "channel %d invalid for range %s\n",
512 info->start_channel, range->name);
513 return -EINVAL;
514 }
515
516 if (info->pacing_mode > 3) {
517 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
518 info->pacing_mode, range->name);
519 return -EINVAL;
520 }
521
522 pdsp = knav_find_pdsp(kdev, info->pdsp_id);
523 if (!pdsp) {
524 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
525 info->pdsp_id, range->name);
526 return -EINVAL;
527 }
528
529 info->pdsp = pdsp;
530 channels = range->num_queues;
531 if (of_get_property(node, "multi-queue", NULL)) {
532 range->flags |= RANGE_MULTI_QUEUE;
533 channels = 1;
534 if (range->queue_base & (32 - 1)) {
535 dev_err(kdev->dev,
536 "misaligned multi-queue accumulator range %s\n",
537 range->name);
538 return -EINVAL;
539 }
540 if (range->num_queues > 32) {
541 dev_err(kdev->dev,
542 "too many queues in accumulator range %s\n",
543 range->name);
544 return -EINVAL;
545 }
546 }
547
548
549 list_size = info->list_entries;
550 list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
551 info->list_size = list_size;
552 mem_size = PAGE_ALIGN(list_size * 2);
553 info->mem_size = mem_size;
554 range->acc = devm_kzalloc(kdev->dev, channels * sizeof(*range->acc),
555 GFP_KERNEL);
556 if (!range->acc)
557 return -ENOMEM;
558
559 for (channel = 0; channel < channels; channel++) {
560 acc = range->acc + channel;
561 acc->channel = info->start_channel + channel;
562
563
564 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
565 if (!list_mem)
566 return -ENOMEM;
567
568 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
569 DMA_BIDIRECTIONAL);
570 if (dma_mapping_error(kdev->dev, list_dma)) {
571 free_pages_exact(list_mem, mem_size);
572 return -ENOMEM;
573 }
574
575 memset(list_mem, 0, mem_size);
576 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
577 DMA_TO_DEVICE);
578 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
579 acc->channel);
580 acc->list_cpu[0] = list_mem;
581 acc->list_cpu[1] = list_mem + list_size;
582 acc->list_dma[0] = list_dma;
583 acc->list_dma[1] = list_dma + list_size;
584 dev_dbg(kdev->dev, "%s: channel %d, phys %08x, virt %8p\n",
585 acc->name, acc->channel, list_dma, list_mem);
586 }
587
588 range->ops = &knav_acc_range_ops;
589 return 0;
590}
591EXPORT_SYMBOL_GPL(knav_init_acc_range);
592
