1
2
3
4
5
6
7#include <linux/types.h>
8#include <linux/bits.h>
9#include <linux/bitfield.h>
10#include <linux/refcount.h>
11#include <linux/scatterlist.h>
12#include <linux/dma-direction.h>
13
14#include "gsi.h"
15#include "gsi_private.h"
16#include "gsi_trans.h"
17#include "ipa_gsi.h"
18#include "ipa_data.h"
19#include "ipa_cmd.h"
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69enum gsi_tre_type {
70 GSI_RE_XFER = 0x2,
71 GSI_RE_IMMD_CMD = 0x3,
72};
73
74
75struct gsi_tre {
76 __le64 addr;
77 __le16 len_opcode;
78 __le16 reserved;
79 __le32 flags;
80};
81
82
83#define TRE_FLAGS_CHAIN_FMASK GENMASK(0, 0)
84#define TRE_FLAGS_IEOT_FMASK GENMASK(9, 9)
85#define TRE_FLAGS_BEI_FMASK GENMASK(10, 10)
86#define TRE_FLAGS_TYPE_FMASK GENMASK(23, 16)
87
88int gsi_trans_pool_init(struct gsi_trans_pool *pool, size_t size, u32 count,
89 u32 max_alloc)
90{
91 void *virt;
92
93 if (!size)
94 return -EINVAL;
95 if (count < max_alloc)
96 return -EINVAL;
97 if (!max_alloc)
98 return -EINVAL;
99
100
101
102
103
104
105
106
107 virt = kcalloc(count + max_alloc - 1, size, GFP_KERNEL);
108 if (!virt)
109 return -ENOMEM;
110
111 pool->base = virt;
112
113 pool->count = ksize(pool->base) / size;
114 pool->free = 0;
115 pool->max_alloc = max_alloc;
116 pool->size = size;
117 pool->addr = 0;
118
119 return 0;
120}
121
122void gsi_trans_pool_exit(struct gsi_trans_pool *pool)
123{
124 kfree(pool->base);
125 memset(pool, 0, sizeof(*pool));
126}
127
128
129
130
131
132
133
134int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool,
135 size_t size, u32 count, u32 max_alloc)
136{
137 size_t total_size;
138 dma_addr_t addr;
139 void *virt;
140
141 if (!size)
142 return -EINVAL;
143 if (count < max_alloc)
144 return -EINVAL;
145 if (!max_alloc)
146 return -EINVAL;
147
148
149 size = __roundup_pow_of_two(size);
150 total_size = (count + max_alloc - 1) * size;
151
152
153
154
155
156
157
158 total_size = get_order(total_size) << PAGE_SHIFT;
159
160 virt = dma_alloc_coherent(dev, total_size, &addr, GFP_KERNEL);
161 if (!virt)
162 return -ENOMEM;
163
164 pool->base = virt;
165 pool->count = total_size / size;
166 pool->free = 0;
167 pool->size = size;
168 pool->max_alloc = max_alloc;
169 pool->addr = addr;
170
171 return 0;
172}
173
174void gsi_trans_pool_exit_dma(struct device *dev, struct gsi_trans_pool *pool)
175{
176 size_t total_size = pool->count * pool->size;
177
178 dma_free_coherent(dev, total_size, pool->base, pool->addr);
179 memset(pool, 0, sizeof(*pool));
180}
181
182
183static u32 gsi_trans_pool_alloc_common(struct gsi_trans_pool *pool, u32 count)
184{
185 u32 offset;
186
187 WARN_ON(!count);
188 WARN_ON(count > pool->max_alloc);
189
190
191 if (count > pool->count - pool->free)
192 pool->free = 0;
193
194 offset = pool->free * pool->size;
195 pool->free += count;
196 memset(pool->base + offset, 0, count * pool->size);
197
198 return offset;
199}
200
201
202void *gsi_trans_pool_alloc(struct gsi_trans_pool *pool, u32 count)
203{
204 return pool->base + gsi_trans_pool_alloc_common(pool, count);
205}
206
207
208void *gsi_trans_pool_alloc_dma(struct gsi_trans_pool *pool, dma_addr_t *addr)
209{
210 u32 offset = gsi_trans_pool_alloc_common(pool, 1);
211
212 *addr = pool->addr + offset;
213
214 return pool->base + offset;
215}
216
217
218
219
220void *gsi_trans_pool_next(struct gsi_trans_pool *pool, void *element)
221{
222 void *end = pool->base + pool->count * pool->size;
223
224 WARN_ON(element < pool->base);
225 WARN_ON(element >= end);
226 WARN_ON(pool->max_alloc != 1);
227
228 element += pool->size;
229
230 return element < end ? element : pool->base;
231}
232
233
234static void gsi_channel_trans_map(struct gsi_channel *channel, u32 index,
235 struct gsi_trans *trans)
236{
237
238 channel->trans_info.map[index % channel->tre_ring.count] = trans;
239}
240
241
242struct gsi_trans *
243gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index)
244{
245
246 return channel->trans_info.map[index % channel->tre_ring.count];
247}
248
249
250struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel)
251{
252 return list_first_entry_or_null(&channel->trans_info.complete,
253 struct gsi_trans, links);
254}
255
256
257static void gsi_trans_move_pending(struct gsi_trans *trans)
258{
259 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
260 struct gsi_trans_info *trans_info = &channel->trans_info;
261
262 spin_lock_bh(&trans_info->spinlock);
263
264 list_move_tail(&trans->links, &trans_info->pending);
265
266 spin_unlock_bh(&trans_info->spinlock);
267}
268
269
270
271
272void gsi_trans_move_complete(struct gsi_trans *trans)
273{
274 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
275 struct gsi_trans_info *trans_info = &channel->trans_info;
276 struct list_head list;
277
278 spin_lock_bh(&trans_info->spinlock);
279
280
281 list_cut_position(&list, &trans_info->pending, &trans->links);
282 list_splice_tail(&list, &trans_info->complete);
283
284 spin_unlock_bh(&trans_info->spinlock);
285}
286
287
288void gsi_trans_move_polled(struct gsi_trans *trans)
289{
290 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
291 struct gsi_trans_info *trans_info = &channel->trans_info;
292
293 spin_lock_bh(&trans_info->spinlock);
294
295 list_move_tail(&trans->links, &trans_info->polled);
296
297 spin_unlock_bh(&trans_info->spinlock);
298}
299
300
301static bool
302gsi_trans_tre_reserve(struct gsi_trans_info *trans_info, u32 tre_count)
303{
304 int avail = atomic_read(&trans_info->tre_avail);
305 int new;
306
307 do {
308 new = avail - (int)tre_count;
309 if (unlikely(new < 0))
310 return false;
311 } while (!atomic_try_cmpxchg(&trans_info->tre_avail, &avail, new));
312
313 return true;
314}
315
316
317static void
318gsi_trans_tre_release(struct gsi_trans_info *trans_info, u32 tre_count)
319{
320 atomic_add(tre_count, &trans_info->tre_avail);
321}
322
323
324struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id,
325 u32 tre_count,
326 enum dma_data_direction direction)
327{
328 struct gsi_channel *channel = &gsi->channel[channel_id];
329 struct gsi_trans_info *trans_info;
330 struct gsi_trans *trans;
331
332 if (WARN_ON(tre_count > gsi_channel_trans_tre_max(gsi, channel_id)))
333 return NULL;
334
335 trans_info = &channel->trans_info;
336
337
338
339
340 if (!gsi_trans_tre_reserve(trans_info, tre_count))
341 return NULL;
342
343
344 trans = gsi_trans_pool_alloc(&trans_info->pool, 1);
345 trans->gsi = gsi;
346 trans->channel_id = channel_id;
347 trans->tre_count = tre_count;
348 init_completion(&trans->completion);
349
350
351 trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count);
352 sg_init_marker(trans->sgl, tre_count);
353
354 trans->direction = direction;
355
356 spin_lock_bh(&trans_info->spinlock);
357
358 list_add_tail(&trans->links, &trans_info->alloc);
359
360 spin_unlock_bh(&trans_info->spinlock);
361
362 refcount_set(&trans->refcount, 1);
363
364 return trans;
365}
366
367
368void gsi_trans_free(struct gsi_trans *trans)
369{
370 refcount_t *refcount = &trans->refcount;
371 struct gsi_trans_info *trans_info;
372 bool last;
373
374
375 if (refcount_dec_not_one(refcount))
376 return;
377
378 trans_info = &trans->gsi->channel[trans->channel_id].trans_info;
379
380 spin_lock_bh(&trans_info->spinlock);
381
382
383 last = refcount_dec_and_test(refcount);
384 if (last)
385 list_del(&trans->links);
386
387 spin_unlock_bh(&trans_info->spinlock);
388
389 if (!last)
390 return;
391
392 ipa_gsi_trans_release(trans);
393
394
395
396
397 gsi_trans_tre_release(trans_info, trans->tre_count);
398}
399
400
401void gsi_trans_cmd_add(struct gsi_trans *trans, void *buf, u32 size,
402 dma_addr_t addr, enum dma_data_direction direction,
403 enum ipa_cmd_opcode opcode)
404{
405 struct ipa_cmd_info *info;
406 u32 which = trans->used++;
407 struct scatterlist *sg;
408
409 WARN_ON(which >= trans->tre_count);
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425 sg = &trans->sgl[which];
426 sg_assign_page(sg, NULL);
427 sg_dma_address(sg) = addr;
428 sg_dma_len(sg) = size;
429
430 info = &trans->info[which];
431 info->opcode = opcode;
432 info->direction = direction;
433}
434
435
436int gsi_trans_page_add(struct gsi_trans *trans, struct page *page, u32 size,
437 u32 offset)
438{
439 struct scatterlist *sg = &trans->sgl[0];
440 int ret;
441
442 if (WARN_ON(trans->tre_count != 1))
443 return -EINVAL;
444 if (WARN_ON(trans->used))
445 return -EINVAL;
446
447 sg_set_page(sg, page, size, offset);
448 ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction);
449 if (!ret)
450 return -ENOMEM;
451
452 trans->used++;
453
454 return 0;
455}
456
457
458int gsi_trans_skb_add(struct gsi_trans *trans, struct sk_buff *skb)
459{
460 struct scatterlist *sg = &trans->sgl[0];
461 u32 used;
462 int ret;
463
464 if (WARN_ON(trans->tre_count != 1))
465 return -EINVAL;
466 if (WARN_ON(trans->used))
467 return -EINVAL;
468
469
470 ret = skb_to_sgvec(skb, sg, 0, skb->len);
471 if (ret < 0)
472 return ret;
473 used = ret;
474
475 ret = dma_map_sg(trans->gsi->dev, sg, used, trans->direction);
476 if (!ret)
477 return -ENOMEM;
478
479 trans->used += used;
480
481 return 0;
482}
483
484
485static __le16 gsi_tre_len_opcode(enum ipa_cmd_opcode opcode, u32 len)
486{
487 return opcode == IPA_CMD_NONE ? cpu_to_le16((u16)len)
488 : cpu_to_le16((u16)opcode);
489}
490
491
492static __le32 gsi_tre_flags(bool last_tre, bool bei, enum ipa_cmd_opcode opcode)
493{
494 enum gsi_tre_type tre_type;
495 u32 tre_flags;
496
497 tre_type = opcode == IPA_CMD_NONE ? GSI_RE_XFER : GSI_RE_IMMD_CMD;
498 tre_flags = u32_encode_bits(tre_type, TRE_FLAGS_TYPE_FMASK);
499
500
501 if (last_tre) {
502
503 tre_flags |= TRE_FLAGS_IEOT_FMASK;
504
505 if (bei)
506 tre_flags |= TRE_FLAGS_BEI_FMASK;
507 } else {
508 tre_flags |= TRE_FLAGS_CHAIN_FMASK;
509 }
510
511 return cpu_to_le32(tre_flags);
512}
513
514static void gsi_trans_tre_fill(struct gsi_tre *dest_tre, dma_addr_t addr,
515 u32 len, bool last_tre, bool bei,
516 enum ipa_cmd_opcode opcode)
517{
518 struct gsi_tre tre;
519
520 tre.addr = cpu_to_le64(addr);
521 tre.len_opcode = gsi_tre_len_opcode(opcode, len);
522 tre.reserved = 0;
523 tre.flags = gsi_tre_flags(last_tre, bei, opcode);
524
525
526
527
528 *dest_tre = tre;
529}
530
531
532
533
534
535
536
537
538
539
540
541
542static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db)
543{
544 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
545 struct gsi_ring *ring = &channel->tre_ring;
546 enum ipa_cmd_opcode opcode = IPA_CMD_NONE;
547 bool bei = channel->toward_ipa;
548 struct ipa_cmd_info *info;
549 struct gsi_tre *dest_tre;
550 struct scatterlist *sg;
551 u32 byte_count = 0;
552 u32 avail;
553 u32 i;
554
555 WARN_ON(!trans->used);
556
557
558
559
560
561
562 info = trans->info ? &trans->info[0] : NULL;
563 avail = ring->count - ring->index % ring->count;
564 dest_tre = gsi_ring_virt(ring, ring->index);
565 for_each_sg(trans->sgl, sg, trans->used, i) {
566 bool last_tre = i == trans->used - 1;
567 dma_addr_t addr = sg_dma_address(sg);
568 u32 len = sg_dma_len(sg);
569
570 byte_count += len;
571 if (!avail--)
572 dest_tre = gsi_ring_virt(ring, 0);
573 if (info)
574 opcode = info++->opcode;
575
576 gsi_trans_tre_fill(dest_tre, addr, len, last_tre, bei, opcode);
577 dest_tre++;
578 }
579 ring->index += trans->used;
580
581 if (channel->toward_ipa) {
582
583 trans->len = byte_count;
584 trans->trans_count = channel->trans_count;
585 trans->byte_count = channel->byte_count;
586 channel->trans_count++;
587 channel->byte_count += byte_count;
588 }
589
590
591 gsi_channel_trans_map(channel, ring->index - 1, trans);
592
593 gsi_trans_move_pending(trans);
594
595
596 if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) {
597
598 if (channel->toward_ipa)
599 gsi_channel_tx_queued(channel);
600 gsi_channel_doorbell(channel);
601 }
602}
603
604
605void gsi_trans_commit(struct gsi_trans *trans, bool ring_db)
606{
607 if (trans->used)
608 __gsi_trans_commit(trans, ring_db);
609 else
610 gsi_trans_free(trans);
611}
612
613
614void gsi_trans_commit_wait(struct gsi_trans *trans)
615{
616 if (!trans->used)
617 goto out_trans_free;
618
619 refcount_inc(&trans->refcount);
620
621 __gsi_trans_commit(trans, true);
622
623 wait_for_completion(&trans->completion);
624
625out_trans_free:
626 gsi_trans_free(trans);
627}
628
629
630int gsi_trans_commit_wait_timeout(struct gsi_trans *trans,
631 unsigned long timeout)
632{
633 unsigned long timeout_jiffies = msecs_to_jiffies(timeout);
634 unsigned long remaining = 1;
635
636 if (!trans->used)
637 goto out_trans_free;
638
639 refcount_inc(&trans->refcount);
640
641 __gsi_trans_commit(trans, true);
642
643 remaining = wait_for_completion_timeout(&trans->completion,
644 timeout_jiffies);
645out_trans_free:
646 gsi_trans_free(trans);
647
648 return remaining ? 0 : -ETIMEDOUT;
649}
650
651
652void gsi_trans_complete(struct gsi_trans *trans)
653{
654
655 if (trans->direction != DMA_NONE)
656 dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used,
657 trans->direction);
658
659 ipa_gsi_trans_complete(trans);
660
661 complete(&trans->completion);
662
663 gsi_trans_free(trans);
664}
665
666
667void gsi_channel_trans_cancel_pending(struct gsi_channel *channel)
668{
669 struct gsi_trans_info *trans_info = &channel->trans_info;
670 struct gsi_trans *trans;
671 bool cancelled;
672
673
674 spin_lock_bh(&trans_info->spinlock);
675
676 cancelled = !list_empty(&trans_info->pending);
677 list_for_each_entry(trans, &trans_info->pending, links)
678 trans->cancelled = true;
679
680 list_splice_tail_init(&trans_info->pending, &trans_info->complete);
681
682 spin_unlock_bh(&trans_info->spinlock);
683
684
685 if (cancelled)
686 napi_schedule(&channel->napi);
687}
688
689
690int gsi_trans_read_byte(struct gsi *gsi, u32 channel_id, dma_addr_t addr)
691{
692 struct gsi_channel *channel = &gsi->channel[channel_id];
693 struct gsi_ring *ring = &channel->tre_ring;
694 struct gsi_trans_info *trans_info;
695 struct gsi_tre *dest_tre;
696
697 trans_info = &channel->trans_info;
698
699
700 if (!gsi_trans_tre_reserve(trans_info, 1))
701 return -EBUSY;
702
703
704
705 dest_tre = gsi_ring_virt(ring, ring->index);
706 gsi_trans_tre_fill(dest_tre, addr, 1, true, false, IPA_CMD_NONE);
707
708 ring->index++;
709 gsi_channel_doorbell(channel);
710
711 return 0;
712}
713
714
715void gsi_trans_read_byte_done(struct gsi *gsi, u32 channel_id)
716{
717 struct gsi_channel *channel = &gsi->channel[channel_id];
718
719 gsi_trans_tre_release(&channel->trans_info, 1);
720}
721
722
723int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id)
724{
725 struct gsi_channel *channel = &gsi->channel[channel_id];
726 struct gsi_trans_info *trans_info;
727 u32 tre_max;
728 int ret;
729
730
731 BUILD_BUG_ON(sizeof(struct gsi_tre) != GSI_RING_ELEMENT_SIZE);
732
733
734
735
736
737 trans_info = &channel->trans_info;
738 trans_info->map = kcalloc(channel->tre_count, sizeof(*trans_info->map),
739 GFP_KERNEL);
740 if (!trans_info->map)
741 return -ENOMEM;
742
743
744
745
746
747
748
749
750 tre_max = gsi_channel_tre_max(channel->gsi, channel_id);
751
752
753 ret = gsi_trans_pool_init(&trans_info->pool, sizeof(struct gsi_trans),
754 tre_max, 1);
755 if (ret)
756 goto err_kfree;
757
758
759
760
761
762
763
764
765
766
767
768 ret = gsi_trans_pool_init(&trans_info->sg_pool,
769 sizeof(struct scatterlist),
770 tre_max, channel->tlv_count);
771 if (ret)
772 goto err_trans_pool_exit;
773
774
775
776
777
778
779
780
781 atomic_set(&trans_info->tre_avail, tre_max);
782
783 spin_lock_init(&trans_info->spinlock);
784 INIT_LIST_HEAD(&trans_info->alloc);
785 INIT_LIST_HEAD(&trans_info->pending);
786 INIT_LIST_HEAD(&trans_info->complete);
787 INIT_LIST_HEAD(&trans_info->polled);
788
789 return 0;
790
791err_trans_pool_exit:
792 gsi_trans_pool_exit(&trans_info->pool);
793err_kfree:
794 kfree(trans_info->map);
795
796 dev_err(gsi->dev, "error %d initializing channel %u transactions\n",
797 ret, channel_id);
798
799 return ret;
800}
801
802
803void gsi_channel_trans_exit(struct gsi_channel *channel)
804{
805 struct gsi_trans_info *trans_info = &channel->trans_info;
806
807 gsi_trans_pool_exit(&trans_info->sg_pool);
808 gsi_trans_pool_exit(&trans_info->pool);
809 kfree(trans_info->map);
810}
811