1
2
3
4
5#include <linux/completion.h>
6#include <linux/device.h>
7#include <linux/dma-buf.h>
8#include <linux/dma-mapping.h>
9#include <linux/idr.h>
10#include <linux/list.h>
11#include <linux/miscdevice.h>
12#include <linux/module.h>
13#include <linux/of_address.h>
14#include <linux/of.h>
15#include <linux/sort.h>
16#include <linux/of_platform.h>
17#include <linux/rpmsg.h>
18#include <linux/scatterlist.h>
19#include <linux/slab.h>
20#include <uapi/misc/fastrpc.h>
21
22#define ADSP_DOMAIN_ID (0)
23#define MDSP_DOMAIN_ID (1)
24#define SDSP_DOMAIN_ID (2)
25#define CDSP_DOMAIN_ID (3)
26#define FASTRPC_DEV_MAX 4
27#define FASTRPC_MAX_SESSIONS 9
28#define FASTRPC_ALIGN 128
29#define FASTRPC_MAX_FDLIST 16
30#define FASTRPC_MAX_CRCLIST 64
31#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
32#define FASTRPC_CTX_MAX (256)
33#define FASTRPC_INIT_HANDLE 1
34#define FASTRPC_CTXID_MASK (0xFF0)
35#define INIT_FILELEN_MAX (2 * 1024 * 1024)
36#define FASTRPC_DEVICE_NAME "fastrpc"
37#define ADSP_MMAP_ADD_PAGES 0x1000
38
39
40#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
41
42
43#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
44
45
46#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
47
48
49#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
50
51#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
52 REMOTE_SCALARS_OUTBUFS(sc) + \
53 REMOTE_SCALARS_INHANDLES(sc)+ \
54 REMOTE_SCALARS_OUTHANDLES(sc))
55#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
56 (((attr & 0x07) << 29) | \
57 ((method & 0x1f) << 24) | \
58 ((in & 0xff) << 16) | \
59 ((out & 0xff) << 8) | \
60 ((oin & 0x0f) << 4) | \
61 (oout & 0x0f))
62
63#define FASTRPC_SCALARS(method, in, out) \
64 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
65
66#define FASTRPC_CREATE_PROCESS_NARGS 6
67
68#define FASTRPC_RMID_INIT_ATTACH 0
69#define FASTRPC_RMID_INIT_RELEASE 1
70#define FASTRPC_RMID_INIT_MMAP 4
71#define FASTRPC_RMID_INIT_MUNMAP 5
72#define FASTRPC_RMID_INIT_CREATE 6
73#define FASTRPC_RMID_INIT_CREATE_ATTR 7
74#define FASTRPC_RMID_INIT_CREATE_STATIC 8
75
76
77#define AUDIO_PD (0)
78#define USER_PD (1)
79#define SENSORS_PD (2)
80
81#define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev)
82
83static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
84 "sdsp", "cdsp"};
85struct fastrpc_phy_page {
86 u64 addr;
87 u64 size;
88};
89
90struct fastrpc_invoke_buf {
91 u32 num;
92 u32 pgidx;
93};
94
95struct fastrpc_remote_arg {
96 u64 pv;
97 u64 len;
98};
99
100struct fastrpc_mmap_rsp_msg {
101 u64 vaddr;
102};
103
104struct fastrpc_mmap_req_msg {
105 s32 pgid;
106 u32 flags;
107 u64 vaddr;
108 s32 num;
109};
110
111struct fastrpc_munmap_req_msg {
112 s32 pgid;
113 u64 vaddr;
114 u64 size;
115};
116
117struct fastrpc_msg {
118 int pid;
119 int tid;
120 u64 ctx;
121 u32 handle;
122 u32 sc;
123 u64 addr;
124 u64 size;
125};
126
127struct fastrpc_invoke_rsp {
128 u64 ctx;
129 int retval;
130};
131
132struct fastrpc_buf_overlap {
133 u64 start;
134 u64 end;
135 int raix;
136 u64 mstart;
137 u64 mend;
138 u64 offset;
139};
140
141struct fastrpc_buf {
142 struct fastrpc_user *fl;
143 struct dma_buf *dmabuf;
144 struct device *dev;
145 void *virt;
146 u64 phys;
147 u64 size;
148
149 struct mutex lock;
150 struct list_head attachments;
151
152 struct list_head node;
153 uintptr_t raddr;
154};
155
156struct fastrpc_dma_buf_attachment {
157 struct device *dev;
158 struct sg_table sgt;
159 struct list_head node;
160};
161
162struct fastrpc_map {
163 struct list_head node;
164 struct fastrpc_user *fl;
165 int fd;
166 struct dma_buf *buf;
167 struct sg_table *table;
168 struct dma_buf_attachment *attach;
169 u64 phys;
170 u64 size;
171 void *va;
172 u64 len;
173 struct kref refcount;
174};
175
176struct fastrpc_invoke_ctx {
177 int nscalars;
178 int nbufs;
179 int retval;
180 int pid;
181 int tgid;
182 u32 sc;
183 u32 *crc;
184 u64 ctxid;
185 u64 msg_sz;
186 struct kref refcount;
187 struct list_head node;
188 struct completion work;
189 struct work_struct put_work;
190 struct fastrpc_msg msg;
191 struct fastrpc_user *fl;
192 struct fastrpc_remote_arg *rpra;
193 struct fastrpc_map **maps;
194 struct fastrpc_buf *buf;
195 struct fastrpc_invoke_args *args;
196 struct fastrpc_buf_overlap *olaps;
197 struct fastrpc_channel_ctx *cctx;
198};
199
200struct fastrpc_session_ctx {
201 struct device *dev;
202 int sid;
203 bool used;
204 bool valid;
205};
206
207struct fastrpc_channel_ctx {
208 int domain_id;
209 int sesscount;
210 struct rpmsg_device *rpdev;
211 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
212 spinlock_t lock;
213 struct idr ctx_idr;
214 struct list_head users;
215 struct miscdevice miscdev;
216 struct kref refcount;
217};
218
219struct fastrpc_user {
220 struct list_head user;
221 struct list_head maps;
222 struct list_head pending;
223 struct list_head mmaps;
224
225 struct fastrpc_channel_ctx *cctx;
226 struct fastrpc_session_ctx *sctx;
227 struct fastrpc_buf *init_mem;
228
229 int tgid;
230 int pd;
231
232 spinlock_t lock;
233
234 struct mutex mutex;
235};
236
237static void fastrpc_free_map(struct kref *ref)
238{
239 struct fastrpc_map *map;
240
241 map = container_of(ref, struct fastrpc_map, refcount);
242
243 if (map->table) {
244 dma_buf_unmap_attachment(map->attach, map->table,
245 DMA_BIDIRECTIONAL);
246 dma_buf_detach(map->buf, map->attach);
247 dma_buf_put(map->buf);
248 }
249
250 kfree(map);
251}
252
253static void fastrpc_map_put(struct fastrpc_map *map)
254{
255 if (map)
256 kref_put(&map->refcount, fastrpc_free_map);
257}
258
259static void fastrpc_map_get(struct fastrpc_map *map)
260{
261 if (map)
262 kref_get(&map->refcount);
263}
264
265static int fastrpc_map_find(struct fastrpc_user *fl, int fd,
266 struct fastrpc_map **ppmap)
267{
268 struct fastrpc_map *map = NULL;
269
270 mutex_lock(&fl->mutex);
271 list_for_each_entry(map, &fl->maps, node) {
272 if (map->fd == fd) {
273 fastrpc_map_get(map);
274 *ppmap = map;
275 mutex_unlock(&fl->mutex);
276 return 0;
277 }
278 }
279 mutex_unlock(&fl->mutex);
280
281 return -ENOENT;
282}
283
284static void fastrpc_buf_free(struct fastrpc_buf *buf)
285{
286 dma_free_coherent(buf->dev, buf->size, buf->virt,
287 FASTRPC_PHYS(buf->phys));
288 kfree(buf);
289}
290
291static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
292 u64 size, struct fastrpc_buf **obuf)
293{
294 struct fastrpc_buf *buf;
295
296 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
297 if (!buf)
298 return -ENOMEM;
299
300 INIT_LIST_HEAD(&buf->attachments);
301 INIT_LIST_HEAD(&buf->node);
302 mutex_init(&buf->lock);
303
304 buf->fl = fl;
305 buf->virt = NULL;
306 buf->phys = 0;
307 buf->size = size;
308 buf->dev = dev;
309 buf->raddr = 0;
310
311 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
312 GFP_KERNEL);
313 if (!buf->virt) {
314 mutex_destroy(&buf->lock);
315 kfree(buf);
316 return -ENOMEM;
317 }
318
319 if (fl->sctx && fl->sctx->sid)
320 buf->phys += ((u64)fl->sctx->sid << 32);
321
322 *obuf = buf;
323
324 return 0;
325}
326
327static void fastrpc_channel_ctx_free(struct kref *ref)
328{
329 struct fastrpc_channel_ctx *cctx;
330
331 cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
332
333 kfree(cctx);
334}
335
336static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
337{
338 kref_get(&cctx->refcount);
339}
340
341static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
342{
343 kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
344}
345
346static void fastrpc_context_free(struct kref *ref)
347{
348 struct fastrpc_invoke_ctx *ctx;
349 struct fastrpc_channel_ctx *cctx;
350 unsigned long flags;
351 int i;
352
353 ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
354 cctx = ctx->cctx;
355
356 for (i = 0; i < ctx->nscalars; i++)
357 fastrpc_map_put(ctx->maps[i]);
358
359 if (ctx->buf)
360 fastrpc_buf_free(ctx->buf);
361
362 spin_lock_irqsave(&cctx->lock, flags);
363 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
364 spin_unlock_irqrestore(&cctx->lock, flags);
365
366 kfree(ctx->maps);
367 kfree(ctx->olaps);
368 kfree(ctx);
369
370 fastrpc_channel_ctx_put(cctx);
371}
372
373static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
374{
375 kref_get(&ctx->refcount);
376}
377
378static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
379{
380 kref_put(&ctx->refcount, fastrpc_context_free);
381}
382
383static void fastrpc_context_put_wq(struct work_struct *work)
384{
385 struct fastrpc_invoke_ctx *ctx =
386 container_of(work, struct fastrpc_invoke_ctx, put_work);
387
388 fastrpc_context_put(ctx);
389}
390
391#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
392static int olaps_cmp(const void *a, const void *b)
393{
394 struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
395 struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
396
397 int st = CMP(pa->start, pb->start);
398
399 int ed = CMP(pb->end, pa->end);
400
401 return st == 0 ? ed : st;
402}
403
404static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
405{
406 u64 max_end = 0;
407 int i;
408
409 for (i = 0; i < ctx->nbufs; ++i) {
410 ctx->olaps[i].start = ctx->args[i].ptr;
411 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
412 ctx->olaps[i].raix = i;
413 }
414
415 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
416
417 for (i = 0; i < ctx->nbufs; ++i) {
418
419 if (ctx->olaps[i].start < max_end) {
420 ctx->olaps[i].mstart = max_end;
421 ctx->olaps[i].mend = ctx->olaps[i].end;
422 ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
423
424 if (ctx->olaps[i].end > max_end) {
425 max_end = ctx->olaps[i].end;
426 } else {
427 ctx->olaps[i].mend = 0;
428 ctx->olaps[i].mstart = 0;
429 }
430
431 } else {
432 ctx->olaps[i].mend = ctx->olaps[i].end;
433 ctx->olaps[i].mstart = ctx->olaps[i].start;
434 ctx->olaps[i].offset = 0;
435 max_end = ctx->olaps[i].end;
436 }
437 }
438}
439
440static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
441 struct fastrpc_user *user, u32 kernel, u32 sc,
442 struct fastrpc_invoke_args *args)
443{
444 struct fastrpc_channel_ctx *cctx = user->cctx;
445 struct fastrpc_invoke_ctx *ctx = NULL;
446 unsigned long flags;
447 int ret;
448
449 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
450 if (!ctx)
451 return ERR_PTR(-ENOMEM);
452
453 INIT_LIST_HEAD(&ctx->node);
454 ctx->fl = user;
455 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
456 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
457 REMOTE_SCALARS_OUTBUFS(sc);
458
459 if (ctx->nscalars) {
460 ctx->maps = kcalloc(ctx->nscalars,
461 sizeof(*ctx->maps), GFP_KERNEL);
462 if (!ctx->maps) {
463 kfree(ctx);
464 return ERR_PTR(-ENOMEM);
465 }
466 ctx->olaps = kcalloc(ctx->nscalars,
467 sizeof(*ctx->olaps), GFP_KERNEL);
468 if (!ctx->olaps) {
469 kfree(ctx->maps);
470 kfree(ctx);
471 return ERR_PTR(-ENOMEM);
472 }
473 ctx->args = args;
474 fastrpc_get_buff_overlaps(ctx);
475 }
476
477
478 fastrpc_channel_ctx_get(cctx);
479
480 ctx->sc = sc;
481 ctx->retval = -1;
482 ctx->pid = current->pid;
483 ctx->tgid = user->tgid;
484 ctx->cctx = cctx;
485 init_completion(&ctx->work);
486 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
487
488 spin_lock(&user->lock);
489 list_add_tail(&ctx->node, &user->pending);
490 spin_unlock(&user->lock);
491
492 spin_lock_irqsave(&cctx->lock, flags);
493 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
494 FASTRPC_CTX_MAX, GFP_ATOMIC);
495 if (ret < 0) {
496 spin_unlock_irqrestore(&cctx->lock, flags);
497 goto err_idr;
498 }
499 ctx->ctxid = ret << 4;
500 spin_unlock_irqrestore(&cctx->lock, flags);
501
502 kref_init(&ctx->refcount);
503
504 return ctx;
505err_idr:
506 spin_lock(&user->lock);
507 list_del(&ctx->node);
508 spin_unlock(&user->lock);
509 fastrpc_channel_ctx_put(cctx);
510 kfree(ctx->maps);
511 kfree(ctx->olaps);
512 kfree(ctx);
513
514 return ERR_PTR(ret);
515}
516
517static struct sg_table *
518fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
519 enum dma_data_direction dir)
520{
521 struct fastrpc_dma_buf_attachment *a = attachment->priv;
522 struct sg_table *table;
523
524 table = &a->sgt;
525
526 if (!dma_map_sgtable(attachment->dev, table, dir, 0))
527 return ERR_PTR(-ENOMEM);
528
529 return table;
530}
531
532static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
533 struct sg_table *table,
534 enum dma_data_direction dir)
535{
536 dma_unmap_sgtable(attach->dev, table, dir, 0);
537}
538
539static void fastrpc_release(struct dma_buf *dmabuf)
540{
541 struct fastrpc_buf *buffer = dmabuf->priv;
542
543 fastrpc_buf_free(buffer);
544}
545
546static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
547 struct dma_buf_attachment *attachment)
548{
549 struct fastrpc_dma_buf_attachment *a;
550 struct fastrpc_buf *buffer = dmabuf->priv;
551 int ret;
552
553 a = kzalloc(sizeof(*a), GFP_KERNEL);
554 if (!a)
555 return -ENOMEM;
556
557 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
558 FASTRPC_PHYS(buffer->phys), buffer->size);
559 if (ret < 0) {
560 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
561 kfree(a);
562 return -EINVAL;
563 }
564
565 a->dev = attachment->dev;
566 INIT_LIST_HEAD(&a->node);
567 attachment->priv = a;
568
569 mutex_lock(&buffer->lock);
570 list_add(&a->node, &buffer->attachments);
571 mutex_unlock(&buffer->lock);
572
573 return 0;
574}
575
576static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
577 struct dma_buf_attachment *attachment)
578{
579 struct fastrpc_dma_buf_attachment *a = attachment->priv;
580 struct fastrpc_buf *buffer = dmabuf->priv;
581
582 mutex_lock(&buffer->lock);
583 list_del(&a->node);
584 mutex_unlock(&buffer->lock);
585 sg_free_table(&a->sgt);
586 kfree(a);
587}
588
589static int fastrpc_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
590{
591 struct fastrpc_buf *buf = dmabuf->priv;
592
593 dma_buf_map_set_vaddr(map, buf->virt);
594
595 return 0;
596}
597
598static int fastrpc_mmap(struct dma_buf *dmabuf,
599 struct vm_area_struct *vma)
600{
601 struct fastrpc_buf *buf = dmabuf->priv;
602 size_t size = vma->vm_end - vma->vm_start;
603
604 return dma_mmap_coherent(buf->dev, vma, buf->virt,
605 FASTRPC_PHYS(buf->phys), size);
606}
607
608static const struct dma_buf_ops fastrpc_dma_buf_ops = {
609 .attach = fastrpc_dma_buf_attach,
610 .detach = fastrpc_dma_buf_detatch,
611 .map_dma_buf = fastrpc_map_dma_buf,
612 .unmap_dma_buf = fastrpc_unmap_dma_buf,
613 .mmap = fastrpc_mmap,
614 .vmap = fastrpc_vmap,
615 .release = fastrpc_release,
616};
617
618static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
619 u64 len, struct fastrpc_map **ppmap)
620{
621 struct fastrpc_session_ctx *sess = fl->sctx;
622 struct fastrpc_map *map = NULL;
623 int err = 0;
624
625 if (!fastrpc_map_find(fl, fd, ppmap))
626 return 0;
627
628 map = kzalloc(sizeof(*map), GFP_KERNEL);
629 if (!map)
630 return -ENOMEM;
631
632 INIT_LIST_HEAD(&map->node);
633 map->fl = fl;
634 map->fd = fd;
635 map->buf = dma_buf_get(fd);
636 if (IS_ERR(map->buf)) {
637 err = PTR_ERR(map->buf);
638 goto get_err;
639 }
640
641 map->attach = dma_buf_attach(map->buf, sess->dev);
642 if (IS_ERR(map->attach)) {
643 dev_err(sess->dev, "Failed to attach dmabuf\n");
644 err = PTR_ERR(map->attach);
645 goto attach_err;
646 }
647
648 map->table = dma_buf_map_attachment(map->attach, DMA_BIDIRECTIONAL);
649 if (IS_ERR(map->table)) {
650 err = PTR_ERR(map->table);
651 goto map_err;
652 }
653
654 map->phys = sg_dma_address(map->table->sgl);
655 map->phys += ((u64)fl->sctx->sid << 32);
656 map->size = len;
657 map->va = sg_virt(map->table->sgl);
658 map->len = len;
659 kref_init(&map->refcount);
660
661 spin_lock(&fl->lock);
662 list_add_tail(&map->node, &fl->maps);
663 spin_unlock(&fl->lock);
664 *ppmap = map;
665
666 return 0;
667
668map_err:
669 dma_buf_detach(map->buf, map->attach);
670attach_err:
671 dma_buf_put(map->buf);
672get_err:
673 kfree(map);
674
675 return err;
676}
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
706{
707 int size = 0;
708
709 size = (sizeof(struct fastrpc_remote_arg) +
710 sizeof(struct fastrpc_invoke_buf) +
711 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
712 sizeof(u64) * FASTRPC_MAX_FDLIST +
713 sizeof(u32) * FASTRPC_MAX_CRCLIST;
714
715 return size;
716}
717
718static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
719{
720 u64 size = 0;
721 int i;
722
723 size = ALIGN(metalen, FASTRPC_ALIGN);
724 for (i = 0; i < ctx->nscalars; i++) {
725 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
726
727 if (ctx->olaps[i].offset == 0)
728 size = ALIGN(size, FASTRPC_ALIGN);
729
730 size += (ctx->olaps[i].mend - ctx->olaps[i].mstart);
731 }
732 }
733
734 return size;
735}
736
737static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
738{
739 struct device *dev = ctx->fl->sctx->dev;
740 int i, err;
741
742 for (i = 0; i < ctx->nscalars; ++i) {
743
744 if (ctx->args[i].reserved)
745 return -EINVAL;
746
747 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
748 ctx->args[i].length == 0)
749 continue;
750
751 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
752 ctx->args[i].length, &ctx->maps[i]);
753 if (err) {
754 dev_err(dev, "Error Creating map %d\n", err);
755 return -EINVAL;
756 }
757
758 }
759 return 0;
760}
761
762static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
763{
764 struct device *dev = ctx->fl->sctx->dev;
765 struct fastrpc_remote_arg *rpra;
766 struct fastrpc_invoke_buf *list;
767 struct fastrpc_phy_page *pages;
768 int inbufs, i, oix, err = 0;
769 u64 len, rlen, pkt_size;
770 u64 pg_start, pg_end;
771 uintptr_t args;
772 int metalen;
773
774 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
775 metalen = fastrpc_get_meta_size(ctx);
776 pkt_size = fastrpc_get_payload_size(ctx, metalen);
777
778 err = fastrpc_create_maps(ctx);
779 if (err)
780 return err;
781
782 ctx->msg_sz = pkt_size;
783
784 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
785 if (err)
786 return err;
787
788 rpra = ctx->buf->virt;
789 list = ctx->buf->virt + ctx->nscalars * sizeof(*rpra);
790 pages = ctx->buf->virt + ctx->nscalars * (sizeof(*list) +
791 sizeof(*rpra));
792 args = (uintptr_t)ctx->buf->virt + metalen;
793 rlen = pkt_size - metalen;
794 ctx->rpra = rpra;
795
796 for (oix = 0; oix < ctx->nbufs; ++oix) {
797 int mlen;
798
799 i = ctx->olaps[oix].raix;
800 len = ctx->args[i].length;
801
802 rpra[i].pv = 0;
803 rpra[i].len = len;
804 list[i].num = len ? 1 : 0;
805 list[i].pgidx = i;
806
807 if (!len)
808 continue;
809
810 if (ctx->maps[i]) {
811 struct vm_area_struct *vma = NULL;
812
813 rpra[i].pv = (u64) ctx->args[i].ptr;
814 pages[i].addr = ctx->maps[i]->phys;
815
816 vma = find_vma(current->mm, ctx->args[i].ptr);
817 if (vma)
818 pages[i].addr += ctx->args[i].ptr -
819 vma->vm_start;
820
821 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
822 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
823 PAGE_SHIFT;
824 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
825
826 } else {
827
828 if (ctx->olaps[oix].offset == 0) {
829 rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
830 args = ALIGN(args, FASTRPC_ALIGN);
831 }
832
833 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
834
835 if (rlen < mlen)
836 goto bail;
837
838 rpra[i].pv = args - ctx->olaps[oix].offset;
839 pages[i].addr = ctx->buf->phys -
840 ctx->olaps[oix].offset +
841 (pkt_size - rlen);
842 pages[i].addr = pages[i].addr & PAGE_MASK;
843
844 pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
845 pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
846 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
847 args = args + mlen;
848 rlen -= mlen;
849 }
850
851 if (i < inbufs && !ctx->maps[i]) {
852 void *dst = (void *)(uintptr_t)rpra[i].pv;
853 void *src = (void *)(uintptr_t)ctx->args[i].ptr;
854
855 if (!kernel) {
856 if (copy_from_user(dst, (void __user *)src,
857 len)) {
858 err = -EFAULT;
859 goto bail;
860 }
861 } else {
862 memcpy(dst, src, len);
863 }
864 }
865 }
866
867 for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
868 rpra[i].pv = (u64) ctx->args[i].ptr;
869 rpra[i].len = ctx->args[i].length;
870 list[i].num = ctx->args[i].length ? 1 : 0;
871 list[i].pgidx = i;
872 pages[i].addr = ctx->maps[i]->phys;
873 pages[i].size = ctx->maps[i]->size;
874 }
875
876bail:
877 if (err)
878 dev_err(dev, "Error: get invoke args failed:%d\n", err);
879
880 return err;
881}
882
883static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
884 u32 kernel)
885{
886 struct fastrpc_remote_arg *rpra = ctx->rpra;
887 int i, inbufs;
888
889 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
890
891 for (i = inbufs; i < ctx->nbufs; ++i) {
892 void *src = (void *)(uintptr_t)rpra[i].pv;
893 void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
894 u64 len = rpra[i].len;
895
896 if (!kernel) {
897 if (copy_to_user((void __user *)dst, src, len))
898 return -EFAULT;
899 } else {
900 memcpy(dst, src, len);
901 }
902 }
903
904 return 0;
905}
906
907static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
908 struct fastrpc_invoke_ctx *ctx,
909 u32 kernel, uint32_t handle)
910{
911 struct fastrpc_channel_ctx *cctx;
912 struct fastrpc_user *fl = ctx->fl;
913 struct fastrpc_msg *msg = &ctx->msg;
914 int ret;
915
916 cctx = fl->cctx;
917 msg->pid = fl->tgid;
918 msg->tid = current->pid;
919
920 if (kernel)
921 msg->pid = 0;
922
923 msg->ctx = ctx->ctxid | fl->pd;
924 msg->handle = handle;
925 msg->sc = ctx->sc;
926 msg->addr = ctx->buf ? ctx->buf->phys : 0;
927 msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
928 fastrpc_context_get(ctx);
929
930 ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
931
932 if (ret)
933 fastrpc_context_put(ctx);
934
935 return ret;
936
937}
938
939static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
940 u32 handle, u32 sc,
941 struct fastrpc_invoke_args *args)
942{
943 struct fastrpc_invoke_ctx *ctx = NULL;
944 int err = 0;
945
946 if (!fl->sctx)
947 return -EINVAL;
948
949 if (!fl->cctx->rpdev)
950 return -EPIPE;
951
952 ctx = fastrpc_context_alloc(fl, kernel, sc, args);
953 if (IS_ERR(ctx))
954 return PTR_ERR(ctx);
955
956 if (ctx->nscalars) {
957 err = fastrpc_get_args(kernel, ctx);
958 if (err)
959 goto bail;
960 }
961
962
963 dma_wmb();
964
965 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
966 if (err)
967 goto bail;
968
969 if (kernel) {
970 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
971 err = -ETIMEDOUT;
972 } else {
973 err = wait_for_completion_interruptible(&ctx->work);
974 }
975
976 if (err)
977 goto bail;
978
979
980 err = ctx->retval;
981 if (err)
982 goto bail;
983
984 if (ctx->nscalars) {
985
986 dma_rmb();
987
988 err = fastrpc_put_args(ctx, kernel);
989 if (err)
990 goto bail;
991 }
992
993bail:
994 if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
995
996 spin_lock(&fl->lock);
997 list_del(&ctx->node);
998 spin_unlock(&fl->lock);
999 fastrpc_context_put(ctx);
1000 }
1001 if (err)
1002 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1003
1004 return err;
1005}
1006
1007static int fastrpc_init_create_process(struct fastrpc_user *fl,
1008 char __user *argp)
1009{
1010 struct fastrpc_init_create init;
1011 struct fastrpc_invoke_args *args;
1012 struct fastrpc_phy_page pages[1];
1013 struct fastrpc_map *map = NULL;
1014 struct fastrpc_buf *imem = NULL;
1015 int memlen;
1016 int err;
1017 struct {
1018 int pgid;
1019 u32 namelen;
1020 u32 filelen;
1021 u32 pageslen;
1022 u32 attrs;
1023 u32 siglen;
1024 } inbuf;
1025 u32 sc;
1026
1027 args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1028 if (!args)
1029 return -ENOMEM;
1030
1031 if (copy_from_user(&init, argp, sizeof(init))) {
1032 err = -EFAULT;
1033 goto err;
1034 }
1035
1036 if (init.filelen > INIT_FILELEN_MAX) {
1037 err = -EINVAL;
1038 goto err;
1039 }
1040
1041 inbuf.pgid = fl->tgid;
1042 inbuf.namelen = strlen(current->comm) + 1;
1043 inbuf.filelen = init.filelen;
1044 inbuf.pageslen = 1;
1045 inbuf.attrs = init.attrs;
1046 inbuf.siglen = init.siglen;
1047 fl->pd = USER_PD;
1048
1049 if (init.filelen && init.filefd) {
1050 err = fastrpc_map_create(fl, init.filefd, init.filelen, &map);
1051 if (err)
1052 goto err;
1053 }
1054
1055 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1056 1024 * 1024);
1057 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1058 &imem);
1059 if (err)
1060 goto err_alloc;
1061
1062 fl->init_mem = imem;
1063 args[0].ptr = (u64)(uintptr_t)&inbuf;
1064 args[0].length = sizeof(inbuf);
1065 args[0].fd = -1;
1066
1067 args[1].ptr = (u64)(uintptr_t)current->comm;
1068 args[1].length = inbuf.namelen;
1069 args[1].fd = -1;
1070
1071 args[2].ptr = (u64) init.file;
1072 args[2].length = inbuf.filelen;
1073 args[2].fd = init.filefd;
1074
1075 pages[0].addr = imem->phys;
1076 pages[0].size = imem->size;
1077
1078 args[3].ptr = (u64)(uintptr_t) pages;
1079 args[3].length = 1 * sizeof(*pages);
1080 args[3].fd = -1;
1081
1082 args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1083 args[4].length = sizeof(inbuf.attrs);
1084 args[4].fd = -1;
1085
1086 args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1087 args[5].length = sizeof(inbuf.siglen);
1088 args[5].fd = -1;
1089
1090 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1091 if (init.attrs)
1092 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 6, 0);
1093
1094 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1095 sc, args);
1096 if (err)
1097 goto err_invoke;
1098
1099 kfree(args);
1100
1101 return 0;
1102
1103err_invoke:
1104 fl->init_mem = NULL;
1105 fastrpc_buf_free(imem);
1106err_alloc:
1107 if (map) {
1108 spin_lock(&fl->lock);
1109 list_del(&map->node);
1110 spin_unlock(&fl->lock);
1111 fastrpc_map_put(map);
1112 }
1113err:
1114 kfree(args);
1115
1116 return err;
1117}
1118
1119static struct fastrpc_session_ctx *fastrpc_session_alloc(
1120 struct fastrpc_channel_ctx *cctx)
1121{
1122 struct fastrpc_session_ctx *session = NULL;
1123 unsigned long flags;
1124 int i;
1125
1126 spin_lock_irqsave(&cctx->lock, flags);
1127 for (i = 0; i < cctx->sesscount; i++) {
1128 if (!cctx->session[i].used && cctx->session[i].valid) {
1129 cctx->session[i].used = true;
1130 session = &cctx->session[i];
1131 break;
1132 }
1133 }
1134 spin_unlock_irqrestore(&cctx->lock, flags);
1135
1136 return session;
1137}
1138
1139static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1140 struct fastrpc_session_ctx *session)
1141{
1142 unsigned long flags;
1143
1144 spin_lock_irqsave(&cctx->lock, flags);
1145 session->used = false;
1146 spin_unlock_irqrestore(&cctx->lock, flags);
1147}
1148
1149static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1150{
1151 struct fastrpc_invoke_args args[1];
1152 int tgid = 0;
1153 u32 sc;
1154
1155 tgid = fl->tgid;
1156 args[0].ptr = (u64)(uintptr_t) &tgid;
1157 args[0].length = sizeof(tgid);
1158 args[0].fd = -1;
1159 args[0].reserved = 0;
1160 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1161
1162 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1163 sc, &args[0]);
1164}
1165
1166static int fastrpc_device_release(struct inode *inode, struct file *file)
1167{
1168 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1169 struct fastrpc_channel_ctx *cctx = fl->cctx;
1170 struct fastrpc_invoke_ctx *ctx, *n;
1171 struct fastrpc_map *map, *m;
1172 struct fastrpc_buf *buf, *b;
1173 unsigned long flags;
1174
1175 fastrpc_release_current_dsp_process(fl);
1176
1177 spin_lock_irqsave(&cctx->lock, flags);
1178 list_del(&fl->user);
1179 spin_unlock_irqrestore(&cctx->lock, flags);
1180
1181 if (fl->init_mem)
1182 fastrpc_buf_free(fl->init_mem);
1183
1184 list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1185 list_del(&ctx->node);
1186 fastrpc_context_put(ctx);
1187 }
1188
1189 list_for_each_entry_safe(map, m, &fl->maps, node) {
1190 list_del(&map->node);
1191 fastrpc_map_put(map);
1192 }
1193
1194 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1195 list_del(&buf->node);
1196 fastrpc_buf_free(buf);
1197 }
1198
1199 fastrpc_session_free(cctx, fl->sctx);
1200 fastrpc_channel_ctx_put(cctx);
1201
1202 mutex_destroy(&fl->mutex);
1203 kfree(fl);
1204 file->private_data = NULL;
1205
1206 return 0;
1207}
1208
1209static int fastrpc_device_open(struct inode *inode, struct file *filp)
1210{
1211 struct fastrpc_channel_ctx *cctx = miscdev_to_cctx(filp->private_data);
1212 struct fastrpc_user *fl = NULL;
1213 unsigned long flags;
1214
1215 fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1216 if (!fl)
1217 return -ENOMEM;
1218
1219
1220 fastrpc_channel_ctx_get(cctx);
1221
1222 filp->private_data = fl;
1223 spin_lock_init(&fl->lock);
1224 mutex_init(&fl->mutex);
1225 INIT_LIST_HEAD(&fl->pending);
1226 INIT_LIST_HEAD(&fl->maps);
1227 INIT_LIST_HEAD(&fl->mmaps);
1228 INIT_LIST_HEAD(&fl->user);
1229 fl->tgid = current->tgid;
1230 fl->cctx = cctx;
1231
1232 fl->sctx = fastrpc_session_alloc(cctx);
1233 if (!fl->sctx) {
1234 dev_err(&cctx->rpdev->dev, "No session available\n");
1235 mutex_destroy(&fl->mutex);
1236 kfree(fl);
1237
1238 return -EBUSY;
1239 }
1240
1241 spin_lock_irqsave(&cctx->lock, flags);
1242 list_add_tail(&fl->user, &cctx->users);
1243 spin_unlock_irqrestore(&cctx->lock, flags);
1244
1245 return 0;
1246}
1247
1248static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1249{
1250 struct fastrpc_alloc_dma_buf bp;
1251 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1252 struct fastrpc_buf *buf = NULL;
1253 int err;
1254
1255 if (copy_from_user(&bp, argp, sizeof(bp)))
1256 return -EFAULT;
1257
1258 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1259 if (err)
1260 return err;
1261 exp_info.ops = &fastrpc_dma_buf_ops;
1262 exp_info.size = bp.size;
1263 exp_info.flags = O_RDWR;
1264 exp_info.priv = buf;
1265 buf->dmabuf = dma_buf_export(&exp_info);
1266 if (IS_ERR(buf->dmabuf)) {
1267 err = PTR_ERR(buf->dmabuf);
1268 fastrpc_buf_free(buf);
1269 return err;
1270 }
1271
1272 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1273 if (bp.fd < 0) {
1274 dma_buf_put(buf->dmabuf);
1275 return -EINVAL;
1276 }
1277
1278 if (copy_to_user(argp, &bp, sizeof(bp))) {
1279 dma_buf_put(buf->dmabuf);
1280 return -EFAULT;
1281 }
1282
1283 return 0;
1284}
1285
1286static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1287{
1288 struct fastrpc_invoke_args args[1];
1289 int tgid = fl->tgid;
1290 u32 sc;
1291
1292 args[0].ptr = (u64)(uintptr_t) &tgid;
1293 args[0].length = sizeof(tgid);
1294 args[0].fd = -1;
1295 args[0].reserved = 0;
1296 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1297 fl->pd = pd;
1298
1299 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1300 sc, &args[0]);
1301}
1302
1303static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1304{
1305 struct fastrpc_invoke_args *args = NULL;
1306 struct fastrpc_invoke inv;
1307 u32 nscalars;
1308 int err;
1309
1310 if (copy_from_user(&inv, argp, sizeof(inv)))
1311 return -EFAULT;
1312
1313
1314 nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1315 if (nscalars) {
1316 args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1317 if (!args)
1318 return -ENOMEM;
1319
1320 if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1321 nscalars * sizeof(*args))) {
1322 kfree(args);
1323 return -EFAULT;
1324 }
1325 }
1326
1327 err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1328 kfree(args);
1329
1330 return err;
1331}
1332
1333static int fastrpc_req_munmap_impl(struct fastrpc_user *fl,
1334 struct fastrpc_req_munmap *req)
1335{
1336 struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1337 struct fastrpc_buf *buf, *b;
1338 struct fastrpc_munmap_req_msg req_msg;
1339 struct device *dev = fl->sctx->dev;
1340 int err;
1341 u32 sc;
1342
1343 spin_lock(&fl->lock);
1344 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1345 if ((buf->raddr == req->vaddrout) && (buf->size == req->size))
1346 break;
1347 buf = NULL;
1348 }
1349 spin_unlock(&fl->lock);
1350
1351 if (!buf) {
1352 dev_err(dev, "mmap not in list\n");
1353 return -EINVAL;
1354 }
1355
1356 req_msg.pgid = fl->tgid;
1357 req_msg.size = buf->size;
1358 req_msg.vaddr = buf->raddr;
1359
1360 args[0].ptr = (u64) (uintptr_t) &req_msg;
1361 args[0].length = sizeof(req_msg);
1362
1363 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1364 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1365 &args[0]);
1366 if (!err) {
1367 dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1368 spin_lock(&fl->lock);
1369 list_del(&buf->node);
1370 spin_unlock(&fl->lock);
1371 fastrpc_buf_free(buf);
1372 } else {
1373 dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1374 }
1375
1376 return err;
1377}
1378
1379static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1380{
1381 struct fastrpc_req_munmap req;
1382
1383 if (copy_from_user(&req, argp, sizeof(req)))
1384 return -EFAULT;
1385
1386 return fastrpc_req_munmap_impl(fl, &req);
1387}
1388
1389static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1390{
1391 struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1392 struct fastrpc_buf *buf = NULL;
1393 struct fastrpc_mmap_req_msg req_msg;
1394 struct fastrpc_mmap_rsp_msg rsp_msg;
1395 struct fastrpc_req_munmap req_unmap;
1396 struct fastrpc_phy_page pages;
1397 struct fastrpc_req_mmap req;
1398 struct device *dev = fl->sctx->dev;
1399 int err;
1400 u32 sc;
1401
1402 if (copy_from_user(&req, argp, sizeof(req)))
1403 return -EFAULT;
1404
1405 if (req.flags != ADSP_MMAP_ADD_PAGES) {
1406 dev_err(dev, "flag not supported 0x%x\n", req.flags);
1407 return -EINVAL;
1408 }
1409
1410 if (req.vaddrin) {
1411 dev_err(dev, "adding user allocated pages is not supported\n");
1412 return -EINVAL;
1413 }
1414
1415 err = fastrpc_buf_alloc(fl, fl->sctx->dev, req.size, &buf);
1416 if (err) {
1417 dev_err(dev, "failed to allocate buffer\n");
1418 return err;
1419 }
1420
1421 req_msg.pgid = fl->tgid;
1422 req_msg.flags = req.flags;
1423 req_msg.vaddr = req.vaddrin;
1424 req_msg.num = sizeof(pages);
1425
1426 args[0].ptr = (u64) (uintptr_t) &req_msg;
1427 args[0].length = sizeof(req_msg);
1428
1429 pages.addr = buf->phys;
1430 pages.size = buf->size;
1431
1432 args[1].ptr = (u64) (uintptr_t) &pages;
1433 args[1].length = sizeof(pages);
1434
1435 args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1436 args[2].length = sizeof(rsp_msg);
1437
1438 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1439 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1440 &args[0]);
1441 if (err) {
1442 dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1443 goto err_invoke;
1444 }
1445
1446
1447 buf->raddr = (uintptr_t) rsp_msg.vaddr;
1448
1449
1450 req.vaddrout = rsp_msg.vaddr;
1451
1452 spin_lock(&fl->lock);
1453 list_add_tail(&buf->node, &fl->mmaps);
1454 spin_unlock(&fl->lock);
1455
1456 if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
1457
1458 req_unmap.vaddrout = buf->raddr;
1459 req_unmap.size = buf->size;
1460 fastrpc_req_munmap_impl(fl, &req_unmap);
1461 return -EFAULT;
1462 }
1463
1464 dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1465 buf->raddr, buf->size);
1466
1467 return 0;
1468
1469err_invoke:
1470 fastrpc_buf_free(buf);
1471
1472 return err;
1473}
1474
1475static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
1476 unsigned long arg)
1477{
1478 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1479 char __user *argp = (char __user *)arg;
1480 int err;
1481
1482 switch (cmd) {
1483 case FASTRPC_IOCTL_INVOKE:
1484 err = fastrpc_invoke(fl, argp);
1485 break;
1486 case FASTRPC_IOCTL_INIT_ATTACH:
1487 err = fastrpc_init_attach(fl, AUDIO_PD);
1488 break;
1489 case FASTRPC_IOCTL_INIT_ATTACH_SNS:
1490 err = fastrpc_init_attach(fl, SENSORS_PD);
1491 break;
1492 case FASTRPC_IOCTL_INIT_CREATE:
1493 err = fastrpc_init_create_process(fl, argp);
1494 break;
1495 case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
1496 err = fastrpc_dmabuf_alloc(fl, argp);
1497 break;
1498 case FASTRPC_IOCTL_MMAP:
1499 err = fastrpc_req_mmap(fl, argp);
1500 break;
1501 case FASTRPC_IOCTL_MUNMAP:
1502 err = fastrpc_req_munmap(fl, argp);
1503 break;
1504 default:
1505 err = -ENOTTY;
1506 break;
1507 }
1508
1509 return err;
1510}
1511
1512static const struct file_operations fastrpc_fops = {
1513 .open = fastrpc_device_open,
1514 .release = fastrpc_device_release,
1515 .unlocked_ioctl = fastrpc_device_ioctl,
1516 .compat_ioctl = fastrpc_device_ioctl,
1517};
1518
1519static int fastrpc_cb_probe(struct platform_device *pdev)
1520{
1521 struct fastrpc_channel_ctx *cctx;
1522 struct fastrpc_session_ctx *sess;
1523 struct device *dev = &pdev->dev;
1524 int i, sessions = 0;
1525 unsigned long flags;
1526 int rc;
1527
1528 cctx = dev_get_drvdata(dev->parent);
1529 if (!cctx)
1530 return -EINVAL;
1531
1532 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
1533
1534 spin_lock_irqsave(&cctx->lock, flags);
1535 sess = &cctx->session[cctx->sesscount];
1536 sess->used = false;
1537 sess->valid = true;
1538 sess->dev = dev;
1539 dev_set_drvdata(dev, sess);
1540
1541 if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
1542 dev_info(dev, "FastRPC Session ID not specified in DT\n");
1543
1544 if (sessions > 0) {
1545 struct fastrpc_session_ctx *dup_sess;
1546
1547 for (i = 1; i < sessions; i++) {
1548 if (cctx->sesscount++ >= FASTRPC_MAX_SESSIONS)
1549 break;
1550 dup_sess = &cctx->session[cctx->sesscount];
1551 memcpy(dup_sess, sess, sizeof(*dup_sess));
1552 }
1553 }
1554 cctx->sesscount++;
1555 spin_unlock_irqrestore(&cctx->lock, flags);
1556 rc = dma_set_mask(dev, DMA_BIT_MASK(32));
1557 if (rc) {
1558 dev_err(dev, "32-bit DMA enable failed\n");
1559 return rc;
1560 }
1561
1562 return 0;
1563}
1564
1565static int fastrpc_cb_remove(struct platform_device *pdev)
1566{
1567 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
1568 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
1569 unsigned long flags;
1570 int i;
1571
1572 spin_lock_irqsave(&cctx->lock, flags);
1573 for (i = 1; i < FASTRPC_MAX_SESSIONS; i++) {
1574 if (cctx->session[i].sid == sess->sid) {
1575 cctx->session[i].valid = false;
1576 cctx->sesscount--;
1577 }
1578 }
1579 spin_unlock_irqrestore(&cctx->lock, flags);
1580
1581 return 0;
1582}
1583
1584static const struct of_device_id fastrpc_match_table[] = {
1585 { .compatible = "qcom,fastrpc-compute-cb", },
1586 {}
1587};
1588
1589static struct platform_driver fastrpc_cb_driver = {
1590 .probe = fastrpc_cb_probe,
1591 .remove = fastrpc_cb_remove,
1592 .driver = {
1593 .name = "qcom,fastrpc-cb",
1594 .of_match_table = fastrpc_match_table,
1595 .suppress_bind_attrs = true,
1596 },
1597};
1598
1599static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
1600{
1601 struct device *rdev = &rpdev->dev;
1602 struct fastrpc_channel_ctx *data;
1603 int i, err, domain_id = -1;
1604 const char *domain;
1605
1606 err = of_property_read_string(rdev->of_node, "label", &domain);
1607 if (err) {
1608 dev_info(rdev, "FastRPC Domain not specified in DT\n");
1609 return err;
1610 }
1611
1612 for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
1613 if (!strcmp(domains[i], domain)) {
1614 domain_id = i;
1615 break;
1616 }
1617 }
1618
1619 if (domain_id < 0) {
1620 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
1621 return -EINVAL;
1622 }
1623
1624 data = kzalloc(sizeof(*data), GFP_KERNEL);
1625 if (!data)
1626 return -ENOMEM;
1627
1628 data->miscdev.minor = MISC_DYNAMIC_MINOR;
1629 data->miscdev.name = devm_kasprintf(rdev, GFP_KERNEL, "fastrpc-%s",
1630 domains[domain_id]);
1631 data->miscdev.fops = &fastrpc_fops;
1632 err = misc_register(&data->miscdev);
1633 if (err) {
1634 kfree(data);
1635 return err;
1636 }
1637
1638 kref_init(&data->refcount);
1639
1640 dev_set_drvdata(&rpdev->dev, data);
1641 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
1642 INIT_LIST_HEAD(&data->users);
1643 spin_lock_init(&data->lock);
1644 idr_init(&data->ctx_idr);
1645 data->domain_id = domain_id;
1646 data->rpdev = rpdev;
1647
1648 return of_platform_populate(rdev->of_node, NULL, NULL, rdev);
1649}
1650
1651static void fastrpc_notify_users(struct fastrpc_user *user)
1652{
1653 struct fastrpc_invoke_ctx *ctx;
1654
1655 spin_lock(&user->lock);
1656 list_for_each_entry(ctx, &user->pending, node)
1657 complete(&ctx->work);
1658 spin_unlock(&user->lock);
1659}
1660
1661static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
1662{
1663 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1664 struct fastrpc_user *user;
1665 unsigned long flags;
1666
1667 spin_lock_irqsave(&cctx->lock, flags);
1668 list_for_each_entry(user, &cctx->users, user)
1669 fastrpc_notify_users(user);
1670 spin_unlock_irqrestore(&cctx->lock, flags);
1671
1672 misc_deregister(&cctx->miscdev);
1673 of_platform_depopulate(&rpdev->dev);
1674
1675 cctx->rpdev = NULL;
1676 fastrpc_channel_ctx_put(cctx);
1677}
1678
1679static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
1680 int len, void *priv, u32 addr)
1681{
1682 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1683 struct fastrpc_invoke_rsp *rsp = data;
1684 struct fastrpc_invoke_ctx *ctx;
1685 unsigned long flags;
1686 unsigned long ctxid;
1687
1688 if (len < sizeof(*rsp))
1689 return -EINVAL;
1690
1691 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
1692
1693 spin_lock_irqsave(&cctx->lock, flags);
1694 ctx = idr_find(&cctx->ctx_idr, ctxid);
1695 spin_unlock_irqrestore(&cctx->lock, flags);
1696
1697 if (!ctx) {
1698 dev_err(&rpdev->dev, "No context ID matches response\n");
1699 return -ENOENT;
1700 }
1701
1702 ctx->retval = rsp->retval;
1703 complete(&ctx->work);
1704
1705
1706
1707
1708
1709
1710 schedule_work(&ctx->put_work);
1711
1712 return 0;
1713}
1714
1715static const struct of_device_id fastrpc_rpmsg_of_match[] = {
1716 { .compatible = "qcom,fastrpc" },
1717 { },
1718};
1719MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
1720
1721static struct rpmsg_driver fastrpc_driver = {
1722 .probe = fastrpc_rpmsg_probe,
1723 .remove = fastrpc_rpmsg_remove,
1724 .callback = fastrpc_rpmsg_callback,
1725 .drv = {
1726 .name = "qcom,fastrpc",
1727 .of_match_table = fastrpc_rpmsg_of_match,
1728 },
1729};
1730
1731static int fastrpc_init(void)
1732{
1733 int ret;
1734
1735 ret = platform_driver_register(&fastrpc_cb_driver);
1736 if (ret < 0) {
1737 pr_err("fastrpc: failed to register cb driver\n");
1738 return ret;
1739 }
1740
1741 ret = register_rpmsg_driver(&fastrpc_driver);
1742 if (ret < 0) {
1743 pr_err("fastrpc: failed to register rpmsg driver\n");
1744 platform_driver_unregister(&fastrpc_cb_driver);
1745 return ret;
1746 }
1747
1748 return 0;
1749}
1750module_init(fastrpc_init);
1751
1752static void fastrpc_exit(void)
1753{
1754 platform_driver_unregister(&fastrpc_cb_driver);
1755 unregister_rpmsg_driver(&fastrpc_driver);
1756}
1757module_exit(fastrpc_exit);
1758
1759MODULE_LICENSE("GPL v2");
1760
