1
2
3
4
5#include <linux/completion.h>
6#include <linux/device.h>
7#include <linux/dma-buf.h>
8#include <linux/dma-mapping.h>
9#include <linux/idr.h>
10#include <linux/list.h>
11#include <linux/miscdevice.h>
12#include <linux/module.h>
13#include <linux/of_address.h>
14#include <linux/of.h>
15#include <linux/sort.h>
16#include <linux/of_platform.h>
17#include <linux/rpmsg.h>
18#include <linux/scatterlist.h>
19#include <linux/slab.h>
20#include <uapi/misc/fastrpc.h>
21
22#define ADSP_DOMAIN_ID (0)
23#define MDSP_DOMAIN_ID (1)
24#define SDSP_DOMAIN_ID (2)
25#define CDSP_DOMAIN_ID (3)
26#define FASTRPC_DEV_MAX 4
27#define FASTRPC_MAX_SESSIONS 13
28#define FASTRPC_ALIGN 128
29#define FASTRPC_MAX_FDLIST 16
30#define FASTRPC_MAX_CRCLIST 64
31#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
32#define FASTRPC_CTX_MAX (256)
33#define FASTRPC_INIT_HANDLE 1
34#define FASTRPC_CTXID_MASK (0xFF0)
35#define INIT_FILELEN_MAX (2 * 1024 * 1024)
36#define FASTRPC_DEVICE_NAME "fastrpc"
37#define ADSP_MMAP_ADD_PAGES 0x1000
38
39
40#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
41
42
43#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
44
45
46#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
47
48
49#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
50
51#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
52 REMOTE_SCALARS_OUTBUFS(sc) + \
53 REMOTE_SCALARS_INHANDLES(sc)+ \
54 REMOTE_SCALARS_OUTHANDLES(sc))
55#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
56 (((attr & 0x07) << 29) | \
57 ((method & 0x1f) << 24) | \
58 ((in & 0xff) << 16) | \
59 ((out & 0xff) << 8) | \
60 ((oin & 0x0f) << 4) | \
61 (oout & 0x0f))
62
63#define FASTRPC_SCALARS(method, in, out) \
64 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
65
66#define FASTRPC_CREATE_PROCESS_NARGS 6
67
68#define FASTRPC_RMID_INIT_ATTACH 0
69#define FASTRPC_RMID_INIT_RELEASE 1
70#define FASTRPC_RMID_INIT_MMAP 4
71#define FASTRPC_RMID_INIT_MUNMAP 5
72#define FASTRPC_RMID_INIT_CREATE 6
73#define FASTRPC_RMID_INIT_CREATE_ATTR 7
74#define FASTRPC_RMID_INIT_CREATE_STATIC 8
75
76
77#define AUDIO_PD (0)
78#define USER_PD (1)
79#define SENSORS_PD (2)
80
81#define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev)
82
83static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
84 "sdsp", "cdsp"};
85struct fastrpc_phy_page {
86 u64 addr;
87 u64 size;
88};
89
90struct fastrpc_invoke_buf {
91 u32 num;
92 u32 pgidx;
93};
94
95struct fastrpc_remote_arg {
96 u64 pv;
97 u64 len;
98};
99
100struct fastrpc_mmap_rsp_msg {
101 u64 vaddr;
102};
103
104struct fastrpc_mmap_req_msg {
105 s32 pgid;
106 u32 flags;
107 u64 vaddr;
108 s32 num;
109};
110
111struct fastrpc_munmap_req_msg {
112 s32 pgid;
113 u64 vaddr;
114 u64 size;
115};
116
117struct fastrpc_msg {
118 int pid;
119 int tid;
120 u64 ctx;
121 u32 handle;
122 u32 sc;
123 u64 addr;
124 u64 size;
125};
126
127struct fastrpc_invoke_rsp {
128 u64 ctx;
129 int retval;
130};
131
132struct fastrpc_buf_overlap {
133 u64 start;
134 u64 end;
135 int raix;
136 u64 mstart;
137 u64 mend;
138 u64 offset;
139};
140
141struct fastrpc_buf {
142 struct fastrpc_user *fl;
143 struct dma_buf *dmabuf;
144 struct device *dev;
145 void *virt;
146 u64 phys;
147 u64 size;
148
149 struct mutex lock;
150 struct list_head attachments;
151
152 struct list_head node;
153 uintptr_t raddr;
154};
155
156struct fastrpc_dma_buf_attachment {
157 struct device *dev;
158 struct sg_table sgt;
159 struct list_head node;
160};
161
162struct fastrpc_map {
163 struct list_head node;
164 struct fastrpc_user *fl;
165 int fd;
166 struct dma_buf *buf;
167 struct sg_table *table;
168 struct dma_buf_attachment *attach;
169 u64 phys;
170 u64 size;
171 void *va;
172 u64 len;
173 struct kref refcount;
174};
175
176struct fastrpc_invoke_ctx {
177 int nscalars;
178 int nbufs;
179 int retval;
180 int pid;
181 int tgid;
182 u32 sc;
183 u32 *crc;
184 u64 ctxid;
185 u64 msg_sz;
186 struct kref refcount;
187 struct list_head node;
188 struct completion work;
189 struct work_struct put_work;
190 struct fastrpc_msg msg;
191 struct fastrpc_user *fl;
192 struct fastrpc_remote_arg *rpra;
193 struct fastrpc_map **maps;
194 struct fastrpc_buf *buf;
195 struct fastrpc_invoke_args *args;
196 struct fastrpc_buf_overlap *olaps;
197 struct fastrpc_channel_ctx *cctx;
198};
199
200struct fastrpc_session_ctx {
201 struct device *dev;
202 int sid;
203 bool used;
204 bool valid;
205};
206
207struct fastrpc_channel_ctx {
208 int domain_id;
209 int sesscount;
210 struct rpmsg_device *rpdev;
211 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
212 spinlock_t lock;
213 struct idr ctx_idr;
214 struct list_head users;
215 struct miscdevice miscdev;
216 struct kref refcount;
217};
218
219struct fastrpc_user {
220 struct list_head user;
221 struct list_head maps;
222 struct list_head pending;
223 struct list_head mmaps;
224
225 struct fastrpc_channel_ctx *cctx;
226 struct fastrpc_session_ctx *sctx;
227 struct fastrpc_buf *init_mem;
228
229 int tgid;
230 int pd;
231
232 spinlock_t lock;
233
234 struct mutex mutex;
235};
236
237static void fastrpc_free_map(struct kref *ref)
238{
239 struct fastrpc_map *map;
240
241 map = container_of(ref, struct fastrpc_map, refcount);
242
243 if (map->table) {
244 dma_buf_unmap_attachment(map->attach, map->table,
245 DMA_BIDIRECTIONAL);
246 dma_buf_detach(map->buf, map->attach);
247 dma_buf_put(map->buf);
248 }
249
250 kfree(map);
251}
252
253static void fastrpc_map_put(struct fastrpc_map *map)
254{
255 if (map)
256 kref_put(&map->refcount, fastrpc_free_map);
257}
258
259static void fastrpc_map_get(struct fastrpc_map *map)
260{
261 if (map)
262 kref_get(&map->refcount);
263}
264
265static int fastrpc_map_find(struct fastrpc_user *fl, int fd,
266 struct fastrpc_map **ppmap)
267{
268 struct fastrpc_map *map = NULL;
269
270 mutex_lock(&fl->mutex);
271 list_for_each_entry(map, &fl->maps, node) {
272 if (map->fd == fd) {
273 fastrpc_map_get(map);
274 *ppmap = map;
275 mutex_unlock(&fl->mutex);
276 return 0;
277 }
278 }
279 mutex_unlock(&fl->mutex);
280
281 return -ENOENT;
282}
283
284static void fastrpc_buf_free(struct fastrpc_buf *buf)
285{
286 dma_free_coherent(buf->dev, buf->size, buf->virt,
287 FASTRPC_PHYS(buf->phys));
288 kfree(buf);
289}
290
291static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
292 u64 size, struct fastrpc_buf **obuf)
293{
294 struct fastrpc_buf *buf;
295
296 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
297 if (!buf)
298 return -ENOMEM;
299
300 INIT_LIST_HEAD(&buf->attachments);
301 INIT_LIST_HEAD(&buf->node);
302 mutex_init(&buf->lock);
303
304 buf->fl = fl;
305 buf->virt = NULL;
306 buf->phys = 0;
307 buf->size = size;
308 buf->dev = dev;
309 buf->raddr = 0;
310
311 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
312 GFP_KERNEL);
313 if (!buf->virt) {
314 mutex_destroy(&buf->lock);
315 kfree(buf);
316 return -ENOMEM;
317 }
318
319 if (fl->sctx && fl->sctx->sid)
320 buf->phys += ((u64)fl->sctx->sid << 32);
321
322 *obuf = buf;
323
324 return 0;
325}
326
327static void fastrpc_channel_ctx_free(struct kref *ref)
328{
329 struct fastrpc_channel_ctx *cctx;
330
331 cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
332
333 kfree(cctx);
334}
335
336static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
337{
338 kref_get(&cctx->refcount);
339}
340
341static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
342{
343 kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
344}
345
346static void fastrpc_context_free(struct kref *ref)
347{
348 struct fastrpc_invoke_ctx *ctx;
349 struct fastrpc_channel_ctx *cctx;
350 unsigned long flags;
351 int i;
352
353 ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
354 cctx = ctx->cctx;
355
356 for (i = 0; i < ctx->nscalars; i++)
357 fastrpc_map_put(ctx->maps[i]);
358
359 if (ctx->buf)
360 fastrpc_buf_free(ctx->buf);
361
362 spin_lock_irqsave(&cctx->lock, flags);
363 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
364 spin_unlock_irqrestore(&cctx->lock, flags);
365
366 kfree(ctx->maps);
367 kfree(ctx->olaps);
368 kfree(ctx);
369
370 fastrpc_channel_ctx_put(cctx);
371}
372
373static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
374{
375 kref_get(&ctx->refcount);
376}
377
378static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
379{
380 kref_put(&ctx->refcount, fastrpc_context_free);
381}
382
383static void fastrpc_context_put_wq(struct work_struct *work)
384{
385 struct fastrpc_invoke_ctx *ctx =
386 container_of(work, struct fastrpc_invoke_ctx, put_work);
387
388 fastrpc_context_put(ctx);
389}
390
391#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
392static int olaps_cmp(const void *a, const void *b)
393{
394 struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
395 struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
396
397 int st = CMP(pa->start, pb->start);
398
399 int ed = CMP(pb->end, pa->end);
400
401 return st == 0 ? ed : st;
402}
403
404static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
405{
406 u64 max_end = 0;
407 int i;
408
409 for (i = 0; i < ctx->nbufs; ++i) {
410 ctx->olaps[i].start = ctx->args[i].ptr;
411 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
412 ctx->olaps[i].raix = i;
413 }
414
415 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
416
417 for (i = 0; i < ctx->nbufs; ++i) {
418
419 if (ctx->olaps[i].start < max_end) {
420 ctx->olaps[i].mstart = max_end;
421 ctx->olaps[i].mend = ctx->olaps[i].end;
422 ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
423
424 if (ctx->olaps[i].end > max_end) {
425 max_end = ctx->olaps[i].end;
426 } else {
427 ctx->olaps[i].mend = 0;
428 ctx->olaps[i].mstart = 0;
429 }
430
431 } else {
432 ctx->olaps[i].mend = ctx->olaps[i].end;
433 ctx->olaps[i].mstart = ctx->olaps[i].start;
434 ctx->olaps[i].offset = 0;
435 max_end = ctx->olaps[i].end;
436 }
437 }
438}
439
440static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
441 struct fastrpc_user *user, u32 kernel, u32 sc,
442 struct fastrpc_invoke_args *args)
443{
444 struct fastrpc_channel_ctx *cctx = user->cctx;
445 struct fastrpc_invoke_ctx *ctx = NULL;
446 unsigned long flags;
447 int ret;
448
449 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
450 if (!ctx)
451 return ERR_PTR(-ENOMEM);
452
453 INIT_LIST_HEAD(&ctx->node);
454 ctx->fl = user;
455 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
456 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
457 REMOTE_SCALARS_OUTBUFS(sc);
458
459 if (ctx->nscalars) {
460 ctx->maps = kcalloc(ctx->nscalars,
461 sizeof(*ctx->maps), GFP_KERNEL);
462 if (!ctx->maps) {
463 kfree(ctx);
464 return ERR_PTR(-ENOMEM);
465 }
466 ctx->olaps = kcalloc(ctx->nscalars,
467 sizeof(*ctx->olaps), GFP_KERNEL);
468 if (!ctx->olaps) {
469 kfree(ctx->maps);
470 kfree(ctx);
471 return ERR_PTR(-ENOMEM);
472 }
473 ctx->args = args;
474 fastrpc_get_buff_overlaps(ctx);
475 }
476
477
478 fastrpc_channel_ctx_get(cctx);
479
480 ctx->sc = sc;
481 ctx->retval = -1;
482 ctx->pid = current->pid;
483 ctx->tgid = user->tgid;
484 ctx->cctx = cctx;
485 init_completion(&ctx->work);
486 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
487
488 spin_lock(&user->lock);
489 list_add_tail(&ctx->node, &user->pending);
490 spin_unlock(&user->lock);
491
492 spin_lock_irqsave(&cctx->lock, flags);
493 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
494 FASTRPC_CTX_MAX, GFP_ATOMIC);
495 if (ret < 0) {
496 spin_unlock_irqrestore(&cctx->lock, flags);
497 goto err_idr;
498 }
499 ctx->ctxid = ret << 4;
500 spin_unlock_irqrestore(&cctx->lock, flags);
501
502 kref_init(&ctx->refcount);
503
504 return ctx;
505err_idr:
506 spin_lock(&user->lock);
507 list_del(&ctx->node);
508 spin_unlock(&user->lock);
509 fastrpc_channel_ctx_put(cctx);
510 kfree(ctx->maps);
511 kfree(ctx->olaps);
512 kfree(ctx);
513
514 return ERR_PTR(ret);
515}
516
517static struct sg_table *
518fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
519 enum dma_data_direction dir)
520{
521 struct fastrpc_dma_buf_attachment *a = attachment->priv;
522 struct sg_table *table;
523 int ret;
524
525 table = &a->sgt;
526
527 ret = dma_map_sgtable(attachment->dev, table, dir, 0);
528 if (ret)
529 table = ERR_PTR(ret);
530 return table;
531}
532
533static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
534 struct sg_table *table,
535 enum dma_data_direction dir)
536{
537 dma_unmap_sgtable(attach->dev, table, dir, 0);
538}
539
540static void fastrpc_release(struct dma_buf *dmabuf)
541{
542 struct fastrpc_buf *buffer = dmabuf->priv;
543
544 fastrpc_buf_free(buffer);
545}
546
547static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
548 struct dma_buf_attachment *attachment)
549{
550 struct fastrpc_dma_buf_attachment *a;
551 struct fastrpc_buf *buffer = dmabuf->priv;
552 int ret;
553
554 a = kzalloc(sizeof(*a), GFP_KERNEL);
555 if (!a)
556 return -ENOMEM;
557
558 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
559 FASTRPC_PHYS(buffer->phys), buffer->size);
560 if (ret < 0) {
561 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
562 kfree(a);
563 return -EINVAL;
564 }
565
566 a->dev = attachment->dev;
567 INIT_LIST_HEAD(&a->node);
568 attachment->priv = a;
569
570 mutex_lock(&buffer->lock);
571 list_add(&a->node, &buffer->attachments);
572 mutex_unlock(&buffer->lock);
573
574 return 0;
575}
576
577static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
578 struct dma_buf_attachment *attachment)
579{
580 struct fastrpc_dma_buf_attachment *a = attachment->priv;
581 struct fastrpc_buf *buffer = dmabuf->priv;
582
583 mutex_lock(&buffer->lock);
584 list_del(&a->node);
585 mutex_unlock(&buffer->lock);
586 sg_free_table(&a->sgt);
587 kfree(a);
588}
589
590static int fastrpc_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
591{
592 struct fastrpc_buf *buf = dmabuf->priv;
593
594 dma_buf_map_set_vaddr(map, buf->virt);
595
596 return 0;
597}
598
599static int fastrpc_mmap(struct dma_buf *dmabuf,
600 struct vm_area_struct *vma)
601{
602 struct fastrpc_buf *buf = dmabuf->priv;
603 size_t size = vma->vm_end - vma->vm_start;
604
605 return dma_mmap_coherent(buf->dev, vma, buf->virt,
606 FASTRPC_PHYS(buf->phys), size);
607}
608
609static const struct dma_buf_ops fastrpc_dma_buf_ops = {
610 .attach = fastrpc_dma_buf_attach,
611 .detach = fastrpc_dma_buf_detatch,
612 .map_dma_buf = fastrpc_map_dma_buf,
613 .unmap_dma_buf = fastrpc_unmap_dma_buf,
614 .mmap = fastrpc_mmap,
615 .vmap = fastrpc_vmap,
616 .release = fastrpc_release,
617};
618
619static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
620 u64 len, struct fastrpc_map **ppmap)
621{
622 struct fastrpc_session_ctx *sess = fl->sctx;
623 struct fastrpc_map *map = NULL;
624 int err = 0;
625
626 if (!fastrpc_map_find(fl, fd, ppmap))
627 return 0;
628
629 map = kzalloc(sizeof(*map), GFP_KERNEL);
630 if (!map)
631 return -ENOMEM;
632
633 INIT_LIST_HEAD(&map->node);
634 map->fl = fl;
635 map->fd = fd;
636 map->buf = dma_buf_get(fd);
637 if (IS_ERR(map->buf)) {
638 err = PTR_ERR(map->buf);
639 goto get_err;
640 }
641
642 map->attach = dma_buf_attach(map->buf, sess->dev);
643 if (IS_ERR(map->attach)) {
644 dev_err(sess->dev, "Failed to attach dmabuf\n");
645 err = PTR_ERR(map->attach);
646 goto attach_err;
647 }
648
649 map->table = dma_buf_map_attachment(map->attach, DMA_BIDIRECTIONAL);
650 if (IS_ERR(map->table)) {
651 err = PTR_ERR(map->table);
652 goto map_err;
653 }
654
655 map->phys = sg_dma_address(map->table->sgl);
656 map->phys += ((u64)fl->sctx->sid << 32);
657 map->size = len;
658 map->va = sg_virt(map->table->sgl);
659 map->len = len;
660 kref_init(&map->refcount);
661
662 spin_lock(&fl->lock);
663 list_add_tail(&map->node, &fl->maps);
664 spin_unlock(&fl->lock);
665 *ppmap = map;
666
667 return 0;
668
669map_err:
670 dma_buf_detach(map->buf, map->attach);
671attach_err:
672 dma_buf_put(map->buf);
673get_err:
674 kfree(map);
675
676 return err;
677}
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
707{
708 int size = 0;
709
710 size = (sizeof(struct fastrpc_remote_arg) +
711 sizeof(struct fastrpc_invoke_buf) +
712 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
713 sizeof(u64) * FASTRPC_MAX_FDLIST +
714 sizeof(u32) * FASTRPC_MAX_CRCLIST;
715
716 return size;
717}
718
719static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
720{
721 u64 size = 0;
722 int oix;
723
724 size = ALIGN(metalen, FASTRPC_ALIGN);
725 for (oix = 0; oix < ctx->nbufs; oix++) {
726 int i = ctx->olaps[oix].raix;
727
728 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
729
730 if (ctx->olaps[oix].offset == 0)
731 size = ALIGN(size, FASTRPC_ALIGN);
732
733 size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
734 }
735 }
736
737 return size;
738}
739
740static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
741{
742 struct device *dev = ctx->fl->sctx->dev;
743 int i, err;
744
745 for (i = 0; i < ctx->nscalars; ++i) {
746
747 if (ctx->args[i].reserved)
748 return -EINVAL;
749
750 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
751 ctx->args[i].length == 0)
752 continue;
753
754 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
755 ctx->args[i].length, &ctx->maps[i]);
756 if (err) {
757 dev_err(dev, "Error Creating map %d\n", err);
758 return -EINVAL;
759 }
760
761 }
762 return 0;
763}
764
765static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
766{
767 struct device *dev = ctx->fl->sctx->dev;
768 struct fastrpc_remote_arg *rpra;
769 struct fastrpc_invoke_buf *list;
770 struct fastrpc_phy_page *pages;
771 int inbufs, i, oix, err = 0;
772 u64 len, rlen, pkt_size;
773 u64 pg_start, pg_end;
774 uintptr_t args;
775 int metalen;
776
777 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
778 metalen = fastrpc_get_meta_size(ctx);
779 pkt_size = fastrpc_get_payload_size(ctx, metalen);
780
781 err = fastrpc_create_maps(ctx);
782 if (err)
783 return err;
784
785 ctx->msg_sz = pkt_size;
786
787 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
788 if (err)
789 return err;
790
791 rpra = ctx->buf->virt;
792 list = ctx->buf->virt + ctx->nscalars * sizeof(*rpra);
793 pages = ctx->buf->virt + ctx->nscalars * (sizeof(*list) +
794 sizeof(*rpra));
795 args = (uintptr_t)ctx->buf->virt + metalen;
796 rlen = pkt_size - metalen;
797 ctx->rpra = rpra;
798
799 for (oix = 0; oix < ctx->nbufs; ++oix) {
800 int mlen;
801
802 i = ctx->olaps[oix].raix;
803 len = ctx->args[i].length;
804
805 rpra[i].pv = 0;
806 rpra[i].len = len;
807 list[i].num = len ? 1 : 0;
808 list[i].pgidx = i;
809
810 if (!len)
811 continue;
812
813 if (ctx->maps[i]) {
814 struct vm_area_struct *vma = NULL;
815
816 rpra[i].pv = (u64) ctx->args[i].ptr;
817 pages[i].addr = ctx->maps[i]->phys;
818
819 mmap_read_lock(current->mm);
820 vma = find_vma(current->mm, ctx->args[i].ptr);
821 if (vma)
822 pages[i].addr += ctx->args[i].ptr -
823 vma->vm_start;
824 mmap_read_unlock(current->mm);
825
826 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
827 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
828 PAGE_SHIFT;
829 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
830
831 } else {
832
833 if (ctx->olaps[oix].offset == 0) {
834 rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
835 args = ALIGN(args, FASTRPC_ALIGN);
836 }
837
838 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
839
840 if (rlen < mlen)
841 goto bail;
842
843 rpra[i].pv = args - ctx->olaps[oix].offset;
844 pages[i].addr = ctx->buf->phys -
845 ctx->olaps[oix].offset +
846 (pkt_size - rlen);
847 pages[i].addr = pages[i].addr & PAGE_MASK;
848
849 pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
850 pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
851 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
852 args = args + mlen;
853 rlen -= mlen;
854 }
855
856 if (i < inbufs && !ctx->maps[i]) {
857 void *dst = (void *)(uintptr_t)rpra[i].pv;
858 void *src = (void *)(uintptr_t)ctx->args[i].ptr;
859
860 if (!kernel) {
861 if (copy_from_user(dst, (void __user *)src,
862 len)) {
863 err = -EFAULT;
864 goto bail;
865 }
866 } else {
867 memcpy(dst, src, len);
868 }
869 }
870 }
871
872 for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
873 rpra[i].pv = (u64) ctx->args[i].ptr;
874 rpra[i].len = ctx->args[i].length;
875 list[i].num = ctx->args[i].length ? 1 : 0;
876 list[i].pgidx = i;
877 pages[i].addr = ctx->maps[i]->phys;
878 pages[i].size = ctx->maps[i]->size;
879 }
880
881bail:
882 if (err)
883 dev_err(dev, "Error: get invoke args failed:%d\n", err);
884
885 return err;
886}
887
888static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
889 u32 kernel)
890{
891 struct fastrpc_remote_arg *rpra = ctx->rpra;
892 int i, inbufs;
893
894 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
895
896 for (i = inbufs; i < ctx->nbufs; ++i) {
897 if (!ctx->maps[i]) {
898 void *src = (void *)(uintptr_t)rpra[i].pv;
899 void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
900 u64 len = rpra[i].len;
901
902 if (!kernel) {
903 if (copy_to_user((void __user *)dst, src, len))
904 return -EFAULT;
905 } else {
906 memcpy(dst, src, len);
907 }
908 }
909 }
910
911 return 0;
912}
913
914static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
915 struct fastrpc_invoke_ctx *ctx,
916 u32 kernel, uint32_t handle)
917{
918 struct fastrpc_channel_ctx *cctx;
919 struct fastrpc_user *fl = ctx->fl;
920 struct fastrpc_msg *msg = &ctx->msg;
921 int ret;
922
923 cctx = fl->cctx;
924 msg->pid = fl->tgid;
925 msg->tid = current->pid;
926
927 if (kernel)
928 msg->pid = 0;
929
930 msg->ctx = ctx->ctxid | fl->pd;
931 msg->handle = handle;
932 msg->sc = ctx->sc;
933 msg->addr = ctx->buf ? ctx->buf->phys : 0;
934 msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
935 fastrpc_context_get(ctx);
936
937 ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
938
939 if (ret)
940 fastrpc_context_put(ctx);
941
942 return ret;
943
944}
945
946static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
947 u32 handle, u32 sc,
948 struct fastrpc_invoke_args *args)
949{
950 struct fastrpc_invoke_ctx *ctx = NULL;
951 int err = 0;
952
953 if (!fl->sctx)
954 return -EINVAL;
955
956 if (!fl->cctx->rpdev)
957 return -EPIPE;
958
959 if (handle == FASTRPC_INIT_HANDLE && !kernel) {
960 dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);
961 return -EPERM;
962 }
963
964 ctx = fastrpc_context_alloc(fl, kernel, sc, args);
965 if (IS_ERR(ctx))
966 return PTR_ERR(ctx);
967
968 if (ctx->nscalars) {
969 err = fastrpc_get_args(kernel, ctx);
970 if (err)
971 goto bail;
972 }
973
974
975 dma_wmb();
976
977 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
978 if (err)
979 goto bail;
980
981 if (kernel) {
982 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
983 err = -ETIMEDOUT;
984 } else {
985 err = wait_for_completion_interruptible(&ctx->work);
986 }
987
988 if (err)
989 goto bail;
990
991
992 err = ctx->retval;
993 if (err)
994 goto bail;
995
996 if (ctx->nscalars) {
997
998 dma_rmb();
999
1000 err = fastrpc_put_args(ctx, kernel);
1001 if (err)
1002 goto bail;
1003 }
1004
1005bail:
1006 if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1007
1008 spin_lock(&fl->lock);
1009 list_del(&ctx->node);
1010 spin_unlock(&fl->lock);
1011 fastrpc_context_put(ctx);
1012 }
1013 if (err)
1014 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1015
1016 return err;
1017}
1018
1019static int fastrpc_init_create_process(struct fastrpc_user *fl,
1020 char __user *argp)
1021{
1022 struct fastrpc_init_create init;
1023 struct fastrpc_invoke_args *args;
1024 struct fastrpc_phy_page pages[1];
1025 struct fastrpc_map *map = NULL;
1026 struct fastrpc_buf *imem = NULL;
1027 int memlen;
1028 int err;
1029 struct {
1030 int pgid;
1031 u32 namelen;
1032 u32 filelen;
1033 u32 pageslen;
1034 u32 attrs;
1035 u32 siglen;
1036 } inbuf;
1037 u32 sc;
1038
1039 args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1040 if (!args)
1041 return -ENOMEM;
1042
1043 if (copy_from_user(&init, argp, sizeof(init))) {
1044 err = -EFAULT;
1045 goto err;
1046 }
1047
1048 if (init.filelen > INIT_FILELEN_MAX) {
1049 err = -EINVAL;
1050 goto err;
1051 }
1052
1053 inbuf.pgid = fl->tgid;
1054 inbuf.namelen = strlen(current->comm) + 1;
1055 inbuf.filelen = init.filelen;
1056 inbuf.pageslen = 1;
1057 inbuf.attrs = init.attrs;
1058 inbuf.siglen = init.siglen;
1059 fl->pd = USER_PD;
1060
1061 if (init.filelen && init.filefd) {
1062 err = fastrpc_map_create(fl, init.filefd, init.filelen, &map);
1063 if (err)
1064 goto err;
1065 }
1066
1067 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1068 1024 * 1024);
1069 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1070 &imem);
1071 if (err)
1072 goto err_alloc;
1073
1074 fl->init_mem = imem;
1075 args[0].ptr = (u64)(uintptr_t)&inbuf;
1076 args[0].length = sizeof(inbuf);
1077 args[0].fd = -1;
1078
1079 args[1].ptr = (u64)(uintptr_t)current->comm;
1080 args[1].length = inbuf.namelen;
1081 args[1].fd = -1;
1082
1083 args[2].ptr = (u64) init.file;
1084 args[2].length = inbuf.filelen;
1085 args[2].fd = init.filefd;
1086
1087 pages[0].addr = imem->phys;
1088 pages[0].size = imem->size;
1089
1090 args[3].ptr = (u64)(uintptr_t) pages;
1091 args[3].length = 1 * sizeof(*pages);
1092 args[3].fd = -1;
1093
1094 args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1095 args[4].length = sizeof(inbuf.attrs);
1096 args[4].fd = -1;
1097
1098 args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1099 args[5].length = sizeof(inbuf.siglen);
1100 args[5].fd = -1;
1101
1102 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1103 if (init.attrs)
1104 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 6, 0);
1105
1106 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1107 sc, args);
1108 if (err)
1109 goto err_invoke;
1110
1111 kfree(args);
1112
1113 return 0;
1114
1115err_invoke:
1116 fl->init_mem = NULL;
1117 fastrpc_buf_free(imem);
1118err_alloc:
1119 if (map) {
1120 spin_lock(&fl->lock);
1121 list_del(&map->node);
1122 spin_unlock(&fl->lock);
1123 fastrpc_map_put(map);
1124 }
1125err:
1126 kfree(args);
1127
1128 return err;
1129}
1130
1131static struct fastrpc_session_ctx *fastrpc_session_alloc(
1132 struct fastrpc_channel_ctx *cctx)
1133{
1134 struct fastrpc_session_ctx *session = NULL;
1135 unsigned long flags;
1136 int i;
1137
1138 spin_lock_irqsave(&cctx->lock, flags);
1139 for (i = 0; i < cctx->sesscount; i++) {
1140 if (!cctx->session[i].used && cctx->session[i].valid) {
1141 cctx->session[i].used = true;
1142 session = &cctx->session[i];
1143 break;
1144 }
1145 }
1146 spin_unlock_irqrestore(&cctx->lock, flags);
1147
1148 return session;
1149}
1150
1151static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1152 struct fastrpc_session_ctx *session)
1153{
1154 unsigned long flags;
1155
1156 spin_lock_irqsave(&cctx->lock, flags);
1157 session->used = false;
1158 spin_unlock_irqrestore(&cctx->lock, flags);
1159}
1160
1161static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1162{
1163 struct fastrpc_invoke_args args[1];
1164 int tgid = 0;
1165 u32 sc;
1166
1167 tgid = fl->tgid;
1168 args[0].ptr = (u64)(uintptr_t) &tgid;
1169 args[0].length = sizeof(tgid);
1170 args[0].fd = -1;
1171 args[0].reserved = 0;
1172 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1173
1174 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1175 sc, &args[0]);
1176}
1177
1178static int fastrpc_device_release(struct inode *inode, struct file *file)
1179{
1180 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1181 struct fastrpc_channel_ctx *cctx = fl->cctx;
1182 struct fastrpc_invoke_ctx *ctx, *n;
1183 struct fastrpc_map *map, *m;
1184 struct fastrpc_buf *buf, *b;
1185 unsigned long flags;
1186
1187 fastrpc_release_current_dsp_process(fl);
1188
1189 spin_lock_irqsave(&cctx->lock, flags);
1190 list_del(&fl->user);
1191 spin_unlock_irqrestore(&cctx->lock, flags);
1192
1193 if (fl->init_mem)
1194 fastrpc_buf_free(fl->init_mem);
1195
1196 list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1197 list_del(&ctx->node);
1198 fastrpc_context_put(ctx);
1199 }
1200
1201 list_for_each_entry_safe(map, m, &fl->maps, node) {
1202 list_del(&map->node);
1203 fastrpc_map_put(map);
1204 }
1205
1206 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1207 list_del(&buf->node);
1208 fastrpc_buf_free(buf);
1209 }
1210
1211 fastrpc_session_free(cctx, fl->sctx);
1212 fastrpc_channel_ctx_put(cctx);
1213
1214 mutex_destroy(&fl->mutex);
1215 kfree(fl);
1216 file->private_data = NULL;
1217
1218 return 0;
1219}
1220
1221static int fastrpc_device_open(struct inode *inode, struct file *filp)
1222{
1223 struct fastrpc_channel_ctx *cctx = miscdev_to_cctx(filp->private_data);
1224 struct fastrpc_user *fl = NULL;
1225 unsigned long flags;
1226
1227 fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1228 if (!fl)
1229 return -ENOMEM;
1230
1231
1232 fastrpc_channel_ctx_get(cctx);
1233
1234 filp->private_data = fl;
1235 spin_lock_init(&fl->lock);
1236 mutex_init(&fl->mutex);
1237 INIT_LIST_HEAD(&fl->pending);
1238 INIT_LIST_HEAD(&fl->maps);
1239 INIT_LIST_HEAD(&fl->mmaps);
1240 INIT_LIST_HEAD(&fl->user);
1241 fl->tgid = current->tgid;
1242 fl->cctx = cctx;
1243
1244 fl->sctx = fastrpc_session_alloc(cctx);
1245 if (!fl->sctx) {
1246 dev_err(&cctx->rpdev->dev, "No session available\n");
1247 mutex_destroy(&fl->mutex);
1248 kfree(fl);
1249
1250 return -EBUSY;
1251 }
1252
1253 spin_lock_irqsave(&cctx->lock, flags);
1254 list_add_tail(&fl->user, &cctx->users);
1255 spin_unlock_irqrestore(&cctx->lock, flags);
1256
1257 return 0;
1258}
1259
1260static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1261{
1262 struct fastrpc_alloc_dma_buf bp;
1263 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1264 struct fastrpc_buf *buf = NULL;
1265 int err;
1266
1267 if (copy_from_user(&bp, argp, sizeof(bp)))
1268 return -EFAULT;
1269
1270 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1271 if (err)
1272 return err;
1273 exp_info.ops = &fastrpc_dma_buf_ops;
1274 exp_info.size = bp.size;
1275 exp_info.flags = O_RDWR;
1276 exp_info.priv = buf;
1277 buf->dmabuf = dma_buf_export(&exp_info);
1278 if (IS_ERR(buf->dmabuf)) {
1279 err = PTR_ERR(buf->dmabuf);
1280 fastrpc_buf_free(buf);
1281 return err;
1282 }
1283
1284 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1285 if (bp.fd < 0) {
1286 dma_buf_put(buf->dmabuf);
1287 return -EINVAL;
1288 }
1289
1290 if (copy_to_user(argp, &bp, sizeof(bp))) {
1291 dma_buf_put(buf->dmabuf);
1292 return -EFAULT;
1293 }
1294
1295 return 0;
1296}
1297
1298static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1299{
1300 struct fastrpc_invoke_args args[1];
1301 int tgid = fl->tgid;
1302 u32 sc;
1303
1304 args[0].ptr = (u64)(uintptr_t) &tgid;
1305 args[0].length = sizeof(tgid);
1306 args[0].fd = -1;
1307 args[0].reserved = 0;
1308 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1309 fl->pd = pd;
1310
1311 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1312 sc, &args[0]);
1313}
1314
1315static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1316{
1317 struct fastrpc_invoke_args *args = NULL;
1318 struct fastrpc_invoke inv;
1319 u32 nscalars;
1320 int err;
1321
1322 if (copy_from_user(&inv, argp, sizeof(inv)))
1323 return -EFAULT;
1324
1325
1326 nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1327 if (nscalars) {
1328 args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1329 if (!args)
1330 return -ENOMEM;
1331
1332 if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1333 nscalars * sizeof(*args))) {
1334 kfree(args);
1335 return -EFAULT;
1336 }
1337 }
1338
1339 err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1340 kfree(args);
1341
1342 return err;
1343}
1344
1345static int fastrpc_req_munmap_impl(struct fastrpc_user *fl,
1346 struct fastrpc_req_munmap *req)
1347{
1348 struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1349 struct fastrpc_buf *buf, *b;
1350 struct fastrpc_munmap_req_msg req_msg;
1351 struct device *dev = fl->sctx->dev;
1352 int err;
1353 u32 sc;
1354
1355 spin_lock(&fl->lock);
1356 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1357 if ((buf->raddr == req->vaddrout) && (buf->size == req->size))
1358 break;
1359 buf = NULL;
1360 }
1361 spin_unlock(&fl->lock);
1362
1363 if (!buf) {
1364 dev_err(dev, "mmap not in list\n");
1365 return -EINVAL;
1366 }
1367
1368 req_msg.pgid = fl->tgid;
1369 req_msg.size = buf->size;
1370 req_msg.vaddr = buf->raddr;
1371
1372 args[0].ptr = (u64) (uintptr_t) &req_msg;
1373 args[0].length = sizeof(req_msg);
1374
1375 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1376 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1377 &args[0]);
1378 if (!err) {
1379 dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1380 spin_lock(&fl->lock);
1381 list_del(&buf->node);
1382 spin_unlock(&fl->lock);
1383 fastrpc_buf_free(buf);
1384 } else {
1385 dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1386 }
1387
1388 return err;
1389}
1390
1391static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1392{
1393 struct fastrpc_req_munmap req;
1394
1395 if (copy_from_user(&req, argp, sizeof(req)))
1396 return -EFAULT;
1397
1398 return fastrpc_req_munmap_impl(fl, &req);
1399}
1400
1401static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1402{
1403 struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1404 struct fastrpc_buf *buf = NULL;
1405 struct fastrpc_mmap_req_msg req_msg;
1406 struct fastrpc_mmap_rsp_msg rsp_msg;
1407 struct fastrpc_req_munmap req_unmap;
1408 struct fastrpc_phy_page pages;
1409 struct fastrpc_req_mmap req;
1410 struct device *dev = fl->sctx->dev;
1411 int err;
1412 u32 sc;
1413
1414 if (copy_from_user(&req, argp, sizeof(req)))
1415 return -EFAULT;
1416
1417 if (req.flags != ADSP_MMAP_ADD_PAGES) {
1418 dev_err(dev, "flag not supported 0x%x\n", req.flags);
1419 return -EINVAL;
1420 }
1421
1422 if (req.vaddrin) {
1423 dev_err(dev, "adding user allocated pages is not supported\n");
1424 return -EINVAL;
1425 }
1426
1427 err = fastrpc_buf_alloc(fl, fl->sctx->dev, req.size, &buf);
1428 if (err) {
1429 dev_err(dev, "failed to allocate buffer\n");
1430 return err;
1431 }
1432
1433 req_msg.pgid = fl->tgid;
1434 req_msg.flags = req.flags;
1435 req_msg.vaddr = req.vaddrin;
1436 req_msg.num = sizeof(pages);
1437
1438 args[0].ptr = (u64) (uintptr_t) &req_msg;
1439 args[0].length = sizeof(req_msg);
1440
1441 pages.addr = buf->phys;
1442 pages.size = buf->size;
1443
1444 args[1].ptr = (u64) (uintptr_t) &pages;
1445 args[1].length = sizeof(pages);
1446
1447 args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1448 args[2].length = sizeof(rsp_msg);
1449
1450 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1451 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1452 &args[0]);
1453 if (err) {
1454 dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1455 goto err_invoke;
1456 }
1457
1458
1459 buf->raddr = (uintptr_t) rsp_msg.vaddr;
1460
1461
1462 req.vaddrout = rsp_msg.vaddr;
1463
1464 spin_lock(&fl->lock);
1465 list_add_tail(&buf->node, &fl->mmaps);
1466 spin_unlock(&fl->lock);
1467
1468 if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
1469
1470 req_unmap.vaddrout = buf->raddr;
1471 req_unmap.size = buf->size;
1472 fastrpc_req_munmap_impl(fl, &req_unmap);
1473 return -EFAULT;
1474 }
1475
1476 dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1477 buf->raddr, buf->size);
1478
1479 return 0;
1480
1481err_invoke:
1482 fastrpc_buf_free(buf);
1483
1484 return err;
1485}
1486
1487static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
1488 unsigned long arg)
1489{
1490 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1491 char __user *argp = (char __user *)arg;
1492 int err;
1493
1494 switch (cmd) {
1495 case FASTRPC_IOCTL_INVOKE:
1496 err = fastrpc_invoke(fl, argp);
1497 break;
1498 case FASTRPC_IOCTL_INIT_ATTACH:
1499 err = fastrpc_init_attach(fl, AUDIO_PD);
1500 break;
1501 case FASTRPC_IOCTL_INIT_ATTACH_SNS:
1502 err = fastrpc_init_attach(fl, SENSORS_PD);
1503 break;
1504 case FASTRPC_IOCTL_INIT_CREATE:
1505 err = fastrpc_init_create_process(fl, argp);
1506 break;
1507 case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
1508 err = fastrpc_dmabuf_alloc(fl, argp);
1509 break;
1510 case FASTRPC_IOCTL_MMAP:
1511 err = fastrpc_req_mmap(fl, argp);
1512 break;
1513 case FASTRPC_IOCTL_MUNMAP:
1514 err = fastrpc_req_munmap(fl, argp);
1515 break;
1516 default:
1517 err = -ENOTTY;
1518 break;
1519 }
1520
1521 return err;
1522}
1523
1524static const struct file_operations fastrpc_fops = {
1525 .open = fastrpc_device_open,
1526 .release = fastrpc_device_release,
1527 .unlocked_ioctl = fastrpc_device_ioctl,
1528 .compat_ioctl = fastrpc_device_ioctl,
1529};
1530
1531static int fastrpc_cb_probe(struct platform_device *pdev)
1532{
1533 struct fastrpc_channel_ctx *cctx;
1534 struct fastrpc_session_ctx *sess;
1535 struct device *dev = &pdev->dev;
1536 int i, sessions = 0;
1537 unsigned long flags;
1538 int rc;
1539
1540 cctx = dev_get_drvdata(dev->parent);
1541 if (!cctx)
1542 return -EINVAL;
1543
1544 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
1545
1546 spin_lock_irqsave(&cctx->lock, flags);
1547 sess = &cctx->session[cctx->sesscount];
1548 sess->used = false;
1549 sess->valid = true;
1550 sess->dev = dev;
1551 dev_set_drvdata(dev, sess);
1552
1553 if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
1554 dev_info(dev, "FastRPC Session ID not specified in DT\n");
1555
1556 if (sessions > 0) {
1557 struct fastrpc_session_ctx *dup_sess;
1558
1559 for (i = 1; i < sessions; i++) {
1560 if (cctx->sesscount++ >= FASTRPC_MAX_SESSIONS)
1561 break;
1562 dup_sess = &cctx->session[cctx->sesscount];
1563 memcpy(dup_sess, sess, sizeof(*dup_sess));
1564 }
1565 }
1566 cctx->sesscount++;
1567 spin_unlock_irqrestore(&cctx->lock, flags);
1568 rc = dma_set_mask(dev, DMA_BIT_MASK(32));
1569 if (rc) {
1570 dev_err(dev, "32-bit DMA enable failed\n");
1571 return rc;
1572 }
1573
1574 return 0;
1575}
1576
1577static int fastrpc_cb_remove(struct platform_device *pdev)
1578{
1579 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
1580 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
1581 unsigned long flags;
1582 int i;
1583
1584 spin_lock_irqsave(&cctx->lock, flags);
1585 for (i = 1; i < FASTRPC_MAX_SESSIONS; i++) {
1586 if (cctx->session[i].sid == sess->sid) {
1587 cctx->session[i].valid = false;
1588 cctx->sesscount--;
1589 }
1590 }
1591 spin_unlock_irqrestore(&cctx->lock, flags);
1592
1593 return 0;
1594}
1595
1596static const struct of_device_id fastrpc_match_table[] = {
1597 { .compatible = "qcom,fastrpc-compute-cb", },
1598 {}
1599};
1600
1601static struct platform_driver fastrpc_cb_driver = {
1602 .probe = fastrpc_cb_probe,
1603 .remove = fastrpc_cb_remove,
1604 .driver = {
1605 .name = "qcom,fastrpc-cb",
1606 .of_match_table = fastrpc_match_table,
1607 .suppress_bind_attrs = true,
1608 },
1609};
1610
1611static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
1612{
1613 struct device *rdev = &rpdev->dev;
1614 struct fastrpc_channel_ctx *data;
1615 int i, err, domain_id = -1;
1616 const char *domain;
1617
1618 err = of_property_read_string(rdev->of_node, "label", &domain);
1619 if (err) {
1620 dev_info(rdev, "FastRPC Domain not specified in DT\n");
1621 return err;
1622 }
1623
1624 for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
1625 if (!strcmp(domains[i], domain)) {
1626 domain_id = i;
1627 break;
1628 }
1629 }
1630
1631 if (domain_id < 0) {
1632 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
1633 return -EINVAL;
1634 }
1635
1636 data = kzalloc(sizeof(*data), GFP_KERNEL);
1637 if (!data)
1638 return -ENOMEM;
1639
1640 data->miscdev.minor = MISC_DYNAMIC_MINOR;
1641 data->miscdev.name = devm_kasprintf(rdev, GFP_KERNEL, "fastrpc-%s",
1642 domains[domain_id]);
1643 data->miscdev.fops = &fastrpc_fops;
1644 err = misc_register(&data->miscdev);
1645 if (err) {
1646 kfree(data);
1647 return err;
1648 }
1649
1650 kref_init(&data->refcount);
1651
1652 dev_set_drvdata(&rpdev->dev, data);
1653 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
1654 INIT_LIST_HEAD(&data->users);
1655 spin_lock_init(&data->lock);
1656 idr_init(&data->ctx_idr);
1657 data->domain_id = domain_id;
1658 data->rpdev = rpdev;
1659
1660 return of_platform_populate(rdev->of_node, NULL, NULL, rdev);
1661}
1662
1663static void fastrpc_notify_users(struct fastrpc_user *user)
1664{
1665 struct fastrpc_invoke_ctx *ctx;
1666
1667 spin_lock(&user->lock);
1668 list_for_each_entry(ctx, &user->pending, node)
1669 complete(&ctx->work);
1670 spin_unlock(&user->lock);
1671}
1672
1673static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
1674{
1675 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1676 struct fastrpc_user *user;
1677 unsigned long flags;
1678
1679 spin_lock_irqsave(&cctx->lock, flags);
1680 list_for_each_entry(user, &cctx->users, user)
1681 fastrpc_notify_users(user);
1682 spin_unlock_irqrestore(&cctx->lock, flags);
1683
1684 misc_deregister(&cctx->miscdev);
1685 of_platform_depopulate(&rpdev->dev);
1686
1687 cctx->rpdev = NULL;
1688 fastrpc_channel_ctx_put(cctx);
1689}
1690
1691static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
1692 int len, void *priv, u32 addr)
1693{
1694 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1695 struct fastrpc_invoke_rsp *rsp = data;
1696 struct fastrpc_invoke_ctx *ctx;
1697 unsigned long flags;
1698 unsigned long ctxid;
1699
1700 if (len < sizeof(*rsp))
1701 return -EINVAL;
1702
1703 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
1704
1705 spin_lock_irqsave(&cctx->lock, flags);
1706 ctx = idr_find(&cctx->ctx_idr, ctxid);
1707 spin_unlock_irqrestore(&cctx->lock, flags);
1708
1709 if (!ctx) {
1710 dev_err(&rpdev->dev, "No context ID matches response\n");
1711 return -ENOENT;
1712 }
1713
1714 ctx->retval = rsp->retval;
1715 complete(&ctx->work);
1716
1717
1718
1719
1720
1721
1722 schedule_work(&ctx->put_work);
1723
1724 return 0;
1725}
1726
1727static const struct of_device_id fastrpc_rpmsg_of_match[] = {
1728 { .compatible = "qcom,fastrpc" },
1729 { },
1730};
1731MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
1732
1733static struct rpmsg_driver fastrpc_driver = {
1734 .probe = fastrpc_rpmsg_probe,
1735 .remove = fastrpc_rpmsg_remove,
1736 .callback = fastrpc_rpmsg_callback,
1737 .drv = {
1738 .name = "qcom,fastrpc",
1739 .of_match_table = fastrpc_rpmsg_of_match,
1740 },
1741};
1742
1743static int fastrpc_init(void)
1744{
1745 int ret;
1746
1747 ret = platform_driver_register(&fastrpc_cb_driver);
1748 if (ret < 0) {
1749 pr_err("fastrpc: failed to register cb driver\n");
1750 return ret;
1751 }
1752
1753 ret = register_rpmsg_driver(&fastrpc_driver);
1754 if (ret < 0) {
1755 pr_err("fastrpc: failed to register rpmsg driver\n");
1756 platform_driver_unregister(&fastrpc_cb_driver);
1757 return ret;
1758 }
1759
1760 return 0;
1761}
1762module_init(fastrpc_init);
1763
1764static void fastrpc_exit(void)
1765{
1766 platform_driver_unregister(&fastrpc_cb_driver);
1767 unregister_rpmsg_driver(&fastrpc_driver);
1768}
1769module_exit(fastrpc_exit);
1770
1771MODULE_LICENSE("GPL v2");
1772MODULE_IMPORT_NS(DMA_BUF);
1773
