1
2
3
4
5#include <linux/completion.h>
6#include <linux/device.h>
7#include <linux/dma-buf.h>
8#include <linux/dma-mapping.h>
9#include <linux/idr.h>
10#include <linux/list.h>
11#include <linux/miscdevice.h>
12#include <linux/module.h>
13#include <linux/of_address.h>
14#include <linux/of.h>
15#include <linux/sort.h>
16#include <linux/of_platform.h>
17#include <linux/rpmsg.h>
18#include <linux/scatterlist.h>
19#include <linux/slab.h>
20#include <uapi/misc/fastrpc.h>
21
22#define ADSP_DOMAIN_ID (0)
23#define MDSP_DOMAIN_ID (1)
24#define SDSP_DOMAIN_ID (2)
25#define CDSP_DOMAIN_ID (3)
26#define FASTRPC_DEV_MAX 4
27#define FASTRPC_MAX_SESSIONS 9
28#define FASTRPC_ALIGN 128
29#define FASTRPC_MAX_FDLIST 16
30#define FASTRPC_MAX_CRCLIST 64
31#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
32#define FASTRPC_CTX_MAX (256)
33#define FASTRPC_INIT_HANDLE 1
34#define FASTRPC_CTXID_MASK (0xFF0)
35#define INIT_FILELEN_MAX (2 * 1024 * 1024)
36#define FASTRPC_DEVICE_NAME "fastrpc"
37#define ADSP_MMAP_ADD_PAGES 0x1000
38
39
40#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
41
42
43#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
44
45
46#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
47
48
49#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
50
51#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
52 REMOTE_SCALARS_OUTBUFS(sc) + \
53 REMOTE_SCALARS_INHANDLES(sc)+ \
54 REMOTE_SCALARS_OUTHANDLES(sc))
55#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
56 (((attr & 0x07) << 29) | \
57 ((method & 0x1f) << 24) | \
58 ((in & 0xff) << 16) | \
59 ((out & 0xff) << 8) | \
60 ((oin & 0x0f) << 4) | \
61 (oout & 0x0f))
62
63#define FASTRPC_SCALARS(method, in, out) \
64 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
65
66#define FASTRPC_CREATE_PROCESS_NARGS 6
67
68#define FASTRPC_RMID_INIT_ATTACH 0
69#define FASTRPC_RMID_INIT_RELEASE 1
70#define FASTRPC_RMID_INIT_MMAP 4
71#define FASTRPC_RMID_INIT_MUNMAP 5
72#define FASTRPC_RMID_INIT_CREATE 6
73#define FASTRPC_RMID_INIT_CREATE_ATTR 7
74#define FASTRPC_RMID_INIT_CREATE_STATIC 8
75
76
77#define AUDIO_PD (0)
78#define USER_PD (1)
79#define SENSORS_PD (2)
80
81#define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev)
82
83static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
84 "sdsp", "cdsp"};
85struct fastrpc_phy_page {
86 u64 addr;
87 u64 size;
88};
89
90struct fastrpc_invoke_buf {
91 u32 num;
92 u32 pgidx;
93};
94
95struct fastrpc_remote_arg {
96 u64 pv;
97 u64 len;
98};
99
100struct fastrpc_mmap_rsp_msg {
101 u64 vaddr;
102};
103
104struct fastrpc_mmap_req_msg {
105 s32 pgid;
106 u32 flags;
107 u64 vaddr;
108 s32 num;
109};
110
111struct fastrpc_munmap_req_msg {
112 s32 pgid;
113 u64 vaddr;
114 u64 size;
115};
116
117struct fastrpc_msg {
118 int pid;
119 int tid;
120 u64 ctx;
121 u32 handle;
122 u32 sc;
123 u64 addr;
124 u64 size;
125};
126
127struct fastrpc_invoke_rsp {
128 u64 ctx;
129 int retval;
130};
131
132struct fastrpc_buf_overlap {
133 u64 start;
134 u64 end;
135 int raix;
136 u64 mstart;
137 u64 mend;
138 u64 offset;
139};
140
141struct fastrpc_buf {
142 struct fastrpc_user *fl;
143 struct dma_buf *dmabuf;
144 struct device *dev;
145 void *virt;
146 u64 phys;
147 u64 size;
148
149 struct mutex lock;
150 struct list_head attachments;
151
152 struct list_head node;
153 uintptr_t raddr;
154};
155
156struct fastrpc_dma_buf_attachment {
157 struct device *dev;
158 struct sg_table sgt;
159 struct list_head node;
160};
161
162struct fastrpc_map {
163 struct list_head node;
164 struct fastrpc_user *fl;
165 int fd;
166 struct dma_buf *buf;
167 struct sg_table *table;
168 struct dma_buf_attachment *attach;
169 u64 phys;
170 u64 size;
171 void *va;
172 u64 len;
173 struct kref refcount;
174};
175
176struct fastrpc_invoke_ctx {
177 int nscalars;
178 int nbufs;
179 int retval;
180 int pid;
181 int tgid;
182 u32 sc;
183 u32 *crc;
184 u64 ctxid;
185 u64 msg_sz;
186 struct kref refcount;
187 struct list_head node;
188 struct completion work;
189 struct work_struct put_work;
190 struct fastrpc_msg msg;
191 struct fastrpc_user *fl;
192 struct fastrpc_remote_arg *rpra;
193 struct fastrpc_map **maps;
194 struct fastrpc_buf *buf;
195 struct fastrpc_invoke_args *args;
196 struct fastrpc_buf_overlap *olaps;
197 struct fastrpc_channel_ctx *cctx;
198};
199
200struct fastrpc_session_ctx {
201 struct device *dev;
202 int sid;
203 bool used;
204 bool valid;
205};
206
207struct fastrpc_channel_ctx {
208 int domain_id;
209 int sesscount;
210 struct rpmsg_device *rpdev;
211 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
212 spinlock_t lock;
213 struct idr ctx_idr;
214 struct list_head users;
215 struct miscdevice miscdev;
216 struct kref refcount;
217};
218
219struct fastrpc_user {
220 struct list_head user;
221 struct list_head maps;
222 struct list_head pending;
223 struct list_head mmaps;
224
225 struct fastrpc_channel_ctx *cctx;
226 struct fastrpc_session_ctx *sctx;
227 struct fastrpc_buf *init_mem;
228
229 int tgid;
230 int pd;
231
232 spinlock_t lock;
233
234 struct mutex mutex;
235};
236
237static void fastrpc_free_map(struct kref *ref)
238{
239 struct fastrpc_map *map;
240
241 map = container_of(ref, struct fastrpc_map, refcount);
242
243 if (map->table) {
244 dma_buf_unmap_attachment(map->attach, map->table,
245 DMA_BIDIRECTIONAL);
246 dma_buf_detach(map->buf, map->attach);
247 dma_buf_put(map->buf);
248 }
249
250 kfree(map);
251}
252
253static void fastrpc_map_put(struct fastrpc_map *map)
254{
255 if (map)
256 kref_put(&map->refcount, fastrpc_free_map);
257}
258
259static void fastrpc_map_get(struct fastrpc_map *map)
260{
261 if (map)
262 kref_get(&map->refcount);
263}
264
265static int fastrpc_map_find(struct fastrpc_user *fl, int fd,
266 struct fastrpc_map **ppmap)
267{
268 struct fastrpc_map *map = NULL;
269
270 mutex_lock(&fl->mutex);
271 list_for_each_entry(map, &fl->maps, node) {
272 if (map->fd == fd) {
273 fastrpc_map_get(map);
274 *ppmap = map;
275 mutex_unlock(&fl->mutex);
276 return 0;
277 }
278 }
279 mutex_unlock(&fl->mutex);
280
281 return -ENOENT;
282}
283
284static void fastrpc_buf_free(struct fastrpc_buf *buf)
285{
286 dma_free_coherent(buf->dev, buf->size, buf->virt,
287 FASTRPC_PHYS(buf->phys));
288 kfree(buf);
289}
290
291static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
292 u64 size, struct fastrpc_buf **obuf)
293{
294 struct fastrpc_buf *buf;
295
296 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
297 if (!buf)
298 return -ENOMEM;
299
300 INIT_LIST_HEAD(&buf->attachments);
301 INIT_LIST_HEAD(&buf->node);
302 mutex_init(&buf->lock);
303
304 buf->fl = fl;
305 buf->virt = NULL;
306 buf->phys = 0;
307 buf->size = size;
308 buf->dev = dev;
309 buf->raddr = 0;
310
311 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
312 GFP_KERNEL);
313 if (!buf->virt) {
314 mutex_destroy(&buf->lock);
315 kfree(buf);
316 return -ENOMEM;
317 }
318
319 if (fl->sctx && fl->sctx->sid)
320 buf->phys += ((u64)fl->sctx->sid << 32);
321
322 *obuf = buf;
323
324 return 0;
325}
326
327static void fastrpc_channel_ctx_free(struct kref *ref)
328{
329 struct fastrpc_channel_ctx *cctx;
330
331 cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
332
333 kfree(cctx);
334}
335
336static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
337{
338 kref_get(&cctx->refcount);
339}
340
341static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
342{
343 kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
344}
345
346static void fastrpc_context_free(struct kref *ref)
347{
348 struct fastrpc_invoke_ctx *ctx;
349 struct fastrpc_channel_ctx *cctx;
350 unsigned long flags;
351 int i;
352
353 ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
354 cctx = ctx->cctx;
355
356 for (i = 0; i < ctx->nscalars; i++)
357 fastrpc_map_put(ctx->maps[i]);
358
359 if (ctx->buf)
360 fastrpc_buf_free(ctx->buf);
361
362 spin_lock_irqsave(&cctx->lock, flags);
363 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
364 spin_unlock_irqrestore(&cctx->lock, flags);
365
366 kfree(ctx->maps);
367 kfree(ctx->olaps);
368 kfree(ctx);
369
370 fastrpc_channel_ctx_put(cctx);
371}
372
373static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
374{
375 kref_get(&ctx->refcount);
376}
377
378static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
379{
380 kref_put(&ctx->refcount, fastrpc_context_free);
381}
382
383static void fastrpc_context_put_wq(struct work_struct *work)
384{
385 struct fastrpc_invoke_ctx *ctx =
386 container_of(work, struct fastrpc_invoke_ctx, put_work);
387
388 fastrpc_context_put(ctx);
389}
390
391#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
392static int olaps_cmp(const void *a, const void *b)
393{
394 struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
395 struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
396
397 int st = CMP(pa->start, pb->start);
398
399 int ed = CMP(pb->end, pa->end);
400
401 return st == 0 ? ed : st;
402}
403
404static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
405{
406 u64 max_end = 0;
407 int i;
408
409 for (i = 0; i < ctx->nbufs; ++i) {
410 ctx->olaps[i].start = ctx->args[i].ptr;
411 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
412 ctx->olaps[i].raix = i;
413 }
414
415 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
416
417 for (i = 0; i < ctx->nbufs; ++i) {
418
419 if (ctx->olaps[i].start < max_end) {
420 ctx->olaps[i].mstart = max_end;
421 ctx->olaps[i].mend = ctx->olaps[i].end;
422 ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
423
424 if (ctx->olaps[i].end > max_end) {
425 max_end = ctx->olaps[i].end;
426 } else {
427 ctx->olaps[i].mend = 0;
428 ctx->olaps[i].mstart = 0;
429 }
430
431 } else {
432 ctx->olaps[i].mend = ctx->olaps[i].end;
433 ctx->olaps[i].mstart = ctx->olaps[i].start;
434 ctx->olaps[i].offset = 0;
435 max_end = ctx->olaps[i].end;
436 }
437 }
438}
439
440static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
441 struct fastrpc_user *user, u32 kernel, u32 sc,
442 struct fastrpc_invoke_args *args)
443{
444 struct fastrpc_channel_ctx *cctx = user->cctx;
445 struct fastrpc_invoke_ctx *ctx = NULL;
446 unsigned long flags;
447 int ret;
448
449 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
450 if (!ctx)
451 return ERR_PTR(-ENOMEM);
452
453 INIT_LIST_HEAD(&ctx->node);
454 ctx->fl = user;
455 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
456 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
457 REMOTE_SCALARS_OUTBUFS(sc);
458
459 if (ctx->nscalars) {
460 ctx->maps = kcalloc(ctx->nscalars,
461 sizeof(*ctx->maps), GFP_KERNEL);
462 if (!ctx->maps) {
463 kfree(ctx);
464 return ERR_PTR(-ENOMEM);
465 }
466 ctx->olaps = kcalloc(ctx->nscalars,
467 sizeof(*ctx->olaps), GFP_KERNEL);
468 if (!ctx->olaps) {
469 kfree(ctx->maps);
470 kfree(ctx);
471 return ERR_PTR(-ENOMEM);
472 }
473 ctx->args = args;
474 fastrpc_get_buff_overlaps(ctx);
475 }
476
477
478 fastrpc_channel_ctx_get(cctx);
479
480 ctx->sc = sc;
481 ctx->retval = -1;
482 ctx->pid = current->pid;
483 ctx->tgid = user->tgid;
484 ctx->cctx = cctx;
485 init_completion(&ctx->work);
486 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
487
488 spin_lock(&user->lock);
489 list_add_tail(&ctx->node, &user->pending);
490 spin_unlock(&user->lock);
491
492 spin_lock_irqsave(&cctx->lock, flags);
493 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
494 FASTRPC_CTX_MAX, GFP_ATOMIC);
495 if (ret < 0) {
496 spin_unlock_irqrestore(&cctx->lock, flags);
497 goto err_idr;
498 }
499 ctx->ctxid = ret << 4;
500 spin_unlock_irqrestore(&cctx->lock, flags);
501
502 kref_init(&ctx->refcount);
503
504 return ctx;
505err_idr:
506 spin_lock(&user->lock);
507 list_del(&ctx->node);
508 spin_unlock(&user->lock);
509 fastrpc_channel_ctx_put(cctx);
510 kfree(ctx->maps);
511 kfree(ctx->olaps);
512 kfree(ctx);
513
514 return ERR_PTR(ret);
515}
516
517static struct sg_table *
518fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
519 enum dma_data_direction dir)
520{
521 struct fastrpc_dma_buf_attachment *a = attachment->priv;
522 struct sg_table *table;
523 int ret;
524
525 table = &a->sgt;
526
527 ret = dma_map_sgtable(attachment->dev, table, dir, 0);
528 if (ret)
529 table = ERR_PTR(ret);
530 return table;
531}
532
533static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
534 struct sg_table *table,
535 enum dma_data_direction dir)
536{
537 dma_unmap_sgtable(attach->dev, table, dir, 0);
538}
539
540static void fastrpc_release(struct dma_buf *dmabuf)
541{
542 struct fastrpc_buf *buffer = dmabuf->priv;
543
544 fastrpc_buf_free(buffer);
545}
546
547static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
548 struct dma_buf_attachment *attachment)
549{
550 struct fastrpc_dma_buf_attachment *a;
551 struct fastrpc_buf *buffer = dmabuf->priv;
552 int ret;
553
554 a = kzalloc(sizeof(*a), GFP_KERNEL);
555 if (!a)
556 return -ENOMEM;
557
558 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
559 FASTRPC_PHYS(buffer->phys), buffer->size);
560 if (ret < 0) {
561 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
562 kfree(a);
563 return -EINVAL;
564 }
565
566 a->dev = attachment->dev;
567 INIT_LIST_HEAD(&a->node);
568 attachment->priv = a;
569
570 mutex_lock(&buffer->lock);
571 list_add(&a->node, &buffer->attachments);
572 mutex_unlock(&buffer->lock);
573
574 return 0;
575}
576
577static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
578 struct dma_buf_attachment *attachment)
579{
580 struct fastrpc_dma_buf_attachment *a = attachment->priv;
581 struct fastrpc_buf *buffer = dmabuf->priv;
582
583 mutex_lock(&buffer->lock);
584 list_del(&a->node);
585 mutex_unlock(&buffer->lock);
586 sg_free_table(&a->sgt);
587 kfree(a);
588}
589
590static int fastrpc_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
591{
592 struct fastrpc_buf *buf = dmabuf->priv;
593
594 dma_buf_map_set_vaddr(map, buf->virt);
595
596 return 0;
597}
598
599static int fastrpc_mmap(struct dma_buf *dmabuf,
600 struct vm_area_struct *vma)
601{
602 struct fastrpc_buf *buf = dmabuf->priv;
603 size_t size = vma->vm_end - vma->vm_start;
604
605 return dma_mmap_coherent(buf->dev, vma, buf->virt,
606 FASTRPC_PHYS(buf->phys), size);
607}
608
609static const struct dma_buf_ops fastrpc_dma_buf_ops = {
610 .attach = fastrpc_dma_buf_attach,
611 .detach = fastrpc_dma_buf_detatch,
612 .map_dma_buf = fastrpc_map_dma_buf,
613 .unmap_dma_buf = fastrpc_unmap_dma_buf,
614 .mmap = fastrpc_mmap,
615 .vmap = fastrpc_vmap,
616 .release = fastrpc_release,
617};
618
619static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
620 u64 len, struct fastrpc_map **ppmap)
621{
622 struct fastrpc_session_ctx *sess = fl->sctx;
623 struct fastrpc_map *map = NULL;
624 int err = 0;
625
626 if (!fastrpc_map_find(fl, fd, ppmap))
627 return 0;
628
629 map = kzalloc(sizeof(*map), GFP_KERNEL);
630 if (!map)
631 return -ENOMEM;
632
633 INIT_LIST_HEAD(&map->node);
634 map->fl = fl;
635 map->fd = fd;
636 map->buf = dma_buf_get(fd);
637 if (IS_ERR(map->buf)) {
638 err = PTR_ERR(map->buf);
639 goto get_err;
640 }
641
642 map->attach = dma_buf_attach(map->buf, sess->dev);
643 if (IS_ERR(map->attach)) {
644 dev_err(sess->dev, "Failed to attach dmabuf\n");
645 err = PTR_ERR(map->attach);
646 goto attach_err;
647 }
648
649 map->table = dma_buf_map_attachment(map->attach, DMA_BIDIRECTIONAL);
650 if (IS_ERR(map->table)) {
651 err = PTR_ERR(map->table);
652 goto map_err;
653 }
654
655 map->phys = sg_dma_address(map->table->sgl);
656 map->phys += ((u64)fl->sctx->sid << 32);
657 map->size = len;
658 map->va = sg_virt(map->table->sgl);
659 map->len = len;
660 kref_init(&map->refcount);
661
662 spin_lock(&fl->lock);
663 list_add_tail(&map->node, &fl->maps);
664 spin_unlock(&fl->lock);
665 *ppmap = map;
666
667 return 0;
668
669map_err:
670 dma_buf_detach(map->buf, map->attach);
671attach_err:
672 dma_buf_put(map->buf);
673get_err:
674 kfree(map);
675
676 return err;
677}
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
707{
708 int size = 0;
709
710 size = (sizeof(struct fastrpc_remote_arg) +
711 sizeof(struct fastrpc_invoke_buf) +
712 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
713 sizeof(u64) * FASTRPC_MAX_FDLIST +
714 sizeof(u32) * FASTRPC_MAX_CRCLIST;
715
716 return size;
717}
718
719static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
720{
721 u64 size = 0;
722 int i;
723
724 size = ALIGN(metalen, FASTRPC_ALIGN);
725 for (i = 0; i < ctx->nscalars; i++) {
726 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
727
728 if (ctx->olaps[i].offset == 0)
729 size = ALIGN(size, FASTRPC_ALIGN);
730
731 size += (ctx->olaps[i].mend - ctx->olaps[i].mstart);
732 }
733 }
734
735 return size;
736}
737
738static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
739{
740 struct device *dev = ctx->fl->sctx->dev;
741 int i, err;
742
743 for (i = 0; i < ctx->nscalars; ++i) {
744
745 if (ctx->args[i].reserved)
746 return -EINVAL;
747
748 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
749 ctx->args[i].length == 0)
750 continue;
751
752 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
753 ctx->args[i].length, &ctx->maps[i]);
754 if (err) {
755 dev_err(dev, "Error Creating map %d\n", err);
756 return -EINVAL;
757 }
758
759 }
760 return 0;
761}
762
763static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
764{
765 struct device *dev = ctx->fl->sctx->dev;
766 struct fastrpc_remote_arg *rpra;
767 struct fastrpc_invoke_buf *list;
768 struct fastrpc_phy_page *pages;
769 int inbufs, i, oix, err = 0;
770 u64 len, rlen, pkt_size;
771 u64 pg_start, pg_end;
772 uintptr_t args;
773 int metalen;
774
775 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
776 metalen = fastrpc_get_meta_size(ctx);
777 pkt_size = fastrpc_get_payload_size(ctx, metalen);
778
779 err = fastrpc_create_maps(ctx);
780 if (err)
781 return err;
782
783 ctx->msg_sz = pkt_size;
784
785 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
786 if (err)
787 return err;
788
789 rpra = ctx->buf->virt;
790 list = ctx->buf->virt + ctx->nscalars * sizeof(*rpra);
791 pages = ctx->buf->virt + ctx->nscalars * (sizeof(*list) +
792 sizeof(*rpra));
793 args = (uintptr_t)ctx->buf->virt + metalen;
794 rlen = pkt_size - metalen;
795 ctx->rpra = rpra;
796
797 for (oix = 0; oix < ctx->nbufs; ++oix) {
798 int mlen;
799
800 i = ctx->olaps[oix].raix;
801 len = ctx->args[i].length;
802
803 rpra[i].pv = 0;
804 rpra[i].len = len;
805 list[i].num = len ? 1 : 0;
806 list[i].pgidx = i;
807
808 if (!len)
809 continue;
810
811 if (ctx->maps[i]) {
812 struct vm_area_struct *vma = NULL;
813
814 rpra[i].pv = (u64) ctx->args[i].ptr;
815 pages[i].addr = ctx->maps[i]->phys;
816
817 mmap_read_lock(current->mm);
818 vma = find_vma(current->mm, ctx->args[i].ptr);
819 if (vma)
820 pages[i].addr += ctx->args[i].ptr -
821 vma->vm_start;
822 mmap_read_unlock(current->mm);
823
824 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
825 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
826 PAGE_SHIFT;
827 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
828
829 } else {
830
831 if (ctx->olaps[oix].offset == 0) {
832 rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
833 args = ALIGN(args, FASTRPC_ALIGN);
834 }
835
836 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
837
838 if (rlen < mlen)
839 goto bail;
840
841 rpra[i].pv = args - ctx->olaps[oix].offset;
842 pages[i].addr = ctx->buf->phys -
843 ctx->olaps[oix].offset +
844 (pkt_size - rlen);
845 pages[i].addr = pages[i].addr & PAGE_MASK;
846
847 pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
848 pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
849 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
850 args = args + mlen;
851 rlen -= mlen;
852 }
853
854 if (i < inbufs && !ctx->maps[i]) {
855 void *dst = (void *)(uintptr_t)rpra[i].pv;
856 void *src = (void *)(uintptr_t)ctx->args[i].ptr;
857
858 if (!kernel) {
859 if (copy_from_user(dst, (void __user *)src,
860 len)) {
861 err = -EFAULT;
862 goto bail;
863 }
864 } else {
865 memcpy(dst, src, len);
866 }
867 }
868 }
869
870 for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
871 rpra[i].pv = (u64) ctx->args[i].ptr;
872 rpra[i].len = ctx->args[i].length;
873 list[i].num = ctx->args[i].length ? 1 : 0;
874 list[i].pgidx = i;
875 pages[i].addr = ctx->maps[i]->phys;
876 pages[i].size = ctx->maps[i]->size;
877 }
878
879bail:
880 if (err)
881 dev_err(dev, "Error: get invoke args failed:%d\n", err);
882
883 return err;
884}
885
886static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
887 u32 kernel)
888{
889 struct fastrpc_remote_arg *rpra = ctx->rpra;
890 int i, inbufs;
891
892 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
893
894 for (i = inbufs; i < ctx->nbufs; ++i) {
895 void *src = (void *)(uintptr_t)rpra[i].pv;
896 void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
897 u64 len = rpra[i].len;
898
899 if (!kernel) {
900 if (copy_to_user((void __user *)dst, src, len))
901 return -EFAULT;
902 } else {
903 memcpy(dst, src, len);
904 }
905 }
906
907 return 0;
908}
909
910static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
911 struct fastrpc_invoke_ctx *ctx,
912 u32 kernel, uint32_t handle)
913{
914 struct fastrpc_channel_ctx *cctx;
915 struct fastrpc_user *fl = ctx->fl;
916 struct fastrpc_msg *msg = &ctx->msg;
917 int ret;
918
919 cctx = fl->cctx;
920 msg->pid = fl->tgid;
921 msg->tid = current->pid;
922
923 if (kernel)
924 msg->pid = 0;
925
926 msg->ctx = ctx->ctxid | fl->pd;
927 msg->handle = handle;
928 msg->sc = ctx->sc;
929 msg->addr = ctx->buf ? ctx->buf->phys : 0;
930 msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
931 fastrpc_context_get(ctx);
932
933 ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
934
935 if (ret)
936 fastrpc_context_put(ctx);
937
938 return ret;
939
940}
941
942static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
943 u32 handle, u32 sc,
944 struct fastrpc_invoke_args *args)
945{
946 struct fastrpc_invoke_ctx *ctx = NULL;
947 int err = 0;
948
949 if (!fl->sctx)
950 return -EINVAL;
951
952 if (!fl->cctx->rpdev)
953 return -EPIPE;
954
955 if (handle == FASTRPC_INIT_HANDLE && !kernel) {
956 dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);
957 return -EPERM;
958 }
959
960 ctx = fastrpc_context_alloc(fl, kernel, sc, args);
961 if (IS_ERR(ctx))
962 return PTR_ERR(ctx);
963
964 if (ctx->nscalars) {
965 err = fastrpc_get_args(kernel, ctx);
966 if (err)
967 goto bail;
968 }
969
970
971 dma_wmb();
972
973 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
974 if (err)
975 goto bail;
976
977 if (kernel) {
978 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
979 err = -ETIMEDOUT;
980 } else {
981 err = wait_for_completion_interruptible(&ctx->work);
982 }
983
984 if (err)
985 goto bail;
986
987
988 err = ctx->retval;
989 if (err)
990 goto bail;
991
992 if (ctx->nscalars) {
993
994 dma_rmb();
995
996 err = fastrpc_put_args(ctx, kernel);
997 if (err)
998 goto bail;
999 }
1000
1001bail:
1002 if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1003
1004 spin_lock(&fl->lock);
1005 list_del(&ctx->node);
1006 spin_unlock(&fl->lock);
1007 fastrpc_context_put(ctx);
1008 }
1009 if (err)
1010 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1011
1012 return err;
1013}
1014
1015static int fastrpc_init_create_process(struct fastrpc_user *fl,
1016 char __user *argp)
1017{
1018 struct fastrpc_init_create init;
1019 struct fastrpc_invoke_args *args;
1020 struct fastrpc_phy_page pages[1];
1021 struct fastrpc_map *map = NULL;
1022 struct fastrpc_buf *imem = NULL;
1023 int memlen;
1024 int err;
1025 struct {
1026 int pgid;
1027 u32 namelen;
1028 u32 filelen;
1029 u32 pageslen;
1030 u32 attrs;
1031 u32 siglen;
1032 } inbuf;
1033 u32 sc;
1034
1035 args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1036 if (!args)
1037 return -ENOMEM;
1038
1039 if (copy_from_user(&init, argp, sizeof(init))) {
1040 err = -EFAULT;
1041 goto err;
1042 }
1043
1044 if (init.filelen > INIT_FILELEN_MAX) {
1045 err = -EINVAL;
1046 goto err;
1047 }
1048
1049 inbuf.pgid = fl->tgid;
1050 inbuf.namelen = strlen(current->comm) + 1;
1051 inbuf.filelen = init.filelen;
1052 inbuf.pageslen = 1;
1053 inbuf.attrs = init.attrs;
1054 inbuf.siglen = init.siglen;
1055 fl->pd = USER_PD;
1056
1057 if (init.filelen && init.filefd) {
1058 err = fastrpc_map_create(fl, init.filefd, init.filelen, &map);
1059 if (err)
1060 goto err;
1061 }
1062
1063 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1064 1024 * 1024);
1065 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1066 &imem);
1067 if (err)
1068 goto err_alloc;
1069
1070 fl->init_mem = imem;
1071 args[0].ptr = (u64)(uintptr_t)&inbuf;
1072 args[0].length = sizeof(inbuf);
1073 args[0].fd = -1;
1074
1075 args[1].ptr = (u64)(uintptr_t)current->comm;
1076 args[1].length = inbuf.namelen;
1077 args[1].fd = -1;
1078
1079 args[2].ptr = (u64) init.file;
1080 args[2].length = inbuf.filelen;
1081 args[2].fd = init.filefd;
1082
1083 pages[0].addr = imem->phys;
1084 pages[0].size = imem->size;
1085
1086 args[3].ptr = (u64)(uintptr_t) pages;
1087 args[3].length = 1 * sizeof(*pages);
1088 args[3].fd = -1;
1089
1090 args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1091 args[4].length = sizeof(inbuf.attrs);
1092 args[4].fd = -1;
1093
1094 args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1095 args[5].length = sizeof(inbuf.siglen);
1096 args[5].fd = -1;
1097
1098 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1099 if (init.attrs)
1100 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 6, 0);
1101
1102 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1103 sc, args);
1104 if (err)
1105 goto err_invoke;
1106
1107 kfree(args);
1108
1109 return 0;
1110
1111err_invoke:
1112 fl->init_mem = NULL;
1113 fastrpc_buf_free(imem);
1114err_alloc:
1115 if (map) {
1116 spin_lock(&fl->lock);
1117 list_del(&map->node);
1118 spin_unlock(&fl->lock);
1119 fastrpc_map_put(map);
1120 }
1121err:
1122 kfree(args);
1123
1124 return err;
1125}
1126
1127static struct fastrpc_session_ctx *fastrpc_session_alloc(
1128 struct fastrpc_channel_ctx *cctx)
1129{
1130 struct fastrpc_session_ctx *session = NULL;
1131 unsigned long flags;
1132 int i;
1133
1134 spin_lock_irqsave(&cctx->lock, flags);
1135 for (i = 0; i < cctx->sesscount; i++) {
1136 if (!cctx->session[i].used && cctx->session[i].valid) {
1137 cctx->session[i].used = true;
1138 session = &cctx->session[i];
1139 break;
1140 }
1141 }
1142 spin_unlock_irqrestore(&cctx->lock, flags);
1143
1144 return session;
1145}
1146
1147static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1148 struct fastrpc_session_ctx *session)
1149{
1150 unsigned long flags;
1151
1152 spin_lock_irqsave(&cctx->lock, flags);
1153 session->used = false;
1154 spin_unlock_irqrestore(&cctx->lock, flags);
1155}
1156
1157static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1158{
1159 struct fastrpc_invoke_args args[1];
1160 int tgid = 0;
1161 u32 sc;
1162
1163 tgid = fl->tgid;
1164 args[0].ptr = (u64)(uintptr_t) &tgid;
1165 args[0].length = sizeof(tgid);
1166 args[0].fd = -1;
1167 args[0].reserved = 0;
1168 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1169
1170 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1171 sc, &args[0]);
1172}
1173
1174static int fastrpc_device_release(struct inode *inode, struct file *file)
1175{
1176 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1177 struct fastrpc_channel_ctx *cctx = fl->cctx;
1178 struct fastrpc_invoke_ctx *ctx, *n;
1179 struct fastrpc_map *map, *m;
1180 struct fastrpc_buf *buf, *b;
1181 unsigned long flags;
1182
1183 fastrpc_release_current_dsp_process(fl);
1184
1185 spin_lock_irqsave(&cctx->lock, flags);
1186 list_del(&fl->user);
1187 spin_unlock_irqrestore(&cctx->lock, flags);
1188
1189 if (fl->init_mem)
1190 fastrpc_buf_free(fl->init_mem);
1191
1192 list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1193 list_del(&ctx->node);
1194 fastrpc_context_put(ctx);
1195 }
1196
1197 list_for_each_entry_safe(map, m, &fl->maps, node) {
1198 list_del(&map->node);
1199 fastrpc_map_put(map);
1200 }
1201
1202 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1203 list_del(&buf->node);
1204 fastrpc_buf_free(buf);
1205 }
1206
1207 fastrpc_session_free(cctx, fl->sctx);
1208 fastrpc_channel_ctx_put(cctx);
1209
1210 mutex_destroy(&fl->mutex);
1211 kfree(fl);
1212 file->private_data = NULL;
1213
1214 return 0;
1215}
1216
1217static int fastrpc_device_open(struct inode *inode, struct file *filp)
1218{
1219 struct fastrpc_channel_ctx *cctx = miscdev_to_cctx(filp->private_data);
1220 struct fastrpc_user *fl = NULL;
1221 unsigned long flags;
1222
1223 fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1224 if (!fl)
1225 return -ENOMEM;
1226
1227
1228 fastrpc_channel_ctx_get(cctx);
1229
1230 filp->private_data = fl;
1231 spin_lock_init(&fl->lock);
1232 mutex_init(&fl->mutex);
1233 INIT_LIST_HEAD(&fl->pending);
1234 INIT_LIST_HEAD(&fl->maps);
1235 INIT_LIST_HEAD(&fl->mmaps);
1236 INIT_LIST_HEAD(&fl->user);
1237 fl->tgid = current->tgid;
1238 fl->cctx = cctx;
1239
1240 fl->sctx = fastrpc_session_alloc(cctx);
1241 if (!fl->sctx) {
1242 dev_err(&cctx->rpdev->dev, "No session available\n");
1243 mutex_destroy(&fl->mutex);
1244 kfree(fl);
1245
1246 return -EBUSY;
1247 }
1248
1249 spin_lock_irqsave(&cctx->lock, flags);
1250 list_add_tail(&fl->user, &cctx->users);
1251 spin_unlock_irqrestore(&cctx->lock, flags);
1252
1253 return 0;
1254}
1255
1256static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1257{
1258 struct fastrpc_alloc_dma_buf bp;
1259 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1260 struct fastrpc_buf *buf = NULL;
1261 int err;
1262
1263 if (copy_from_user(&bp, argp, sizeof(bp)))
1264 return -EFAULT;
1265
1266 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1267 if (err)
1268 return err;
1269 exp_info.ops = &fastrpc_dma_buf_ops;
1270 exp_info.size = bp.size;
1271 exp_info.flags = O_RDWR;
1272 exp_info.priv = buf;
1273 buf->dmabuf = dma_buf_export(&exp_info);
1274 if (IS_ERR(buf->dmabuf)) {
1275 err = PTR_ERR(buf->dmabuf);
1276 fastrpc_buf_free(buf);
1277 return err;
1278 }
1279
1280 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1281 if (bp.fd < 0) {
1282 dma_buf_put(buf->dmabuf);
1283 return -EINVAL;
1284 }
1285
1286 if (copy_to_user(argp, &bp, sizeof(bp))) {
1287 dma_buf_put(buf->dmabuf);
1288 return -EFAULT;
1289 }
1290
1291 return 0;
1292}
1293
1294static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1295{
1296 struct fastrpc_invoke_args args[1];
1297 int tgid = fl->tgid;
1298 u32 sc;
1299
1300 args[0].ptr = (u64)(uintptr_t) &tgid;
1301 args[0].length = sizeof(tgid);
1302 args[0].fd = -1;
1303 args[0].reserved = 0;
1304 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1305 fl->pd = pd;
1306
1307 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1308 sc, &args[0]);
1309}
1310
1311static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1312{
1313 struct fastrpc_invoke_args *args = NULL;
1314 struct fastrpc_invoke inv;
1315 u32 nscalars;
1316 int err;
1317
1318 if (copy_from_user(&inv, argp, sizeof(inv)))
1319 return -EFAULT;
1320
1321
1322 nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1323 if (nscalars) {
1324 args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1325 if (!args)
1326 return -ENOMEM;
1327
1328 if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1329 nscalars * sizeof(*args))) {
1330 kfree(args);
1331 return -EFAULT;
1332 }
1333 }
1334
1335 err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1336 kfree(args);
1337
1338 return err;
1339}
1340
1341static int fastrpc_req_munmap_impl(struct fastrpc_user *fl,
1342 struct fastrpc_req_munmap *req)
1343{
1344 struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1345 struct fastrpc_buf *buf, *b;
1346 struct fastrpc_munmap_req_msg req_msg;
1347 struct device *dev = fl->sctx->dev;
1348 int err;
1349 u32 sc;
1350
1351 spin_lock(&fl->lock);
1352 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1353 if ((buf->raddr == req->vaddrout) && (buf->size == req->size))
1354 break;
1355 buf = NULL;
1356 }
1357 spin_unlock(&fl->lock);
1358
1359 if (!buf) {
1360 dev_err(dev, "mmap not in list\n");
1361 return -EINVAL;
1362 }
1363
1364 req_msg.pgid = fl->tgid;
1365 req_msg.size = buf->size;
1366 req_msg.vaddr = buf->raddr;
1367
1368 args[0].ptr = (u64) (uintptr_t) &req_msg;
1369 args[0].length = sizeof(req_msg);
1370
1371 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1372 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1373 &args[0]);
1374 if (!err) {
1375 dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1376 spin_lock(&fl->lock);
1377 list_del(&buf->node);
1378 spin_unlock(&fl->lock);
1379 fastrpc_buf_free(buf);
1380 } else {
1381 dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1382 }
1383
1384 return err;
1385}
1386
1387static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1388{
1389 struct fastrpc_req_munmap req;
1390
1391 if (copy_from_user(&req, argp, sizeof(req)))
1392 return -EFAULT;
1393
1394 return fastrpc_req_munmap_impl(fl, &req);
1395}
1396
1397static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1398{
1399 struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1400 struct fastrpc_buf *buf = NULL;
1401 struct fastrpc_mmap_req_msg req_msg;
1402 struct fastrpc_mmap_rsp_msg rsp_msg;
1403 struct fastrpc_req_munmap req_unmap;
1404 struct fastrpc_phy_page pages;
1405 struct fastrpc_req_mmap req;
1406 struct device *dev = fl->sctx->dev;
1407 int err;
1408 u32 sc;
1409
1410 if (copy_from_user(&req, argp, sizeof(req)))
1411 return -EFAULT;
1412
1413 if (req.flags != ADSP_MMAP_ADD_PAGES) {
1414 dev_err(dev, "flag not supported 0x%x\n", req.flags);
1415 return -EINVAL;
1416 }
1417
1418 if (req.vaddrin) {
1419 dev_err(dev, "adding user allocated pages is not supported\n");
1420 return -EINVAL;
1421 }
1422
1423 err = fastrpc_buf_alloc(fl, fl->sctx->dev, req.size, &buf);
1424 if (err) {
1425 dev_err(dev, "failed to allocate buffer\n");
1426 return err;
1427 }
1428
1429 req_msg.pgid = fl->tgid;
1430 req_msg.flags = req.flags;
1431 req_msg.vaddr = req.vaddrin;
1432 req_msg.num = sizeof(pages);
1433
1434 args[0].ptr = (u64) (uintptr_t) &req_msg;
1435 args[0].length = sizeof(req_msg);
1436
1437 pages.addr = buf->phys;
1438 pages.size = buf->size;
1439
1440 args[1].ptr = (u64) (uintptr_t) &pages;
1441 args[1].length = sizeof(pages);
1442
1443 args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1444 args[2].length = sizeof(rsp_msg);
1445
1446 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1447 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1448 &args[0]);
1449 if (err) {
1450 dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1451 goto err_invoke;
1452 }
1453
1454
1455 buf->raddr = (uintptr_t) rsp_msg.vaddr;
1456
1457
1458 req.vaddrout = rsp_msg.vaddr;
1459
1460 spin_lock(&fl->lock);
1461 list_add_tail(&buf->node, &fl->mmaps);
1462 spin_unlock(&fl->lock);
1463
1464 if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
1465
1466 req_unmap.vaddrout = buf->raddr;
1467 req_unmap.size = buf->size;
1468 fastrpc_req_munmap_impl(fl, &req_unmap);
1469 return -EFAULT;
1470 }
1471
1472 dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1473 buf->raddr, buf->size);
1474
1475 return 0;
1476
1477err_invoke:
1478 fastrpc_buf_free(buf);
1479
1480 return err;
1481}
1482
1483static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
1484 unsigned long arg)
1485{
1486 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1487 char __user *argp = (char __user *)arg;
1488 int err;
1489
1490 switch (cmd) {
1491 case FASTRPC_IOCTL_INVOKE:
1492 err = fastrpc_invoke(fl, argp);
1493 break;
1494 case FASTRPC_IOCTL_INIT_ATTACH:
1495 err = fastrpc_init_attach(fl, AUDIO_PD);
1496 break;
1497 case FASTRPC_IOCTL_INIT_ATTACH_SNS:
1498 err = fastrpc_init_attach(fl, SENSORS_PD);
1499 break;
1500 case FASTRPC_IOCTL_INIT_CREATE:
1501 err = fastrpc_init_create_process(fl, argp);
1502 break;
1503 case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
1504 err = fastrpc_dmabuf_alloc(fl, argp);
1505 break;
1506 case FASTRPC_IOCTL_MMAP:
1507 err = fastrpc_req_mmap(fl, argp);
1508 break;
1509 case FASTRPC_IOCTL_MUNMAP:
1510 err = fastrpc_req_munmap(fl, argp);
1511 break;
1512 default:
1513 err = -ENOTTY;
1514 break;
1515 }
1516
1517 return err;
1518}
1519
1520static const struct file_operations fastrpc_fops = {
1521 .open = fastrpc_device_open,
1522 .release = fastrpc_device_release,
1523 .unlocked_ioctl = fastrpc_device_ioctl,
1524 .compat_ioctl = fastrpc_device_ioctl,
1525};
1526
1527static int fastrpc_cb_probe(struct platform_device *pdev)
1528{
1529 struct fastrpc_channel_ctx *cctx;
1530 struct fastrpc_session_ctx *sess;
1531 struct device *dev = &pdev->dev;
1532 int i, sessions = 0;
1533 unsigned long flags;
1534 int rc;
1535
1536 cctx = dev_get_drvdata(dev->parent);
1537 if (!cctx)
1538 return -EINVAL;
1539
1540 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
1541
1542 spin_lock_irqsave(&cctx->lock, flags);
1543 sess = &cctx->session[cctx->sesscount];
1544 sess->used = false;
1545 sess->valid = true;
1546 sess->dev = dev;
1547 dev_set_drvdata(dev, sess);
1548
1549 if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
1550 dev_info(dev, "FastRPC Session ID not specified in DT\n");
1551
1552 if (sessions > 0) {
1553 struct fastrpc_session_ctx *dup_sess;
1554
1555 for (i = 1; i < sessions; i++) {
1556 if (cctx->sesscount++ >= FASTRPC_MAX_SESSIONS)
1557 break;
1558 dup_sess = &cctx->session[cctx->sesscount];
1559 memcpy(dup_sess, sess, sizeof(*dup_sess));
1560 }
1561 }
1562 cctx->sesscount++;
1563 spin_unlock_irqrestore(&cctx->lock, flags);
1564 rc = dma_set_mask(dev, DMA_BIT_MASK(32));
1565 if (rc) {
1566 dev_err(dev, "32-bit DMA enable failed\n");
1567 return rc;
1568 }
1569
1570 return 0;
1571}
1572
1573static int fastrpc_cb_remove(struct platform_device *pdev)
1574{
1575 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
1576 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
1577 unsigned long flags;
1578 int i;
1579
1580 spin_lock_irqsave(&cctx->lock, flags);
1581 for (i = 1; i < FASTRPC_MAX_SESSIONS; i++) {
1582 if (cctx->session[i].sid == sess->sid) {
1583 cctx->session[i].valid = false;
1584 cctx->sesscount--;
1585 }
1586 }
1587 spin_unlock_irqrestore(&cctx->lock, flags);
1588
1589 return 0;
1590}
1591
1592static const struct of_device_id fastrpc_match_table[] = {
1593 { .compatible = "qcom,fastrpc-compute-cb", },
1594 {}
1595};
1596
1597static struct platform_driver fastrpc_cb_driver = {
1598 .probe = fastrpc_cb_probe,
1599 .remove = fastrpc_cb_remove,
1600 .driver = {
1601 .name = "qcom,fastrpc-cb",
1602 .of_match_table = fastrpc_match_table,
1603 .suppress_bind_attrs = true,
1604 },
1605};
1606
1607static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
1608{
1609 struct device *rdev = &rpdev->dev;
1610 struct fastrpc_channel_ctx *data;
1611 int i, err, domain_id = -1;
1612 const char *domain;
1613
1614 err = of_property_read_string(rdev->of_node, "label", &domain);
1615 if (err) {
1616 dev_info(rdev, "FastRPC Domain not specified in DT\n");
1617 return err;
1618 }
1619
1620 for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
1621 if (!strcmp(domains[i], domain)) {
1622 domain_id = i;
1623 break;
1624 }
1625 }
1626
1627 if (domain_id < 0) {
1628 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
1629 return -EINVAL;
1630 }
1631
1632 data = kzalloc(sizeof(*data), GFP_KERNEL);
1633 if (!data)
1634 return -ENOMEM;
1635
1636 data->miscdev.minor = MISC_DYNAMIC_MINOR;
1637 data->miscdev.name = devm_kasprintf(rdev, GFP_KERNEL, "fastrpc-%s",
1638 domains[domain_id]);
1639 data->miscdev.fops = &fastrpc_fops;
1640 err = misc_register(&data->miscdev);
1641 if (err) {
1642 kfree(data);
1643 return err;
1644 }
1645
1646 kref_init(&data->refcount);
1647
1648 dev_set_drvdata(&rpdev->dev, data);
1649 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
1650 INIT_LIST_HEAD(&data->users);
1651 spin_lock_init(&data->lock);
1652 idr_init(&data->ctx_idr);
1653 data->domain_id = domain_id;
1654 data->rpdev = rpdev;
1655
1656 return of_platform_populate(rdev->of_node, NULL, NULL, rdev);
1657}
1658
1659static void fastrpc_notify_users(struct fastrpc_user *user)
1660{
1661 struct fastrpc_invoke_ctx *ctx;
1662
1663 spin_lock(&user->lock);
1664 list_for_each_entry(ctx, &user->pending, node)
1665 complete(&ctx->work);
1666 spin_unlock(&user->lock);
1667}
1668
1669static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
1670{
1671 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1672 struct fastrpc_user *user;
1673 unsigned long flags;
1674
1675 spin_lock_irqsave(&cctx->lock, flags);
1676 list_for_each_entry(user, &cctx->users, user)
1677 fastrpc_notify_users(user);
1678 spin_unlock_irqrestore(&cctx->lock, flags);
1679
1680 misc_deregister(&cctx->miscdev);
1681 of_platform_depopulate(&rpdev->dev);
1682
1683 cctx->rpdev = NULL;
1684 fastrpc_channel_ctx_put(cctx);
1685}
1686
1687static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
1688 int len, void *priv, u32 addr)
1689{
1690 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1691 struct fastrpc_invoke_rsp *rsp = data;
1692 struct fastrpc_invoke_ctx *ctx;
1693 unsigned long flags;
1694 unsigned long ctxid;
1695
1696 if (len < sizeof(*rsp))
1697 return -EINVAL;
1698
1699 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
1700
1701 spin_lock_irqsave(&cctx->lock, flags);
1702 ctx = idr_find(&cctx->ctx_idr, ctxid);
1703 spin_unlock_irqrestore(&cctx->lock, flags);
1704
1705 if (!ctx) {
1706 dev_err(&rpdev->dev, "No context ID matches response\n");
1707 return -ENOENT;
1708 }
1709
1710 ctx->retval = rsp->retval;
1711 complete(&ctx->work);
1712
1713
1714
1715
1716
1717
1718 schedule_work(&ctx->put_work);
1719
1720 return 0;
1721}
1722
1723static const struct of_device_id fastrpc_rpmsg_of_match[] = {
1724 { .compatible = "qcom,fastrpc" },
1725 { },
1726};
1727MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
1728
1729static struct rpmsg_driver fastrpc_driver = {
1730 .probe = fastrpc_rpmsg_probe,
1731 .remove = fastrpc_rpmsg_remove,
1732 .callback = fastrpc_rpmsg_callback,
1733 .drv = {
1734 .name = "qcom,fastrpc",
1735 .of_match_table = fastrpc_rpmsg_of_match,
1736 },
1737};
1738
1739static int fastrpc_init(void)
1740{
1741 int ret;
1742
1743 ret = platform_driver_register(&fastrpc_cb_driver);
1744 if (ret < 0) {
1745 pr_err("fastrpc: failed to register cb driver\n");
1746 return ret;
1747 }
1748
1749 ret = register_rpmsg_driver(&fastrpc_driver);
1750 if (ret < 0) {
1751 pr_err("fastrpc: failed to register rpmsg driver\n");
1752 platform_driver_unregister(&fastrpc_cb_driver);
1753 return ret;
1754 }
1755
1756 return 0;
1757}
1758module_init(fastrpc_init);
1759
1760static void fastrpc_exit(void)
1761{
1762 platform_driver_unregister(&fastrpc_cb_driver);
1763 unregister_rpmsg_driver(&fastrpc_driver);
1764}
1765module_exit(fastrpc_exit);
1766
1767MODULE_LICENSE("GPL v2");
1768
