1
2
3
4#include <linux/dma-fence-array.h>
5#include <linux/dma-mapping.h>
6#include <linux/file.h>
7#include <linux/host1x.h>
8#include <linux/iommu.h>
9#include <linux/kref.h>
10#include <linux/list.h>
11#include <linux/nospec.h>
12#include <linux/pm_runtime.h>
13#include <linux/scatterlist.h>
14#include <linux/slab.h>
15#include <linux/sync_file.h>
16
17#include <drm/drm_drv.h>
18#include <drm/drm_file.h>
19#include <drm/drm_syncobj.h>
20
21#include "drm.h"
22#include "gem.h"
23#include "submit.h"
24#include "uapi.h"
25
26#define SUBMIT_ERR(context, fmt, ...) \
27 dev_err_ratelimited(context->client->base.dev, \
28 "%s: job submission failed: " fmt "\n", \
29 current->comm, ##__VA_ARGS__)
30
31struct gather_bo {
32 struct host1x_bo base;
33
34 struct kref ref;
35
36 struct device *dev;
37 u32 *gather_data;
38 dma_addr_t gather_data_dma;
39 size_t gather_data_words;
40};
41
42static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo)
43{
44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
45
46 kref_get(&bo->ref);
47
48 return host_bo;
49}
50
51static void gather_bo_release(struct kref *ref)
52{
53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref);
54
55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma,
56 0);
57 kfree(bo);
58}
59
60static void gather_bo_put(struct host1x_bo *host_bo)
61{
62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
63
64 kref_put(&bo->ref, gather_bo_release);
65}
66
67static struct host1x_bo_mapping *
68gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction)
69{
70 struct gather_bo *gather = container_of(bo, struct gather_bo, base);
71 struct host1x_bo_mapping *map;
72 int err;
73
74 map = kzalloc(sizeof(*map), GFP_KERNEL);
75 if (!map)
76 return ERR_PTR(-ENOMEM);
77
78 kref_init(&map->ref);
79 map->bo = host1x_bo_get(bo);
80 map->direction = direction;
81 map->dev = dev;
82
83 map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL);
84 if (!map->sgt) {
85 err = -ENOMEM;
86 goto free;
87 }
88
89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma,
90 gather->gather_data_words * 4);
91 if (err)
92 goto free_sgt;
93
94 err = dma_map_sgtable(dev, map->sgt, direction, 0);
95 if (err)
96 goto free_sgt;
97
98 map->phys = sg_dma_address(map->sgt->sgl);
99 map->size = gather->gather_data_words * 4;
100 map->chunks = err;
101
102 return map;
103
104free_sgt:
105 sg_free_table(map->sgt);
106 kfree(map->sgt);
107free:
108 kfree(map);
109 return ERR_PTR(err);
110}
111
112static void gather_bo_unpin(struct host1x_bo_mapping *map)
113{
114 if (!map)
115 return;
116
117 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);
118 sg_free_table(map->sgt);
119 kfree(map->sgt);
120 host1x_bo_put(map->bo);
121
122 kfree(map);
123}
124
125static void *gather_bo_mmap(struct host1x_bo *host_bo)
126{
127 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
128
129 return bo->gather_data;
130}
131
132static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr)
133{
134}
135
136const struct host1x_bo_ops gather_bo_ops = {
137 .get = gather_bo_get,
138 .put = gather_bo_put,
139 .pin = gather_bo_pin,
140 .unpin = gather_bo_unpin,
141 .mmap = gather_bo_mmap,
142 .munmap = gather_bo_munmap,
143};
144
145static struct tegra_drm_mapping *
146tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id)
147{
148 struct tegra_drm_mapping *mapping;
149
150 xa_lock(&context->mappings);
151
152 mapping = xa_load(&context->mappings, id);
153 if (mapping)
154 kref_get(&mapping->ref);
155
156 xa_unlock(&context->mappings);
157
158 return mapping;
159}
160
161static void *alloc_copy_user_array(void __user *from, size_t count, size_t size)
162{
163 size_t copy_len;
164 void *data;
165
166 if (check_mul_overflow(count, size, ©_len))
167 return ERR_PTR(-EINVAL);
168
169 if (copy_len > 0x4000)
170 return ERR_PTR(-E2BIG);
171
172 data = kvmalloc(copy_len, GFP_KERNEL);
173 if (!data)
174 return ERR_PTR(-ENOMEM);
175
176 if (copy_from_user(data, from, copy_len)) {
177 kvfree(data);
178 return ERR_PTR(-EFAULT);
179 }
180
181 return data;
182}
183
184static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev,
185 struct tegra_drm_context *context,
186 struct drm_tegra_channel_submit *args)
187{
188 struct gather_bo *bo;
189 size_t copy_len;
190
191 if (args->gather_data_words == 0) {
192 SUBMIT_ERR(context, "gather_data_words cannot be zero");
193 return -EINVAL;
194 }
195
196 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) {
197 SUBMIT_ERR(context, "gather_data_words is too large");
198 return -EINVAL;
199 }
200
201 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
202 if (!bo) {
203 SUBMIT_ERR(context, "failed to allocate memory for bo info");
204 return -ENOMEM;
205 }
206
207 host1x_bo_init(&bo->base, &gather_bo_ops);
208 kref_init(&bo->ref);
209 bo->dev = dev;
210
211 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma,
212 GFP_KERNEL | __GFP_NOWARN, 0);
213 if (!bo->gather_data) {
214 SUBMIT_ERR(context, "failed to allocate memory for gather data");
215 kfree(bo);
216 return -ENOMEM;
217 }
218
219 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) {
220 SUBMIT_ERR(context, "failed to copy gather data from userspace");
221 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0);
222 kfree(bo);
223 return -EFAULT;
224 }
225
226 bo->gather_data_words = args->gather_data_words;
227
228 *pbo = bo;
229
230 return 0;
231}
232
233static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo,
234 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping)
235{
236
237 dma_addr_t iova = mapping->iova + buf->reloc.target_offset;
238 u32 written_ptr;
239
240#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
241 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT)
242 iova |= BIT_ULL(39);
243#endif
244
245 written_ptr = iova >> buf->reloc.shift;
246
247 if (buf->reloc.gather_offset_words >= bo->gather_data_words) {
248 SUBMIT_ERR(context,
249 "relocation has too large gather offset (%u vs gather length %zu)",
250 buf->reloc.gather_offset_words, bo->gather_data_words);
251 return -EINVAL;
252 }
253
254 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words,
255 bo->gather_data_words);
256
257 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;
258
259 return 0;
260}
261
262static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo,
263 struct drm_tegra_channel_submit *args,
264 struct tegra_drm_submit_data *job_data)
265{
266 struct tegra_drm_used_mapping *mappings;
267 struct drm_tegra_submit_buf *bufs;
268 int err;
269 u32 i;
270
271 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs,
272 sizeof(*bufs));
273 if (IS_ERR(bufs)) {
274 SUBMIT_ERR(context, "failed to copy bufs array from userspace");
275 return PTR_ERR(bufs);
276 }
277
278 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL);
279 if (!mappings) {
280 SUBMIT_ERR(context, "failed to allocate memory for mapping info");
281 err = -ENOMEM;
282 goto done;
283 }
284
285 for (i = 0; i < args->num_bufs; i++) {
286 struct drm_tegra_submit_buf *buf = &bufs[i];
287 struct tegra_drm_mapping *mapping;
288
289 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) {
290 SUBMIT_ERR(context, "invalid flag specified for buffer");
291 err = -EINVAL;
292 goto drop_refs;
293 }
294
295 mapping = tegra_drm_mapping_get(context, buf->mapping);
296 if (!mapping) {
297 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping);
298 err = -EINVAL;
299 goto drop_refs;
300 }
301
302 err = submit_write_reloc(context, bo, buf, mapping);
303 if (err) {
304 tegra_drm_mapping_put(mapping);
305 goto drop_refs;
306 }
307
308 mappings[i].mapping = mapping;
309 mappings[i].flags = buf->flags;
310 }
311
312 job_data->used_mappings = mappings;
313 job_data->num_used_mappings = i;
314
315 err = 0;
316
317 goto done;
318
319drop_refs:
320 while (i--)
321 tegra_drm_mapping_put(mappings[i].mapping);
322
323 kfree(mappings);
324 job_data->used_mappings = NULL;
325
326done:
327 kvfree(bufs);
328
329 return err;
330}
331
332static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job,
333 struct xarray *syncpoints, struct drm_tegra_channel_submit *args)
334{
335 struct host1x_syncpt *sp;
336
337 if (args->syncpt.flags) {
338 SUBMIT_ERR(context, "invalid flag specified for syncpt");
339 return -EINVAL;
340 }
341
342
343 sp = xa_load(syncpoints, args->syncpt.id);
344 if (!sp) {
345 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated");
346 return -EINVAL;
347 }
348
349 job->syncpt = host1x_syncpt_get(sp);
350 job->syncpt_incrs = args->syncpt.increments;
351
352 return 0;
353}
354
355static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context,
356 struct drm_tegra_submit_cmd_gather_uptr *cmd,
357 struct gather_bo *bo, u32 *offset,
358 struct tegra_drm_submit_data *job_data,
359 u32 *class)
360{
361 u32 next_offset;
362
363 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) {
364 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command");
365 return -EINVAL;
366 }
367
368
369 if (cmd->words > 16383) {
370 SUBMIT_ERR(context, "too many words in GATHER_UPTR command");
371 return -EINVAL;
372 }
373
374 if (check_add_overflow(*offset, cmd->words, &next_offset)) {
375 SUBMIT_ERR(context, "too many total words in job");
376 return -EINVAL;
377 }
378
379 if (next_offset > bo->gather_data_words) {
380 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data");
381 return -EINVAL;
382 }
383
384 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset,
385 cmd->words, job_data, class)) {
386 SUBMIT_ERR(context, "job was rejected by firewall");
387 return -EINVAL;
388 }
389
390 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4);
391
392 *offset = next_offset;
393
394 return 0;
395}
396
397static struct host1x_job *
398submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo,
399 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data,
400 struct xarray *syncpoints)
401{
402 struct drm_tegra_submit_cmd *cmds;
403 u32 i, gather_offset = 0, class;
404 struct host1x_job *job;
405 int err;
406
407
408 class = context->client->base.class;
409
410 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds,
411 sizeof(*cmds));
412 if (IS_ERR(cmds)) {
413 SUBMIT_ERR(context, "failed to copy cmds array from userspace");
414 return ERR_CAST(cmds);
415 }
416
417 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true);
418 if (!job) {
419 SUBMIT_ERR(context, "failed to allocate memory for job");
420 job = ERR_PTR(-ENOMEM);
421 goto done;
422 }
423
424 err = submit_get_syncpt(context, job, syncpoints, args);
425 if (err < 0)
426 goto free_job;
427
428 job->client = &context->client->base;
429 job->class = context->client->base.class;
430 job->serialize = true;
431
432 for (i = 0; i < args->num_cmds; i++) {
433 struct drm_tegra_submit_cmd *cmd = &cmds[i];
434
435 if (cmd->flags) {
436 SUBMIT_ERR(context, "unknown flags given for cmd");
437 err = -EINVAL;
438 goto free_job;
439 }
440
441 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) {
442 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo,
443 &gather_offset, job_data, &class);
444 if (err)
445 goto free_job;
446 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) {
447 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
448 SUBMIT_ERR(context, "non-zero reserved value");
449 err = -EINVAL;
450 goto free_job;
451 }
452
453 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
454 false, class);
455 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) {
456 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
457 SUBMIT_ERR(context, "non-zero reserved value");
458 err = -EINVAL;
459 goto free_job;
460 }
461
462 if (cmd->wait_syncpt.id != args->syncpt.id) {
463 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job");
464 err = -EINVAL;
465 goto free_job;
466 }
467
468 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
469 true, class);
470 } else {
471 SUBMIT_ERR(context, "unknown cmd type");
472 err = -EINVAL;
473 goto free_job;
474 }
475 }
476
477 if (gather_offset == 0) {
478 SUBMIT_ERR(context, "job must have at least one gather");
479 err = -EINVAL;
480 goto free_job;
481 }
482
483 goto done;
484
485free_job:
486 host1x_job_put(job);
487 job = ERR_PTR(err);
488
489done:
490 kvfree(cmds);
491
492 return job;
493}
494
495static void release_job(struct host1x_job *job)
496{
497 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base);
498 struct tegra_drm_submit_data *job_data = job->user_data;
499 u32 i;
500
501 for (i = 0; i < job_data->num_used_mappings; i++)
502 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
503
504 kfree(job_data->used_mappings);
505 kfree(job_data);
506
507 pm_runtime_mark_last_busy(client->base.dev);
508 pm_runtime_put_autosuspend(client->base.dev);
509}
510
511int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,
512 struct drm_file *file)
513{
514 struct tegra_drm_file *fpriv = file->driver_priv;
515 struct drm_tegra_channel_submit *args = data;
516 struct tegra_drm_submit_data *job_data;
517 struct drm_syncobj *syncobj = NULL;
518 struct tegra_drm_context *context;
519 struct host1x_job *job;
520 struct gather_bo *bo;
521 u32 i;
522 int err;
523
524 mutex_lock(&fpriv->lock);
525
526 context = xa_load(&fpriv->contexts, args->context);
527 if (!context) {
528 mutex_unlock(&fpriv->lock);
529 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__,
530 current->comm, args->context);
531 return -EINVAL;
532 }
533
534 if (args->syncobj_in) {
535 struct dma_fence *fence;
536
537 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence);
538 if (err) {
539 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in);
540 goto unlock;
541 }
542
543 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000));
544 dma_fence_put(fence);
545 if (err) {
546 SUBMIT_ERR(context, "wait for syncobj_in timed out");
547 goto unlock;
548 }
549 }
550
551 if (args->syncobj_out) {
552 syncobj = drm_syncobj_find(file, args->syncobj_out);
553 if (!syncobj) {
554 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out);
555 err = -ENOENT;
556 goto unlock;
557 }
558 }
559
560
561 err = submit_copy_gather_data(&bo, drm->dev, context, args);
562 if (err)
563 goto unlock;
564
565 job_data = kzalloc(sizeof(*job_data), GFP_KERNEL);
566 if (!job_data) {
567 SUBMIT_ERR(context, "failed to allocate memory for job data");
568 err = -ENOMEM;
569 goto put_bo;
570 }
571
572
573 err = submit_process_bufs(context, bo, args, job_data);
574 if (err)
575 goto free_job_data;
576
577
578 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints);
579 if (IS_ERR(job)) {
580 err = PTR_ERR(job);
581 goto free_job_data;
582 }
583
584
585 err = host1x_job_pin(job, context->client->base.dev);
586 if (err) {
587 SUBMIT_ERR(context, "failed to pin job: %d", err);
588 goto put_job;
589 }
590
591
592 err = pm_runtime_resume_and_get(context->client->base.dev);
593 if (err < 0) {
594 SUBMIT_ERR(context, "could not power up engine: %d", err);
595 goto unpin_job;
596 }
597
598 job->user_data = job_data;
599 job->release = release_job;
600 job->timeout = 10000;
601
602
603
604
605
606 job_data = NULL;
607
608
609 err = host1x_job_submit(job);
610 if (err) {
611 SUBMIT_ERR(context, "host1x job submission failed: %d", err);
612 goto unpin_job;
613 }
614
615
616 args->syncpt.value = job->syncpt_end;
617
618 if (syncobj) {
619 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end);
620 if (IS_ERR(fence)) {
621 err = PTR_ERR(fence);
622 SUBMIT_ERR(context, "failed to create postfence: %d", err);
623 }
624
625 drm_syncobj_replace_fence(syncobj, fence);
626 }
627
628 goto put_job;
629
630unpin_job:
631 host1x_job_unpin(job);
632put_job:
633 host1x_job_put(job);
634free_job_data:
635 if (job_data && job_data->used_mappings) {
636 for (i = 0; i < job_data->num_used_mappings; i++)
637 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
638
639 kfree(job_data->used_mappings);
640 }
641
642 if (job_data)
643 kfree(job_data);
644put_bo:
645 gather_bo_put(&bo->base);
646unlock:
647 if (syncobj)
648 drm_syncobj_put(syncobj);
649
650 mutex_unlock(&fpriv->lock);
651 return err;
652}
653
