1
2
3
4#include <linux/dma-fence-array.h>
5#include <linux/dma-mapping.h>
6#include <linux/file.h>
7#include <linux/host1x.h>
8#include <linux/iommu.h>
9#include <linux/kref.h>
10#include <linux/list.h>
11#include <linux/nospec.h>
12#include <linux/pm_runtime.h>
13#include <linux/scatterlist.h>
14#include <linux/slab.h>
15#include <linux/sync_file.h>
16
17#include <drm/drm_drv.h>
18#include <drm/drm_file.h>
19#include <drm/drm_syncobj.h>
20
21#include "drm.h"
22#include "gem.h"
23#include "submit.h"
24#include "uapi.h"
25
26#define SUBMIT_ERR(context, fmt, ...) \
27 dev_err_ratelimited(context->client->base.dev, \
28 "%s: job submission failed: " fmt "\n", \
29 current->comm, ##__VA_ARGS__)
30
31struct gather_bo {
32 struct host1x_bo base;
33
34 struct kref ref;
35
36 struct device *dev;
37 u32 *gather_data;
38 dma_addr_t gather_data_dma;
39 size_t gather_data_words;
40};
41
42static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo)
43{
44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
45
46 kref_get(&bo->ref);
47
48 return host_bo;
49}
50
51static void gather_bo_release(struct kref *ref)
52{
53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref);
54
55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma,
56 0);
57 kfree(bo);
58}
59
60static void gather_bo_put(struct host1x_bo *host_bo)
61{
62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
63
64 kref_put(&bo->ref, gather_bo_release);
65}
66
67static struct sg_table *
68gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys)
69{
70 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
71 struct sg_table *sgt;
72 int err;
73
74 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
75 if (!sgt)
76 return ERR_PTR(-ENOMEM);
77
78 err = dma_get_sgtable(bo->dev, sgt, bo->gather_data, bo->gather_data_dma,
79 bo->gather_data_words * 4);
80 if (err) {
81 kfree(sgt);
82 return ERR_PTR(err);
83 }
84
85 return sgt;
86}
87
88static void gather_bo_unpin(struct device *dev, struct sg_table *sgt)
89{
90 if (sgt) {
91 sg_free_table(sgt);
92 kfree(sgt);
93 }
94}
95
96static void *gather_bo_mmap(struct host1x_bo *host_bo)
97{
98 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
99
100 return bo->gather_data;
101}
102
103static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr)
104{
105}
106
107const struct host1x_bo_ops gather_bo_ops = {
108 .get = gather_bo_get,
109 .put = gather_bo_put,
110 .pin = gather_bo_pin,
111 .unpin = gather_bo_unpin,
112 .mmap = gather_bo_mmap,
113 .munmap = gather_bo_munmap,
114};
115
116static struct tegra_drm_mapping *
117tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id)
118{
119 struct tegra_drm_mapping *mapping;
120
121 xa_lock(&context->mappings);
122
123 mapping = xa_load(&context->mappings, id);
124 if (mapping)
125 kref_get(&mapping->ref);
126
127 xa_unlock(&context->mappings);
128
129 return mapping;
130}
131
132static void *alloc_copy_user_array(void __user *from, size_t count, size_t size)
133{
134 size_t copy_len;
135 void *data;
136
137 if (check_mul_overflow(count, size, ©_len))
138 return ERR_PTR(-EINVAL);
139
140 if (copy_len > 0x4000)
141 return ERR_PTR(-E2BIG);
142
143 data = kvmalloc(copy_len, GFP_KERNEL);
144 if (!data)
145 return ERR_PTR(-ENOMEM);
146
147 if (copy_from_user(data, from, copy_len)) {
148 kvfree(data);
149 return ERR_PTR(-EFAULT);
150 }
151
152 return data;
153}
154
155static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev,
156 struct tegra_drm_context *context,
157 struct drm_tegra_channel_submit *args)
158{
159 struct gather_bo *bo;
160 size_t copy_len;
161
162 if (args->gather_data_words == 0) {
163 SUBMIT_ERR(context, "gather_data_words cannot be zero");
164 return -EINVAL;
165 }
166
167 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) {
168 SUBMIT_ERR(context, "gather_data_words is too large");
169 return -EINVAL;
170 }
171
172 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
173 if (!bo) {
174 SUBMIT_ERR(context, "failed to allocate memory for bo info");
175 return -ENOMEM;
176 }
177
178 host1x_bo_init(&bo->base, &gather_bo_ops);
179 kref_init(&bo->ref);
180 bo->dev = dev;
181
182 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma,
183 GFP_KERNEL | __GFP_NOWARN, 0);
184 if (!bo->gather_data) {
185 SUBMIT_ERR(context, "failed to allocate memory for gather data");
186 kfree(bo);
187 return -ENOMEM;
188 }
189
190 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) {
191 SUBMIT_ERR(context, "failed to copy gather data from userspace");
192 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0);
193 kfree(bo);
194 return -EFAULT;
195 }
196
197 bo->gather_data_words = args->gather_data_words;
198
199 *pbo = bo;
200
201 return 0;
202}
203
204static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo,
205 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping)
206{
207
208 dma_addr_t iova = mapping->iova + buf->reloc.target_offset;
209 u32 written_ptr;
210
211#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
212 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT)
213 iova |= BIT_ULL(39);
214#endif
215
216 written_ptr = iova >> buf->reloc.shift;
217
218 if (buf->reloc.gather_offset_words >= bo->gather_data_words) {
219 SUBMIT_ERR(context,
220 "relocation has too large gather offset (%u vs gather length %zu)",
221 buf->reloc.gather_offset_words, bo->gather_data_words);
222 return -EINVAL;
223 }
224
225 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words,
226 bo->gather_data_words);
227
228 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;
229
230 return 0;
231}
232
233static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo,
234 struct drm_tegra_channel_submit *args,
235 struct tegra_drm_submit_data *job_data)
236{
237 struct tegra_drm_used_mapping *mappings;
238 struct drm_tegra_submit_buf *bufs;
239 int err;
240 u32 i;
241
242 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs,
243 sizeof(*bufs));
244 if (IS_ERR(bufs)) {
245 SUBMIT_ERR(context, "failed to copy bufs array from userspace");
246 return PTR_ERR(bufs);
247 }
248
249 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL);
250 if (!mappings) {
251 SUBMIT_ERR(context, "failed to allocate memory for mapping info");
252 err = -ENOMEM;
253 goto done;
254 }
255
256 for (i = 0; i < args->num_bufs; i++) {
257 struct drm_tegra_submit_buf *buf = &bufs[i];
258 struct tegra_drm_mapping *mapping;
259
260 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) {
261 SUBMIT_ERR(context, "invalid flag specified for buffer");
262 err = -EINVAL;
263 goto drop_refs;
264 }
265
266 mapping = tegra_drm_mapping_get(context, buf->mapping);
267 if (!mapping) {
268 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping);
269 err = -EINVAL;
270 goto drop_refs;
271 }
272
273 err = submit_write_reloc(context, bo, buf, mapping);
274 if (err) {
275 tegra_drm_mapping_put(mapping);
276 goto drop_refs;
277 }
278
279 mappings[i].mapping = mapping;
280 mappings[i].flags = buf->flags;
281 }
282
283 job_data->used_mappings = mappings;
284 job_data->num_used_mappings = i;
285
286 err = 0;
287
288 goto done;
289
290drop_refs:
291 while (i--)
292 tegra_drm_mapping_put(mappings[i].mapping);
293
294 kfree(mappings);
295 job_data->used_mappings = NULL;
296
297done:
298 kvfree(bufs);
299
300 return err;
301}
302
303static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job,
304 struct xarray *syncpoints, struct drm_tegra_channel_submit *args)
305{
306 struct host1x_syncpt *sp;
307
308 if (args->syncpt.flags) {
309 SUBMIT_ERR(context, "invalid flag specified for syncpt");
310 return -EINVAL;
311 }
312
313
314 sp = xa_load(syncpoints, args->syncpt.id);
315 if (!sp) {
316 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated");
317 return -EINVAL;
318 }
319
320 job->syncpt = host1x_syncpt_get(sp);
321 job->syncpt_incrs = args->syncpt.increments;
322
323 return 0;
324}
325
326static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context,
327 struct drm_tegra_submit_cmd_gather_uptr *cmd,
328 struct gather_bo *bo, u32 *offset,
329 struct tegra_drm_submit_data *job_data,
330 u32 *class)
331{
332 u32 next_offset;
333
334 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) {
335 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command");
336 return -EINVAL;
337 }
338
339
340 if (cmd->words > 16383) {
341 SUBMIT_ERR(context, "too many words in GATHER_UPTR command");
342 return -EINVAL;
343 }
344
345 if (check_add_overflow(*offset, cmd->words, &next_offset)) {
346 SUBMIT_ERR(context, "too many total words in job");
347 return -EINVAL;
348 }
349
350 if (next_offset > bo->gather_data_words) {
351 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data");
352 return -EINVAL;
353 }
354
355 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset,
356 cmd->words, job_data, class)) {
357 SUBMIT_ERR(context, "job was rejected by firewall");
358 return -EINVAL;
359 }
360
361 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4);
362
363 *offset = next_offset;
364
365 return 0;
366}
367
368static struct host1x_job *
369submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo,
370 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data,
371 struct xarray *syncpoints)
372{
373 struct drm_tegra_submit_cmd *cmds;
374 u32 i, gather_offset = 0, class;
375 struct host1x_job *job;
376 int err;
377
378
379 class = context->client->base.class;
380
381 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds,
382 sizeof(*cmds));
383 if (IS_ERR(cmds)) {
384 SUBMIT_ERR(context, "failed to copy cmds array from userspace");
385 return ERR_CAST(cmds);
386 }
387
388 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true);
389 if (!job) {
390 SUBMIT_ERR(context, "failed to allocate memory for job");
391 job = ERR_PTR(-ENOMEM);
392 goto done;
393 }
394
395 err = submit_get_syncpt(context, job, syncpoints, args);
396 if (err < 0)
397 goto free_job;
398
399 job->client = &context->client->base;
400 job->class = context->client->base.class;
401 job->serialize = true;
402
403 for (i = 0; i < args->num_cmds; i++) {
404 struct drm_tegra_submit_cmd *cmd = &cmds[i];
405
406 if (cmd->flags) {
407 SUBMIT_ERR(context, "unknown flags given for cmd");
408 err = -EINVAL;
409 goto free_job;
410 }
411
412 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) {
413 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo,
414 &gather_offset, job_data, &class);
415 if (err)
416 goto free_job;
417 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) {
418 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
419 SUBMIT_ERR(context, "non-zero reserved value");
420 err = -EINVAL;
421 goto free_job;
422 }
423
424 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
425 false, class);
426 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) {
427 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
428 SUBMIT_ERR(context, "non-zero reserved value");
429 err = -EINVAL;
430 goto free_job;
431 }
432
433 if (cmd->wait_syncpt.id != args->syncpt.id) {
434 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job");
435 err = -EINVAL;
436 goto free_job;
437 }
438
439 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
440 true, class);
441 } else {
442 SUBMIT_ERR(context, "unknown cmd type");
443 err = -EINVAL;
444 goto free_job;
445 }
446 }
447
448 if (gather_offset == 0) {
449 SUBMIT_ERR(context, "job must have at least one gather");
450 err = -EINVAL;
451 goto free_job;
452 }
453
454 goto done;
455
456free_job:
457 host1x_job_put(job);
458 job = ERR_PTR(err);
459
460done:
461 kvfree(cmds);
462
463 return job;
464}
465
466static void release_job(struct host1x_job *job)
467{
468 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base);
469 struct tegra_drm_submit_data *job_data = job->user_data;
470 u32 i;
471
472 for (i = 0; i < job_data->num_used_mappings; i++)
473 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
474
475 kfree(job_data->used_mappings);
476 kfree(job_data);
477
478 if (pm_runtime_enabled(client->base.dev))
479 pm_runtime_put_autosuspend(client->base.dev);
480}
481
482int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,
483 struct drm_file *file)
484{
485 struct tegra_drm_file *fpriv = file->driver_priv;
486 struct drm_tegra_channel_submit *args = data;
487 struct tegra_drm_submit_data *job_data;
488 struct drm_syncobj *syncobj = NULL;
489 struct tegra_drm_context *context;
490 struct host1x_job *job;
491 struct gather_bo *bo;
492 u32 i;
493 int err;
494
495 mutex_lock(&fpriv->lock);
496
497 context = xa_load(&fpriv->contexts, args->context);
498 if (!context) {
499 mutex_unlock(&fpriv->lock);
500 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__,
501 current->comm, args->context);
502 return -EINVAL;
503 }
504
505 if (args->syncobj_in) {
506 struct dma_fence *fence;
507
508 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence);
509 if (err) {
510 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in);
511 goto unlock;
512 }
513
514 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000));
515 dma_fence_put(fence);
516 if (err) {
517 SUBMIT_ERR(context, "wait for syncobj_in timed out");
518 goto unlock;
519 }
520 }
521
522 if (args->syncobj_out) {
523 syncobj = drm_syncobj_find(file, args->syncobj_out);
524 if (!syncobj) {
525 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out);
526 err = -ENOENT;
527 goto unlock;
528 }
529 }
530
531
532 err = submit_copy_gather_data(&bo, drm->dev, context, args);
533 if (err)
534 goto unlock;
535
536 job_data = kzalloc(sizeof(*job_data), GFP_KERNEL);
537 if (!job_data) {
538 SUBMIT_ERR(context, "failed to allocate memory for job data");
539 err = -ENOMEM;
540 goto put_bo;
541 }
542
543
544 err = submit_process_bufs(context, bo, args, job_data);
545 if (err)
546 goto free_job_data;
547
548
549 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints);
550 if (IS_ERR(job)) {
551 err = PTR_ERR(job);
552 goto free_job_data;
553 }
554
555
556 err = host1x_job_pin(job, context->client->base.dev);
557 if (err) {
558 SUBMIT_ERR(context, "failed to pin job: %d", err);
559 goto put_job;
560 }
561
562
563 if (pm_runtime_enabled(context->client->base.dev)) {
564 err = pm_runtime_resume_and_get(context->client->base.dev);
565 if (err < 0) {
566 SUBMIT_ERR(context, "could not power up engine: %d", err);
567 goto unpin_job;
568 }
569 }
570
571 job->user_data = job_data;
572 job->release = release_job;
573 job->timeout = 10000;
574
575
576
577
578
579 job_data = NULL;
580
581
582 err = host1x_job_submit(job);
583 if (err) {
584 SUBMIT_ERR(context, "host1x job submission failed: %d", err);
585 goto unpin_job;
586 }
587
588
589 args->syncpt.value = job->syncpt_end;
590
591 if (syncobj) {
592 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end);
593 if (IS_ERR(fence)) {
594 err = PTR_ERR(fence);
595 SUBMIT_ERR(context, "failed to create postfence: %d", err);
596 }
597
598 drm_syncobj_replace_fence(syncobj, fence);
599 }
600
601 goto put_job;
602
603unpin_job:
604 host1x_job_unpin(job);
605put_job:
606 host1x_job_put(job);
607free_job_data:
608 if (job_data && job_data->used_mappings) {
609 for (i = 0; i < job_data->num_used_mappings; i++)
610 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
611
612 kfree(job_data->used_mappings);
613 }
614
615 if (job_data)
616 kfree(job_data);
617put_bo:
618 gather_bo_put(&bo->base);
619unlock:
620 if (syncobj)
621 drm_syncobj_put(syncobj);
622
623 mutex_unlock(&fpriv->lock);
624 return err;
625}
626
