1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/pagemap.h>
28#include <drm/drmP.h>
29#include <drm/amdgpu_drm.h>
30#include "amdgpu.h"
31#include "amdgpu_trace.h"
32
33int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
34 u32 ip_instance, u32 ring,
35 struct amdgpu_ring **out_ring)
36{
37
38 if (ip_instance != 0) {
39 DRM_ERROR("invalid ip instance: %d\n", ip_instance);
40 return -EINVAL;
41 }
42
43 switch (ip_type) {
44 default:
45 DRM_ERROR("unknown ip type: %d\n", ip_type);
46 return -EINVAL;
47 case AMDGPU_HW_IP_GFX:
48 if (ring < adev->gfx.num_gfx_rings) {
49 *out_ring = &adev->gfx.gfx_ring[ring];
50 } else {
51 DRM_ERROR("only %d gfx rings are supported now\n",
52 adev->gfx.num_gfx_rings);
53 return -EINVAL;
54 }
55 break;
56 case AMDGPU_HW_IP_COMPUTE:
57 if (ring < adev->gfx.num_compute_rings) {
58 *out_ring = &adev->gfx.compute_ring[ring];
59 } else {
60 DRM_ERROR("only %d compute rings are supported now\n",
61 adev->gfx.num_compute_rings);
62 return -EINVAL;
63 }
64 break;
65 case AMDGPU_HW_IP_DMA:
66 if (ring < adev->sdma.num_instances) {
67 *out_ring = &adev->sdma.instance[ring].ring;
68 } else {
69 DRM_ERROR("only %d SDMA rings are supported\n",
70 adev->sdma.num_instances);
71 return -EINVAL;
72 }
73 break;
74 case AMDGPU_HW_IP_UVD:
75 *out_ring = &adev->uvd.ring;
76 break;
77 case AMDGPU_HW_IP_VCE:
78 if (ring < 2){
79 *out_ring = &adev->vce.ring[ring];
80 } else {
81 DRM_ERROR("only two VCE rings are supported\n");
82 return -EINVAL;
83 }
84 break;
85 }
86 return 0;
87}
88
89static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
90 struct drm_amdgpu_cs_chunk_fence *data,
91 uint32_t *offset)
92{
93 struct drm_gem_object *gobj;
94
95 gobj = drm_gem_object_lookup(p->filp, data->handle);
96 if (gobj == NULL)
97 return -EINVAL;
98
99 p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
100 p->uf_entry.priority = 0;
101 p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
102 p->uf_entry.tv.shared = true;
103 p->uf_entry.user_pages = NULL;
104 *offset = data->offset;
105
106 drm_gem_object_unreference_unlocked(gobj);
107
108 if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
109 amdgpu_bo_unref(&p->uf_entry.robj);
110 return -EINVAL;
111 }
112
113 return 0;
114}
115
116int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
117{
118 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
119 struct amdgpu_vm *vm = &fpriv->vm;
120 union drm_amdgpu_cs *cs = data;
121 uint64_t *chunk_array_user;
122 uint64_t *chunk_array;
123 unsigned size, num_ibs = 0;
124 uint32_t uf_offset = 0;
125 int i;
126 int ret;
127
128 if (cs->in.num_chunks == 0)
129 return 0;
130
131 chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
132 if (!chunk_array)
133 return -ENOMEM;
134
135 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
136 if (!p->ctx) {
137 ret = -EINVAL;
138 goto free_chunk;
139 }
140
141
142 chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks);
143 if (copy_from_user(chunk_array, chunk_array_user,
144 sizeof(uint64_t)*cs->in.num_chunks)) {
145 ret = -EFAULT;
146 goto put_ctx;
147 }
148
149 p->nchunks = cs->in.num_chunks;
150 p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
151 GFP_KERNEL);
152 if (!p->chunks) {
153 ret = -ENOMEM;
154 goto put_ctx;
155 }
156
157 for (i = 0; i < p->nchunks; i++) {
158 struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
159 struct drm_amdgpu_cs_chunk user_chunk;
160 uint32_t __user *cdata;
161
162 chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
163 if (copy_from_user(&user_chunk, chunk_ptr,
164 sizeof(struct drm_amdgpu_cs_chunk))) {
165 ret = -EFAULT;
166 i--;
167 goto free_partial_kdata;
168 }
169 p->chunks[i].chunk_id = user_chunk.chunk_id;
170 p->chunks[i].length_dw = user_chunk.length_dw;
171
172 size = p->chunks[i].length_dw;
173 cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
174
175 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
176 if (p->chunks[i].kdata == NULL) {
177 ret = -ENOMEM;
178 i--;
179 goto free_partial_kdata;
180 }
181 size *= sizeof(uint32_t);
182 if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
183 ret = -EFAULT;
184 goto free_partial_kdata;
185 }
186
187 switch (p->chunks[i].chunk_id) {
188 case AMDGPU_CHUNK_ID_IB:
189 ++num_ibs;
190 break;
191
192 case AMDGPU_CHUNK_ID_FENCE:
193 size = sizeof(struct drm_amdgpu_cs_chunk_fence);
194 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
195 ret = -EINVAL;
196 goto free_partial_kdata;
197 }
198
199 ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
200 &uf_offset);
201 if (ret)
202 goto free_partial_kdata;
203
204 break;
205
206 case AMDGPU_CHUNK_ID_DEPENDENCIES:
207 break;
208
209 default:
210 ret = -EINVAL;
211 goto free_partial_kdata;
212 }
213 }
214
215 ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
216 if (ret)
217 goto free_all_kdata;
218
219 if (p->uf_entry.robj) {
220 p->job->uf_bo = amdgpu_bo_ref(p->uf_entry.robj);
221 p->job->uf_offset = uf_offset;
222 }
223
224 kfree(chunk_array);
225 return 0;
226
227free_all_kdata:
228 i = p->nchunks - 1;
229free_partial_kdata:
230 for (; i >= 0; i--)
231 drm_free_large(p->chunks[i].kdata);
232 kfree(p->chunks);
233put_ctx:
234 amdgpu_ctx_put(p->ctx);
235free_chunk:
236 kfree(chunk_array);
237
238 return ret;
239}
240
241
242
243static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
244{
245 u64 real_vram_size = adev->mc.real_vram_size;
246 u64 vram_usage = atomic64_read(&adev->vram_usage);
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287 u64 half_vram = real_vram_size >> 1;
288 u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
289 u64 bytes_moved_threshold = half_free_vram >> 1;
290 return max(bytes_moved_threshold, 1024*1024ull);
291}
292
293int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
294 struct list_head *validated)
295{
296 struct amdgpu_bo_list_entry *lobj;
297 u64 initial_bytes_moved;
298 int r;
299
300 list_for_each_entry(lobj, validated, tv.head) {
301 struct amdgpu_bo *bo = lobj->robj;
302 bool binding_userptr = false;
303 struct mm_struct *usermm;
304 uint32_t domain;
305
306 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
307 if (usermm && usermm != current->mm)
308 return -EPERM;
309
310
311 if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
312 size_t size = sizeof(struct page *);
313
314 size *= bo->tbo.ttm->num_pages;
315 memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
316 binding_userptr = true;
317 }
318
319 if (bo->pin_count)
320 continue;
321
322
323
324
325
326
327
328
329
330 if (p->bytes_moved <= p->bytes_moved_threshold)
331 domain = bo->prefered_domains;
332 else
333 domain = bo->allowed_domains;
334
335 retry:
336 amdgpu_ttm_placement_from_domain(bo, domain);
337 initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
338 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
339 p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
340 initial_bytes_moved;
341
342 if (unlikely(r)) {
343 if (r != -ERESTARTSYS && domain != bo->allowed_domains) {
344 domain = bo->allowed_domains;
345 goto retry;
346 }
347 return r;
348 }
349
350 if (binding_userptr) {
351 drm_free_large(lobj->user_pages);
352 lobj->user_pages = NULL;
353 }
354 }
355 return 0;
356}
357
358static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
359 union drm_amdgpu_cs *cs)
360{
361 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
362 struct amdgpu_bo_list_entry *e;
363 struct list_head duplicates;
364 bool need_mmap_lock = false;
365 unsigned i, tries = 10;
366 int r;
367
368 INIT_LIST_HEAD(&p->validated);
369
370 p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
371 if (p->bo_list) {
372 need_mmap_lock = p->bo_list->first_userptr !=
373 p->bo_list->num_entries;
374 amdgpu_bo_list_get_list(p->bo_list, &p->validated);
375 }
376
377 INIT_LIST_HEAD(&duplicates);
378 amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
379
380 if (p->uf_entry.robj)
381 list_add(&p->uf_entry.tv.head, &p->validated);
382
383 if (need_mmap_lock)
384 down_read(¤t->mm->mmap_sem);
385
386 while (1) {
387 struct list_head need_pages;
388 unsigned i;
389
390 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
391 &duplicates);
392 if (unlikely(r != 0))
393 goto error_free_pages;
394
395
396 if (!p->bo_list)
397 break;
398
399 INIT_LIST_HEAD(&need_pages);
400 for (i = p->bo_list->first_userptr;
401 i < p->bo_list->num_entries; ++i) {
402
403 e = &p->bo_list->array[i];
404
405 if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
406 &e->user_invalidated) && e->user_pages) {
407
408
409
410
411 release_pages(e->user_pages,
412 e->robj->tbo.ttm->num_pages,
413 false);
414 drm_free_large(e->user_pages);
415 e->user_pages = NULL;
416 }
417
418 if (e->robj->tbo.ttm->state != tt_bound &&
419 !e->user_pages) {
420 list_del(&e->tv.head);
421 list_add(&e->tv.head, &need_pages);
422
423 amdgpu_bo_unreserve(e->robj);
424 }
425 }
426
427 if (list_empty(&need_pages))
428 break;
429
430
431 ttm_eu_backoff_reservation(&p->ticket, &p->validated);
432
433
434 if (!--tries) {
435 r = -EDEADLK;
436 goto error_free_pages;
437 }
438
439
440 list_for_each_entry(e, &need_pages, tv.head) {
441 struct ttm_tt *ttm = e->robj->tbo.ttm;
442
443 e->user_pages = drm_calloc_large(ttm->num_pages,
444 sizeof(struct page*));
445 if (!e->user_pages) {
446 r = -ENOMEM;
447 goto error_free_pages;
448 }
449
450 r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
451 if (r) {
452 drm_free_large(e->user_pages);
453 e->user_pages = NULL;
454 goto error_free_pages;
455 }
456 }
457
458
459 list_splice(&need_pages, &p->validated);
460 }
461
462 amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates);
463
464 p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
465 p->bytes_moved = 0;
466
467 r = amdgpu_cs_list_validate(p, &duplicates);
468 if (r)
469 goto error_validate;
470
471 r = amdgpu_cs_list_validate(p, &p->validated);
472 if (r)
473 goto error_validate;
474
475 if (p->bo_list) {
476 struct amdgpu_bo *gds = p->bo_list->gds_obj;
477 struct amdgpu_bo *gws = p->bo_list->gws_obj;
478 struct amdgpu_bo *oa = p->bo_list->oa_obj;
479 struct amdgpu_vm *vm = &fpriv->vm;
480 unsigned i;
481
482 for (i = 0; i < p->bo_list->num_entries; i++) {
483 struct amdgpu_bo *bo = p->bo_list->array[i].robj;
484
485 p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
486 }
487
488 if (gds) {
489 p->job->gds_base = amdgpu_bo_gpu_offset(gds);
490 p->job->gds_size = amdgpu_bo_size(gds);
491 }
492 if (gws) {
493 p->job->gws_base = amdgpu_bo_gpu_offset(gws);
494 p->job->gws_size = amdgpu_bo_size(gws);
495 }
496 if (oa) {
497 p->job->oa_base = amdgpu_bo_gpu_offset(oa);
498 p->job->oa_size = amdgpu_bo_size(oa);
499 }
500 }
501
502error_validate:
503 if (r) {
504 amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
505 ttm_eu_backoff_reservation(&p->ticket, &p->validated);
506 }
507
508error_free_pages:
509
510 if (need_mmap_lock)
511 up_read(¤t->mm->mmap_sem);
512
513 if (p->bo_list) {
514 for (i = p->bo_list->first_userptr;
515 i < p->bo_list->num_entries; ++i) {
516 e = &p->bo_list->array[i];
517
518 if (!e->user_pages)
519 continue;
520
521 release_pages(e->user_pages,
522 e->robj->tbo.ttm->num_pages,
523 false);
524 drm_free_large(e->user_pages);
525 }
526 }
527
528 return r;
529}
530
531static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
532{
533 struct amdgpu_bo_list_entry *e;
534 int r;
535
536 list_for_each_entry(e, &p->validated, tv.head) {
537 struct reservation_object *resv = e->robj->tbo.resv;
538 r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp);
539
540 if (r)
541 return r;
542 }
543 return 0;
544}
545
546
547
548
549
550
551
552
553
554static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
555{
556 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
557 unsigned i;
558
559 if (!error) {
560 amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);
561
562 ttm_eu_fence_buffer_objects(&parser->ticket,
563 &parser->validated,
564 parser->fence);
565 } else if (backoff) {
566 ttm_eu_backoff_reservation(&parser->ticket,
567 &parser->validated);
568 }
569 fence_put(parser->fence);
570
571 if (parser->ctx)
572 amdgpu_ctx_put(parser->ctx);
573 if (parser->bo_list)
574 amdgpu_bo_list_put(parser->bo_list);
575
576 for (i = 0; i < parser->nchunks; i++)
577 drm_free_large(parser->chunks[i].kdata);
578 kfree(parser->chunks);
579 if (parser->job)
580 amdgpu_job_free(parser->job);
581 amdgpu_bo_unref(&parser->uf_entry.robj);
582}
583
584static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
585 struct amdgpu_vm *vm)
586{
587 struct amdgpu_device *adev = p->adev;
588 struct amdgpu_bo_va *bo_va;
589 struct amdgpu_bo *bo;
590 int i, r;
591
592 r = amdgpu_vm_update_page_directory(adev, vm);
593 if (r)
594 return r;
595
596 r = amdgpu_sync_fence(adev, &p->job->sync, vm->page_directory_fence);
597 if (r)
598 return r;
599
600 r = amdgpu_vm_clear_freed(adev, vm);
601 if (r)
602 return r;
603
604 if (p->bo_list) {
605 for (i = 0; i < p->bo_list->num_entries; i++) {
606 struct fence *f;
607
608
609 bo = p->bo_list->array[i].robj;
610 if (!bo)
611 continue;
612
613 bo_va = p->bo_list->array[i].bo_va;
614 if (bo_va == NULL)
615 continue;
616
617 r = amdgpu_vm_bo_update(adev, bo_va, &bo->tbo.mem);
618 if (r)
619 return r;
620
621 f = bo_va->last_pt_update;
622 r = amdgpu_sync_fence(adev, &p->job->sync, f);
623 if (r)
624 return r;
625 }
626
627 }
628
629 r = amdgpu_vm_clear_invalids(adev, vm, &p->job->sync);
630
631 if (amdgpu_vm_debug && p->bo_list) {
632
633 for (i = 0; i < p->bo_list->num_entries; i++) {
634
635 bo = p->bo_list->array[i].robj;
636 if (!bo)
637 continue;
638
639 amdgpu_vm_bo_invalidate(adev, bo);
640 }
641 }
642
643 return r;
644}
645
646static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
647 struct amdgpu_cs_parser *p)
648{
649 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
650 struct amdgpu_vm *vm = &fpriv->vm;
651 struct amdgpu_ring *ring = p->job->ring;
652 int i, r;
653
654
655 if (ring->funcs->parse_cs) {
656 for (i = 0; i < p->job->num_ibs; i++) {
657 r = amdgpu_ring_parse_cs(ring, p, i);
658 if (r)
659 return r;
660 }
661 }
662
663 r = amdgpu_bo_vm_update_pte(p, vm);
664 if (!r)
665 amdgpu_cs_sync_rings(p);
666
667 return r;
668}
669
670static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
671{
672 if (r == -EDEADLK) {
673 r = amdgpu_gpu_reset(adev);
674 if (!r)
675 r = -EAGAIN;
676 }
677 return r;
678}
679
680static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
681 struct amdgpu_cs_parser *parser)
682{
683 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
684 struct amdgpu_vm *vm = &fpriv->vm;
685 int i, j;
686 int r;
687
688 for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
689 struct amdgpu_cs_chunk *chunk;
690 struct amdgpu_ib *ib;
691 struct drm_amdgpu_cs_chunk_ib *chunk_ib;
692 struct amdgpu_ring *ring;
693
694 chunk = &parser->chunks[i];
695 ib = &parser->job->ibs[j];
696 chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
697
698 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
699 continue;
700
701 r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type,
702 chunk_ib->ip_instance, chunk_ib->ring,
703 &ring);
704 if (r)
705 return r;
706
707 if (parser->job->ring && parser->job->ring != ring)
708 return -EINVAL;
709
710 parser->job->ring = ring;
711
712 if (ring->funcs->parse_cs) {
713 struct amdgpu_bo_va_mapping *m;
714 struct amdgpu_bo *aobj = NULL;
715 uint64_t offset;
716 uint8_t *kptr;
717
718 m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
719 &aobj);
720 if (!aobj) {
721 DRM_ERROR("IB va_start is invalid\n");
722 return -EINVAL;
723 }
724
725 if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
726 (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) {
727 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
728 return -EINVAL;
729 }
730
731
732 r = amdgpu_bo_kmap(aobj, (void **)&kptr);
733 if (r) {
734 return r;
735 }
736
737 offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE;
738 kptr += chunk_ib->va_start - offset;
739
740 r = amdgpu_ib_get(adev, NULL, chunk_ib->ib_bytes, ib);
741 if (r) {
742 DRM_ERROR("Failed to get ib !\n");
743 return r;
744 }
745
746 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
747 amdgpu_bo_kunmap(aobj);
748 } else {
749 r = amdgpu_ib_get(adev, vm, 0, ib);
750 if (r) {
751 DRM_ERROR("Failed to get ib !\n");
752 return r;
753 }
754
755 ib->gpu_addr = chunk_ib->va_start;
756 }
757
758 ib->length_dw = chunk_ib->ib_bytes / 4;
759 ib->flags = chunk_ib->flags;
760 j++;
761 }
762
763
764 if (parser->job->uf_bo && (
765 parser->job->ring->type == AMDGPU_RING_TYPE_UVD ||
766 parser->job->ring->type == AMDGPU_RING_TYPE_VCE))
767 return -EINVAL;
768
769 return 0;
770}
771
772static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
773 struct amdgpu_cs_parser *p)
774{
775 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
776 int i, j, r;
777
778 for (i = 0; i < p->nchunks; ++i) {
779 struct drm_amdgpu_cs_chunk_dep *deps;
780 struct amdgpu_cs_chunk *chunk;
781 unsigned num_deps;
782
783 chunk = &p->chunks[i];
784
785 if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES)
786 continue;
787
788 deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
789 num_deps = chunk->length_dw * 4 /
790 sizeof(struct drm_amdgpu_cs_chunk_dep);
791
792 for (j = 0; j < num_deps; ++j) {
793 struct amdgpu_ring *ring;
794 struct amdgpu_ctx *ctx;
795 struct fence *fence;
796
797 r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
798 deps[j].ip_instance,
799 deps[j].ring, &ring);
800 if (r)
801 return r;
802
803 ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id);
804 if (ctx == NULL)
805 return -EINVAL;
806
807 fence = amdgpu_ctx_get_fence(ctx, ring,
808 deps[j].handle);
809 if (IS_ERR(fence)) {
810 r = PTR_ERR(fence);
811 amdgpu_ctx_put(ctx);
812 return r;
813
814 } else if (fence) {
815 r = amdgpu_sync_fence(adev, &p->job->sync,
816 fence);
817 fence_put(fence);
818 amdgpu_ctx_put(ctx);
819 if (r)
820 return r;
821 }
822 }
823 }
824
825 return 0;
826}
827
828static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
829 union drm_amdgpu_cs *cs)
830{
831 struct amdgpu_ring *ring = p->job->ring;
832 struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
833 struct fence *fence;
834 struct amdgpu_job *job;
835 int r;
836
837 job = p->job;
838 p->job = NULL;
839
840 r = amd_sched_job_init(&job->base, &ring->sched,
841 entity, amdgpu_job_timeout_func,
842 amdgpu_job_free_func,
843 p->filp, &fence);
844 if (r) {
845 amdgpu_job_free(job);
846 return r;
847 }
848
849 job->owner = p->filp;
850 job->ctx = entity->fence_context;
851 p->fence = fence_get(fence);
852 cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, fence);
853 job->uf_sequence = cs->out.handle;
854
855 trace_amdgpu_cs_ioctl(job);
856 amd_sched_entity_push_job(&job->base);
857
858 return 0;
859}
860
861int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
862{
863 struct amdgpu_device *adev = dev->dev_private;
864 union drm_amdgpu_cs *cs = data;
865 struct amdgpu_cs_parser parser = {};
866 bool reserved_buffers = false;
867 int i, r;
868
869 if (!adev->accel_working)
870 return -EBUSY;
871
872 parser.adev = adev;
873 parser.filp = filp;
874
875 r = amdgpu_cs_parser_init(&parser, data);
876 if (r) {
877 DRM_ERROR("Failed to initialize parser !\n");
878 amdgpu_cs_parser_fini(&parser, r, false);
879 r = amdgpu_cs_handle_lockup(adev, r);
880 return r;
881 }
882 r = amdgpu_cs_parser_bos(&parser, data);
883 if (r == -ENOMEM)
884 DRM_ERROR("Not enough memory for command submission!\n");
885 else if (r && r != -ERESTARTSYS)
886 DRM_ERROR("Failed to process the buffer list %d!\n", r);
887 else if (!r) {
888 reserved_buffers = true;
889 r = amdgpu_cs_ib_fill(adev, &parser);
890 }
891
892 if (!r) {
893 r = amdgpu_cs_dependencies(adev, &parser);
894 if (r)
895 DRM_ERROR("Failed in the dependencies handling %d!\n", r);
896 }
897
898 if (r)
899 goto out;
900
901 for (i = 0; i < parser.job->num_ibs; i++)
902 trace_amdgpu_cs(&parser, i);
903
904 r = amdgpu_cs_ib_vm_chunk(adev, &parser);
905 if (r)
906 goto out;
907
908 r = amdgpu_cs_submit(&parser, cs);
909
910out:
911 amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
912 r = amdgpu_cs_handle_lockup(adev, r);
913 return r;
914}
915
916
917
918
919
920
921
922
923
924
925int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
926 struct drm_file *filp)
927{
928 union drm_amdgpu_wait_cs *wait = data;
929 struct amdgpu_device *adev = dev->dev_private;
930 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
931 struct amdgpu_ring *ring = NULL;
932 struct amdgpu_ctx *ctx;
933 struct fence *fence;
934 long r;
935
936 r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
937 wait->in.ring, &ring);
938 if (r)
939 return r;
940
941 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
942 if (ctx == NULL)
943 return -EINVAL;
944
945 fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
946 if (IS_ERR(fence))
947 r = PTR_ERR(fence);
948 else if (fence) {
949 r = fence_wait_timeout(fence, true, timeout);
950 fence_put(fence);
951 } else
952 r = 1;
953
954 amdgpu_ctx_put(ctx);
955 if (r < 0)
956 return r;
957
958 memset(wait, 0, sizeof(*wait));
959 wait->out.status = (r == 0);
960
961 return 0;
962}
963
964
965
966
967
968
969
970
971
972
973
974
975struct amdgpu_bo_va_mapping *
976amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
977 uint64_t addr, struct amdgpu_bo **bo)
978{
979 struct amdgpu_bo_va_mapping *mapping;
980 unsigned i;
981
982 if (!parser->bo_list)
983 return NULL;
984
985 addr /= AMDGPU_GPU_PAGE_SIZE;
986
987 for (i = 0; i < parser->bo_list->num_entries; i++) {
988 struct amdgpu_bo_list_entry *lobj;
989
990 lobj = &parser->bo_list->array[i];
991 if (!lobj->bo_va)
992 continue;
993
994 list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
995 if (mapping->it.start > addr ||
996 addr > mapping->it.last)
997 continue;
998
999 *bo = lobj->bo_va->bo;
1000 return mapping;
1001 }
1002
1003 list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
1004 if (mapping->it.start > addr ||
1005 addr > mapping->it.last)
1006 continue;
1007
1008 *bo = lobj->bo_va->bo;
1009 return mapping;
1010 }
1011 }
1012
1013 return NULL;
1014}
1015