1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include "nouveau_drv.h"
28#include "nouveau_dma.h"
29#include "nouveau_fence.h"
30#include "nouveau_abi16.h"
31
32#include "nouveau_ttm.h"
33#include "nouveau_gem.h"
34
35void
36nouveau_gem_object_del(struct drm_gem_object *gem)
37{
38 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
39 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
40 struct ttm_buffer_object *bo = &nvbo->bo;
41 struct device *dev = drm->dev->dev;
42 int ret;
43
44 ret = pm_runtime_get_sync(dev);
45 if (WARN_ON(ret < 0 && ret != -EACCES))
46 return;
47
48 if (gem->import_attach)
49 drm_prime_gem_destroy(gem, nvbo->bo.sg);
50
51 drm_gem_object_release(gem);
52
53
54 gem->filp = NULL;
55 ttm_bo_unref(&bo);
56
57 pm_runtime_mark_last_busy(dev);
58 pm_runtime_put_autosuspend(dev);
59}
60
61int
62nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
63{
64 struct nouveau_cli *cli = nouveau_cli(file_priv);
65 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
66 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
67 struct nvkm_vma *vma;
68 struct device *dev = drm->dev->dev;
69 int ret;
70
71 if (!cli->vm)
72 return 0;
73
74 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
75 if (ret)
76 return ret;
77
78 vma = nouveau_bo_vma_find(nvbo, cli->vm);
79 if (!vma) {
80 vma = kzalloc(sizeof(*vma), GFP_KERNEL);
81 if (!vma) {
82 ret = -ENOMEM;
83 goto out;
84 }
85
86 ret = pm_runtime_get_sync(dev);
87 if (ret < 0 && ret != -EACCES) {
88 kfree(vma);
89 goto out;
90 }
91
92 ret = nouveau_bo_vma_add(nvbo, cli->vm, vma);
93 if (ret)
94 kfree(vma);
95
96 pm_runtime_mark_last_busy(dev);
97 pm_runtime_put_autosuspend(dev);
98 } else {
99 vma->refcount++;
100 }
101
102out:
103 ttm_bo_unreserve(&nvbo->bo);
104 return ret;
105}
106
107static void
108nouveau_gem_object_delete(void *data)
109{
110 struct nvkm_vma *vma = data;
111 nvkm_vm_unmap(vma);
112 nvkm_vm_put(vma);
113 kfree(vma);
114}
115
116static void
117nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nvkm_vma *vma)
118{
119 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM;
120 struct reservation_object *resv = nvbo->bo.resv;
121 struct reservation_object_list *fobj;
122 struct dma_fence *fence = NULL;
123
124 fobj = reservation_object_get_list(resv);
125
126 list_del(&vma->head);
127
128 if (fobj && fobj->shared_count > 1)
129 ttm_bo_wait(&nvbo->bo, false, false);
130 else if (fobj && fobj->shared_count == 1)
131 fence = rcu_dereference_protected(fobj->shared[0],
132 reservation_object_held(resv));
133 else
134 fence = reservation_object_get_excl(nvbo->bo.resv);
135
136 if (fence && mapped) {
137 nouveau_fence_work(fence, nouveau_gem_object_delete, vma);
138 } else {
139 if (mapped)
140 nvkm_vm_unmap(vma);
141 nvkm_vm_put(vma);
142 kfree(vma);
143 }
144}
145
146void
147nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
148{
149 struct nouveau_cli *cli = nouveau_cli(file_priv);
150 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
151 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
152 struct device *dev = drm->dev->dev;
153 struct nvkm_vma *vma;
154 int ret;
155
156 if (!cli->vm)
157 return;
158
159 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
160 if (ret)
161 return;
162
163 vma = nouveau_bo_vma_find(nvbo, cli->vm);
164 if (vma) {
165 if (--vma->refcount == 0) {
166 ret = pm_runtime_get_sync(dev);
167 if (!WARN_ON(ret < 0 && ret != -EACCES)) {
168 nouveau_gem_object_unmap(nvbo, vma);
169 pm_runtime_mark_last_busy(dev);
170 pm_runtime_put_autosuspend(dev);
171 }
172 }
173 }
174 ttm_bo_unreserve(&nvbo->bo);
175}
176
177int
178nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
179 uint32_t tile_mode, uint32_t tile_flags,
180 struct nouveau_bo **pnvbo)
181{
182 struct nouveau_drm *drm = nouveau_drm(cli->dev);
183 struct nouveau_bo *nvbo;
184 u32 flags = 0;
185 int ret;
186
187 if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
188 flags |= TTM_PL_FLAG_VRAM;
189 if (domain & NOUVEAU_GEM_DOMAIN_GART)
190 flags |= TTM_PL_FLAG_TT;
191 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
192 flags |= TTM_PL_FLAG_SYSTEM;
193
194 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT)
195 flags |= TTM_PL_FLAG_UNCACHED;
196
197 ret = nouveau_bo_new(cli, size, align, flags, tile_mode,
198 tile_flags, NULL, NULL, pnvbo);
199 if (ret)
200 return ret;
201 nvbo = *pnvbo;
202
203
204
205
206
207 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
208 NOUVEAU_GEM_DOMAIN_GART;
209 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
210 nvbo->valid_domains &= domain;
211
212
213
214 ret = drm_gem_object_init(drm->dev, &nvbo->gem, nvbo->bo.mem.size);
215 if (ret) {
216 nouveau_bo_ref(NULL, pnvbo);
217 return -ENOMEM;
218 }
219
220 nvbo->bo.persistent_swap_storage = nvbo->gem.filp;
221 return 0;
222}
223
224static int
225nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
226 struct drm_nouveau_gem_info *rep)
227{
228 struct nouveau_cli *cli = nouveau_cli(file_priv);
229 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
230 struct nvkm_vma *vma;
231
232 if (is_power_of_2(nvbo->valid_domains))
233 rep->domain = nvbo->valid_domains;
234 else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
235 rep->domain = NOUVEAU_GEM_DOMAIN_GART;
236 else
237 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
238 rep->offset = nvbo->bo.offset;
239 if (cli->vm) {
240 vma = nouveau_bo_vma_find(nvbo, cli->vm);
241 if (!vma)
242 return -EINVAL;
243
244 rep->offset = vma->offset;
245 }
246
247 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
248 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node);
249 rep->tile_mode = nvbo->tile_mode;
250 rep->tile_flags = nvbo->tile_flags;
251 return 0;
252}
253
254int
255nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
256 struct drm_file *file_priv)
257{
258 struct nouveau_drm *drm = nouveau_drm(dev);
259 struct nouveau_cli *cli = nouveau_cli(file_priv);
260 struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
261 struct drm_nouveau_gem_new *req = data;
262 struct nouveau_bo *nvbo = NULL;
263 int ret = 0;
264
265 if (!nvkm_fb_memtype_valid(fb, req->info.tile_flags)) {
266 NV_PRINTK(err, cli, "bad page flags: 0x%08x\n", req->info.tile_flags);
267 return -EINVAL;
268 }
269
270 ret = nouveau_gem_new(cli, req->info.size, req->align,
271 req->info.domain, req->info.tile_mode,
272 req->info.tile_flags, &nvbo);
273 if (ret)
274 return ret;
275
276 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle);
277 if (ret == 0) {
278 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info);
279 if (ret)
280 drm_gem_handle_delete(file_priv, req->info.handle);
281 }
282
283
284 drm_gem_object_unreference_unlocked(&nvbo->gem);
285 return ret;
286}
287
288static int
289nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
290 uint32_t write_domains, uint32_t valid_domains)
291{
292 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
293 struct ttm_buffer_object *bo = &nvbo->bo;
294 uint32_t domains = valid_domains & nvbo->valid_domains &
295 (write_domains ? write_domains : read_domains);
296 uint32_t pref_flags = 0, valid_flags = 0;
297
298 if (!domains)
299 return -EINVAL;
300
301 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
302 valid_flags |= TTM_PL_FLAG_VRAM;
303
304 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
305 valid_flags |= TTM_PL_FLAG_TT;
306
307 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
308 bo->mem.mem_type == TTM_PL_VRAM)
309 pref_flags |= TTM_PL_FLAG_VRAM;
310
311 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
312 bo->mem.mem_type == TTM_PL_TT)
313 pref_flags |= TTM_PL_FLAG_TT;
314
315 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
316 pref_flags |= TTM_PL_FLAG_VRAM;
317
318 else
319 pref_flags |= TTM_PL_FLAG_TT;
320
321 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
322
323 return 0;
324}
325
326struct validate_op {
327 struct list_head list;
328 struct ww_acquire_ctx ticket;
329};
330
331static void
332validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence,
333 struct drm_nouveau_gem_pushbuf_bo *pbbo)
334{
335 struct nouveau_bo *nvbo;
336 struct drm_nouveau_gem_pushbuf_bo *b;
337
338 while (!list_empty(&op->list)) {
339 nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
340 b = &pbbo[nvbo->pbbo_index];
341
342 if (likely(fence))
343 nouveau_bo_fence(nvbo, fence, !!b->write_domains);
344
345 if (unlikely(nvbo->validate_mapped)) {
346 ttm_bo_kunmap(&nvbo->kmap);
347 nvbo->validate_mapped = false;
348 }
349
350 list_del(&nvbo->entry);
351 nvbo->reserved_by = NULL;
352 ttm_bo_unreserve_ticket(&nvbo->bo, &op->ticket);
353 drm_gem_object_unreference_unlocked(&nvbo->gem);
354 }
355}
356
357static void
358validate_fini(struct validate_op *op, struct nouveau_fence *fence,
359 struct drm_nouveau_gem_pushbuf_bo *pbbo)
360{
361 validate_fini_no_ticket(op, fence, pbbo);
362 ww_acquire_fini(&op->ticket);
363}
364
365static int
366validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
367 struct drm_nouveau_gem_pushbuf_bo *pbbo,
368 int nr_buffers, struct validate_op *op)
369{
370 struct nouveau_cli *cli = nouveau_cli(file_priv);
371 int trycnt = 0;
372 int ret = -EINVAL, i;
373 struct nouveau_bo *res_bo = NULL;
374 LIST_HEAD(gart_list);
375 LIST_HEAD(vram_list);
376 LIST_HEAD(both_list);
377
378 ww_acquire_init(&op->ticket, &reservation_ww_class);
379retry:
380 if (++trycnt > 100000) {
381 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
382 return -EINVAL;
383 }
384
385 for (i = 0; i < nr_buffers; i++) {
386 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
387 struct drm_gem_object *gem;
388 struct nouveau_bo *nvbo;
389
390 gem = drm_gem_object_lookup(file_priv, b->handle);
391 if (!gem) {
392 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
393 ret = -ENOENT;
394 break;
395 }
396 nvbo = nouveau_gem_object(gem);
397 if (nvbo == res_bo) {
398 res_bo = NULL;
399 drm_gem_object_unreference_unlocked(gem);
400 continue;
401 }
402
403 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
404 NV_PRINTK(err, cli, "multiple instances of buffer %d on "
405 "validation list\n", b->handle);
406 drm_gem_object_unreference_unlocked(gem);
407 ret = -EINVAL;
408 break;
409 }
410
411 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
412 if (ret) {
413 list_splice_tail_init(&vram_list, &op->list);
414 list_splice_tail_init(&gart_list, &op->list);
415 list_splice_tail_init(&both_list, &op->list);
416 validate_fini_no_ticket(op, NULL, NULL);
417 if (unlikely(ret == -EDEADLK)) {
418 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
419 &op->ticket);
420 if (!ret)
421 res_bo = nvbo;
422 }
423 if (unlikely(ret)) {
424 if (ret != -ERESTARTSYS)
425 NV_PRINTK(err, cli, "fail reserve\n");
426 break;
427 }
428 }
429
430 b->user_priv = (uint64_t)(unsigned long)nvbo;
431 nvbo->reserved_by = file_priv;
432 nvbo->pbbo_index = i;
433 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
434 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
435 list_add_tail(&nvbo->entry, &both_list);
436 else
437 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
438 list_add_tail(&nvbo->entry, &vram_list);
439 else
440 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
441 list_add_tail(&nvbo->entry, &gart_list);
442 else {
443 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
444 b->valid_domains);
445 list_add_tail(&nvbo->entry, &both_list);
446 ret = -EINVAL;
447 break;
448 }
449 if (nvbo == res_bo)
450 goto retry;
451 }
452
453 ww_acquire_done(&op->ticket);
454 list_splice_tail(&vram_list, &op->list);
455 list_splice_tail(&gart_list, &op->list);
456 list_splice_tail(&both_list, &op->list);
457 if (ret)
458 validate_fini(op, NULL, NULL);
459 return ret;
460
461}
462
463static int
464validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
465 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo,
466 uint64_t user_pbbo_ptr)
467{
468 struct nouveau_drm *drm = chan->drm;
469 struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
470 (void __force __user *)(uintptr_t)user_pbbo_ptr;
471 struct nouveau_bo *nvbo;
472 int ret, relocs = 0;
473
474 list_for_each_entry(nvbo, list, entry) {
475 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
476
477 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains,
478 b->write_domains,
479 b->valid_domains);
480 if (unlikely(ret)) {
481 NV_PRINTK(err, cli, "fail set_domain\n");
482 return ret;
483 }
484
485 ret = nouveau_bo_validate(nvbo, true, false);
486 if (unlikely(ret)) {
487 if (ret != -ERESTARTSYS)
488 NV_PRINTK(err, cli, "fail ttm_validate\n");
489 return ret;
490 }
491
492 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
493 if (unlikely(ret)) {
494 if (ret != -ERESTARTSYS)
495 NV_PRINTK(err, cli, "fail post-validate sync\n");
496 return ret;
497 }
498
499 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
500 if (nvbo->bo.offset == b->presumed.offset &&
501 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
502 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
503 (nvbo->bo.mem.mem_type == TTM_PL_TT &&
504 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
505 continue;
506
507 if (nvbo->bo.mem.mem_type == TTM_PL_TT)
508 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
509 else
510 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
511 b->presumed.offset = nvbo->bo.offset;
512 b->presumed.valid = 0;
513 relocs++;
514
515 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed,
516 &b->presumed, sizeof(b->presumed)))
517 return -EFAULT;
518 }
519 }
520
521 return relocs;
522}
523
524static int
525nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
526 struct drm_file *file_priv,
527 struct drm_nouveau_gem_pushbuf_bo *pbbo,
528 uint64_t user_buffers, int nr_buffers,
529 struct validate_op *op, int *apply_relocs)
530{
531 struct nouveau_cli *cli = nouveau_cli(file_priv);
532 int ret;
533
534 INIT_LIST_HEAD(&op->list);
535
536 if (nr_buffers == 0)
537 return 0;
538
539 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
540 if (unlikely(ret)) {
541 if (ret != -ERESTARTSYS)
542 NV_PRINTK(err, cli, "validate_init\n");
543 return ret;
544 }
545
546 ret = validate_list(chan, cli, &op->list, pbbo, user_buffers);
547 if (unlikely(ret < 0)) {
548 if (ret != -ERESTARTSYS)
549 NV_PRINTK(err, cli, "validating bo list\n");
550 validate_fini(op, NULL, NULL);
551 return ret;
552 }
553 *apply_relocs = ret;
554 return 0;
555}
556
557static inline void
558u_free(void *addr)
559{
560 kvfree(addr);
561}
562
563static inline void *
564u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
565{
566 void *mem;
567 void __user *userptr = (void __force __user *)(uintptr_t)user;
568
569 size *= nmemb;
570
571 mem = kvmalloc(size, GFP_KERNEL);
572 if (!mem)
573 return ERR_PTR(-ENOMEM);
574
575 if (copy_from_user(mem, userptr, size)) {
576 u_free(mem);
577 return ERR_PTR(-EFAULT);
578 }
579
580 return mem;
581}
582
583static int
584nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
585 struct drm_nouveau_gem_pushbuf *req,
586 struct drm_nouveau_gem_pushbuf_bo *bo)
587{
588 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
589 int ret = 0;
590 unsigned i;
591
592 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
593 if (IS_ERR(reloc))
594 return PTR_ERR(reloc);
595
596 for (i = 0; i < req->nr_relocs; i++) {
597 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
598 struct drm_nouveau_gem_pushbuf_bo *b;
599 struct nouveau_bo *nvbo;
600 uint32_t data;
601
602 if (unlikely(r->bo_index > req->nr_buffers)) {
603 NV_PRINTK(err, cli, "reloc bo index invalid\n");
604 ret = -EINVAL;
605 break;
606 }
607
608 b = &bo[r->bo_index];
609 if (b->presumed.valid)
610 continue;
611
612 if (unlikely(r->reloc_bo_index > req->nr_buffers)) {
613 NV_PRINTK(err, cli, "reloc container bo index invalid\n");
614 ret = -EINVAL;
615 break;
616 }
617 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
618
619 if (unlikely(r->reloc_bo_offset + 4 >
620 nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
621 NV_PRINTK(err, cli, "reloc outside of bo\n");
622 ret = -EINVAL;
623 break;
624 }
625
626 if (!nvbo->kmap.virtual) {
627 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
628 &nvbo->kmap);
629 if (ret) {
630 NV_PRINTK(err, cli, "failed kmap for reloc\n");
631 break;
632 }
633 nvbo->validate_mapped = true;
634 }
635
636 if (r->flags & NOUVEAU_GEM_RELOC_LOW)
637 data = b->presumed.offset + r->data;
638 else
639 if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
640 data = (b->presumed.offset + r->data) >> 32;
641 else
642 data = r->data;
643
644 if (r->flags & NOUVEAU_GEM_RELOC_OR) {
645 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
646 data |= r->tor;
647 else
648 data |= r->vor;
649 }
650
651 ret = ttm_bo_wait(&nvbo->bo, false, false);
652 if (ret) {
653 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret);
654 break;
655 }
656
657 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
658 }
659
660 u_free(reloc);
661 return ret;
662}
663
664int
665nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
666 struct drm_file *file_priv)
667{
668 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
669 struct nouveau_cli *cli = nouveau_cli(file_priv);
670 struct nouveau_abi16_chan *temp;
671 struct nouveau_drm *drm = nouveau_drm(dev);
672 struct drm_nouveau_gem_pushbuf *req = data;
673 struct drm_nouveau_gem_pushbuf_push *push;
674 struct drm_nouveau_gem_pushbuf_bo *bo;
675 struct nouveau_channel *chan = NULL;
676 struct validate_op op;
677 struct nouveau_fence *fence = NULL;
678 int i, j, ret = 0, do_reloc = 0;
679
680 if (unlikely(!abi16))
681 return -ENOMEM;
682
683 list_for_each_entry(temp, &abi16->channels, head) {
684 if (temp->chan->chid == req->channel) {
685 chan = temp->chan;
686 break;
687 }
688 }
689
690 if (!chan)
691 return nouveau_abi16_put(abi16, -ENOENT);
692
693 req->vram_available = drm->gem.vram_available;
694 req->gart_available = drm->gem.gart_available;
695 if (unlikely(req->nr_push == 0))
696 goto out_next;
697
698 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
699 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
700 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
701 return nouveau_abi16_put(abi16, -EINVAL);
702 }
703
704 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
705 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
706 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
707 return nouveau_abi16_put(abi16, -EINVAL);
708 }
709
710 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
711 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
712 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
713 return nouveau_abi16_put(abi16, -EINVAL);
714 }
715
716 push = u_memcpya(req->push, req->nr_push, sizeof(*push));
717 if (IS_ERR(push))
718 return nouveau_abi16_put(abi16, PTR_ERR(push));
719
720 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
721 if (IS_ERR(bo)) {
722 u_free(push);
723 return nouveau_abi16_put(abi16, PTR_ERR(bo));
724 }
725
726
727 for (i = 0; i < req->nr_push; i++) {
728 if (push[i].bo_index >= req->nr_buffers) {
729 NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
730 ret = -EINVAL;
731 goto out_prevalid;
732 }
733 }
734
735
736 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
737 req->nr_buffers, &op, &do_reloc);
738 if (ret) {
739 if (ret != -ERESTARTSYS)
740 NV_PRINTK(err, cli, "validate: %d\n", ret);
741 goto out_prevalid;
742 }
743
744
745 if (do_reloc) {
746 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo);
747 if (ret) {
748 NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
749 goto out;
750 }
751 }
752
753 if (chan->dma.ib_max) {
754 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
755 if (ret) {
756 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
757 goto out;
758 }
759
760 for (i = 0; i < req->nr_push; i++) {
761 struct nouveau_bo *nvbo = (void *)(unsigned long)
762 bo[push[i].bo_index].user_priv;
763
764 nv50_dma_push(chan, nvbo, push[i].offset,
765 push[i].length);
766 }
767 } else
768 if (drm->client.device.info.chipset >= 0x25) {
769 ret = RING_SPACE(chan, req->nr_push * 2);
770 if (ret) {
771 NV_PRINTK(err, cli, "cal_space: %d\n", ret);
772 goto out;
773 }
774
775 for (i = 0; i < req->nr_push; i++) {
776 struct nouveau_bo *nvbo = (void *)(unsigned long)
777 bo[push[i].bo_index].user_priv;
778
779 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
780 OUT_RING(chan, 0);
781 }
782 } else {
783 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
784 if (ret) {
785 NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
786 goto out;
787 }
788
789 for (i = 0; i < req->nr_push; i++) {
790 struct nouveau_bo *nvbo = (void *)(unsigned long)
791 bo[push[i].bo_index].user_priv;
792 uint32_t cmd;
793
794 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2);
795 cmd |= 0x20000000;
796 if (unlikely(cmd != req->suffix0)) {
797 if (!nvbo->kmap.virtual) {
798 ret = ttm_bo_kmap(&nvbo->bo, 0,
799 nvbo->bo.mem.
800 num_pages,
801 &nvbo->kmap);
802 if (ret) {
803 WIND_RING(chan);
804 goto out;
805 }
806 nvbo->validate_mapped = true;
807 }
808
809 nouveau_bo_wr32(nvbo, (push[i].offset +
810 push[i].length - 8) / 4, cmd);
811 }
812
813 OUT_RING(chan, 0x20000000 |
814 (nvbo->bo.offset + push[i].offset));
815 OUT_RING(chan, 0);
816 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
817 OUT_RING(chan, 0);
818 }
819 }
820
821 ret = nouveau_fence_new(chan, false, &fence);
822 if (ret) {
823 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
824 WIND_RING(chan);
825 goto out;
826 }
827
828out:
829 validate_fini(&op, fence, bo);
830 nouveau_fence_unref(&fence);
831
832out_prevalid:
833 u_free(bo);
834 u_free(push);
835
836out_next:
837 if (chan->dma.ib_max) {
838 req->suffix0 = 0x00000000;
839 req->suffix1 = 0x00000000;
840 } else
841 if (drm->client.device.info.chipset >= 0x25) {
842 req->suffix0 = 0x00020000;
843 req->suffix1 = 0x00000000;
844 } else {
845 req->suffix0 = 0x20000000 |
846 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2));
847 req->suffix1 = 0x00000000;
848 }
849
850 return nouveau_abi16_put(abi16, ret);
851}
852
853int
854nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
855 struct drm_file *file_priv)
856{
857 struct drm_nouveau_gem_cpu_prep *req = data;
858 struct drm_gem_object *gem;
859 struct nouveau_bo *nvbo;
860 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
861 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
862 long lret;
863 int ret;
864
865 gem = drm_gem_object_lookup(file_priv, req->handle);
866 if (!gem)
867 return -ENOENT;
868 nvbo = nouveau_gem_object(gem);
869
870 lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true,
871 no_wait ? 0 : 30 * HZ);
872 if (!lret)
873 ret = -EBUSY;
874 else if (lret > 0)
875 ret = 0;
876 else
877 ret = lret;
878
879 nouveau_bo_sync_for_cpu(nvbo);
880 drm_gem_object_unreference_unlocked(gem);
881
882 return ret;
883}
884
885int
886nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
887 struct drm_file *file_priv)
888{
889 struct drm_nouveau_gem_cpu_fini *req = data;
890 struct drm_gem_object *gem;
891 struct nouveau_bo *nvbo;
892
893 gem = drm_gem_object_lookup(file_priv, req->handle);
894 if (!gem)
895 return -ENOENT;
896 nvbo = nouveau_gem_object(gem);
897
898 nouveau_bo_sync_for_device(nvbo);
899 drm_gem_object_unreference_unlocked(gem);
900 return 0;
901}
902
903int
904nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
905 struct drm_file *file_priv)
906{
907 struct drm_nouveau_gem_info *req = data;
908 struct drm_gem_object *gem;
909 int ret;
910
911 gem = drm_gem_object_lookup(file_priv, req->handle);
912 if (!gem)
913 return -ENOENT;
914
915 ret = nouveau_gem_info(file_priv, gem, req);
916 drm_gem_object_unreference_unlocked(gem);
917 return ret;
918}
919
920
