1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#include <linux/dma-mapping.h>
31#include <linux/swiotlb.h>
32
33#include "nouveau_drv.h"
34#include "nouveau_dma.h"
35#include "nouveau_fence.h"
36
37#include "nouveau_bo.h"
38#include "nouveau_ttm.h"
39#include "nouveau_gem.h"
40#include "nouveau_mem.h"
41#include "nouveau_vmm.h"
42
43#include <nvif/class.h>
44#include <nvif/if500b.h>
45#include <nvif/if900b.h>
46
47
48
49
50
51static void
52nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
53 u32 addr, u32 size, u32 pitch, u32 flags)
54{
55 struct nouveau_drm *drm = nouveau_drm(dev);
56 int i = reg - drm->tile.reg;
57 struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
58 struct nvkm_fb_tile *tile = &fb->tile.region[i];
59
60 nouveau_fence_unref(®->fence);
61
62 if (tile->pitch)
63 nvkm_fb_tile_fini(fb, i, tile);
64
65 if (pitch)
66 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile);
67
68 nvkm_fb_tile_prog(fb, i, tile);
69}
70
71static struct nouveau_drm_tile *
72nv10_bo_get_tile_region(struct drm_device *dev, int i)
73{
74 struct nouveau_drm *drm = nouveau_drm(dev);
75 struct nouveau_drm_tile *tile = &drm->tile.reg[i];
76
77 spin_lock(&drm->tile.lock);
78
79 if (!tile->used &&
80 (!tile->fence || nouveau_fence_done(tile->fence)))
81 tile->used = true;
82 else
83 tile = NULL;
84
85 spin_unlock(&drm->tile.lock);
86 return tile;
87}
88
89static void
90nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
91 struct dma_fence *fence)
92{
93 struct nouveau_drm *drm = nouveau_drm(dev);
94
95 if (tile) {
96 spin_lock(&drm->tile.lock);
97 tile->fence = (struct nouveau_fence *)dma_fence_get(fence);
98 tile->used = false;
99 spin_unlock(&drm->tile.lock);
100 }
101}
102
103static struct nouveau_drm_tile *
104nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
105 u32 size, u32 pitch, u32 zeta)
106{
107 struct nouveau_drm *drm = nouveau_drm(dev);
108 struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
109 struct nouveau_drm_tile *tile, *found = NULL;
110 int i;
111
112 for (i = 0; i < fb->tile.regions; i++) {
113 tile = nv10_bo_get_tile_region(dev, i);
114
115 if (pitch && !found) {
116 found = tile;
117 continue;
118
119 } else if (tile && fb->tile.region[i].pitch) {
120
121 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
122 }
123
124 nv10_bo_put_tile_region(dev, tile, NULL);
125 }
126
127 if (found)
128 nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta);
129 return found;
130}
131
132static void
133nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
134{
135 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
136 struct drm_device *dev = drm->dev;
137 struct nouveau_bo *nvbo = nouveau_bo(bo);
138
139 WARN_ON(nvbo->pin_refcnt > 0);
140 nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
141
142
143
144
145
146 if (bo->base.dev)
147 drm_gem_object_release(&bo->base);
148
149 kfree(nvbo);
150}
151
152static inline u64
153roundup_64(u64 x, u32 y)
154{
155 x += y - 1;
156 do_div(x, y);
157 return x * y;
158}
159
160static void
161nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
162 int *align, u64 *size)
163{
164 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
165 struct nvif_device *device = &drm->client.device;
166
167 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
168 if (nvbo->mode) {
169 if (device->info.chipset >= 0x40) {
170 *align = 65536;
171 *size = roundup_64(*size, 64 * nvbo->mode);
172
173 } else if (device->info.chipset >= 0x30) {
174 *align = 32768;
175 *size = roundup_64(*size, 64 * nvbo->mode);
176
177 } else if (device->info.chipset >= 0x20) {
178 *align = 16384;
179 *size = roundup_64(*size, 64 * nvbo->mode);
180
181 } else if (device->info.chipset >= 0x10) {
182 *align = 16384;
183 *size = roundup_64(*size, 32 * nvbo->mode);
184 }
185 }
186 } else {
187 *size = roundup_64(*size, (1 << nvbo->page));
188 *align = max((1 << nvbo->page), *align);
189 }
190
191 *size = roundup_64(*size, PAGE_SIZE);
192}
193
194struct nouveau_bo *
195nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 flags,
196 u32 tile_mode, u32 tile_flags)
197{
198 struct nouveau_drm *drm = cli->drm;
199 struct nouveau_bo *nvbo;
200 struct nvif_mmu *mmu = &cli->mmu;
201 struct nvif_vmm *vmm = cli->svm.cli ? &cli->svm.vmm : &cli->vmm.vmm;
202 int i, pi = -1;
203
204 if (!*size) {
205 NV_WARN(drm, "skipped size %016llx\n", *size);
206 return ERR_PTR(-EINVAL);
207 }
208
209 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
210 if (!nvbo)
211 return ERR_PTR(-ENOMEM);
212 INIT_LIST_HEAD(&nvbo->head);
213 INIT_LIST_HEAD(&nvbo->entry);
214 INIT_LIST_HEAD(&nvbo->vma_list);
215 nvbo->bo.bdev = &drm->ttm.bdev;
216
217
218
219
220
221 if (flags & TTM_PL_FLAG_UNCACHED) {
222
223
224
225 if (!nouveau_drm_use_coherent_gpu_mapping(drm))
226 nvbo->force_coherent = true;
227 }
228
229 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) {
230 nvbo->kind = (tile_flags & 0x0000ff00) >> 8;
231 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
232 kfree(nvbo);
233 return ERR_PTR(-EINVAL);
234 }
235
236 nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;
237 } else
238 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
239 nvbo->kind = (tile_flags & 0x00007f00) >> 8;
240 nvbo->comp = (tile_flags & 0x00030000) >> 16;
241 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
242 kfree(nvbo);
243 return ERR_PTR(-EINVAL);
244 }
245 } else {
246 nvbo->zeta = (tile_flags & 0x00000007);
247 }
248 nvbo->mode = tile_mode;
249 nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG);
250
251
252 for (i = 0; i < vmm->page_nr; i++) {
253
254
255
256
257
258
259
260 if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE &&
261 (flags & TTM_PL_FLAG_VRAM) && !vmm->page[i].vram)
262 continue;
263 if ((flags & TTM_PL_FLAG_TT) &&
264 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT))
265 continue;
266
267
268
269
270
271 if (pi < 0 || !nvbo->comp || vmm->page[i].comp)
272 pi = i;
273
274
275 if (*size >= 1ULL << vmm->page[i].shift)
276 break;
277 }
278
279 if (WARN_ON(pi < 0))
280 return ERR_PTR(-EINVAL);
281
282
283 if (nvbo->comp && !vmm->page[pi].comp) {
284 if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100)
285 nvbo->kind = mmu->kind[nvbo->kind];
286 nvbo->comp = 0;
287 }
288 nvbo->page = vmm->page[pi].shift;
289
290 nouveau_bo_fixup_align(nvbo, flags, align, size);
291
292 return nvbo;
293}
294
295int
296nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 flags,
297 struct sg_table *sg, struct dma_resv *robj)
298{
299 int type = sg ? ttm_bo_type_sg : ttm_bo_type_device;
300 size_t acc_size;
301 int ret;
302
303 acc_size = ttm_bo_dma_acc_size(nvbo->bo.bdev, size, sizeof(*nvbo));
304
305 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
306 nouveau_bo_placement_set(nvbo, flags, 0);
307
308 ret = ttm_bo_init(nvbo->bo.bdev, &nvbo->bo, size, type,
309 &nvbo->placement, align >> PAGE_SHIFT, false,
310 acc_size, sg, robj, nouveau_bo_del_ttm);
311 if (ret) {
312
313 return ret;
314 }
315
316 return 0;
317}
318
319int
320nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align,
321 uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
322 struct sg_table *sg, struct dma_resv *robj,
323 struct nouveau_bo **pnvbo)
324{
325 struct nouveau_bo *nvbo;
326 int ret;
327
328 nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode,
329 tile_flags);
330 if (IS_ERR(nvbo))
331 return PTR_ERR(nvbo);
332
333 ret = nouveau_bo_init(nvbo, size, align, flags, sg, robj);
334 if (ret)
335 return ret;
336
337 *pnvbo = nvbo;
338 return 0;
339}
340
341static void
342set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t type, uint32_t flags)
343{
344 *n = 0;
345
346 if (type & TTM_PL_FLAG_VRAM)
347 pl[(*n)++].flags = TTM_PL_FLAG_VRAM | flags;
348 if (type & TTM_PL_FLAG_TT)
349 pl[(*n)++].flags = TTM_PL_FLAG_TT | flags;
350 if (type & TTM_PL_FLAG_SYSTEM)
351 pl[(*n)++].flags = TTM_PL_FLAG_SYSTEM | flags;
352}
353
354static void
355set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
356{
357 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
358 u32 vram_pages = drm->client.device.info.ram_size >> PAGE_SHIFT;
359 unsigned i, fpfn, lpfn;
360
361 if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
362 nvbo->mode && (type & TTM_PL_FLAG_VRAM) &&
363 nvbo->bo.mem.num_pages < vram_pages / 4) {
364
365
366
367
368
369
370 if (nvbo->zeta) {
371 fpfn = vram_pages / 2;
372 lpfn = ~0;
373 } else {
374 fpfn = 0;
375 lpfn = vram_pages / 2;
376 }
377 for (i = 0; i < nvbo->placement.num_placement; ++i) {
378 nvbo->placements[i].fpfn = fpfn;
379 nvbo->placements[i].lpfn = lpfn;
380 }
381 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
382 nvbo->busy_placements[i].fpfn = fpfn;
383 nvbo->busy_placements[i].lpfn = lpfn;
384 }
385 }
386}
387
388void
389nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
390{
391 struct ttm_placement *pl = &nvbo->placement;
392 uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
393 TTM_PL_MASK_CACHING) |
394 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
395
396 pl->placement = nvbo->placements;
397 set_placement_list(nvbo->placements, &pl->num_placement,
398 type, flags);
399
400 pl->busy_placement = nvbo->busy_placements;
401 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
402 type | busy, flags);
403
404 set_placement_range(nvbo, type);
405}
406
407int
408nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig)
409{
410 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
411 struct ttm_buffer_object *bo = &nvbo->bo;
412 bool force = false, evict = false;
413 int ret;
414
415 ret = ttm_bo_reserve(bo, false, false, NULL);
416 if (ret)
417 return ret;
418
419 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
420 memtype == TTM_PL_FLAG_VRAM && contig) {
421 if (!nvbo->contig) {
422 nvbo->contig = true;
423 force = true;
424 evict = true;
425 }
426 }
427
428 if (nvbo->pin_refcnt) {
429 if (!(memtype & (1 << bo->mem.mem_type)) || evict) {
430 NV_ERROR(drm, "bo %p pinned elsewhere: "
431 "0x%08x vs 0x%08x\n", bo,
432 1 << bo->mem.mem_type, memtype);
433 ret = -EBUSY;
434 }
435 nvbo->pin_refcnt++;
436 goto out;
437 }
438
439 if (evict) {
440 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0);
441 ret = nouveau_bo_validate(nvbo, false, false);
442 if (ret)
443 goto out;
444 }
445
446 nvbo->pin_refcnt++;
447 nouveau_bo_placement_set(nvbo, memtype, 0);
448
449
450
451
452
453 nvbo->pin_refcnt--;
454 ret = nouveau_bo_validate(nvbo, false, false);
455 if (ret)
456 goto out;
457 nvbo->pin_refcnt++;
458
459 switch (bo->mem.mem_type) {
460 case TTM_PL_VRAM:
461 drm->gem.vram_available -= bo->mem.size;
462 break;
463 case TTM_PL_TT:
464 drm->gem.gart_available -= bo->mem.size;
465 break;
466 default:
467 break;
468 }
469
470out:
471 if (force && ret)
472 nvbo->contig = false;
473 ttm_bo_unreserve(bo);
474 return ret;
475}
476
477int
478nouveau_bo_unpin(struct nouveau_bo *nvbo)
479{
480 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
481 struct ttm_buffer_object *bo = &nvbo->bo;
482 int ret, ref;
483
484 ret = ttm_bo_reserve(bo, false, false, NULL);
485 if (ret)
486 return ret;
487
488 ref = --nvbo->pin_refcnt;
489 WARN_ON_ONCE(ref < 0);
490 if (ref)
491 goto out;
492
493 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
494
495 ret = nouveau_bo_validate(nvbo, false, false);
496 if (ret == 0) {
497 switch (bo->mem.mem_type) {
498 case TTM_PL_VRAM:
499 drm->gem.vram_available += bo->mem.size;
500 break;
501 case TTM_PL_TT:
502 drm->gem.gart_available += bo->mem.size;
503 break;
504 default:
505 break;
506 }
507 }
508
509out:
510 ttm_bo_unreserve(bo);
511 return ret;
512}
513
514int
515nouveau_bo_map(struct nouveau_bo *nvbo)
516{
517 int ret;
518
519 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
520 if (ret)
521 return ret;
522
523 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
524
525 ttm_bo_unreserve(&nvbo->bo);
526 return ret;
527}
528
529void
530nouveau_bo_unmap(struct nouveau_bo *nvbo)
531{
532 if (!nvbo)
533 return;
534
535 ttm_bo_kunmap(&nvbo->kmap);
536}
537
538void
539nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
540{
541 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
542 struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
543 int i;
544
545 if (!ttm_dma)
546 return;
547
548
549 if (nvbo->force_coherent)
550 return;
551
552 for (i = 0; i < ttm_dma->ttm.num_pages; i++)
553 dma_sync_single_for_device(drm->dev->dev,
554 ttm_dma->dma_address[i],
555 PAGE_SIZE, DMA_TO_DEVICE);
556}
557
558void
559nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
560{
561 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
562 struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
563 int i;
564
565 if (!ttm_dma)
566 return;
567
568
569 if (nvbo->force_coherent)
570 return;
571
572 for (i = 0; i < ttm_dma->ttm.num_pages; i++)
573 dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i],
574 PAGE_SIZE, DMA_FROM_DEVICE);
575}
576
577int
578nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
579 bool no_wait_gpu)
580{
581 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
582 int ret;
583
584 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx);
585 if (ret)
586 return ret;
587
588 nouveau_bo_sync_for_device(nvbo);
589
590 return 0;
591}
592
593void
594nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
595{
596 bool is_iomem;
597 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
598
599 mem += index;
600
601 if (is_iomem)
602 iowrite16_native(val, (void __force __iomem *)mem);
603 else
604 *mem = val;
605}
606
607u32
608nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
609{
610 bool is_iomem;
611 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
612
613 mem += index;
614
615 if (is_iomem)
616 return ioread32_native((void __force __iomem *)mem);
617 else
618 return *mem;
619}
620
621void
622nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
623{
624 bool is_iomem;
625 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
626
627 mem += index;
628
629 if (is_iomem)
630 iowrite32_native(val, (void __force __iomem *)mem);
631 else
632 *mem = val;
633}
634
635static struct ttm_tt *
636nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags)
637{
638#if IS_ENABLED(CONFIG_AGP)
639 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
640
641 if (drm->agp.bridge) {
642 return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags);
643 }
644#endif
645
646 return nouveau_sgdma_create_ttm(bo, page_flags);
647}
648
649static int
650nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
651 struct ttm_mem_type_manager *man)
652{
653 struct nouveau_drm *drm = nouveau_bdev(bdev);
654 struct nvif_mmu *mmu = &drm->client.mmu;
655
656 switch (type) {
657 case TTM_PL_SYSTEM:
658 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
659 man->available_caching = TTM_PL_MASK_CACHING;
660 man->default_caching = TTM_PL_FLAG_CACHED;
661 break;
662 case TTM_PL_VRAM:
663 man->flags = TTM_MEMTYPE_FLAG_FIXED |
664 TTM_MEMTYPE_FLAG_MAPPABLE;
665 man->available_caching = TTM_PL_FLAG_UNCACHED |
666 TTM_PL_FLAG_WC;
667 man->default_caching = TTM_PL_FLAG_WC;
668
669 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
670
671 const u8 type = mmu->type[drm->ttm.type_vram].type;
672 if (type & NVIF_MEM_UNCACHED) {
673 man->available_caching = TTM_PL_FLAG_UNCACHED;
674 man->default_caching = TTM_PL_FLAG_UNCACHED;
675 }
676
677 man->func = &nouveau_vram_manager;
678 man->io_reserve_fastpath = false;
679 man->use_io_reserve_lru = true;
680 } else {
681 man->func = &ttm_bo_manager_func;
682 }
683 break;
684 case TTM_PL_TT:
685 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
686 man->func = &nouveau_gart_manager;
687 else
688 if (!drm->agp.bridge)
689 man->func = &nv04_gart_manager;
690 else
691 man->func = &ttm_bo_manager_func;
692
693 if (drm->agp.bridge) {
694 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
695 man->available_caching = TTM_PL_FLAG_UNCACHED |
696 TTM_PL_FLAG_WC;
697 man->default_caching = TTM_PL_FLAG_WC;
698 } else {
699 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
700 TTM_MEMTYPE_FLAG_CMA;
701 man->available_caching = TTM_PL_MASK_CACHING;
702 man->default_caching = TTM_PL_FLAG_CACHED;
703 }
704
705 break;
706 default:
707 return -EINVAL;
708 }
709 return 0;
710}
711
712static void
713nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
714{
715 struct nouveau_bo *nvbo = nouveau_bo(bo);
716
717 switch (bo->mem.mem_type) {
718 case TTM_PL_VRAM:
719 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
720 TTM_PL_FLAG_SYSTEM);
721 break;
722 default:
723 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
724 break;
725 }
726
727 *pl = nvbo->placement;
728}
729
730
731static int
732nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
733{
734 int ret = RING_SPACE(chan, 2);
735 if (ret == 0) {
736 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
737 OUT_RING (chan, handle & 0x0000ffff);
738 FIRE_RING (chan);
739 }
740 return ret;
741}
742
743static int
744nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
745 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
746{
747 struct nouveau_mem *mem = nouveau_mem(old_reg);
748 int ret = RING_SPACE(chan, 10);
749 if (ret == 0) {
750 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
751 OUT_RING (chan, upper_32_bits(mem->vma[0].addr));
752 OUT_RING (chan, lower_32_bits(mem->vma[0].addr));
753 OUT_RING (chan, upper_32_bits(mem->vma[1].addr));
754 OUT_RING (chan, lower_32_bits(mem->vma[1].addr));
755 OUT_RING (chan, PAGE_SIZE);
756 OUT_RING (chan, PAGE_SIZE);
757 OUT_RING (chan, PAGE_SIZE);
758 OUT_RING (chan, new_reg->num_pages);
759 BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
760 }
761 return ret;
762}
763
764static int
765nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
766{
767 int ret = RING_SPACE(chan, 2);
768 if (ret == 0) {
769 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
770 OUT_RING (chan, handle);
771 }
772 return ret;
773}
774
775static int
776nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
777 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
778{
779 struct nouveau_mem *mem = nouveau_mem(old_reg);
780 u64 src_offset = mem->vma[0].addr;
781 u64 dst_offset = mem->vma[1].addr;
782 u32 page_count = new_reg->num_pages;
783 int ret;
784
785 page_count = new_reg->num_pages;
786 while (page_count) {
787 int line_count = (page_count > 8191) ? 8191 : page_count;
788
789 ret = RING_SPACE(chan, 11);
790 if (ret)
791 return ret;
792
793 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
794 OUT_RING (chan, upper_32_bits(src_offset));
795 OUT_RING (chan, lower_32_bits(src_offset));
796 OUT_RING (chan, upper_32_bits(dst_offset));
797 OUT_RING (chan, lower_32_bits(dst_offset));
798 OUT_RING (chan, PAGE_SIZE);
799 OUT_RING (chan, PAGE_SIZE);
800 OUT_RING (chan, PAGE_SIZE);
801 OUT_RING (chan, line_count);
802 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
803 OUT_RING (chan, 0x00000110);
804
805 page_count -= line_count;
806 src_offset += (PAGE_SIZE * line_count);
807 dst_offset += (PAGE_SIZE * line_count);
808 }
809
810 return 0;
811}
812
813static int
814nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
815 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
816{
817 struct nouveau_mem *mem = nouveau_mem(old_reg);
818 u64 src_offset = mem->vma[0].addr;
819 u64 dst_offset = mem->vma[1].addr;
820 u32 page_count = new_reg->num_pages;
821 int ret;
822
823 page_count = new_reg->num_pages;
824 while (page_count) {
825 int line_count = (page_count > 2047) ? 2047 : page_count;
826
827 ret = RING_SPACE(chan, 12);
828 if (ret)
829 return ret;
830
831 BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
832 OUT_RING (chan, upper_32_bits(dst_offset));
833 OUT_RING (chan, lower_32_bits(dst_offset));
834 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
835 OUT_RING (chan, upper_32_bits(src_offset));
836 OUT_RING (chan, lower_32_bits(src_offset));
837 OUT_RING (chan, PAGE_SIZE);
838 OUT_RING (chan, PAGE_SIZE);
839 OUT_RING (chan, PAGE_SIZE);
840 OUT_RING (chan, line_count);
841 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
842 OUT_RING (chan, 0x00100110);
843
844 page_count -= line_count;
845 src_offset += (PAGE_SIZE * line_count);
846 dst_offset += (PAGE_SIZE * line_count);
847 }
848
849 return 0;
850}
851
852static int
853nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
854 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
855{
856 struct nouveau_mem *mem = nouveau_mem(old_reg);
857 u64 src_offset = mem->vma[0].addr;
858 u64 dst_offset = mem->vma[1].addr;
859 u32 page_count = new_reg->num_pages;
860 int ret;
861
862 page_count = new_reg->num_pages;
863 while (page_count) {
864 int line_count = (page_count > 8191) ? 8191 : page_count;
865
866 ret = RING_SPACE(chan, 11);
867 if (ret)
868 return ret;
869
870 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
871 OUT_RING (chan, upper_32_bits(src_offset));
872 OUT_RING (chan, lower_32_bits(src_offset));
873 OUT_RING (chan, upper_32_bits(dst_offset));
874 OUT_RING (chan, lower_32_bits(dst_offset));
875 OUT_RING (chan, PAGE_SIZE);
876 OUT_RING (chan, PAGE_SIZE);
877 OUT_RING (chan, PAGE_SIZE);
878 OUT_RING (chan, line_count);
879 BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
880 OUT_RING (chan, 0x00000110);
881
882 page_count -= line_count;
883 src_offset += (PAGE_SIZE * line_count);
884 dst_offset += (PAGE_SIZE * line_count);
885 }
886
887 return 0;
888}
889
890static int
891nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
892 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
893{
894 struct nouveau_mem *mem = nouveau_mem(old_reg);
895 int ret = RING_SPACE(chan, 7);
896 if (ret == 0) {
897 BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
898 OUT_RING (chan, upper_32_bits(mem->vma[0].addr));
899 OUT_RING (chan, lower_32_bits(mem->vma[0].addr));
900 OUT_RING (chan, upper_32_bits(mem->vma[1].addr));
901 OUT_RING (chan, lower_32_bits(mem->vma[1].addr));
902 OUT_RING (chan, 0x00000000 );
903 OUT_RING (chan, new_reg->num_pages << PAGE_SHIFT);
904 }
905 return ret;
906}
907
908static int
909nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
910 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
911{
912 struct nouveau_mem *mem = nouveau_mem(old_reg);
913 int ret = RING_SPACE(chan, 7);
914 if (ret == 0) {
915 BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
916 OUT_RING (chan, new_reg->num_pages << PAGE_SHIFT);
917 OUT_RING (chan, upper_32_bits(mem->vma[0].addr));
918 OUT_RING (chan, lower_32_bits(mem->vma[0].addr));
919 OUT_RING (chan, upper_32_bits(mem->vma[1].addr));
920 OUT_RING (chan, lower_32_bits(mem->vma[1].addr));
921 OUT_RING (chan, 0x00000000 );
922 }
923 return ret;
924}
925
926static int
927nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
928{
929 int ret = RING_SPACE(chan, 6);
930 if (ret == 0) {
931 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
932 OUT_RING (chan, handle);
933 BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
934 OUT_RING (chan, chan->drm->ntfy.handle);
935 OUT_RING (chan, chan->vram.handle);
936 OUT_RING (chan, chan->vram.handle);
937 }
938
939 return ret;
940}
941
942static int
943nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
944 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
945{
946 struct nouveau_mem *mem = nouveau_mem(old_reg);
947 u64 length = (new_reg->num_pages << PAGE_SHIFT);
948 u64 src_offset = mem->vma[0].addr;
949 u64 dst_offset = mem->vma[1].addr;
950 int src_tiled = !!mem->kind;
951 int dst_tiled = !!nouveau_mem(new_reg)->kind;
952 int ret;
953
954 while (length) {
955 u32 amount, stride, height;
956
957 ret = RING_SPACE(chan, 18 + 6 * (src_tiled + dst_tiled));
958 if (ret)
959 return ret;
960
961 amount = min(length, (u64)(4 * 1024 * 1024));
962 stride = 16 * 4;
963 height = amount / stride;
964
965 if (src_tiled) {
966 BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
967 OUT_RING (chan, 0);
968 OUT_RING (chan, 0);
969 OUT_RING (chan, stride);
970 OUT_RING (chan, height);
971 OUT_RING (chan, 1);
972 OUT_RING (chan, 0);
973 OUT_RING (chan, 0);
974 } else {
975 BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
976 OUT_RING (chan, 1);
977 }
978 if (dst_tiled) {
979 BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
980 OUT_RING (chan, 0);
981 OUT_RING (chan, 0);
982 OUT_RING (chan, stride);
983 OUT_RING (chan, height);
984 OUT_RING (chan, 1);
985 OUT_RING (chan, 0);
986 OUT_RING (chan, 0);
987 } else {
988 BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
989 OUT_RING (chan, 1);
990 }
991
992 BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
993 OUT_RING (chan, upper_32_bits(src_offset));
994 OUT_RING (chan, upper_32_bits(dst_offset));
995 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
996 OUT_RING (chan, lower_32_bits(src_offset));
997 OUT_RING (chan, lower_32_bits(dst_offset));
998 OUT_RING (chan, stride);
999 OUT_RING (chan, stride);
1000 OUT_RING (chan, stride);
1001 OUT_RING (chan, height);
1002 OUT_RING (chan, 0x00000101);
1003 OUT_RING (chan, 0x00000000);
1004 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
1005 OUT_RING (chan, 0);
1006
1007 length -= amount;
1008 src_offset += amount;
1009 dst_offset += amount;
1010 }
1011
1012 return 0;
1013}
1014
1015static int
1016nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
1017{
1018 int ret = RING_SPACE(chan, 4);
1019 if (ret == 0) {
1020 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
1021 OUT_RING (chan, handle);
1022 BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
1023 OUT_RING (chan, chan->drm->ntfy.handle);
1024 }
1025
1026 return ret;
1027}
1028
1029static inline uint32_t
1030nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
1031 struct nouveau_channel *chan, struct ttm_mem_reg *reg)
1032{
1033 if (reg->mem_type == TTM_PL_TT)
1034 return NvDmaTT;
1035 return chan->vram.handle;
1036}
1037
1038static int
1039nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
1040 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
1041{
1042 u32 src_offset = old_reg->start << PAGE_SHIFT;
1043 u32 dst_offset = new_reg->start << PAGE_SHIFT;
1044 u32 page_count = new_reg->num_pages;
1045 int ret;
1046
1047 ret = RING_SPACE(chan, 3);
1048 if (ret)
1049 return ret;
1050
1051 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
1052 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_reg));
1053 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_reg));
1054
1055 page_count = new_reg->num_pages;
1056 while (page_count) {
1057 int line_count = (page_count > 2047) ? 2047 : page_count;
1058
1059 ret = RING_SPACE(chan, 11);
1060 if (ret)
1061 return ret;
1062
1063 BEGIN_NV04(chan, NvSubCopy,
1064 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
1065 OUT_RING (chan, src_offset);
1066 OUT_RING (chan, dst_offset);
1067 OUT_RING (chan, PAGE_SIZE);
1068 OUT_RING (chan, PAGE_SIZE);
1069 OUT_RING (chan, PAGE_SIZE);
1070 OUT_RING (chan, line_count);
1071 OUT_RING (chan, 0x00000101);
1072 OUT_RING (chan, 0x00000000);
1073 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
1074 OUT_RING (chan, 0);
1075
1076 page_count -= line_count;
1077 src_offset += (PAGE_SIZE * line_count);
1078 dst_offset += (PAGE_SIZE * line_count);
1079 }
1080
1081 return 0;
1082}
1083
1084static int
1085nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
1086 struct ttm_mem_reg *reg)
1087{
1088 struct nouveau_mem *old_mem = nouveau_mem(&bo->mem);
1089 struct nouveau_mem *new_mem = nouveau_mem(reg);
1090 struct nvif_vmm *vmm = &drm->client.vmm.vmm;
1091 int ret;
1092
1093 ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0,
1094 old_mem->mem.size, &old_mem->vma[0]);
1095 if (ret)
1096 return ret;
1097
1098 ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0,
1099 new_mem->mem.size, &old_mem->vma[1]);
1100 if (ret)
1101 goto done;
1102
1103 ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]);
1104 if (ret)
1105 goto done;
1106
1107 ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]);
1108done:
1109 if (ret) {
1110 nvif_vmm_put(vmm, &old_mem->vma[1]);
1111 nvif_vmm_put(vmm, &old_mem->vma[0]);
1112 }
1113 return 0;
1114}
1115
1116static int
1117nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
1118 bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1119{
1120 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1121 struct nouveau_channel *chan = drm->ttm.chan;
1122 struct nouveau_cli *cli = (void *)chan->user.client;
1123 struct nouveau_fence *fence;
1124 int ret;
1125
1126
1127
1128
1129
1130 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1131 ret = nouveau_bo_move_prep(drm, bo, new_reg);
1132 if (ret)
1133 return ret;
1134 }
1135
1136 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
1137 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);
1138 if (ret == 0) {
1139 ret = drm->ttm.move(chan, bo, &bo->mem, new_reg);
1140 if (ret == 0) {
1141 ret = nouveau_fence_new(chan, false, &fence);
1142 if (ret == 0) {
1143 ret = ttm_bo_move_accel_cleanup(bo,
1144 &fence->base,
1145 evict,
1146 new_reg);
1147 nouveau_fence_unref(&fence);
1148 }
1149 }
1150 }
1151 mutex_unlock(&cli->mutex);
1152 return ret;
1153}
1154
1155void
1156nouveau_bo_move_init(struct nouveau_drm *drm)
1157{
1158 static const struct _method_table {
1159 const char *name;
1160 int engine;
1161 s32 oclass;
1162 int (*exec)(struct nouveau_channel *,
1163 struct ttm_buffer_object *,
1164 struct ttm_mem_reg *, struct ttm_mem_reg *);
1165 int (*init)(struct nouveau_channel *, u32 handle);
1166 } _methods[] = {
1167 { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init },
1168 { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init },
1169 { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init },
1170 { "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init },
1171 { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
1172 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
1173 { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
1174 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1175 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
1176 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1177 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
1178 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1179 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
1180 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
1181 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
1182 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
1183 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
1184 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
1185 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1186 {},
1187 { "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
1188 };
1189 const struct _method_table *mthd = _methods;
1190 const char *name = "CPU";
1191 int ret;
1192
1193 do {
1194 struct nouveau_channel *chan;
1195
1196 if (mthd->engine)
1197 chan = drm->cechan;
1198 else
1199 chan = drm->channel;
1200 if (chan == NULL)
1201 continue;
1202
1203 ret = nvif_object_init(&chan->user,
1204 mthd->oclass | (mthd->engine << 16),
1205 mthd->oclass, NULL, 0,
1206 &drm->ttm.copy);
1207 if (ret == 0) {
1208 ret = mthd->init(chan, drm->ttm.copy.handle);
1209 if (ret) {
1210 nvif_object_fini(&drm->ttm.copy);
1211 continue;
1212 }
1213
1214 drm->ttm.move = mthd->exec;
1215 drm->ttm.chan = chan;
1216 name = mthd->name;
1217 break;
1218 }
1219 } while ((++mthd)->exec);
1220
1221 NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1222}
1223
1224static int
1225nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
1226 bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1227{
1228 struct ttm_operation_ctx ctx = { intr, no_wait_gpu };
1229 struct ttm_place placement_memtype = {
1230 .fpfn = 0,
1231 .lpfn = 0,
1232 .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1233 };
1234 struct ttm_placement placement;
1235 struct ttm_mem_reg tmp_reg;
1236 int ret;
1237
1238 placement.num_placement = placement.num_busy_placement = 1;
1239 placement.placement = placement.busy_placement = &placement_memtype;
1240
1241 tmp_reg = *new_reg;
1242 tmp_reg.mm_node = NULL;
1243 ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, &ctx);
1244 if (ret)
1245 return ret;
1246
1247 ret = ttm_tt_bind(bo->ttm, &tmp_reg, &ctx);
1248 if (ret)
1249 goto out;
1250
1251 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_reg);
1252 if (ret)
1253 goto out;
1254
1255 ret = ttm_bo_move_ttm(bo, &ctx, new_reg);
1256out:
1257 ttm_bo_mem_put(bo, &tmp_reg);
1258 return ret;
1259}
1260
1261static int
1262nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1263 bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1264{
1265 struct ttm_operation_ctx ctx = { intr, no_wait_gpu };
1266 struct ttm_place placement_memtype = {
1267 .fpfn = 0,
1268 .lpfn = 0,
1269 .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1270 };
1271 struct ttm_placement placement;
1272 struct ttm_mem_reg tmp_reg;
1273 int ret;
1274
1275 placement.num_placement = placement.num_busy_placement = 1;
1276 placement.placement = placement.busy_placement = &placement_memtype;
1277
1278 tmp_reg = *new_reg;
1279 tmp_reg.mm_node = NULL;
1280 ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, &ctx);
1281 if (ret)
1282 return ret;
1283
1284 ret = ttm_bo_move_ttm(bo, &ctx, &tmp_reg);
1285 if (ret)
1286 goto out;
1287
1288 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_reg);
1289 if (ret)
1290 goto out;
1291
1292out:
1293 ttm_bo_mem_put(bo, &tmp_reg);
1294 return ret;
1295}
1296
1297static void
1298nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict,
1299 struct ttm_mem_reg *new_reg)
1300{
1301 struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL;
1302 struct nouveau_bo *nvbo = nouveau_bo(bo);
1303 struct nouveau_vma *vma;
1304
1305
1306 if (bo->destroy != nouveau_bo_del_ttm)
1307 return;
1308
1309 if (mem && new_reg->mem_type != TTM_PL_SYSTEM &&
1310 mem->mem.page == nvbo->page) {
1311 list_for_each_entry(vma, &nvbo->vma_list, head) {
1312 nouveau_vma_map(vma, mem);
1313 }
1314 } else {
1315 list_for_each_entry(vma, &nvbo->vma_list, head) {
1316 WARN_ON(ttm_bo_wait(bo, false, false));
1317 nouveau_vma_unmap(vma);
1318 }
1319 }
1320}
1321
1322static int
1323nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_reg,
1324 struct nouveau_drm_tile **new_tile)
1325{
1326 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1327 struct drm_device *dev = drm->dev;
1328 struct nouveau_bo *nvbo = nouveau_bo(bo);
1329 u64 offset = new_reg->start << PAGE_SHIFT;
1330
1331 *new_tile = NULL;
1332 if (new_reg->mem_type != TTM_PL_VRAM)
1333 return 0;
1334
1335 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1336 *new_tile = nv10_bo_set_tiling(dev, offset, new_reg->size,
1337 nvbo->mode, nvbo->zeta);
1338 }
1339
1340 return 0;
1341}
1342
1343static void
1344nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1345 struct nouveau_drm_tile *new_tile,
1346 struct nouveau_drm_tile **old_tile)
1347{
1348 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1349 struct drm_device *dev = drm->dev;
1350 struct dma_fence *fence = dma_resv_get_excl(bo->base.resv);
1351
1352 nv10_bo_put_tile_region(dev, *old_tile, fence);
1353 *old_tile = new_tile;
1354}
1355
1356static int
1357nouveau_bo_move(struct ttm_buffer_object *bo, bool evict,
1358 struct ttm_operation_ctx *ctx,
1359 struct ttm_mem_reg *new_reg)
1360{
1361 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1362 struct nouveau_bo *nvbo = nouveau_bo(bo);
1363 struct ttm_mem_reg *old_reg = &bo->mem;
1364 struct nouveau_drm_tile *new_tile = NULL;
1365 int ret = 0;
1366
1367 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
1368 if (ret)
1369 return ret;
1370
1371 if (nvbo->pin_refcnt)
1372 NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
1373
1374 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1375 ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile);
1376 if (ret)
1377 return ret;
1378 }
1379
1380
1381 if (old_reg->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
1382 BUG_ON(bo->mem.mm_node != NULL);
1383 bo->mem = *new_reg;
1384 new_reg->mm_node = NULL;
1385 goto out;
1386 }
1387
1388
1389 if (drm->ttm.move) {
1390 if (new_reg->mem_type == TTM_PL_SYSTEM)
1391 ret = nouveau_bo_move_flipd(bo, evict,
1392 ctx->interruptible,
1393 ctx->no_wait_gpu, new_reg);
1394 else if (old_reg->mem_type == TTM_PL_SYSTEM)
1395 ret = nouveau_bo_move_flips(bo, evict,
1396 ctx->interruptible,
1397 ctx->no_wait_gpu, new_reg);
1398 else
1399 ret = nouveau_bo_move_m2mf(bo, evict,
1400 ctx->interruptible,
1401 ctx->no_wait_gpu, new_reg);
1402 if (!ret)
1403 goto out;
1404 }
1405
1406
1407 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
1408 if (ret == 0)
1409 ret = ttm_bo_move_memcpy(bo, ctx, new_reg);
1410
1411out:
1412 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1413 if (ret)
1414 nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1415 else
1416 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1417 }
1418
1419 return ret;
1420}
1421
1422static int
1423nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
1424{
1425 struct nouveau_bo *nvbo = nouveau_bo(bo);
1426
1427 return drm_vma_node_verify_access(&nvbo->bo.base.vma_node,
1428 filp->private_data);
1429}
1430
1431static int
1432nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg)
1433{
1434 struct ttm_mem_type_manager *man = &bdev->man[reg->mem_type];
1435 struct nouveau_drm *drm = nouveau_bdev(bdev);
1436 struct nvkm_device *device = nvxx_device(&drm->client.device);
1437 struct nouveau_mem *mem = nouveau_mem(reg);
1438
1439 reg->bus.addr = NULL;
1440 reg->bus.offset = 0;
1441 reg->bus.size = reg->num_pages << PAGE_SHIFT;
1442 reg->bus.base = 0;
1443 reg->bus.is_iomem = false;
1444 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1445 return -EINVAL;
1446 switch (reg->mem_type) {
1447 case TTM_PL_SYSTEM:
1448
1449 return 0;
1450 case TTM_PL_TT:
1451#if IS_ENABLED(CONFIG_AGP)
1452 if (drm->agp.bridge) {
1453 reg->bus.offset = reg->start << PAGE_SHIFT;
1454 reg->bus.base = drm->agp.base;
1455 reg->bus.is_iomem = !drm->agp.cma;
1456 }
1457#endif
1458 if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || !mem->kind)
1459
1460 break;
1461
1462 case TTM_PL_VRAM:
1463 reg->bus.offset = reg->start << PAGE_SHIFT;
1464 reg->bus.base = device->func->resource_addr(device, 1);
1465 reg->bus.is_iomem = true;
1466 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
1467 union {
1468 struct nv50_mem_map_v0 nv50;
1469 struct gf100_mem_map_v0 gf100;
1470 } args;
1471 u64 handle, length;
1472 u32 argc = 0;
1473 int ret;
1474
1475 switch (mem->mem.object.oclass) {
1476 case NVIF_CLASS_MEM_NV50:
1477 args.nv50.version = 0;
1478 args.nv50.ro = 0;
1479 args.nv50.kind = mem->kind;
1480 args.nv50.comp = mem->comp;
1481 argc = sizeof(args.nv50);
1482 break;
1483 case NVIF_CLASS_MEM_GF100:
1484 args.gf100.version = 0;
1485 args.gf100.ro = 0;
1486 args.gf100.kind = mem->kind;
1487 argc = sizeof(args.gf100);
1488 break;
1489 default:
1490 WARN_ON(1);
1491 break;
1492 }
1493
1494 ret = nvif_object_map_handle(&mem->mem.object,
1495 &args, argc,
1496 &handle, &length);
1497 if (ret != 1) {
1498 if (WARN_ON(ret == 0))
1499 return -EINVAL;
1500 if (ret == -ENOSPC)
1501 return -EAGAIN;
1502 return ret;
1503 }
1504
1505 reg->bus.base = 0;
1506 reg->bus.offset = handle;
1507 }
1508 break;
1509 default:
1510 return -EINVAL;
1511 }
1512 return 0;
1513}
1514
1515static void
1516nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg)
1517{
1518 struct nouveau_drm *drm = nouveau_bdev(bdev);
1519 struct nouveau_mem *mem = nouveau_mem(reg);
1520
1521 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
1522 switch (reg->mem_type) {
1523 case TTM_PL_TT:
1524 if (mem->kind)
1525 nvif_object_unmap_handle(&mem->mem.object);
1526 break;
1527 case TTM_PL_VRAM:
1528 nvif_object_unmap_handle(&mem->mem.object);
1529 break;
1530 default:
1531 break;
1532 }
1533 }
1534}
1535
1536static int
1537nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1538{
1539 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1540 struct nouveau_bo *nvbo = nouveau_bo(bo);
1541 struct nvkm_device *device = nvxx_device(&drm->client.device);
1542 u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT;
1543 int i, ret;
1544
1545
1546
1547
1548 if (bo->mem.mem_type != TTM_PL_VRAM) {
1549 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1550 !nvbo->kind)
1551 return 0;
1552
1553 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1554 nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);
1555
1556 ret = nouveau_bo_validate(nvbo, false, false);
1557 if (ret)
1558 return ret;
1559 }
1560 return 0;
1561 }
1562
1563
1564 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1565 bo->mem.start + bo->mem.num_pages < mappable)
1566 return 0;
1567
1568 for (i = 0; i < nvbo->placement.num_placement; ++i) {
1569 nvbo->placements[i].fpfn = 0;
1570 nvbo->placements[i].lpfn = mappable;
1571 }
1572
1573 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
1574 nvbo->busy_placements[i].fpfn = 0;
1575 nvbo->busy_placements[i].lpfn = mappable;
1576 }
1577
1578 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1579 return nouveau_bo_validate(nvbo, false, false);
1580}
1581
1582static int
1583nouveau_ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
1584{
1585 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1586 struct nouveau_drm *drm;
1587 struct device *dev;
1588 unsigned i;
1589 int r;
1590 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1591
1592 if (ttm->state != tt_unpopulated)
1593 return 0;
1594
1595 if (slave && ttm->sg) {
1596
1597 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1598 ttm_dma->dma_address, ttm->num_pages);
1599 ttm->state = tt_unbound;
1600 return 0;
1601 }
1602
1603 drm = nouveau_bdev(ttm->bdev);
1604 dev = drm->dev->dev;
1605
1606#if IS_ENABLED(CONFIG_AGP)
1607 if (drm->agp.bridge) {
1608 return ttm_agp_tt_populate(ttm, ctx);
1609 }
1610#endif
1611
1612#if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1613 if (swiotlb_nr_tbl()) {
1614 return ttm_dma_populate((void *)ttm, dev, ctx);
1615 }
1616#endif
1617
1618 r = ttm_pool_populate(ttm, ctx);
1619 if (r) {
1620 return r;
1621 }
1622
1623 for (i = 0; i < ttm->num_pages; i++) {
1624 dma_addr_t addr;
1625
1626 addr = dma_map_page(dev, ttm->pages[i], 0, PAGE_SIZE,
1627 DMA_BIDIRECTIONAL);
1628
1629 if (dma_mapping_error(dev, addr)) {
1630 while (i--) {
1631 dma_unmap_page(dev, ttm_dma->dma_address[i],
1632 PAGE_SIZE, DMA_BIDIRECTIONAL);
1633 ttm_dma->dma_address[i] = 0;
1634 }
1635 ttm_pool_unpopulate(ttm);
1636 return -EFAULT;
1637 }
1638
1639 ttm_dma->dma_address[i] = addr;
1640 }
1641 return 0;
1642}
1643
1644static void
1645nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
1646{
1647 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1648 struct nouveau_drm *drm;
1649 struct device *dev;
1650 unsigned i;
1651 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1652
1653 if (slave)
1654 return;
1655
1656 drm = nouveau_bdev(ttm->bdev);
1657 dev = drm->dev->dev;
1658
1659#if IS_ENABLED(CONFIG_AGP)
1660 if (drm->agp.bridge) {
1661 ttm_agp_tt_unpopulate(ttm);
1662 return;
1663 }
1664#endif
1665
1666#if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1667 if (swiotlb_nr_tbl()) {
1668 ttm_dma_unpopulate((void *)ttm, dev);
1669 return;
1670 }
1671#endif
1672
1673 for (i = 0; i < ttm->num_pages; i++) {
1674 if (ttm_dma->dma_address[i]) {
1675 dma_unmap_page(dev, ttm_dma->dma_address[i], PAGE_SIZE,
1676 DMA_BIDIRECTIONAL);
1677 }
1678 }
1679
1680 ttm_pool_unpopulate(ttm);
1681}
1682
1683void
1684nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1685{
1686 struct dma_resv *resv = nvbo->bo.base.resv;
1687
1688 if (exclusive)
1689 dma_resv_add_excl_fence(resv, &fence->base);
1690 else if (fence)
1691 dma_resv_add_shared_fence(resv, &fence->base);
1692}
1693
1694struct ttm_bo_driver nouveau_bo_driver = {
1695 .ttm_tt_create = &nouveau_ttm_tt_create,
1696 .ttm_tt_populate = &nouveau_ttm_tt_populate,
1697 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1698 .init_mem_type = nouveau_bo_init_mem_type,
1699 .eviction_valuable = ttm_bo_eviction_valuable,
1700 .evict_flags = nouveau_bo_evict_flags,
1701 .move_notify = nouveau_bo_move_ntfy,
1702 .move = nouveau_bo_move,
1703 .verify_access = nouveau_bo_verify_access,
1704 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1705 .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1706 .io_mem_free = &nouveau_ttm_io_mem_free,
1707};
1708
