1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/slab.h>
27
28#include <linux/fault-inject.h>
29#include <linux/log2.h>
30#include <linux/random.h>
31#include <linux/seq_file.h>
32#include <linux/stop_machine.h>
33
34#include <asm/set_memory.h>
35
36#include <drm/drmP.h>
37#include <drm/i915_drm.h>
38
39#include "i915_drv.h"
40#include "i915_vgpu.h"
41#include "i915_trace.h"
42#include "intel_drv.h"
43#include "intel_frontbuffer.h"
44
45#define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108static int
109i915_get_ggtt_vma_pages(struct i915_vma *vma);
110
111static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv)
112{
113
114
115
116 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
117}
118
119static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv)
120{
121 gen6_ggtt_invalidate(dev_priv);
122 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
123}
124
125static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv)
126{
127 intel_gtt_chipset_flush();
128}
129
130static inline void i915_ggtt_invalidate(struct drm_i915_private *i915)
131{
132 i915->ggtt.invalidate(i915);
133}
134
135int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
136 int enable_ppgtt)
137{
138 bool has_full_ppgtt;
139 bool has_full_48bit_ppgtt;
140
141 if (!dev_priv->info.has_aliasing_ppgtt)
142 return 0;
143
144 has_full_ppgtt = dev_priv->info.has_full_ppgtt;
145 has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
146
147 if (intel_vgpu_active(dev_priv)) {
148
149 has_full_ppgtt = false;
150 has_full_48bit_ppgtt = intel_vgpu_has_full_48bit_ppgtt(dev_priv);
151 }
152
153
154
155
156
157 if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
158 return 0;
159
160 if (enable_ppgtt == 1)
161 return 1;
162
163 if (enable_ppgtt == 2 && has_full_ppgtt)
164 return 2;
165
166 if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
167 return 3;
168
169
170 if (IS_GEN6(dev_priv) && intel_vtd_active()) {
171 DRM_INFO("Disabling PPGTT because VT-d is on\n");
172 return 0;
173 }
174
175
176 if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
177 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
178 return 0;
179 }
180
181 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
182 if (has_full_48bit_ppgtt)
183 return 3;
184
185 if (has_full_ppgtt)
186 return 2;
187 }
188
189 return 1;
190}
191
192static int ppgtt_bind_vma(struct i915_vma *vma,
193 enum i915_cache_level cache_level,
194 u32 unused)
195{
196 u32 pte_flags;
197 int ret;
198
199 if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
200 ret = vma->vm->allocate_va_range(vma->vm, vma->node.start,
201 vma->size);
202 if (ret)
203 return ret;
204 }
205
206
207 pte_flags = 0;
208 if (vma->obj->gt_ro)
209 pte_flags |= PTE_READ_ONLY;
210
211 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
212
213 return 0;
214}
215
216static void ppgtt_unbind_vma(struct i915_vma *vma)
217{
218 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
219}
220
221static int ppgtt_set_pages(struct i915_vma *vma)
222{
223 GEM_BUG_ON(vma->pages);
224
225 vma->pages = vma->obj->mm.pages;
226
227 vma->page_sizes = vma->obj->mm.page_sizes;
228
229 return 0;
230}
231
232static void clear_pages(struct i915_vma *vma)
233{
234 GEM_BUG_ON(!vma->pages);
235
236 if (vma->pages != vma->obj->mm.pages) {
237 sg_free_table(vma->pages);
238 kfree(vma->pages);
239 }
240 vma->pages = NULL;
241
242 memset(&vma->page_sizes, 0, sizeof(vma->page_sizes));
243}
244
245static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
246 enum i915_cache_level level)
247{
248 gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
249 pte |= addr;
250
251 switch (level) {
252 case I915_CACHE_NONE:
253 pte |= PPAT_UNCACHED;
254 break;
255 case I915_CACHE_WT:
256 pte |= PPAT_DISPLAY_ELLC;
257 break;
258 default:
259 pte |= PPAT_CACHED;
260 break;
261 }
262
263 return pte;
264}
265
266static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
267 const enum i915_cache_level level)
268{
269 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
270 pde |= addr;
271 if (level != I915_CACHE_NONE)
272 pde |= PPAT_CACHED_PDE;
273 else
274 pde |= PPAT_UNCACHED;
275 return pde;
276}
277
278#define gen8_pdpe_encode gen8_pde_encode
279#define gen8_pml4e_encode gen8_pde_encode
280
281static gen6_pte_t snb_pte_encode(dma_addr_t addr,
282 enum i915_cache_level level,
283 u32 unused)
284{
285 gen6_pte_t pte = GEN6_PTE_VALID;
286 pte |= GEN6_PTE_ADDR_ENCODE(addr);
287
288 switch (level) {
289 case I915_CACHE_L3_LLC:
290 case I915_CACHE_LLC:
291 pte |= GEN6_PTE_CACHE_LLC;
292 break;
293 case I915_CACHE_NONE:
294 pte |= GEN6_PTE_UNCACHED;
295 break;
296 default:
297 MISSING_CASE(level);
298 }
299
300 return pte;
301}
302
303static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
304 enum i915_cache_level level,
305 u32 unused)
306{
307 gen6_pte_t pte = GEN6_PTE_VALID;
308 pte |= GEN6_PTE_ADDR_ENCODE(addr);
309
310 switch (level) {
311 case I915_CACHE_L3_LLC:
312 pte |= GEN7_PTE_CACHE_L3_LLC;
313 break;
314 case I915_CACHE_LLC:
315 pte |= GEN6_PTE_CACHE_LLC;
316 break;
317 case I915_CACHE_NONE:
318 pte |= GEN6_PTE_UNCACHED;
319 break;
320 default:
321 MISSING_CASE(level);
322 }
323
324 return pte;
325}
326
327static gen6_pte_t byt_pte_encode(dma_addr_t addr,
328 enum i915_cache_level level,
329 u32 flags)
330{
331 gen6_pte_t pte = GEN6_PTE_VALID;
332 pte |= GEN6_PTE_ADDR_ENCODE(addr);
333
334 if (!(flags & PTE_READ_ONLY))
335 pte |= BYT_PTE_WRITEABLE;
336
337 if (level != I915_CACHE_NONE)
338 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
339
340 return pte;
341}
342
343static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
344 enum i915_cache_level level,
345 u32 unused)
346{
347 gen6_pte_t pte = GEN6_PTE_VALID;
348 pte |= HSW_PTE_ADDR_ENCODE(addr);
349
350 if (level != I915_CACHE_NONE)
351 pte |= HSW_WB_LLC_AGE3;
352
353 return pte;
354}
355
356static gen6_pte_t iris_pte_encode(dma_addr_t addr,
357 enum i915_cache_level level,
358 u32 unused)
359{
360 gen6_pte_t pte = GEN6_PTE_VALID;
361 pte |= HSW_PTE_ADDR_ENCODE(addr);
362
363 switch (level) {
364 case I915_CACHE_NONE:
365 break;
366 case I915_CACHE_WT:
367 pte |= HSW_WT_ELLC_LLC_AGE3;
368 break;
369 default:
370 pte |= HSW_WB_ELLC_LLC_AGE3;
371 break;
372 }
373
374 return pte;
375}
376
377static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
378{
379 struct pagevec *pvec = &vm->free_pages;
380 struct pagevec stash;
381
382 if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
383 i915_gem_shrink_all(vm->i915);
384
385 if (likely(pvec->nr))
386 return pvec->pages[--pvec->nr];
387
388 if (!vm->pt_kmap_wc)
389 return alloc_page(gfp);
390
391
392 lockdep_assert_held(&vm->i915->drm.struct_mutex);
393
394
395 pvec = &vm->i915->mm.wc_stash;
396 if (likely(pvec->nr))
397 return pvec->pages[--pvec->nr];
398
399
400
401
402
403
404
405
406
407 pagevec_init(&stash);
408 do {
409 struct page *page;
410
411 page = alloc_page(gfp);
412 if (unlikely(!page))
413 break;
414
415 stash.pages[stash.nr++] = page;
416 } while (stash.nr < pagevec_space(pvec));
417
418 if (stash.nr) {
419 int nr = min_t(int, stash.nr, pagevec_space(pvec));
420 struct page **pages = stash.pages + stash.nr - nr;
421
422 if (nr && !set_pages_array_wc(pages, nr)) {
423 memcpy(pvec->pages + pvec->nr,
424 pages, sizeof(pages[0]) * nr);
425 pvec->nr += nr;
426 stash.nr -= nr;
427 }
428
429 pagevec_release(&stash);
430 }
431
432 return likely(pvec->nr) ? pvec->pages[--pvec->nr] : NULL;
433}
434
435static void vm_free_pages_release(struct i915_address_space *vm,
436 bool immediate)
437{
438 struct pagevec *pvec = &vm->free_pages;
439
440 GEM_BUG_ON(!pagevec_count(pvec));
441
442 if (vm->pt_kmap_wc) {
443 struct pagevec *stash = &vm->i915->mm.wc_stash;
444
445
446
447
448
449 lockdep_assert_held(&vm->i915->drm.struct_mutex);
450 if (pagevec_space(stash)) {
451 do {
452 stash->pages[stash->nr++] =
453 pvec->pages[--pvec->nr];
454 if (!pvec->nr)
455 return;
456 } while (pagevec_space(stash));
457
458
459
460
461
462
463 if (!immediate)
464 return;
465 }
466
467 set_pages_array_wb(pvec->pages, pvec->nr);
468 }
469
470 __pagevec_release(pvec);
471}
472
473static void vm_free_page(struct i915_address_space *vm, struct page *page)
474{
475
476
477
478
479
480
481
482 might_sleep();
483 if (!pagevec_add(&vm->free_pages, page))
484 vm_free_pages_release(vm, false);
485}
486
487static int __setup_page_dma(struct i915_address_space *vm,
488 struct i915_page_dma *p,
489 gfp_t gfp)
490{
491 p->page = vm_alloc_page(vm, gfp | __GFP_NOWARN | __GFP_NORETRY);
492 if (unlikely(!p->page))
493 return -ENOMEM;
494
495 p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE,
496 PCI_DMA_BIDIRECTIONAL);
497 if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
498 vm_free_page(vm, p->page);
499 return -ENOMEM;
500 }
501
502 return 0;
503}
504
505static int setup_page_dma(struct i915_address_space *vm,
506 struct i915_page_dma *p)
507{
508 return __setup_page_dma(vm, p, I915_GFP_DMA);
509}
510
511static void cleanup_page_dma(struct i915_address_space *vm,
512 struct i915_page_dma *p)
513{
514 dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
515 vm_free_page(vm, p->page);
516}
517
518#define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
519
520#define setup_px(vm, px) setup_page_dma((vm), px_base(px))
521#define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px))
522#define fill_px(ppgtt, px, v) fill_page_dma((vm), px_base(px), (v))
523#define fill32_px(ppgtt, px, v) fill_page_dma_32((vm), px_base(px), (v))
524
525static void fill_page_dma(struct i915_address_space *vm,
526 struct i915_page_dma *p,
527 const u64 val)
528{
529 u64 * const vaddr = kmap_atomic(p->page);
530
531 memset64(vaddr, val, PAGE_SIZE / sizeof(val));
532
533 kunmap_atomic(vaddr);
534}
535
536static void fill_page_dma_32(struct i915_address_space *vm,
537 struct i915_page_dma *p,
538 const u32 v)
539{
540 fill_page_dma(vm, p, (u64)v << 32 | v);
541}
542
543static int
544setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
545{
546 unsigned long size;
547
548
549
550
551
552
553
554
555
556
557
558
559
560 size = I915_GTT_PAGE_SIZE_4K;
561 if (i915_vm_is_48bit(vm) &&
562 HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) {
563 size = I915_GTT_PAGE_SIZE_64K;
564 gfp |= __GFP_NOWARN;
565 }
566 gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL;
567
568 do {
569 int order = get_order(size);
570 struct page *page;
571 dma_addr_t addr;
572
573 page = alloc_pages(gfp, order);
574 if (unlikely(!page))
575 goto skip;
576
577 addr = dma_map_page(vm->dma, page, 0, size,
578 PCI_DMA_BIDIRECTIONAL);
579 if (unlikely(dma_mapping_error(vm->dma, addr)))
580 goto free_page;
581
582 if (unlikely(!IS_ALIGNED(addr, size)))
583 goto unmap_page;
584
585 vm->scratch_page.page = page;
586 vm->scratch_page.daddr = addr;
587 vm->scratch_page.order = order;
588 return 0;
589
590unmap_page:
591 dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL);
592free_page:
593 __free_pages(page, order);
594skip:
595 if (size == I915_GTT_PAGE_SIZE_4K)
596 return -ENOMEM;
597
598 size = I915_GTT_PAGE_SIZE_4K;
599 gfp &= ~__GFP_NOWARN;
600 } while (1);
601}
602
603static void cleanup_scratch_page(struct i915_address_space *vm)
604{
605 struct i915_page_dma *p = &vm->scratch_page;
606
607 dma_unmap_page(vm->dma, p->daddr, BIT(p->order) << PAGE_SHIFT,
608 PCI_DMA_BIDIRECTIONAL);
609 __free_pages(p->page, p->order);
610}
611
612static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
613{
614 struct i915_page_table *pt;
615
616 pt = kmalloc(sizeof(*pt), GFP_KERNEL | __GFP_NOWARN);
617 if (unlikely(!pt))
618 return ERR_PTR(-ENOMEM);
619
620 if (unlikely(setup_px(vm, pt))) {
621 kfree(pt);
622 return ERR_PTR(-ENOMEM);
623 }
624
625 pt->used_ptes = 0;
626 return pt;
627}
628
629static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
630{
631 cleanup_px(vm, pt);
632 kfree(pt);
633}
634
635static void gen8_initialize_pt(struct i915_address_space *vm,
636 struct i915_page_table *pt)
637{
638 fill_px(vm, pt,
639 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC));
640}
641
642static void gen6_initialize_pt(struct i915_address_space *vm,
643 struct i915_page_table *pt)
644{
645 fill32_px(vm, pt,
646 vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0));
647}
648
649static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
650{
651 struct i915_page_directory *pd;
652
653 pd = kzalloc(sizeof(*pd), GFP_KERNEL | __GFP_NOWARN);
654 if (unlikely(!pd))
655 return ERR_PTR(-ENOMEM);
656
657 if (unlikely(setup_px(vm, pd))) {
658 kfree(pd);
659 return ERR_PTR(-ENOMEM);
660 }
661
662 pd->used_pdes = 0;
663 return pd;
664}
665
666static void free_pd(struct i915_address_space *vm,
667 struct i915_page_directory *pd)
668{
669 cleanup_px(vm, pd);
670 kfree(pd);
671}
672
673static void gen8_initialize_pd(struct i915_address_space *vm,
674 struct i915_page_directory *pd)
675{
676 fill_px(vm, pd,
677 gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
678 memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES);
679}
680
681static int __pdp_init(struct i915_address_space *vm,
682 struct i915_page_directory_pointer *pdp)
683{
684 const unsigned int pdpes = i915_pdpes_per_pdp(vm);
685
686 pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),
687 GFP_KERNEL | __GFP_NOWARN);
688 if (unlikely(!pdp->page_directory))
689 return -ENOMEM;
690
691 memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
692
693 return 0;
694}
695
696static void __pdp_fini(struct i915_page_directory_pointer *pdp)
697{
698 kfree(pdp->page_directory);
699 pdp->page_directory = NULL;
700}
701
702static inline bool use_4lvl(const struct i915_address_space *vm)
703{
704 return i915_vm_is_48bit(vm);
705}
706
707static struct i915_page_directory_pointer *
708alloc_pdp(struct i915_address_space *vm)
709{
710 struct i915_page_directory_pointer *pdp;
711 int ret = -ENOMEM;
712
713 GEM_BUG_ON(!use_4lvl(vm));
714
715 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
716 if (!pdp)
717 return ERR_PTR(-ENOMEM);
718
719 ret = __pdp_init(vm, pdp);
720 if (ret)
721 goto fail_bitmap;
722
723 ret = setup_px(vm, pdp);
724 if (ret)
725 goto fail_page_m;
726
727 return pdp;
728
729fail_page_m:
730 __pdp_fini(pdp);
731fail_bitmap:
732 kfree(pdp);
733
734 return ERR_PTR(ret);
735}
736
737static void free_pdp(struct i915_address_space *vm,
738 struct i915_page_directory_pointer *pdp)
739{
740 __pdp_fini(pdp);
741
742 if (!use_4lvl(vm))
743 return;
744
745 cleanup_px(vm, pdp);
746 kfree(pdp);
747}
748
749static void gen8_initialize_pdp(struct i915_address_space *vm,
750 struct i915_page_directory_pointer *pdp)
751{
752 gen8_ppgtt_pdpe_t scratch_pdpe;
753
754 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
755
756 fill_px(vm, pdp, scratch_pdpe);
757}
758
759static void gen8_initialize_pml4(struct i915_address_space *vm,
760 struct i915_pml4 *pml4)
761{
762 fill_px(vm, pml4,
763 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
764 memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
765}
766
767
768static int gen8_write_pdp(struct i915_request *rq,
769 unsigned entry,
770 dma_addr_t addr)
771{
772 struct intel_engine_cs *engine = rq->engine;
773 u32 *cs;
774
775 BUG_ON(entry >= 4);
776
777 cs = intel_ring_begin(rq, 6);
778 if (IS_ERR(cs))
779 return PTR_ERR(cs);
780
781 *cs++ = MI_LOAD_REGISTER_IMM(1);
782 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, entry));
783 *cs++ = upper_32_bits(addr);
784 *cs++ = MI_LOAD_REGISTER_IMM(1);
785 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry));
786 *cs++ = lower_32_bits(addr);
787 intel_ring_advance(rq, cs);
788
789 return 0;
790}
791
792static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt,
793 struct i915_request *rq)
794{
795 int i, ret;
796
797 for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
798 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
799
800 ret = gen8_write_pdp(rq, i, pd_daddr);
801 if (ret)
802 return ret;
803 }
804
805 return 0;
806}
807
808static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt *ppgtt,
809 struct i915_request *rq)
810{
811 return gen8_write_pdp(rq, 0, px_dma(&ppgtt->pml4));
812}
813
814
815
816
817
818
819static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
820{
821 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask;
822}
823
824
825
826
827static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
828 struct i915_page_table *pt,
829 u64 start, u64 length)
830{
831 unsigned int num_entries = gen8_pte_count(start, length);
832 unsigned int pte = gen8_pte_index(start);
833 unsigned int pte_end = pte + num_entries;
834 const gen8_pte_t scratch_pte =
835 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
836 gen8_pte_t *vaddr;
837
838 GEM_BUG_ON(num_entries > pt->used_ptes);
839
840 pt->used_ptes -= num_entries;
841 if (!pt->used_ptes)
842 return true;
843
844 vaddr = kmap_atomic_px(pt);
845 while (pte < pte_end)
846 vaddr[pte++] = scratch_pte;
847 kunmap_atomic(vaddr);
848
849 return false;
850}
851
852static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
853 struct i915_page_directory *pd,
854 struct i915_page_table *pt,
855 unsigned int pde)
856{
857 gen8_pde_t *vaddr;
858
859 pd->page_table[pde] = pt;
860
861 vaddr = kmap_atomic_px(pd);
862 vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
863 kunmap_atomic(vaddr);
864}
865
866static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
867 struct i915_page_directory *pd,
868 u64 start, u64 length)
869{
870 struct i915_page_table *pt;
871 u32 pde;
872
873 gen8_for_each_pde(pt, pd, start, length, pde) {
874 GEM_BUG_ON(pt == vm->scratch_pt);
875
876 if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
877 continue;
878
879 gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
880 GEM_BUG_ON(!pd->used_pdes);
881 pd->used_pdes--;
882
883 free_pt(vm, pt);
884 }
885
886 return !pd->used_pdes;
887}
888
889static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
890 struct i915_page_directory_pointer *pdp,
891 struct i915_page_directory *pd,
892 unsigned int pdpe)
893{
894 gen8_ppgtt_pdpe_t *vaddr;
895
896 pdp->page_directory[pdpe] = pd;
897 if (!use_4lvl(vm))
898 return;
899
900 vaddr = kmap_atomic_px(pdp);
901 vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
902 kunmap_atomic(vaddr);
903}
904
905
906
907
908static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
909 struct i915_page_directory_pointer *pdp,
910 u64 start, u64 length)
911{
912 struct i915_page_directory *pd;
913 unsigned int pdpe;
914
915 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
916 GEM_BUG_ON(pd == vm->scratch_pd);
917
918 if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
919 continue;
920
921 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
922 GEM_BUG_ON(!pdp->used_pdpes);
923 pdp->used_pdpes--;
924
925 free_pd(vm, pd);
926 }
927
928 return !pdp->used_pdpes;
929}
930
931static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
932 u64 start, u64 length)
933{
934 gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);
935}
936
937static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
938 struct i915_page_directory_pointer *pdp,
939 unsigned int pml4e)
940{
941 gen8_ppgtt_pml4e_t *vaddr;
942
943 pml4->pdps[pml4e] = pdp;
944
945 vaddr = kmap_atomic_px(pml4);
946 vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
947 kunmap_atomic(vaddr);
948}
949
950
951
952
953
954static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
955 u64 start, u64 length)
956{
957 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
958 struct i915_pml4 *pml4 = &ppgtt->pml4;
959 struct i915_page_directory_pointer *pdp;
960 unsigned int pml4e;
961
962 GEM_BUG_ON(!use_4lvl(vm));
963
964 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
965 GEM_BUG_ON(pdp == vm->scratch_pdp);
966
967 if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
968 continue;
969
970 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
971
972 free_pdp(vm, pdp);
973 }
974}
975
976static inline struct sgt_dma {
977 struct scatterlist *sg;
978 dma_addr_t dma, max;
979} sgt_dma(struct i915_vma *vma) {
980 struct scatterlist *sg = vma->pages->sgl;
981 dma_addr_t addr = sg_dma_address(sg);
982 return (struct sgt_dma) { sg, addr, addr + sg->length };
983}
984
985struct gen8_insert_pte {
986 u16 pml4e;
987 u16 pdpe;
988 u16 pde;
989 u16 pte;
990};
991
992static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
993{
994 return (struct gen8_insert_pte) {
995 gen8_pml4e_index(start),
996 gen8_pdpe_index(start),
997 gen8_pde_index(start),
998 gen8_pte_index(start),
999 };
1000}
1001
1002static __always_inline bool
1003gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
1004 struct i915_page_directory_pointer *pdp,
1005 struct sgt_dma *iter,
1006 struct gen8_insert_pte *idx,
1007 enum i915_cache_level cache_level)
1008{
1009 struct i915_page_directory *pd;
1010 const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
1011 gen8_pte_t *vaddr;
1012 bool ret;
1013
1014 GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
1015 pd = pdp->page_directory[idx->pdpe];
1016 vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
1017 do {
1018 vaddr[idx->pte] = pte_encode | iter->dma;
1019
1020 iter->dma += PAGE_SIZE;
1021 if (iter->dma >= iter->max) {
1022 iter->sg = __sg_next(iter->sg);
1023 if (!iter->sg) {
1024 ret = false;
1025 break;
1026 }
1027
1028 iter->dma = sg_dma_address(iter->sg);
1029 iter->max = iter->dma + iter->sg->length;
1030 }
1031
1032 if (++idx->pte == GEN8_PTES) {
1033 idx->pte = 0;
1034
1035 if (++idx->pde == I915_PDES) {
1036 idx->pde = 0;
1037
1038
1039 if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
1040 idx->pdpe = 0;
1041 ret = true;
1042 break;
1043 }
1044
1045 GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
1046 pd = pdp->page_directory[idx->pdpe];
1047 }
1048
1049 kunmap_atomic(vaddr);
1050 vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
1051 }
1052 } while (1);
1053 kunmap_atomic(vaddr);
1054
1055 return ret;
1056}
1057
1058static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
1059 struct i915_vma *vma,
1060 enum i915_cache_level cache_level,
1061 u32 unused)
1062{
1063 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1064 struct sgt_dma iter = sgt_dma(vma);
1065 struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
1066
1067 gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,
1068 cache_level);
1069
1070 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1071}
1072
1073static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
1074 struct i915_page_directory_pointer **pdps,
1075 struct sgt_dma *iter,
1076 enum i915_cache_level cache_level)
1077{
1078 const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
1079 u64 start = vma->node.start;
1080 dma_addr_t rem = iter->sg->length;
1081
1082 do {
1083 struct gen8_insert_pte idx = gen8_insert_pte(start);
1084 struct i915_page_directory_pointer *pdp = pdps[idx.pml4e];
1085 struct i915_page_directory *pd = pdp->page_directory[idx.pdpe];
1086 unsigned int page_size;
1087 bool maybe_64K = false;
1088 gen8_pte_t encode = pte_encode;
1089 gen8_pte_t *vaddr;
1090 u16 index, max;
1091
1092 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M &&
1093 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) &&
1094 rem >= I915_GTT_PAGE_SIZE_2M && !idx.pte) {
1095 index = idx.pde;
1096 max = I915_PDES;
1097 page_size = I915_GTT_PAGE_SIZE_2M;
1098
1099 encode |= GEN8_PDE_PS_2M;
1100
1101 vaddr = kmap_atomic_px(pd);
1102 } else {
1103 struct i915_page_table *pt = pd->page_table[idx.pde];
1104
1105 index = idx.pte;
1106 max = GEN8_PTES;
1107 page_size = I915_GTT_PAGE_SIZE;
1108
1109 if (!index &&
1110 vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
1111 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1112 (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1113 rem >= (max - index) << PAGE_SHIFT))
1114 maybe_64K = true;
1115
1116 vaddr = kmap_atomic_px(pt);
1117 }
1118
1119 do {
1120 GEM_BUG_ON(iter->sg->length < page_size);
1121 vaddr[index++] = encode | iter->dma;
1122
1123 start += page_size;
1124 iter->dma += page_size;
1125 rem -= page_size;
1126 if (iter->dma >= iter->max) {
1127 iter->sg = __sg_next(iter->sg);
1128 if (!iter->sg)
1129 break;
1130
1131 rem = iter->sg->length;
1132 iter->dma = sg_dma_address(iter->sg);
1133 iter->max = iter->dma + rem;
1134
1135 if (maybe_64K && index < max &&
1136 !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1137 (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1138 rem >= (max - index) << PAGE_SHIFT)))
1139 maybe_64K = false;
1140
1141 if (unlikely(!IS_ALIGNED(iter->dma, page_size)))
1142 break;
1143 }
1144 } while (rem >= page_size && index < max);
1145
1146 kunmap_atomic(vaddr);
1147
1148
1149
1150
1151
1152
1153
1154 if (maybe_64K &&
1155 (index == max ||
1156 (i915_vm_has_scratch_64K(vma->vm) &&
1157 !iter->sg && IS_ALIGNED(vma->node.start +
1158 vma->node.size,
1159 I915_GTT_PAGE_SIZE_2M)))) {
1160 vaddr = kmap_atomic_px(pd);
1161 vaddr[idx.pde] |= GEN8_PDE_IPS_64K;
1162 kunmap_atomic(vaddr);
1163 page_size = I915_GTT_PAGE_SIZE_64K;
1164 }
1165
1166 vma->page_sizes.gtt |= page_size;
1167 } while (iter->sg);
1168}
1169
1170static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
1171 struct i915_vma *vma,
1172 enum i915_cache_level cache_level,
1173 u32 unused)
1174{
1175 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1176 struct sgt_dma iter = sgt_dma(vma);
1177 struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
1178
1179 if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
1180 gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level);
1181 } else {
1182 struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
1183
1184 while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++],
1185 &iter, &idx, cache_level))
1186 GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
1187
1188 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1189 }
1190}
1191
1192static void gen8_free_page_tables(struct i915_address_space *vm,
1193 struct i915_page_directory *pd)
1194{
1195 int i;
1196
1197 if (!px_page(pd))
1198 return;
1199
1200 for (i = 0; i < I915_PDES; i++) {
1201 if (pd->page_table[i] != vm->scratch_pt)
1202 free_pt(vm, pd->page_table[i]);
1203 }
1204}
1205
1206static int gen8_init_scratch(struct i915_address_space *vm)
1207{
1208 int ret;
1209
1210 ret = setup_scratch_page(vm, I915_GFP_DMA);
1211 if (ret)
1212 return ret;
1213
1214 vm->scratch_pt = alloc_pt(vm);
1215 if (IS_ERR(vm->scratch_pt)) {
1216 ret = PTR_ERR(vm->scratch_pt);
1217 goto free_scratch_page;
1218 }
1219
1220 vm->scratch_pd = alloc_pd(vm);
1221 if (IS_ERR(vm->scratch_pd)) {
1222 ret = PTR_ERR(vm->scratch_pd);
1223 goto free_pt;
1224 }
1225
1226 if (use_4lvl(vm)) {
1227 vm->scratch_pdp = alloc_pdp(vm);
1228 if (IS_ERR(vm->scratch_pdp)) {
1229 ret = PTR_ERR(vm->scratch_pdp);
1230 goto free_pd;
1231 }
1232 }
1233
1234 gen8_initialize_pt(vm, vm->scratch_pt);
1235 gen8_initialize_pd(vm, vm->scratch_pd);
1236 if (use_4lvl(vm))
1237 gen8_initialize_pdp(vm, vm->scratch_pdp);
1238
1239 return 0;
1240
1241free_pd:
1242 free_pd(vm, vm->scratch_pd);
1243free_pt:
1244 free_pt(vm, vm->scratch_pt);
1245free_scratch_page:
1246 cleanup_scratch_page(vm);
1247
1248 return ret;
1249}
1250
1251static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
1252{
1253 struct i915_address_space *vm = &ppgtt->base;
1254 struct drm_i915_private *dev_priv = vm->i915;
1255 enum vgt_g2v_type msg;
1256 int i;
1257
1258 if (use_4lvl(vm)) {
1259 const u64 daddr = px_dma(&ppgtt->pml4);
1260
1261 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1262 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1263
1264 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1265 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1266 } else {
1267 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1268 const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1269
1270 I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1271 I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1272 }
1273
1274 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1275 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1276 }
1277
1278 I915_WRITE(vgtif_reg(g2v_notify), msg);
1279
1280 return 0;
1281}
1282
1283static void gen8_free_scratch(struct i915_address_space *vm)
1284{
1285 if (use_4lvl(vm))
1286 free_pdp(vm, vm->scratch_pdp);
1287 free_pd(vm, vm->scratch_pd);
1288 free_pt(vm, vm->scratch_pt);
1289 cleanup_scratch_page(vm);
1290}
1291
1292static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
1293 struct i915_page_directory_pointer *pdp)
1294{
1295 const unsigned int pdpes = i915_pdpes_per_pdp(vm);
1296 int i;
1297
1298 for (i = 0; i < pdpes; i++) {
1299 if (pdp->page_directory[i] == vm->scratch_pd)
1300 continue;
1301
1302 gen8_free_page_tables(vm, pdp->page_directory[i]);
1303 free_pd(vm, pdp->page_directory[i]);
1304 }
1305
1306 free_pdp(vm, pdp);
1307}
1308
1309static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1310{
1311 int i;
1312
1313 for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
1314 if (ppgtt->pml4.pdps[i] == ppgtt->base.scratch_pdp)
1315 continue;
1316
1317 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, ppgtt->pml4.pdps[i]);
1318 }
1319
1320 cleanup_px(&ppgtt->base, &ppgtt->pml4);
1321}
1322
1323static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1324{
1325 struct drm_i915_private *dev_priv = vm->i915;
1326 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1327
1328 if (intel_vgpu_active(dev_priv))
1329 gen8_ppgtt_notify_vgt(ppgtt, false);
1330
1331 if (use_4lvl(vm))
1332 gen8_ppgtt_cleanup_4lvl(ppgtt);
1333 else
1334 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, &ppgtt->pdp);
1335
1336 gen8_free_scratch(vm);
1337}
1338
1339static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
1340 struct i915_page_directory *pd,
1341 u64 start, u64 length)
1342{
1343 struct i915_page_table *pt;
1344 u64 from = start;
1345 unsigned int pde;
1346
1347 gen8_for_each_pde(pt, pd, start, length, pde) {
1348 int count = gen8_pte_count(start, length);
1349
1350 if (pt == vm->scratch_pt) {
1351 pd->used_pdes++;
1352
1353 pt = alloc_pt(vm);
1354 if (IS_ERR(pt)) {
1355 pd->used_pdes--;
1356 goto unwind;
1357 }
1358
1359 if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
1360 gen8_initialize_pt(vm, pt);
1361
1362 gen8_ppgtt_set_pde(vm, pd, pt, pde);
1363 GEM_BUG_ON(pd->used_pdes > I915_PDES);
1364 }
1365
1366 pt->used_ptes += count;
1367 }
1368 return 0;
1369
1370unwind:
1371 gen8_ppgtt_clear_pd(vm, pd, from, start - from);
1372 return -ENOMEM;
1373}
1374
1375static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
1376 struct i915_page_directory_pointer *pdp,
1377 u64 start, u64 length)
1378{
1379 struct i915_page_directory *pd;
1380 u64 from = start;
1381 unsigned int pdpe;
1382 int ret;
1383
1384 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1385 if (pd == vm->scratch_pd) {
1386 pdp->used_pdpes++;
1387
1388 pd = alloc_pd(vm);
1389 if (IS_ERR(pd)) {
1390 pdp->used_pdpes--;
1391 goto unwind;
1392 }
1393
1394 gen8_initialize_pd(vm, pd);
1395 gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1396 GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
1397
1398 mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
1399 }
1400
1401 ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
1402 if (unlikely(ret))
1403 goto unwind_pd;
1404 }
1405
1406 return 0;
1407
1408unwind_pd:
1409 if (!pd->used_pdes) {
1410 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1411 GEM_BUG_ON(!pdp->used_pdpes);
1412 pdp->used_pdpes--;
1413 free_pd(vm, pd);
1414 }
1415unwind:
1416 gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
1417 return -ENOMEM;
1418}
1419
1420static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
1421 u64 start, u64 length)
1422{
1423 return gen8_ppgtt_alloc_pdp(vm,
1424 &i915_vm_to_ppgtt(vm)->pdp, start, length);
1425}
1426
1427static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
1428 u64 start, u64 length)
1429{
1430 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1431 struct i915_pml4 *pml4 = &ppgtt->pml4;
1432 struct i915_page_directory_pointer *pdp;
1433 u64 from = start;
1434 u32 pml4e;
1435 int ret;
1436
1437 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1438 if (pml4->pdps[pml4e] == vm->scratch_pdp) {
1439 pdp = alloc_pdp(vm);
1440 if (IS_ERR(pdp))
1441 goto unwind;
1442
1443 gen8_initialize_pdp(vm, pdp);
1444 gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
1445 }
1446
1447 ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
1448 if (unlikely(ret))
1449 goto unwind_pdp;
1450 }
1451
1452 return 0;
1453
1454unwind_pdp:
1455 if (!pdp->used_pdpes) {
1456 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
1457 free_pdp(vm, pdp);
1458 }
1459unwind:
1460 gen8_ppgtt_clear_4lvl(vm, from, start - from);
1461 return -ENOMEM;
1462}
1463
1464static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt,
1465 struct i915_page_directory_pointer *pdp,
1466 u64 start, u64 length,
1467 gen8_pte_t scratch_pte,
1468 struct seq_file *m)
1469{
1470 struct i915_address_space *vm = &ppgtt->base;
1471 struct i915_page_directory *pd;
1472 u32 pdpe;
1473
1474 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1475 struct i915_page_table *pt;
1476 u64 pd_len = length;
1477 u64 pd_start = start;
1478 u32 pde;
1479
1480 if (pdp->page_directory[pdpe] == ppgtt->base.scratch_pd)
1481 continue;
1482
1483 seq_printf(m, "\tPDPE #%d\n", pdpe);
1484 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1485 u32 pte;
1486 gen8_pte_t *pt_vaddr;
1487
1488 if (pd->page_table[pde] == ppgtt->base.scratch_pt)
1489 continue;
1490
1491 pt_vaddr = kmap_atomic_px(pt);
1492 for (pte = 0; pte < GEN8_PTES; pte += 4) {
1493 u64 va = (pdpe << GEN8_PDPE_SHIFT |
1494 pde << GEN8_PDE_SHIFT |
1495 pte << GEN8_PTE_SHIFT);
1496 int i;
1497 bool found = false;
1498
1499 for (i = 0; i < 4; i++)
1500 if (pt_vaddr[pte + i] != scratch_pte)
1501 found = true;
1502 if (!found)
1503 continue;
1504
1505 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1506 for (i = 0; i < 4; i++) {
1507 if (pt_vaddr[pte + i] != scratch_pte)
1508 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1509 else
1510 seq_puts(m, " SCRATCH ");
1511 }
1512 seq_puts(m, "\n");
1513 }
1514 kunmap_atomic(pt_vaddr);
1515 }
1516 }
1517}
1518
1519static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1520{
1521 struct i915_address_space *vm = &ppgtt->base;
1522 const gen8_pte_t scratch_pte =
1523 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
1524 u64 start = 0, length = ppgtt->base.total;
1525
1526 if (use_4lvl(vm)) {
1527 u64 pml4e;
1528 struct i915_pml4 *pml4 = &ppgtt->pml4;
1529 struct i915_page_directory_pointer *pdp;
1530
1531 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1532 if (pml4->pdps[pml4e] == ppgtt->base.scratch_pdp)
1533 continue;
1534
1535 seq_printf(m, " PML4E #%llu\n", pml4e);
1536 gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m);
1537 }
1538 } else {
1539 gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m);
1540 }
1541}
1542
1543static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
1544{
1545 struct i915_address_space *vm = &ppgtt->base;
1546 struct i915_page_directory_pointer *pdp = &ppgtt->pdp;
1547 struct i915_page_directory *pd;
1548 u64 start = 0, length = ppgtt->base.total;
1549 u64 from = start;
1550 unsigned int pdpe;
1551
1552 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1553 pd = alloc_pd(vm);
1554 if (IS_ERR(pd))
1555 goto unwind;
1556
1557 gen8_initialize_pd(vm, pd);
1558 gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1559 pdp->used_pdpes++;
1560 }
1561
1562 pdp->used_pdpes++;
1563 return 0;
1564
1565unwind:
1566 start -= from;
1567 gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
1568 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1569 free_pd(vm, pd);
1570 }
1571 pdp->used_pdpes = 0;
1572 return -ENOMEM;
1573}
1574
1575
1576
1577
1578
1579
1580
1581
1582static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1583{
1584 struct i915_address_space *vm = &ppgtt->base;
1585 struct drm_i915_private *dev_priv = vm->i915;
1586 int ret;
1587
1588 ppgtt->base.total = USES_FULL_48BIT_PPGTT(dev_priv) ?
1589 1ULL << 48 :
1590 1ULL << 32;
1591
1592
1593
1594
1595 if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
1596 ppgtt->base.pt_kmap_wc = true;
1597
1598 ret = gen8_init_scratch(&ppgtt->base);
1599 if (ret) {
1600 ppgtt->base.total = 0;
1601 return ret;
1602 }
1603
1604 if (use_4lvl(vm)) {
1605 ret = setup_px(&ppgtt->base, &ppgtt->pml4);
1606 if (ret)
1607 goto free_scratch;
1608
1609 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1610
1611 ppgtt->switch_mm = gen8_mm_switch_4lvl;
1612 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_4lvl;
1613 ppgtt->base.insert_entries = gen8_ppgtt_insert_4lvl;
1614 ppgtt->base.clear_range = gen8_ppgtt_clear_4lvl;
1615 } else {
1616 ret = __pdp_init(&ppgtt->base, &ppgtt->pdp);
1617 if (ret)
1618 goto free_scratch;
1619
1620 if (intel_vgpu_active(dev_priv)) {
1621 ret = gen8_preallocate_top_level_pdp(ppgtt);
1622 if (ret) {
1623 __pdp_fini(&ppgtt->pdp);
1624 goto free_scratch;
1625 }
1626 }
1627
1628 ppgtt->switch_mm = gen8_mm_switch_3lvl;
1629 ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_3lvl;
1630 ppgtt->base.insert_entries = gen8_ppgtt_insert_3lvl;
1631 ppgtt->base.clear_range = gen8_ppgtt_clear_3lvl;
1632 }
1633
1634 if (intel_vgpu_active(dev_priv))
1635 gen8_ppgtt_notify_vgt(ppgtt, true);
1636
1637 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1638 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1639 ppgtt->base.bind_vma = ppgtt_bind_vma;
1640 ppgtt->base.set_pages = ppgtt_set_pages;
1641 ppgtt->base.clear_pages = clear_pages;
1642 ppgtt->debug_dump = gen8_dump_ppgtt;
1643
1644 return 0;
1645
1646free_scratch:
1647 gen8_free_scratch(&ppgtt->base);
1648 return ret;
1649}
1650
1651static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1652{
1653 struct i915_address_space *vm = &ppgtt->base;
1654 struct i915_page_table *unused;
1655 gen6_pte_t scratch_pte;
1656 u32 pd_entry, pte, pde;
1657 u32 start = 0, length = ppgtt->base.total;
1658
1659 scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1660 I915_CACHE_LLC, 0);
1661
1662 gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
1663 u32 expected;
1664 gen6_pte_t *pt_vaddr;
1665 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1666 pd_entry = readl(ppgtt->pd_addr + pde);
1667 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1668
1669 if (pd_entry != expected)
1670 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1671 pde,
1672 pd_entry,
1673 expected);
1674 seq_printf(m, "\tPDE: %x\n", pd_entry);
1675
1676 pt_vaddr = kmap_atomic_px(ppgtt->pd.page_table[pde]);
1677
1678 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1679 unsigned long va =
1680 (pde * PAGE_SIZE * GEN6_PTES) +
1681 (pte * PAGE_SIZE);
1682 int i;
1683 bool found = false;
1684 for (i = 0; i < 4; i++)
1685 if (pt_vaddr[pte + i] != scratch_pte)
1686 found = true;
1687 if (!found)
1688 continue;
1689
1690 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1691 for (i = 0; i < 4; i++) {
1692 if (pt_vaddr[pte + i] != scratch_pte)
1693 seq_printf(m, " %08x", pt_vaddr[pte + i]);
1694 else
1695 seq_puts(m, " SCRATCH ");
1696 }
1697 seq_puts(m, "\n");
1698 }
1699 kunmap_atomic(pt_vaddr);
1700 }
1701}
1702
1703
1704static inline void gen6_write_pde(const struct i915_hw_ppgtt *ppgtt,
1705 const unsigned int pde,
1706 const struct i915_page_table *pt)
1707{
1708
1709 writel_relaxed(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID,
1710 ppgtt->pd_addr + pde);
1711}
1712
1713
1714
1715static void gen6_write_page_range(struct i915_hw_ppgtt *ppgtt,
1716 u32 start, u32 length)
1717{
1718 struct i915_page_table *pt;
1719 unsigned int pde;
1720
1721 gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde)
1722 gen6_write_pde(ppgtt, pde, pt);
1723
1724 mark_tlbs_dirty(ppgtt);
1725 wmb();
1726}
1727
1728static inline u32 get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1729{
1730 GEM_BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1731 return ppgtt->pd.base.ggtt_offset << 10;
1732}
1733
1734static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1735 struct i915_request *rq)
1736{
1737 struct intel_engine_cs *engine = rq->engine;
1738 u32 *cs;
1739
1740
1741 cs = intel_ring_begin(rq, 6);
1742 if (IS_ERR(cs))
1743 return PTR_ERR(cs);
1744
1745 *cs++ = MI_LOAD_REGISTER_IMM(2);
1746 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1747 *cs++ = PP_DIR_DCLV_2G;
1748 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1749 *cs++ = get_pd_offset(ppgtt);
1750 *cs++ = MI_NOOP;
1751 intel_ring_advance(rq, cs);
1752
1753 return 0;
1754}
1755
1756static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1757 struct i915_request *rq)
1758{
1759 struct intel_engine_cs *engine = rq->engine;
1760 u32 *cs;
1761
1762
1763 cs = intel_ring_begin(rq, 6);
1764 if (IS_ERR(cs))
1765 return PTR_ERR(cs);
1766
1767 *cs++ = MI_LOAD_REGISTER_IMM(2);
1768 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1769 *cs++ = PP_DIR_DCLV_2G;
1770 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1771 *cs++ = get_pd_offset(ppgtt);
1772 *cs++ = MI_NOOP;
1773 intel_ring_advance(rq, cs);
1774
1775 return 0;
1776}
1777
1778static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1779 struct i915_request *rq)
1780{
1781 struct intel_engine_cs *engine = rq->engine;
1782 struct drm_i915_private *dev_priv = rq->i915;
1783
1784 I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1785 I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
1786 return 0;
1787}
1788
1789static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1790{
1791 struct intel_engine_cs *engine;
1792 enum intel_engine_id id;
1793
1794 for_each_engine(engine, dev_priv, id) {
1795 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1796 GEN8_GFX_PPGTT_48B : 0;
1797 I915_WRITE(RING_MODE_GEN7(engine),
1798 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1799 }
1800}
1801
1802static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1803{
1804 struct intel_engine_cs *engine;
1805 u32 ecochk, ecobits;
1806 enum intel_engine_id id;
1807
1808 ecobits = I915_READ(GAC_ECO_BITS);
1809 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1810
1811 ecochk = I915_READ(GAM_ECOCHK);
1812 if (IS_HASWELL(dev_priv)) {
1813 ecochk |= ECOCHK_PPGTT_WB_HSW;
1814 } else {
1815 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1816 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1817 }
1818 I915_WRITE(GAM_ECOCHK, ecochk);
1819
1820 for_each_engine(engine, dev_priv, id) {
1821
1822 I915_WRITE(RING_MODE_GEN7(engine),
1823 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1824 }
1825}
1826
1827static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1828{
1829 u32 ecochk, gab_ctl, ecobits;
1830
1831 ecobits = I915_READ(GAC_ECO_BITS);
1832 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1833 ECOBITS_PPGTT_CACHE64B);
1834
1835 gab_ctl = I915_READ(GAB_CTL);
1836 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1837
1838 ecochk = I915_READ(GAM_ECOCHK);
1839 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1840
1841 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1842}
1843
1844
1845static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1846 u64 start, u64 length)
1847{
1848 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1849 unsigned int first_entry = start >> PAGE_SHIFT;
1850 unsigned int pde = first_entry / GEN6_PTES;
1851 unsigned int pte = first_entry % GEN6_PTES;
1852 unsigned int num_entries = length >> PAGE_SHIFT;
1853 gen6_pte_t scratch_pte =
1854 vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
1855
1856 while (num_entries) {
1857 struct i915_page_table *pt = ppgtt->pd.page_table[pde++];
1858 unsigned int end = min(pte + num_entries, GEN6_PTES);
1859 gen6_pte_t *vaddr;
1860
1861 num_entries -= end - pte;
1862
1863
1864
1865
1866
1867
1868
1869 vaddr = kmap_atomic_px(pt);
1870 do {
1871 vaddr[pte++] = scratch_pte;
1872 } while (pte < end);
1873 kunmap_atomic(vaddr);
1874
1875 pte = 0;
1876 }
1877}
1878
1879static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1880 struct i915_vma *vma,
1881 enum i915_cache_level cache_level,
1882 u32 flags)
1883{
1884 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1885 unsigned first_entry = vma->node.start >> PAGE_SHIFT;
1886 unsigned act_pt = first_entry / GEN6_PTES;
1887 unsigned act_pte = first_entry % GEN6_PTES;
1888 const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
1889 struct sgt_dma iter = sgt_dma(vma);
1890 gen6_pte_t *vaddr;
1891
1892 vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
1893 do {
1894 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
1895
1896 iter.dma += PAGE_SIZE;
1897 if (iter.dma == iter.max) {
1898 iter.sg = __sg_next(iter.sg);
1899 if (!iter.sg)
1900 break;
1901
1902 iter.dma = sg_dma_address(iter.sg);
1903 iter.max = iter.dma + iter.sg->length;
1904 }
1905
1906 if (++act_pte == GEN6_PTES) {
1907 kunmap_atomic(vaddr);
1908 vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]);
1909 act_pte = 0;
1910 }
1911 } while (1);
1912 kunmap_atomic(vaddr);
1913
1914 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1915}
1916
1917static int gen6_alloc_va_range(struct i915_address_space *vm,
1918 u64 start, u64 length)
1919{
1920 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1921 struct i915_page_table *pt;
1922 u64 from = start;
1923 unsigned int pde;
1924 bool flush = false;
1925
1926 gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1927 if (pt == vm->scratch_pt) {
1928 pt = alloc_pt(vm);
1929 if (IS_ERR(pt))
1930 goto unwind_out;
1931
1932 gen6_initialize_pt(vm, pt);
1933 ppgtt->pd.page_table[pde] = pt;
1934 gen6_write_pde(ppgtt, pde, pt);
1935 flush = true;
1936 }
1937 }
1938
1939 if (flush) {
1940 mark_tlbs_dirty(ppgtt);
1941 wmb();
1942 }
1943
1944 return 0;
1945
1946unwind_out:
1947 gen6_ppgtt_clear_range(vm, from, start);
1948 return -ENOMEM;
1949}
1950
1951static int gen6_init_scratch(struct i915_address_space *vm)
1952{
1953 int ret;
1954
1955 ret = setup_scratch_page(vm, I915_GFP_DMA);
1956 if (ret)
1957 return ret;
1958
1959 vm->scratch_pt = alloc_pt(vm);
1960 if (IS_ERR(vm->scratch_pt)) {
1961 cleanup_scratch_page(vm);
1962 return PTR_ERR(vm->scratch_pt);
1963 }
1964
1965 gen6_initialize_pt(vm, vm->scratch_pt);
1966
1967 return 0;
1968}
1969
1970static void gen6_free_scratch(struct i915_address_space *vm)
1971{
1972 free_pt(vm, vm->scratch_pt);
1973 cleanup_scratch_page(vm);
1974}
1975
1976static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1977{
1978 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1979 struct i915_page_directory *pd = &ppgtt->pd;
1980 struct i915_page_table *pt;
1981 u32 pde;
1982
1983 drm_mm_remove_node(&ppgtt->node);
1984
1985 gen6_for_all_pdes(pt, pd, pde)
1986 if (pt != vm->scratch_pt)
1987 free_pt(vm, pt);
1988
1989 gen6_free_scratch(vm);
1990}
1991
1992static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
1993{
1994 struct i915_address_space *vm = &ppgtt->base;
1995 struct drm_i915_private *dev_priv = ppgtt->base.i915;
1996 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1997 int ret;
1998
1999
2000
2001
2002
2003 BUG_ON(!drm_mm_initialized(&ggtt->base.mm));
2004
2005 ret = gen6_init_scratch(vm);
2006 if (ret)
2007 return ret;
2008
2009 ret = i915_gem_gtt_insert(&ggtt->base, &ppgtt->node,
2010 GEN6_PD_SIZE, GEN6_PD_ALIGN,
2011 I915_COLOR_UNEVICTABLE,
2012 0, ggtt->base.total,
2013 PIN_HIGH);
2014 if (ret)
2015 goto err_out;
2016
2017 if (ppgtt->node.start < ggtt->mappable_end)
2018 DRM_DEBUG("Forced to use aperture for PDEs\n");
2019
2020 ppgtt->pd.base.ggtt_offset =
2021 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
2022
2023 ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm +
2024 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
2025
2026 return 0;
2027
2028err_out:
2029 gen6_free_scratch(vm);
2030 return ret;
2031}
2032
2033static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2034{
2035 return gen6_ppgtt_allocate_page_directories(ppgtt);
2036}
2037
2038static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2039 u64 start, u64 length)
2040{
2041 struct i915_page_table *unused;
2042 u32 pde;
2043
2044 gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
2045 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
2046}
2047
2048static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
2049{
2050 struct drm_i915_private *dev_priv = ppgtt->base.i915;
2051 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2052 int ret;
2053
2054 ppgtt->base.pte_encode = ggtt->base.pte_encode;
2055 if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
2056 ppgtt->switch_mm = gen6_mm_switch;
2057 else if (IS_HASWELL(dev_priv))
2058 ppgtt->switch_mm = hsw_mm_switch;
2059 else if (IS_GEN7(dev_priv))
2060 ppgtt->switch_mm = gen7_mm_switch;
2061 else
2062 BUG();
2063
2064 ret = gen6_ppgtt_alloc(ppgtt);
2065 if (ret)
2066 return ret;
2067
2068 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2069
2070 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
2071 gen6_write_page_range(ppgtt, 0, ppgtt->base.total);
2072
2073 ret = gen6_alloc_va_range(&ppgtt->base, 0, ppgtt->base.total);
2074 if (ret) {
2075 gen6_ppgtt_cleanup(&ppgtt->base);
2076 return ret;
2077 }
2078
2079 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2080 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
2081 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2082 ppgtt->base.bind_vma = ppgtt_bind_vma;
2083 ppgtt->base.set_pages = ppgtt_set_pages;
2084 ppgtt->base.clear_pages = clear_pages;
2085 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
2086 ppgtt->debug_dump = gen6_dump_ppgtt;
2087
2088 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
2089 ppgtt->node.size >> 20,
2090 ppgtt->node.start / PAGE_SIZE);
2091
2092 DRM_DEBUG_DRIVER("Adding PPGTT at offset %x\n",
2093 ppgtt->pd.base.ggtt_offset << 10);
2094
2095 return 0;
2096}
2097
2098static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2099 struct drm_i915_private *dev_priv)
2100{
2101 ppgtt->base.i915 = dev_priv;
2102 ppgtt->base.dma = &dev_priv->drm.pdev->dev;
2103
2104 if (INTEL_GEN(dev_priv) < 8)
2105 return gen6_ppgtt_init(ppgtt);
2106 else
2107 return gen8_ppgtt_init(ppgtt);
2108}
2109
2110static void i915_address_space_init(struct i915_address_space *vm,
2111 struct drm_i915_private *dev_priv,
2112 const char *name)
2113{
2114 i915_gem_timeline_init(dev_priv, &vm->timeline, name);
2115
2116 drm_mm_init(&vm->mm, 0, vm->total);
2117 vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
2118
2119 INIT_LIST_HEAD(&vm->active_list);
2120 INIT_LIST_HEAD(&vm->inactive_list);
2121 INIT_LIST_HEAD(&vm->unbound_list);
2122
2123 list_add_tail(&vm->global_link, &dev_priv->vm_list);
2124 pagevec_init(&vm->free_pages);
2125}
2126
2127static void i915_address_space_fini(struct i915_address_space *vm)
2128{
2129 if (pagevec_count(&vm->free_pages))
2130 vm_free_pages_release(vm, true);
2131
2132 i915_gem_timeline_fini(&vm->timeline);
2133 drm_mm_takedown(&vm->mm);
2134 list_del(&vm->global_link);
2135}
2136
2137static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2138{
2139
2140
2141
2142
2143
2144 if (IS_BROADWELL(dev_priv))
2145 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2146 else if (IS_CHERRYVIEW(dev_priv))
2147 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2148 else if (IS_GEN9_BC(dev_priv) || IS_GEN10(dev_priv))
2149 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2150 else if (IS_GEN9_LP(dev_priv))
2151 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164 if (HAS_PAGE_SIZES(dev_priv, I915_GTT_PAGE_SIZE_64K) &&
2165 INTEL_GEN(dev_priv) <= 10)
2166 I915_WRITE(GEN8_GAMW_ECO_DEV_RW_IA,
2167 I915_READ(GEN8_GAMW_ECO_DEV_RW_IA) |
2168 GAMW_ECO_ENABLE_64K_IPS_FIELD);
2169}
2170
2171int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2172{
2173 gtt_write_workarounds(dev_priv);
2174
2175
2176
2177
2178 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv))
2179 return 0;
2180
2181 if (!USES_PPGTT(dev_priv))
2182 return 0;
2183
2184 if (IS_GEN6(dev_priv))
2185 gen6_ppgtt_enable(dev_priv);
2186 else if (IS_GEN7(dev_priv))
2187 gen7_ppgtt_enable(dev_priv);
2188 else if (INTEL_GEN(dev_priv) >= 8)
2189 gen8_ppgtt_enable(dev_priv);
2190 else
2191 MISSING_CASE(INTEL_GEN(dev_priv));
2192
2193 return 0;
2194}
2195
2196struct i915_hw_ppgtt *
2197i915_ppgtt_create(struct drm_i915_private *dev_priv,
2198 struct drm_i915_file_private *fpriv,
2199 const char *name)
2200{
2201 struct i915_hw_ppgtt *ppgtt;
2202 int ret;
2203
2204 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2205 if (!ppgtt)
2206 return ERR_PTR(-ENOMEM);
2207
2208 ret = __hw_ppgtt_init(ppgtt, dev_priv);
2209 if (ret) {
2210 kfree(ppgtt);
2211 return ERR_PTR(ret);
2212 }
2213
2214 kref_init(&ppgtt->ref);
2215 i915_address_space_init(&ppgtt->base, dev_priv, name);
2216 ppgtt->base.file = fpriv;
2217
2218 trace_i915_ppgtt_create(&ppgtt->base);
2219
2220 return ppgtt;
2221}
2222
2223void i915_ppgtt_close(struct i915_address_space *vm)
2224{
2225 struct list_head *phases[] = {
2226 &vm->active_list,
2227 &vm->inactive_list,
2228 &vm->unbound_list,
2229 NULL,
2230 }, **phase;
2231
2232 GEM_BUG_ON(vm->closed);
2233 vm->closed = true;
2234
2235 for (phase = phases; *phase; phase++) {
2236 struct i915_vma *vma, *vn;
2237
2238 list_for_each_entry_safe(vma, vn, *phase, vm_link)
2239 if (!i915_vma_is_closed(vma))
2240 i915_vma_close(vma);
2241 }
2242}
2243
2244void i915_ppgtt_release(struct kref *kref)
2245{
2246 struct i915_hw_ppgtt *ppgtt =
2247 container_of(kref, struct i915_hw_ppgtt, ref);
2248
2249 trace_i915_ppgtt_release(&ppgtt->base);
2250
2251
2252 GEM_BUG_ON(!list_empty(&ppgtt->base.active_list));
2253 GEM_BUG_ON(!list_empty(&ppgtt->base.inactive_list));
2254 GEM_BUG_ON(!list_empty(&ppgtt->base.unbound_list));
2255
2256 ppgtt->base.cleanup(&ppgtt->base);
2257 i915_address_space_fini(&ppgtt->base);
2258 kfree(ppgtt);
2259}
2260
2261
2262
2263
2264static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2265{
2266
2267
2268
2269 return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active();
2270}
2271
2272static void gen6_check_and_clear_faults(struct drm_i915_private *dev_priv)
2273{
2274 struct intel_engine_cs *engine;
2275 enum intel_engine_id id;
2276 u32 fault;
2277
2278 for_each_engine(engine, dev_priv, id) {
2279 fault = I915_READ(RING_FAULT_REG(engine));
2280 if (fault & RING_FAULT_VALID) {
2281 DRM_DEBUG_DRIVER("Unexpected fault\n"
2282 "\tAddr: 0x%08lx\n"
2283 "\tAddress space: %s\n"
2284 "\tSource ID: %d\n"
2285 "\tType: %d\n",
2286 fault & PAGE_MASK,
2287 fault & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2288 RING_FAULT_SRCID(fault),
2289 RING_FAULT_FAULT_TYPE(fault));
2290 I915_WRITE(RING_FAULT_REG(engine),
2291 fault & ~RING_FAULT_VALID);
2292 }
2293 }
2294
2295 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2296}
2297
2298static void gen8_check_and_clear_faults(struct drm_i915_private *dev_priv)
2299{
2300 u32 fault = I915_READ(GEN8_RING_FAULT_REG);
2301
2302 if (fault & RING_FAULT_VALID) {
2303 u32 fault_data0, fault_data1;
2304 u64 fault_addr;
2305
2306 fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
2307 fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
2308 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
2309 ((u64)fault_data0 << 12);
2310
2311 DRM_DEBUG_DRIVER("Unexpected fault\n"
2312 "\tAddr: 0x%08x_%08x\n"
2313 "\tAddress space: %s\n"
2314 "\tEngine ID: %d\n"
2315 "\tSource ID: %d\n"
2316 "\tType: %d\n",
2317 upper_32_bits(fault_addr),
2318 lower_32_bits(fault_addr),
2319 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
2320 GEN8_RING_FAULT_ENGINE_ID(fault),
2321 RING_FAULT_SRCID(fault),
2322 RING_FAULT_FAULT_TYPE(fault));
2323 I915_WRITE(GEN8_RING_FAULT_REG,
2324 fault & ~RING_FAULT_VALID);
2325 }
2326
2327 POSTING_READ(GEN8_RING_FAULT_REG);
2328}
2329
2330void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2331{
2332
2333 if (INTEL_GEN(dev_priv) >= 8)
2334 gen8_check_and_clear_faults(dev_priv);
2335 else if (INTEL_GEN(dev_priv) >= 6)
2336 gen6_check_and_clear_faults(dev_priv);
2337 else
2338 return;
2339}
2340
2341void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2342{
2343 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2344
2345
2346
2347
2348 if (INTEL_GEN(dev_priv) < 6)
2349 return;
2350
2351 i915_check_and_clear_faults(dev_priv);
2352
2353 ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total);
2354
2355 i915_ggtt_invalidate(dev_priv);
2356}
2357
2358int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2359 struct sg_table *pages)
2360{
2361 do {
2362 if (dma_map_sg_attrs(&obj->base.dev->pdev->dev,
2363 pages->sgl, pages->nents,
2364 PCI_DMA_BIDIRECTIONAL,
2365 DMA_ATTR_NO_WARN))
2366 return 0;
2367
2368
2369
2370
2371
2372
2373
2374 GEM_BUG_ON(obj->mm.pages == pages);
2375 } while (i915_gem_shrink(to_i915(obj->base.dev),
2376 obj->base.size >> PAGE_SHIFT, NULL,
2377 I915_SHRINK_BOUND |
2378 I915_SHRINK_UNBOUND |
2379 I915_SHRINK_ACTIVE));
2380
2381 return -ENOSPC;
2382}
2383
2384static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2385{
2386 writeq(pte, addr);
2387}
2388
2389static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2390 dma_addr_t addr,
2391 u64 offset,
2392 enum i915_cache_level level,
2393 u32 unused)
2394{
2395 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2396 gen8_pte_t __iomem *pte =
2397 (gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2398
2399 gen8_set_pte(pte, gen8_pte_encode(addr, level));
2400
2401 ggtt->invalidate(vm->i915);
2402}
2403
2404static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2405 struct i915_vma *vma,
2406 enum i915_cache_level level,
2407 u32 unused)
2408{
2409 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2410 struct sgt_iter sgt_iter;
2411 gen8_pte_t __iomem *gtt_entries;
2412 const gen8_pte_t pte_encode = gen8_pte_encode(0, level);
2413 dma_addr_t addr;
2414
2415 gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
2416 gtt_entries += vma->node.start >> PAGE_SHIFT;
2417 for_each_sgt_dma(addr, sgt_iter, vma->pages)
2418 gen8_set_pte(gtt_entries++, pte_encode | addr);
2419
2420 wmb();
2421
2422
2423
2424
2425
2426 ggtt->invalidate(vm->i915);
2427}
2428
2429static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2430 dma_addr_t addr,
2431 u64 offset,
2432 enum i915_cache_level level,
2433 u32 flags)
2434{
2435 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2436 gen6_pte_t __iomem *pte =
2437 (gen6_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2438
2439 iowrite32(vm->pte_encode(addr, level, flags), pte);
2440
2441 ggtt->invalidate(vm->i915);
2442}
2443
2444
2445
2446
2447
2448
2449
2450static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2451 struct i915_vma *vma,
2452 enum i915_cache_level level,
2453 u32 flags)
2454{
2455 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2456 gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
2457 unsigned int i = vma->node.start >> PAGE_SHIFT;
2458 struct sgt_iter iter;
2459 dma_addr_t addr;
2460 for_each_sgt_dma(addr, iter, vma->pages)
2461 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
2462 wmb();
2463
2464
2465
2466
2467
2468 ggtt->invalidate(vm->i915);
2469}
2470
2471static void nop_clear_range(struct i915_address_space *vm,
2472 u64 start, u64 length)
2473{
2474}
2475
2476static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2477 u64 start, u64 length)
2478{
2479 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2480 unsigned first_entry = start >> PAGE_SHIFT;
2481 unsigned num_entries = length >> PAGE_SHIFT;
2482 const gen8_pte_t scratch_pte =
2483 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
2484 gen8_pte_t __iomem *gtt_base =
2485 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2486 const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2487 int i;
2488
2489 if (WARN(num_entries > max_entries,
2490 "First entry = %d; Num entries = %d (max=%d)\n",
2491 first_entry, num_entries, max_entries))
2492 num_entries = max_entries;
2493
2494 for (i = 0; i < num_entries; i++)
2495 gen8_set_pte(>t_base[i], scratch_pte);
2496}
2497
2498static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
2499{
2500 struct drm_i915_private *dev_priv = vm->i915;
2501
2502
2503
2504
2505
2506
2507
2508
2509 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2510}
2511
2512struct insert_page {
2513 struct i915_address_space *vm;
2514 dma_addr_t addr;
2515 u64 offset;
2516 enum i915_cache_level level;
2517};
2518
2519static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
2520{
2521 struct insert_page *arg = _arg;
2522
2523 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
2524 bxt_vtd_ggtt_wa(arg->vm);
2525
2526 return 0;
2527}
2528
2529static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
2530 dma_addr_t addr,
2531 u64 offset,
2532 enum i915_cache_level level,
2533 u32 unused)
2534{
2535 struct insert_page arg = { vm, addr, offset, level };
2536
2537 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
2538}
2539
2540struct insert_entries {
2541 struct i915_address_space *vm;
2542 struct i915_vma *vma;
2543 enum i915_cache_level level;
2544};
2545
2546static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
2547{
2548 struct insert_entries *arg = _arg;
2549
2550 gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, 0);
2551 bxt_vtd_ggtt_wa(arg->vm);
2552
2553 return 0;
2554}
2555
2556static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2557 struct i915_vma *vma,
2558 enum i915_cache_level level,
2559 u32 unused)
2560{
2561 struct insert_entries arg = { vm, vma, level };
2562
2563 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
2564}
2565
2566struct clear_range {
2567 struct i915_address_space *vm;
2568 u64 start;
2569 u64 length;
2570};
2571
2572static int bxt_vtd_ggtt_clear_range__cb(void *_arg)
2573{
2574 struct clear_range *arg = _arg;
2575
2576 gen8_ggtt_clear_range(arg->vm, arg->start, arg->length);
2577 bxt_vtd_ggtt_wa(arg->vm);
2578
2579 return 0;
2580}
2581
2582static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm,
2583 u64 start,
2584 u64 length)
2585{
2586 struct clear_range arg = { vm, start, length };
2587
2588 stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL);
2589}
2590
2591static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2592 u64 start, u64 length)
2593{
2594 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2595 unsigned first_entry = start >> PAGE_SHIFT;
2596 unsigned num_entries = length >> PAGE_SHIFT;
2597 gen6_pte_t scratch_pte, __iomem *gtt_base =
2598 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2599 const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2600 int i;
2601
2602 if (WARN(num_entries > max_entries,
2603 "First entry = %d; Num entries = %d (max=%d)\n",
2604 first_entry, num_entries, max_entries))
2605 num_entries = max_entries;
2606
2607 scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2608 I915_CACHE_LLC, 0);
2609
2610 for (i = 0; i < num_entries; i++)
2611 iowrite32(scratch_pte, >t_base[i]);
2612}
2613
2614static void i915_ggtt_insert_page(struct i915_address_space *vm,
2615 dma_addr_t addr,
2616 u64 offset,
2617 enum i915_cache_level cache_level,
2618 u32 unused)
2619{
2620 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2621 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2622
2623 intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2624}
2625
2626static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2627 struct i915_vma *vma,
2628 enum i915_cache_level cache_level,
2629 u32 unused)
2630{
2631 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2632 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2633
2634 intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
2635 flags);
2636}
2637
2638static void i915_ggtt_clear_range(struct i915_address_space *vm,
2639 u64 start, u64 length)
2640{
2641 intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2642}
2643
2644static int ggtt_bind_vma(struct i915_vma *vma,
2645 enum i915_cache_level cache_level,
2646 u32 flags)
2647{
2648 struct drm_i915_private *i915 = vma->vm->i915;
2649 struct drm_i915_gem_object *obj = vma->obj;
2650 u32 pte_flags;
2651
2652
2653 pte_flags = 0;
2654 if (obj->gt_ro)
2655 pte_flags |= PTE_READ_ONLY;
2656
2657 intel_runtime_pm_get(i915);
2658 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2659 intel_runtime_pm_put(i915);
2660
2661 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
2662
2663
2664
2665
2666
2667
2668 vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2669
2670 return 0;
2671}
2672
2673static void ggtt_unbind_vma(struct i915_vma *vma)
2674{
2675 struct drm_i915_private *i915 = vma->vm->i915;
2676
2677 intel_runtime_pm_get(i915);
2678 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2679 intel_runtime_pm_put(i915);
2680}
2681
2682static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2683 enum i915_cache_level cache_level,
2684 u32 flags)
2685{
2686 struct drm_i915_private *i915 = vma->vm->i915;
2687 u32 pte_flags;
2688 int ret;
2689
2690
2691 pte_flags = 0;
2692 if (vma->obj->gt_ro)
2693 pte_flags |= PTE_READ_ONLY;
2694
2695 if (flags & I915_VMA_LOCAL_BIND) {
2696 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2697
2698 if (!(vma->flags & I915_VMA_LOCAL_BIND) &&
2699 appgtt->base.allocate_va_range) {
2700 ret = appgtt->base.allocate_va_range(&appgtt->base,
2701 vma->node.start,
2702 vma->size);
2703 if (ret)
2704 return ret;
2705 }
2706
2707 appgtt->base.insert_entries(&appgtt->base, vma, cache_level,
2708 pte_flags);
2709 }
2710
2711 if (flags & I915_VMA_GLOBAL_BIND) {
2712 intel_runtime_pm_get(i915);
2713 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2714 intel_runtime_pm_put(i915);
2715 }
2716
2717 return 0;
2718}
2719
2720static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
2721{
2722 struct drm_i915_private *i915 = vma->vm->i915;
2723
2724 if (vma->flags & I915_VMA_GLOBAL_BIND) {
2725 intel_runtime_pm_get(i915);
2726 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2727 intel_runtime_pm_put(i915);
2728 }
2729
2730 if (vma->flags & I915_VMA_LOCAL_BIND) {
2731 struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->base;
2732
2733 vm->clear_range(vm, vma->node.start, vma->size);
2734 }
2735}
2736
2737void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2738 struct sg_table *pages)
2739{
2740 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2741 struct device *kdev = &dev_priv->drm.pdev->dev;
2742 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2743
2744 if (unlikely(ggtt->do_idle_maps)) {
2745 if (i915_gem_wait_for_idle(dev_priv, 0)) {
2746 DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2747
2748 udelay(10);
2749 }
2750 }
2751
2752 dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2753}
2754
2755static int ggtt_set_pages(struct i915_vma *vma)
2756{
2757 int ret;
2758
2759 GEM_BUG_ON(vma->pages);
2760
2761 ret = i915_get_ggtt_vma_pages(vma);
2762 if (ret)
2763 return ret;
2764
2765 vma->page_sizes = vma->obj->mm.page_sizes;
2766
2767 return 0;
2768}
2769
2770static void i915_gtt_color_adjust(const struct drm_mm_node *node,
2771 unsigned long color,
2772 u64 *start,
2773 u64 *end)
2774{
2775 if (node->allocated && node->color != color)
2776 *start += I915_GTT_PAGE_SIZE;
2777
2778
2779
2780
2781
2782
2783 node = list_next_entry(node, node_list);
2784 if (node->color != color)
2785 *end -= I915_GTT_PAGE_SIZE;
2786}
2787
2788int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
2789{
2790 struct i915_ggtt *ggtt = &i915->ggtt;
2791 struct i915_hw_ppgtt *ppgtt;
2792 int err;
2793
2794 ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM), "[alias]");
2795 if (IS_ERR(ppgtt))
2796 return PTR_ERR(ppgtt);
2797
2798 if (WARN_ON(ppgtt->base.total < ggtt->base.total)) {
2799 err = -ENODEV;
2800 goto err_ppgtt;
2801 }
2802
2803 if (ppgtt->base.allocate_va_range) {
2804
2805
2806
2807
2808
2809 err = ppgtt->base.allocate_va_range(&ppgtt->base,
2810 0, ggtt->base.total);
2811 if (err)
2812 goto err_ppgtt;
2813 }
2814
2815 i915->mm.aliasing_ppgtt = ppgtt;
2816
2817 GEM_BUG_ON(ggtt->base.bind_vma != ggtt_bind_vma);
2818 ggtt->base.bind_vma = aliasing_gtt_bind_vma;
2819
2820 GEM_BUG_ON(ggtt->base.unbind_vma != ggtt_unbind_vma);
2821 ggtt->base.unbind_vma = aliasing_gtt_unbind_vma;
2822
2823 return 0;
2824
2825err_ppgtt:
2826 i915_ppgtt_put(ppgtt);
2827 return err;
2828}
2829
2830void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915)
2831{
2832 struct i915_ggtt *ggtt = &i915->ggtt;
2833 struct i915_hw_ppgtt *ppgtt;
2834
2835 ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt);
2836 if (!ppgtt)
2837 return;
2838
2839 i915_ppgtt_put(ppgtt);
2840
2841 ggtt->base.bind_vma = ggtt_bind_vma;
2842 ggtt->base.unbind_vma = ggtt_unbind_vma;
2843}
2844
2845int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2846{
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2857 unsigned long hole_start, hole_end;
2858 struct drm_mm_node *entry;
2859 int ret;
2860
2861 ret = intel_vgt_balloon(dev_priv);
2862 if (ret)
2863 return ret;
2864
2865
2866 ret = drm_mm_insert_node_in_range(&ggtt->base.mm, &ggtt->error_capture,
2867 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
2868 0, ggtt->mappable_end,
2869 DRM_MM_INSERT_LOW);
2870 if (ret)
2871 return ret;
2872
2873
2874 drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
2875 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2876 hole_start, hole_end);
2877 ggtt->base.clear_range(&ggtt->base, hole_start,
2878 hole_end - hole_start);
2879 }
2880
2881
2882 ggtt->base.clear_range(&ggtt->base,
2883 ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
2884
2885 if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2886 ret = i915_gem_init_aliasing_ppgtt(dev_priv);
2887 if (ret)
2888 goto err;
2889 }
2890
2891 return 0;
2892
2893err:
2894 drm_mm_remove_node(&ggtt->error_capture);
2895 return ret;
2896}
2897
2898
2899
2900
2901
2902void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2903{
2904 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2905 struct i915_vma *vma, *vn;
2906 struct pagevec *pvec;
2907
2908 ggtt->base.closed = true;
2909
2910 mutex_lock(&dev_priv->drm.struct_mutex);
2911 GEM_BUG_ON(!list_empty(&ggtt->base.active_list));
2912 list_for_each_entry_safe(vma, vn, &ggtt->base.inactive_list, vm_link)
2913 WARN_ON(i915_vma_unbind(vma));
2914 mutex_unlock(&dev_priv->drm.struct_mutex);
2915
2916 i915_gem_cleanup_stolen(&dev_priv->drm);
2917
2918 mutex_lock(&dev_priv->drm.struct_mutex);
2919 i915_gem_fini_aliasing_ppgtt(dev_priv);
2920
2921 if (drm_mm_node_allocated(&ggtt->error_capture))
2922 drm_mm_remove_node(&ggtt->error_capture);
2923
2924 if (drm_mm_initialized(&ggtt->base.mm)) {
2925 intel_vgt_deballoon(dev_priv);
2926 i915_address_space_fini(&ggtt->base);
2927 }
2928
2929 ggtt->base.cleanup(&ggtt->base);
2930
2931 pvec = &dev_priv->mm.wc_stash;
2932 if (pvec->nr) {
2933 set_pages_array_wb(pvec->pages, pvec->nr);
2934 __pagevec_release(pvec);
2935 }
2936
2937 mutex_unlock(&dev_priv->drm.struct_mutex);
2938
2939 arch_phys_wc_del(ggtt->mtrr);
2940 io_mapping_fini(&ggtt->iomap);
2941}
2942
2943static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2944{
2945 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2946 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2947 return snb_gmch_ctl << 20;
2948}
2949
2950static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2951{
2952 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2953 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2954 if (bdw_gmch_ctl)
2955 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2956
2957#ifdef CONFIG_X86_32
2958
2959 if (bdw_gmch_ctl > 4)
2960 bdw_gmch_ctl = 4;
2961#endif
2962
2963 return bdw_gmch_ctl << 20;
2964}
2965
2966static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2967{
2968 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2969 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2970
2971 if (gmch_ctrl)
2972 return 1 << (20 + gmch_ctrl);
2973
2974 return 0;
2975}
2976
2977static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2978{
2979 struct drm_i915_private *dev_priv = ggtt->base.i915;
2980 struct pci_dev *pdev = dev_priv->drm.pdev;
2981 phys_addr_t phys_addr;
2982 int ret;
2983
2984
2985 phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2986
2987
2988
2989
2990
2991
2992
2993
2994 if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10)
2995 ggtt->gsm = ioremap_nocache(phys_addr, size);
2996 else
2997 ggtt->gsm = ioremap_wc(phys_addr, size);
2998 if (!ggtt->gsm) {
2999 DRM_ERROR("Failed to map the ggtt page table\n");
3000 return -ENOMEM;
3001 }
3002
3003 ret = setup_scratch_page(&ggtt->base, GFP_DMA32);
3004 if (ret) {
3005 DRM_ERROR("Scratch setup failed\n");
3006
3007 iounmap(ggtt->gsm);
3008 return ret;
3009 }
3010
3011 return 0;
3012}
3013
3014static struct intel_ppat_entry *
3015__alloc_ppat_entry(struct intel_ppat *ppat, unsigned int index, u8 value)
3016{
3017 struct intel_ppat_entry *entry = &ppat->entries[index];
3018
3019 GEM_BUG_ON(index >= ppat->max_entries);
3020 GEM_BUG_ON(test_bit(index, ppat->used));
3021
3022 entry->ppat = ppat;
3023 entry->value = value;
3024 kref_init(&entry->ref);
3025 set_bit(index, ppat->used);
3026 set_bit(index, ppat->dirty);
3027
3028 return entry;
3029}
3030
3031static void __free_ppat_entry(struct intel_ppat_entry *entry)
3032{
3033 struct intel_ppat *ppat = entry->ppat;
3034 unsigned int index = entry - ppat->entries;
3035
3036 GEM_BUG_ON(index >= ppat->max_entries);
3037 GEM_BUG_ON(!test_bit(index, ppat->used));
3038
3039 entry->value = ppat->clear_value;
3040 clear_bit(index, ppat->used);
3041 set_bit(index, ppat->dirty);
3042}
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056const struct intel_ppat_entry *
3057intel_ppat_get(struct drm_i915_private *i915, u8 value)
3058{
3059 struct intel_ppat *ppat = &i915->ppat;
3060 struct intel_ppat_entry *entry = NULL;
3061 unsigned int scanned, best_score;
3062 int i;
3063
3064 GEM_BUG_ON(!ppat->max_entries);
3065
3066 scanned = best_score = 0;
3067 for_each_set_bit(i, ppat->used, ppat->max_entries) {
3068 unsigned int score;
3069
3070 score = ppat->match(ppat->entries[i].value, value);
3071 if (score > best_score) {
3072 entry = &ppat->entries[i];
3073 if (score == INTEL_PPAT_PERFECT_MATCH) {
3074 kref_get(&entry->ref);
3075 return entry;
3076 }
3077 best_score = score;
3078 }
3079 scanned++;
3080 }
3081
3082 if (scanned == ppat->max_entries) {
3083 if (!entry)
3084 return ERR_PTR(-ENOSPC);
3085
3086 kref_get(&entry->ref);
3087 return entry;
3088 }
3089
3090 i = find_first_zero_bit(ppat->used, ppat->max_entries);
3091 entry = __alloc_ppat_entry(ppat, i, value);
3092 ppat->update_hw(i915);
3093 return entry;
3094}
3095
3096static void release_ppat(struct kref *kref)
3097{
3098 struct intel_ppat_entry *entry =
3099 container_of(kref, struct intel_ppat_entry, ref);
3100 struct drm_i915_private *i915 = entry->ppat->i915;
3101
3102 __free_ppat_entry(entry);
3103 entry->ppat->update_hw(i915);
3104}
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114void intel_ppat_put(const struct intel_ppat_entry *entry)
3115{
3116 struct intel_ppat *ppat = entry->ppat;
3117 unsigned int index = entry - ppat->entries;
3118
3119 GEM_BUG_ON(!ppat->max_entries);
3120
3121 kref_put(&ppat->entries[index].ref, release_ppat);
3122}
3123
3124static void cnl_private_pat_update_hw(struct drm_i915_private *dev_priv)
3125{
3126 struct intel_ppat *ppat = &dev_priv->ppat;
3127 int i;
3128
3129 for_each_set_bit(i, ppat->dirty, ppat->max_entries) {
3130 I915_WRITE(GEN10_PAT_INDEX(i), ppat->entries[i].value);
3131 clear_bit(i, ppat->dirty);
3132 }
3133}
3134
3135static void bdw_private_pat_update_hw(struct drm_i915_private *dev_priv)
3136{
3137 struct intel_ppat *ppat = &dev_priv->ppat;
3138 u64 pat = 0;
3139 int i;
3140
3141 for (i = 0; i < ppat->max_entries; i++)
3142 pat |= GEN8_PPAT(i, ppat->entries[i].value);
3143
3144 bitmap_clear(ppat->dirty, 0, ppat->max_entries);
3145
3146 I915_WRITE(GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
3147 I915_WRITE(GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
3148}
3149
3150static unsigned int bdw_private_pat_match(u8 src, u8 dst)
3151{
3152 unsigned int score = 0;
3153 enum {
3154 AGE_MATCH = BIT(0),
3155 TC_MATCH = BIT(1),
3156 CA_MATCH = BIT(2),
3157 };
3158
3159
3160 if (GEN8_PPAT_GET_CA(src) != GEN8_PPAT_GET_CA(dst))
3161 return 0;
3162
3163 score |= CA_MATCH;
3164
3165 if (GEN8_PPAT_GET_TC(src) == GEN8_PPAT_GET_TC(dst))
3166 score |= TC_MATCH;
3167
3168 if (GEN8_PPAT_GET_AGE(src) == GEN8_PPAT_GET_AGE(dst))
3169 score |= AGE_MATCH;
3170
3171 if (score == (AGE_MATCH | TC_MATCH | CA_MATCH))
3172 return INTEL_PPAT_PERFECT_MATCH;
3173
3174 return score;
3175}
3176
3177static unsigned int chv_private_pat_match(u8 src, u8 dst)
3178{
3179 return (CHV_PPAT_GET_SNOOP(src) == CHV_PPAT_GET_SNOOP(dst)) ?
3180 INTEL_PPAT_PERFECT_MATCH : 0;
3181}
3182
3183static void cnl_setup_private_ppat(struct intel_ppat *ppat)
3184{
3185 ppat->max_entries = 8;
3186 ppat->update_hw = cnl_private_pat_update_hw;
3187 ppat->match = bdw_private_pat_match;
3188 ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
3189
3190 __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
3191 __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
3192 __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
3193 __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);
3194 __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
3195 __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
3196 __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
3197 __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3198}
3199
3200
3201
3202
3203static void bdw_setup_private_ppat(struct intel_ppat *ppat)
3204{
3205 ppat->max_entries = 8;
3206 ppat->update_hw = bdw_private_pat_update_hw;
3207 ppat->match = bdw_private_pat_match;
3208 ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
3209
3210 if (!USES_PPGTT(ppat->i915)) {
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224 __alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
3225 return;
3226 }
3227
3228 __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
3229 __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
3230 __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
3231 __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);
3232 __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
3233 __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
3234 __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
3235 __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3236}
3237
3238static void chv_setup_private_ppat(struct intel_ppat *ppat)
3239{
3240 ppat->max_entries = 8;
3241 ppat->update_hw = bdw_private_pat_update_hw;
3242 ppat->match = chv_private_pat_match;
3243 ppat->clear_value = CHV_PPAT_SNOOP;
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264 __alloc_ppat_entry(ppat, 0, CHV_PPAT_SNOOP);
3265 __alloc_ppat_entry(ppat, 1, 0);
3266 __alloc_ppat_entry(ppat, 2, 0);
3267 __alloc_ppat_entry(ppat, 3, 0);
3268 __alloc_ppat_entry(ppat, 4, CHV_PPAT_SNOOP);
3269 __alloc_ppat_entry(ppat, 5, CHV_PPAT_SNOOP);
3270 __alloc_ppat_entry(ppat, 6, CHV_PPAT_SNOOP);
3271 __alloc_ppat_entry(ppat, 7, CHV_PPAT_SNOOP);
3272}
3273
3274static void gen6_gmch_remove(struct i915_address_space *vm)
3275{
3276 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3277
3278 iounmap(ggtt->gsm);
3279 cleanup_scratch_page(vm);
3280}
3281
3282static void setup_private_pat(struct drm_i915_private *dev_priv)
3283{
3284 struct intel_ppat *ppat = &dev_priv->ppat;
3285 int i;
3286
3287 ppat->i915 = dev_priv;
3288
3289 if (INTEL_GEN(dev_priv) >= 10)
3290 cnl_setup_private_ppat(ppat);
3291 else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3292 chv_setup_private_ppat(ppat);
3293 else
3294 bdw_setup_private_ppat(ppat);
3295
3296 GEM_BUG_ON(ppat->max_entries > INTEL_MAX_PPAT_ENTRIES);
3297
3298 for_each_clear_bit(i, ppat->used, ppat->max_entries) {
3299 ppat->entries[i].value = ppat->clear_value;
3300 ppat->entries[i].ppat = ppat;
3301 set_bit(i, ppat->dirty);
3302 }
3303
3304 ppat->update_hw(dev_priv);
3305}
3306
3307static int gen8_gmch_probe(struct i915_ggtt *ggtt)
3308{
3309 struct drm_i915_private *dev_priv = ggtt->base.i915;
3310 struct pci_dev *pdev = dev_priv->drm.pdev;
3311 unsigned int size;
3312 u16 snb_gmch_ctl;
3313 int err;
3314
3315
3316 ggtt->gmadr =
3317 (struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
3318 pci_resource_len(pdev, 2));
3319 ggtt->mappable_end = resource_size(&ggtt->gmadr);
3320
3321 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
3322 if (!err)
3323 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
3324 if (err)
3325 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
3326
3327 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3328
3329 if (INTEL_GEN(dev_priv) >= 9) {
3330 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3331 } else if (IS_CHERRYVIEW(dev_priv)) {
3332 size = chv_get_total_gtt_size(snb_gmch_ctl);
3333 } else {
3334 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3335 }
3336
3337 ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
3338 ggtt->base.cleanup = gen6_gmch_remove;
3339 ggtt->base.bind_vma = ggtt_bind_vma;
3340 ggtt->base.unbind_vma = ggtt_unbind_vma;
3341 ggtt->base.set_pages = ggtt_set_pages;
3342 ggtt->base.clear_pages = clear_pages;
3343 ggtt->base.insert_page = gen8_ggtt_insert_page;
3344 ggtt->base.clear_range = nop_clear_range;
3345 if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
3346 ggtt->base.clear_range = gen8_ggtt_clear_range;
3347
3348 ggtt->base.insert_entries = gen8_ggtt_insert_entries;
3349
3350
3351 if (intel_ggtt_update_needs_vtd_wa(dev_priv)) {
3352 ggtt->base.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
3353 ggtt->base.insert_page = bxt_vtd_ggtt_insert_page__BKL;
3354 if (ggtt->base.clear_range != nop_clear_range)
3355 ggtt->base.clear_range = bxt_vtd_ggtt_clear_range__BKL;
3356 }
3357
3358 ggtt->invalidate = gen6_ggtt_invalidate;
3359
3360 setup_private_pat(dev_priv);
3361
3362 return ggtt_probe_common(ggtt, size);
3363}
3364
3365static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3366{
3367 struct drm_i915_private *dev_priv = ggtt->base.i915;
3368 struct pci_dev *pdev = dev_priv->drm.pdev;
3369 unsigned int size;
3370 u16 snb_gmch_ctl;
3371 int err;
3372
3373 ggtt->gmadr =
3374 (struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
3375 pci_resource_len(pdev, 2));
3376 ggtt->mappable_end = resource_size(&ggtt->gmadr);
3377
3378
3379
3380
3381 if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3382 DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end);
3383 return -ENXIO;
3384 }
3385
3386 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40));
3387 if (!err)
3388 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
3389 if (err)
3390 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
3391 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3392
3393 size = gen6_get_total_gtt_size(snb_gmch_ctl);
3394 ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
3395
3396 ggtt->base.clear_range = gen6_ggtt_clear_range;
3397 ggtt->base.insert_page = gen6_ggtt_insert_page;
3398 ggtt->base.insert_entries = gen6_ggtt_insert_entries;
3399 ggtt->base.bind_vma = ggtt_bind_vma;
3400 ggtt->base.unbind_vma = ggtt_unbind_vma;
3401 ggtt->base.set_pages = ggtt_set_pages;
3402 ggtt->base.clear_pages = clear_pages;
3403 ggtt->base.cleanup = gen6_gmch_remove;
3404
3405 ggtt->invalidate = gen6_ggtt_invalidate;
3406
3407 if (HAS_EDRAM(dev_priv))
3408 ggtt->base.pte_encode = iris_pte_encode;
3409 else if (IS_HASWELL(dev_priv))
3410 ggtt->base.pte_encode = hsw_pte_encode;
3411 else if (IS_VALLEYVIEW(dev_priv))
3412 ggtt->base.pte_encode = byt_pte_encode;
3413 else if (INTEL_GEN(dev_priv) >= 7)
3414 ggtt->base.pte_encode = ivb_pte_encode;
3415 else
3416 ggtt->base.pte_encode = snb_pte_encode;
3417
3418 return ggtt_probe_common(ggtt, size);
3419}
3420
3421static void i915_gmch_remove(struct i915_address_space *vm)
3422{
3423 intel_gmch_remove();
3424}
3425
3426static int i915_gmch_probe(struct i915_ggtt *ggtt)
3427{
3428 struct drm_i915_private *dev_priv = ggtt->base.i915;
3429 phys_addr_t gmadr_base;
3430 int ret;
3431
3432 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3433 if (!ret) {
3434 DRM_ERROR("failed to set up gmch\n");
3435 return -EIO;
3436 }
3437
3438 intel_gtt_get(&ggtt->base.total,
3439 &gmadr_base,
3440 &ggtt->mappable_end);
3441
3442 ggtt->gmadr =
3443 (struct resource) DEFINE_RES_MEM(gmadr_base,
3444 ggtt->mappable_end);
3445
3446 ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3447 ggtt->base.insert_page = i915_ggtt_insert_page;
3448 ggtt->base.insert_entries = i915_ggtt_insert_entries;
3449 ggtt->base.clear_range = i915_ggtt_clear_range;
3450 ggtt->base.bind_vma = ggtt_bind_vma;
3451 ggtt->base.unbind_vma = ggtt_unbind_vma;
3452 ggtt->base.set_pages = ggtt_set_pages;
3453 ggtt->base.clear_pages = clear_pages;
3454 ggtt->base.cleanup = i915_gmch_remove;
3455
3456 ggtt->invalidate = gmch_ggtt_invalidate;
3457
3458 if (unlikely(ggtt->do_idle_maps))
3459 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3460
3461 return 0;
3462}
3463
3464
3465
3466
3467
3468int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
3469{
3470 struct i915_ggtt *ggtt = &dev_priv->ggtt;
3471 int ret;
3472
3473 ggtt->base.i915 = dev_priv;
3474 ggtt->base.dma = &dev_priv->drm.pdev->dev;
3475
3476 if (INTEL_GEN(dev_priv) <= 5)
3477 ret = i915_gmch_probe(ggtt);
3478 else if (INTEL_GEN(dev_priv) < 8)
3479 ret = gen6_gmch_probe(ggtt);
3480 else
3481 ret = gen8_gmch_probe(ggtt);
3482 if (ret)
3483 return ret;
3484
3485
3486
3487
3488
3489
3490 if (USES_GUC(dev_priv)) {
3491 ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP);
3492 ggtt->mappable_end = min_t(u64, ggtt->mappable_end, ggtt->base.total);
3493 }
3494
3495 if ((ggtt->base.total - 1) >> 32) {
3496 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3497 " of address space! Found %lldM!\n",
3498 ggtt->base.total >> 20);
3499 ggtt->base.total = 1ULL << 32;
3500 ggtt->mappable_end = min_t(u64, ggtt->mappable_end, ggtt->base.total);
3501 }
3502
3503 if (ggtt->mappable_end > ggtt->base.total) {
3504 DRM_ERROR("mappable aperture extends past end of GGTT,"
3505 " aperture=%pa, total=%llx\n",
3506 &ggtt->mappable_end, ggtt->base.total);
3507 ggtt->mappable_end = ggtt->base.total;
3508 }
3509
3510
3511 DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->base.total >> 20);
3512 DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20);
3513 DRM_DEBUG_DRIVER("DSM size = %lluM\n",
3514 (u64)resource_size(&intel_graphics_stolen_res) >> 20);
3515 if (intel_vtd_active())
3516 DRM_INFO("VT-d active for gfx access\n");
3517
3518 return 0;
3519}
3520
3521
3522
3523
3524
3525int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3526{
3527 struct i915_ggtt *ggtt = &dev_priv->ggtt;
3528 int ret;
3529
3530 INIT_LIST_HEAD(&dev_priv->vm_list);
3531
3532
3533
3534
3535
3536
3537 mutex_lock(&dev_priv->drm.struct_mutex);
3538 i915_address_space_init(&ggtt->base, dev_priv, "[global]");
3539 if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv))
3540 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
3541 mutex_unlock(&dev_priv->drm.struct_mutex);
3542
3543 if (!io_mapping_init_wc(&dev_priv->ggtt.iomap,
3544 dev_priv->ggtt.gmadr.start,
3545 dev_priv->ggtt.mappable_end)) {
3546 ret = -EIO;
3547 goto out_gtt_cleanup;
3548 }
3549
3550 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, ggtt->mappable_end);
3551
3552
3553
3554
3555
3556 ret = i915_gem_init_stolen(dev_priv);
3557 if (ret)
3558 goto out_gtt_cleanup;
3559
3560 return 0;
3561
3562out_gtt_cleanup:
3563 ggtt->base.cleanup(&ggtt->base);
3564 return ret;
3565}
3566
3567int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3568{
3569 if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3570 return -EIO;
3571
3572 return 0;
3573}
3574
3575void i915_ggtt_enable_guc(struct drm_i915_private *i915)
3576{
3577 GEM_BUG_ON(i915->ggtt.invalidate != gen6_ggtt_invalidate);
3578
3579 i915->ggtt.invalidate = guc_ggtt_invalidate;
3580
3581 i915_ggtt_invalidate(i915);
3582}
3583
3584void i915_ggtt_disable_guc(struct drm_i915_private *i915)
3585{
3586
3587 GEM_BUG_ON(i915->ggtt.invalidate != guc_ggtt_invalidate);
3588
3589 i915->ggtt.invalidate = gen6_ggtt_invalidate;
3590
3591 i915_ggtt_invalidate(i915);
3592}
3593
3594void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3595{
3596 struct i915_ggtt *ggtt = &dev_priv->ggtt;
3597 struct drm_i915_gem_object *obj, *on;
3598
3599 i915_check_and_clear_faults(dev_priv);
3600
3601
3602 ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total);
3603
3604 ggtt->base.closed = true;
3605
3606
3607 list_for_each_entry_safe(obj, on, &dev_priv->mm.bound_list, mm.link) {
3608 bool ggtt_bound = false;
3609 struct i915_vma *vma;
3610
3611 for_each_ggtt_vma(vma, obj) {
3612 if (!i915_vma_unbind(vma))
3613 continue;
3614
3615 WARN_ON(i915_vma_bind(vma, obj->cache_level,
3616 PIN_UPDATE));
3617 ggtt_bound = true;
3618 }
3619
3620 if (ggtt_bound)
3621 WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3622 }
3623
3624 ggtt->base.closed = false;
3625
3626 if (INTEL_GEN(dev_priv) >= 8) {
3627 struct intel_ppat *ppat = &dev_priv->ppat;
3628
3629 bitmap_set(ppat->dirty, 0, ppat->max_entries);
3630 dev_priv->ppat.update_hw(dev_priv);
3631 return;
3632 }
3633
3634 if (USES_PPGTT(dev_priv)) {
3635 struct i915_address_space *vm;
3636
3637 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3638 struct i915_hw_ppgtt *ppgtt;
3639
3640 if (i915_is_ggtt(vm))
3641 ppgtt = dev_priv->mm.aliasing_ppgtt;
3642 else
3643 ppgtt = i915_vm_to_ppgtt(vm);
3644
3645 gen6_write_page_range(ppgtt, 0, ppgtt->base.total);
3646 }
3647 }
3648
3649 i915_ggtt_invalidate(dev_priv);
3650}
3651
3652static struct scatterlist *
3653rotate_pages(const dma_addr_t *in, unsigned int offset,
3654 unsigned int width, unsigned int height,
3655 unsigned int stride,
3656 struct sg_table *st, struct scatterlist *sg)
3657{
3658 unsigned int column, row;
3659 unsigned int src_idx;
3660
3661 for (column = 0; column < width; column++) {
3662 src_idx = stride * (height - 1) + column;
3663 for (row = 0; row < height; row++) {
3664 st->nents++;
3665
3666
3667
3668
3669 sg_set_page(sg, NULL, PAGE_SIZE, 0);
3670 sg_dma_address(sg) = in[offset + src_idx];
3671 sg_dma_len(sg) = PAGE_SIZE;
3672 sg = sg_next(sg);
3673 src_idx -= stride;
3674 }
3675 }
3676
3677 return sg;
3678}
3679
3680static noinline struct sg_table *
3681intel_rotate_pages(struct intel_rotation_info *rot_info,
3682 struct drm_i915_gem_object *obj)
3683{
3684 const unsigned long n_pages = obj->base.size / PAGE_SIZE;
3685 unsigned int size = intel_rotation_info_size(rot_info);
3686 struct sgt_iter sgt_iter;
3687 dma_addr_t dma_addr;
3688 unsigned long i;
3689 dma_addr_t *page_addr_list;
3690 struct sg_table *st;
3691 struct scatterlist *sg;
3692 int ret = -ENOMEM;
3693
3694
3695 page_addr_list = kvmalloc_array(n_pages,
3696 sizeof(dma_addr_t),
3697 GFP_KERNEL);
3698 if (!page_addr_list)
3699 return ERR_PTR(ret);
3700
3701
3702 st = kmalloc(sizeof(*st), GFP_KERNEL);
3703 if (!st)
3704 goto err_st_alloc;
3705
3706 ret = sg_alloc_table(st, size, GFP_KERNEL);
3707 if (ret)
3708 goto err_sg_alloc;
3709
3710
3711 i = 0;
3712 for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
3713 page_addr_list[i++] = dma_addr;
3714
3715 GEM_BUG_ON(i != n_pages);
3716 st->nents = 0;
3717 sg = st->sgl;
3718
3719 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3720 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
3721 rot_info->plane[i].width, rot_info->plane[i].height,
3722 rot_info->plane[i].stride, st, sg);
3723 }
3724
3725 kvfree(page_addr_list);
3726
3727 return st;
3728
3729err_sg_alloc:
3730 kfree(st);
3731err_st_alloc:
3732 kvfree(page_addr_list);
3733
3734 DRM_DEBUG_DRIVER("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3735 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3736
3737 return ERR_PTR(ret);
3738}
3739
3740static noinline struct sg_table *
3741intel_partial_pages(const struct i915_ggtt_view *view,
3742 struct drm_i915_gem_object *obj)
3743{
3744 struct sg_table *st;
3745 struct scatterlist *sg, *iter;
3746 unsigned int count = view->partial.size;
3747 unsigned int offset;
3748 int ret = -ENOMEM;
3749
3750 st = kmalloc(sizeof(*st), GFP_KERNEL);
3751 if (!st)
3752 goto err_st_alloc;
3753
3754 ret = sg_alloc_table(st, count, GFP_KERNEL);
3755 if (ret)
3756 goto err_sg_alloc;
3757
3758 iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset);
3759 GEM_BUG_ON(!iter);
3760
3761 sg = st->sgl;
3762 st->nents = 0;
3763 do {
3764 unsigned int len;
3765
3766 len = min(iter->length - (offset << PAGE_SHIFT),
3767 count << PAGE_SHIFT);
3768 sg_set_page(sg, NULL, len, 0);
3769 sg_dma_address(sg) =
3770 sg_dma_address(iter) + (offset << PAGE_SHIFT);
3771 sg_dma_len(sg) = len;
3772
3773 st->nents++;
3774 count -= len >> PAGE_SHIFT;
3775 if (count == 0) {
3776 sg_mark_end(sg);
3777 return st;
3778 }
3779
3780 sg = __sg_next(sg);
3781 iter = __sg_next(iter);
3782 offset = 0;
3783 } while (1);
3784
3785err_sg_alloc:
3786 kfree(st);
3787err_st_alloc:
3788 return ERR_PTR(ret);
3789}
3790
3791static int
3792i915_get_ggtt_vma_pages(struct i915_vma *vma)
3793{
3794 int ret;
3795
3796
3797
3798
3799
3800
3801 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3802
3803 switch (vma->ggtt_view.type) {
3804 default:
3805 GEM_BUG_ON(vma->ggtt_view.type);
3806
3807 case I915_GGTT_VIEW_NORMAL:
3808 vma->pages = vma->obj->mm.pages;
3809 return 0;
3810
3811 case I915_GGTT_VIEW_ROTATED:
3812 vma->pages =
3813 intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
3814 break;
3815
3816 case I915_GGTT_VIEW_PARTIAL:
3817 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3818 break;
3819 }
3820
3821 ret = 0;
3822 if (unlikely(IS_ERR(vma->pages))) {
3823 ret = PTR_ERR(vma->pages);
3824 vma->pages = NULL;
3825 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3826 vma->ggtt_view.type, ret);
3827 }
3828 return ret;
3829}
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856int i915_gem_gtt_reserve(struct i915_address_space *vm,
3857 struct drm_mm_node *node,
3858 u64 size, u64 offset, unsigned long color,
3859 unsigned int flags)
3860{
3861 int err;
3862
3863 GEM_BUG_ON(!size);
3864 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3865 GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
3866 GEM_BUG_ON(range_overflows(offset, size, vm->total));
3867 GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3868 GEM_BUG_ON(drm_mm_node_allocated(node));
3869
3870 node->size = size;
3871 node->start = offset;
3872 node->color = color;
3873
3874 err = drm_mm_reserve_node(&vm->mm, node);
3875 if (err != -ENOSPC)
3876 return err;
3877
3878 if (flags & PIN_NOEVICT)
3879 return -ENOSPC;
3880
3881 err = i915_gem_evict_for_node(vm, node, flags);
3882 if (err == 0)
3883 err = drm_mm_reserve_node(&vm->mm, node);
3884
3885 return err;
3886}
3887
3888static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
3889{
3890 u64 range, addr;
3891
3892 GEM_BUG_ON(range_overflows(start, len, end));
3893 GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
3894
3895 range = round_down(end - len, align) - round_up(start, align);
3896 if (range) {
3897 if (sizeof(unsigned long) == sizeof(u64)) {
3898 addr = get_random_long();
3899 } else {
3900 addr = get_random_int();
3901 if (range > U32_MAX) {
3902 addr <<= 32;
3903 addr |= get_random_int();
3904 }
3905 }
3906 div64_u64_rem(addr, range, &addr);
3907 start += addr;
3908 }
3909
3910 return round_up(start, align);
3911}
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947int i915_gem_gtt_insert(struct i915_address_space *vm,
3948 struct drm_mm_node *node,
3949 u64 size, u64 alignment, unsigned long color,
3950 u64 start, u64 end, unsigned int flags)
3951{
3952 enum drm_mm_insert_mode mode;
3953 u64 offset;
3954 int err;
3955
3956 lockdep_assert_held(&vm->i915->drm.struct_mutex);
3957 GEM_BUG_ON(!size);
3958 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3959 GEM_BUG_ON(alignment && !is_power_of_2(alignment));
3960 GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
3961 GEM_BUG_ON(start >= end);
3962 GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
3963 GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
3964 GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3965 GEM_BUG_ON(drm_mm_node_allocated(node));
3966
3967 if (unlikely(range_overflows(start, size, end)))
3968 return -ENOSPC;
3969
3970 if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
3971 return -ENOSPC;
3972
3973 mode = DRM_MM_INSERT_BEST;
3974 if (flags & PIN_HIGH)
3975 mode = DRM_MM_INSERT_HIGH;
3976 if (flags & PIN_MAPPABLE)
3977 mode = DRM_MM_INSERT_LOW;
3978
3979
3980
3981
3982
3983
3984
3985 BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
3986 if (alignment <= I915_GTT_MIN_ALIGNMENT)
3987 alignment = 0;
3988
3989 err = drm_mm_insert_node_in_range(&vm->mm, node,
3990 size, alignment, color,
3991 start, end, mode);
3992 if (err != -ENOSPC)
3993 return err;
3994
3995 if (flags & PIN_NOEVICT)
3996 return -ENOSPC;
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020 offset = random_offset(start, end,
4021 size, alignment ?: I915_GTT_MIN_ALIGNMENT);
4022 err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags);
4023 if (err != -ENOSPC)
4024 return err;
4025
4026
4027 err = i915_gem_evict_something(vm, size, alignment, color,
4028 start, end, flags);
4029 if (err)
4030 return err;
4031
4032 return drm_mm_insert_node_in_range(&vm->mm, node,
4033 size, alignment, color,
4034 start, end, DRM_MM_INSERT_EVICT);
4035}
4036
4037#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4038#include "selftests/mock_gtt.c"
4039#include "selftests/i915_gem_gtt.c"
4040#endif
4041