1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include "i915_drv.h"
25#include "i915_scatterlist.h"
26#include "i915_pvinfo.h"
27#include "i915_vgpu.h"
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59#define pipelined 0
60
61static struct drm_i915_private *fence_to_i915(struct i915_fence_reg *fence)
62{
63 return fence->ggtt->vm.i915;
64}
65
66static struct intel_uncore *fence_to_uncore(struct i915_fence_reg *fence)
67{
68 return fence->ggtt->vm.gt->uncore;
69}
70
71static void i965_write_fence_reg(struct i915_fence_reg *fence)
72{
73 i915_reg_t fence_reg_lo, fence_reg_hi;
74 int fence_pitch_shift;
75 u64 val;
76
77 if (INTEL_GEN(fence_to_i915(fence)) >= 6) {
78 fence_reg_lo = FENCE_REG_GEN6_LO(fence->id);
79 fence_reg_hi = FENCE_REG_GEN6_HI(fence->id);
80 fence_pitch_shift = GEN6_FENCE_PITCH_SHIFT;
81
82 } else {
83 fence_reg_lo = FENCE_REG_965_LO(fence->id);
84 fence_reg_hi = FENCE_REG_965_HI(fence->id);
85 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
86 }
87
88 val = 0;
89 if (fence->tiling) {
90 unsigned int stride = fence->stride;
91
92 GEM_BUG_ON(!IS_ALIGNED(stride, 128));
93
94 val = fence->start + fence->size - I965_FENCE_PAGE;
95 val <<= 32;
96 val |= fence->start;
97 val |= (u64)((stride / 128) - 1) << fence_pitch_shift;
98 if (fence->tiling == I915_TILING_Y)
99 val |= BIT(I965_FENCE_TILING_Y_SHIFT);
100 val |= I965_FENCE_REG_VALID;
101 }
102
103 if (!pipelined) {
104 struct intel_uncore *uncore = fence_to_uncore(fence);
105
106
107
108
109
110
111
112
113
114
115
116 intel_uncore_write_fw(uncore, fence_reg_lo, 0);
117 intel_uncore_posting_read_fw(uncore, fence_reg_lo);
118
119 intel_uncore_write_fw(uncore, fence_reg_hi, upper_32_bits(val));
120 intel_uncore_write_fw(uncore, fence_reg_lo, lower_32_bits(val));
121 intel_uncore_posting_read_fw(uncore, fence_reg_lo);
122 }
123}
124
125static void i915_write_fence_reg(struct i915_fence_reg *fence)
126{
127 u32 val;
128
129 val = 0;
130 if (fence->tiling) {
131 unsigned int stride = fence->stride;
132 unsigned int tiling = fence->tiling;
133 bool is_y_tiled = tiling == I915_TILING_Y;
134
135 if (is_y_tiled && HAS_128_BYTE_Y_TILING(fence_to_i915(fence)))
136 stride /= 128;
137 else
138 stride /= 512;
139 GEM_BUG_ON(!is_power_of_2(stride));
140
141 val = fence->start;
142 if (is_y_tiled)
143 val |= BIT(I830_FENCE_TILING_Y_SHIFT);
144 val |= I915_FENCE_SIZE_BITS(fence->size);
145 val |= ilog2(stride) << I830_FENCE_PITCH_SHIFT;
146
147 val |= I830_FENCE_REG_VALID;
148 }
149
150 if (!pipelined) {
151 struct intel_uncore *uncore = fence_to_uncore(fence);
152 i915_reg_t reg = FENCE_REG(fence->id);
153
154 intel_uncore_write_fw(uncore, reg, val);
155 intel_uncore_posting_read_fw(uncore, reg);
156 }
157}
158
159static void i830_write_fence_reg(struct i915_fence_reg *fence)
160{
161 u32 val;
162
163 val = 0;
164 if (fence->tiling) {
165 unsigned int stride = fence->stride;
166
167 val = fence->start;
168 if (fence->tiling == I915_TILING_Y)
169 val |= BIT(I830_FENCE_TILING_Y_SHIFT);
170 val |= I830_FENCE_SIZE_BITS(fence->size);
171 val |= ilog2(stride / 128) << I830_FENCE_PITCH_SHIFT;
172 val |= I830_FENCE_REG_VALID;
173 }
174
175 if (!pipelined) {
176 struct intel_uncore *uncore = fence_to_uncore(fence);
177 i915_reg_t reg = FENCE_REG(fence->id);
178
179 intel_uncore_write_fw(uncore, reg, val);
180 intel_uncore_posting_read_fw(uncore, reg);
181 }
182}
183
184static void fence_write(struct i915_fence_reg *fence)
185{
186 struct drm_i915_private *i915 = fence_to_i915(fence);
187
188
189
190
191
192
193
194 if (IS_GEN(i915, 2))
195 i830_write_fence_reg(fence);
196 else if (IS_GEN(i915, 3))
197 i915_write_fence_reg(fence);
198 else
199 i965_write_fence_reg(fence);
200
201
202
203
204
205}
206
207static bool gpu_uses_fence_registers(struct i915_fence_reg *fence)
208{
209 return INTEL_GEN(fence_to_i915(fence)) < 4;
210}
211
212static int fence_update(struct i915_fence_reg *fence,
213 struct i915_vma *vma)
214{
215 struct i915_ggtt *ggtt = fence->ggtt;
216 struct intel_uncore *uncore = fence_to_uncore(fence);
217 intel_wakeref_t wakeref;
218 struct i915_vma *old;
219 int ret;
220
221 fence->tiling = 0;
222 if (vma) {
223 GEM_BUG_ON(!i915_gem_object_get_stride(vma->obj) ||
224 !i915_gem_object_get_tiling(vma->obj));
225
226 if (!i915_vma_is_map_and_fenceable(vma))
227 return -EINVAL;
228
229 if (gpu_uses_fence_registers(fence)) {
230
231 ret = i915_vma_sync(vma);
232 if (ret)
233 return ret;
234 }
235
236 fence->start = vma->node.start;
237 fence->size = vma->fence_size;
238 fence->stride = i915_gem_object_get_stride(vma->obj);
239 fence->tiling = i915_gem_object_get_tiling(vma->obj);
240 }
241 WRITE_ONCE(fence->dirty, false);
242
243 old = xchg(&fence->vma, NULL);
244 if (old) {
245
246 ret = i915_active_wait(&fence->active);
247 if (ret) {
248 fence->vma = old;
249 return ret;
250 }
251
252 i915_vma_flush_writes(old);
253
254
255
256
257
258 if (old != vma) {
259 GEM_BUG_ON(old->fence != fence);
260 i915_vma_revoke_mmap(old);
261 old->fence = NULL;
262 }
263
264 list_move(&fence->link, &ggtt->fence_list);
265 }
266
267
268
269
270
271
272
273
274
275
276
277 wakeref = intel_runtime_pm_get_if_in_use(uncore->rpm);
278 if (!wakeref) {
279 GEM_BUG_ON(vma);
280 return 0;
281 }
282
283 WRITE_ONCE(fence->vma, vma);
284 fence_write(fence);
285
286 if (vma) {
287 vma->fence = fence;
288 list_move_tail(&fence->link, &ggtt->fence_list);
289 }
290
291 intel_runtime_pm_put(uncore->rpm, wakeref);
292 return 0;
293}
294
295
296
297
298
299
300
301
302void i915_vma_revoke_fence(struct i915_vma *vma)
303{
304 struct i915_fence_reg *fence = vma->fence;
305 intel_wakeref_t wakeref;
306
307 lockdep_assert_held(&vma->vm->mutex);
308 if (!fence)
309 return;
310
311 GEM_BUG_ON(fence->vma != vma);
312 GEM_BUG_ON(!i915_active_is_idle(&fence->active));
313 GEM_BUG_ON(atomic_read(&fence->pin_count));
314
315 fence->tiling = 0;
316 WRITE_ONCE(fence->vma, NULL);
317 vma->fence = NULL;
318
319 with_intel_runtime_pm_if_in_use(fence_to_uncore(fence)->rpm, wakeref)
320 fence_write(fence);
321}
322
323static struct i915_fence_reg *fence_find(struct i915_ggtt *ggtt)
324{
325 struct i915_fence_reg *fence;
326
327 list_for_each_entry(fence, &ggtt->fence_list, link) {
328 GEM_BUG_ON(fence->vma && fence->vma->fence != fence);
329
330 if (atomic_read(&fence->pin_count))
331 continue;
332
333 return fence;
334 }
335
336
337 if (intel_has_pending_fb_unpin(ggtt->vm.i915))
338 return ERR_PTR(-EAGAIN);
339
340 return ERR_PTR(-EDEADLK);
341}
342
343int __i915_vma_pin_fence(struct i915_vma *vma)
344{
345 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm);
346 struct i915_fence_reg *fence;
347 struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL;
348 int err;
349
350 lockdep_assert_held(&vma->vm->mutex);
351
352
353 if (vma->fence) {
354 fence = vma->fence;
355 GEM_BUG_ON(fence->vma != vma);
356 atomic_inc(&fence->pin_count);
357 if (!fence->dirty) {
358 list_move_tail(&fence->link, &ggtt->fence_list);
359 return 0;
360 }
361 } else if (set) {
362 fence = fence_find(ggtt);
363 if (IS_ERR(fence))
364 return PTR_ERR(fence);
365
366 GEM_BUG_ON(atomic_read(&fence->pin_count));
367 atomic_inc(&fence->pin_count);
368 } else {
369 return 0;
370 }
371
372 err = fence_update(fence, set);
373 if (err)
374 goto out_unpin;
375
376 GEM_BUG_ON(fence->vma != set);
377 GEM_BUG_ON(vma->fence != (set ? fence : NULL));
378
379 if (set)
380 return 0;
381
382out_unpin:
383 atomic_dec(&fence->pin_count);
384 return err;
385}
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405int i915_vma_pin_fence(struct i915_vma *vma)
406{
407 int err;
408
409 if (!vma->fence && !i915_gem_object_is_tiled(vma->obj))
410 return 0;
411
412
413
414
415
416 assert_rpm_wakelock_held(vma->vm->gt->uncore->rpm);
417 GEM_BUG_ON(!i915_vma_is_pinned(vma));
418 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
419
420 err = mutex_lock_interruptible(&vma->vm->mutex);
421 if (err)
422 return err;
423
424 err = __i915_vma_pin_fence(vma);
425 mutex_unlock(&vma->vm->mutex);
426
427 return err;
428}
429
430
431
432
433
434
435
436
437struct i915_fence_reg *i915_reserve_fence(struct i915_ggtt *ggtt)
438{
439 struct i915_fence_reg *fence;
440 int count;
441 int ret;
442
443 lockdep_assert_held(&ggtt->vm.mutex);
444
445
446 count = 0;
447 list_for_each_entry(fence, &ggtt->fence_list, link)
448 count += !atomic_read(&fence->pin_count);
449 if (count <= 1)
450 return ERR_PTR(-ENOSPC);
451
452 fence = fence_find(ggtt);
453 if (IS_ERR(fence))
454 return fence;
455
456 if (fence->vma) {
457
458 ret = fence_update(fence, NULL);
459 if (ret)
460 return ERR_PTR(ret);
461 }
462
463 list_del(&fence->link);
464
465 return fence;
466}
467
468
469
470
471
472
473
474void i915_unreserve_fence(struct i915_fence_reg *fence)
475{
476 struct i915_ggtt *ggtt = fence->ggtt;
477
478 lockdep_assert_held(&ggtt->vm.mutex);
479
480 list_add(&fence->link, &ggtt->fence_list);
481}
482
483
484
485
486
487
488
489
490
491void intel_ggtt_restore_fences(struct i915_ggtt *ggtt)
492{
493 int i;
494
495 for (i = 0; i < ggtt->num_fences; i++)
496 fence_write(&ggtt->fence_regs[i]);
497}
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554static void detect_bit_6_swizzle(struct i915_ggtt *ggtt)
555{
556 struct intel_uncore *uncore = ggtt->vm.gt->uncore;
557 struct drm_i915_private *i915 = ggtt->vm.i915;
558 u32 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
559 u32 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
560
561 if (INTEL_GEN(i915) >= 8 || IS_VALLEYVIEW(i915)) {
562
563
564
565
566
567
568
569 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
570 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
571 } else if (INTEL_GEN(i915) >= 6) {
572 if (i915->preserve_bios_swizzle) {
573 if (intel_uncore_read(uncore, DISP_ARB_CTL) &
574 DISP_TILE_SURFACE_SWIZZLING) {
575 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
576 swizzle_y = I915_BIT_6_SWIZZLE_9;
577 } else {
578 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
579 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
580 }
581 } else {
582 u32 dimm_c0, dimm_c1;
583 dimm_c0 = intel_uncore_read(uncore, MAD_DIMM_C0);
584 dimm_c1 = intel_uncore_read(uncore, MAD_DIMM_C1);
585 dimm_c0 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK;
586 dimm_c1 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK;
587
588
589
590
591
592
593
594 if (dimm_c0 == dimm_c1) {
595 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
596 swizzle_y = I915_BIT_6_SWIZZLE_9;
597 } else {
598 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
599 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
600 }
601 }
602 } else if (IS_GEN(i915, 5)) {
603
604
605
606
607 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
608 swizzle_y = I915_BIT_6_SWIZZLE_9;
609 } else if (IS_GEN(i915, 2)) {
610
611
612
613
614 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
615 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
616 } else if (IS_G45(i915) || IS_I965G(i915) || IS_G33(i915)) {
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644 if (intel_uncore_read(uncore, C0DRB3) ==
645 intel_uncore_read(uncore, C1DRB3)) {
646 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
647 swizzle_y = I915_BIT_6_SWIZZLE_9;
648 }
649 } else {
650 u32 dcc = intel_uncore_read(uncore, DCC);
651
652
653
654
655
656
657
658
659
660
661 switch (dcc & DCC_ADDRESSING_MODE_MASK) {
662 case DCC_ADDRESSING_MODE_SINGLE_CHANNEL:
663 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC:
664 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
665 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
666 break;
667 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED:
668 if (dcc & DCC_CHANNEL_XOR_DISABLE) {
669
670
671
672
673 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
674 swizzle_y = I915_BIT_6_SWIZZLE_9;
675 } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) {
676
677 swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
678 swizzle_y = I915_BIT_6_SWIZZLE_9_11;
679 } else {
680
681 swizzle_x = I915_BIT_6_SWIZZLE_9_10_17;
682 swizzle_y = I915_BIT_6_SWIZZLE_9_17;
683 }
684 break;
685 }
686
687
688 if (IS_GEN(i915, 4) &&
689 !(intel_uncore_read(uncore, DCC2) & DCC2_MODIFIED_ENHANCED_DISABLE)) {
690 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
691 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
692 }
693
694 if (dcc == 0xffffffff) {
695 drm_err(&i915->drm, "Couldn't read from MCHBAR. "
696 "Disabling tiling.\n");
697 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
698 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
699 }
700 }
701
702 if (swizzle_x == I915_BIT_6_SWIZZLE_UNKNOWN ||
703 swizzle_y == I915_BIT_6_SWIZZLE_UNKNOWN) {
704
705
706
707
708
709
710
711
712
713
714 i915->quirks |= QUIRK_PIN_SWIZZLED_PAGES;
715 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
716 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
717 }
718
719 i915->ggtt.bit_6_swizzle_x = swizzle_x;
720 i915->ggtt.bit_6_swizzle_y = swizzle_y;
721}
722
723
724
725
726
727
728static void swizzle_page(struct page *page)
729{
730 char temp[64];
731 char *vaddr;
732 int i;
733
734 vaddr = kmap(page);
735
736 for (i = 0; i < PAGE_SIZE; i += 128) {
737 memcpy(temp, &vaddr[i], 64);
738 memcpy(&vaddr[i], &vaddr[i + 64], 64);
739 memcpy(&vaddr[i + 64], temp, 64);
740 }
741
742 kunmap(page);
743}
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758void
759i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
760 struct sg_table *pages)
761{
762 struct sgt_iter sgt_iter;
763 struct page *page;
764 int i;
765
766 if (obj->bit_17 == NULL)
767 return;
768
769 i = 0;
770 for_each_sgt_page(page, sgt_iter, pages) {
771 char new_bit_17 = page_to_phys(page) >> 17;
772 if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) {
773 swizzle_page(page);
774 set_page_dirty(page);
775 }
776 i++;
777 }
778}
779
780
781
782
783
784
785
786
787
788
789void
790i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
791 struct sg_table *pages)
792{
793 const unsigned int page_count = obj->base.size >> PAGE_SHIFT;
794 struct sgt_iter sgt_iter;
795 struct page *page;
796 int i;
797
798 if (obj->bit_17 == NULL) {
799 obj->bit_17 = bitmap_zalloc(page_count, GFP_KERNEL);
800 if (obj->bit_17 == NULL) {
801 DRM_ERROR("Failed to allocate memory for bit 17 "
802 "record\n");
803 return;
804 }
805 }
806
807 i = 0;
808
809 for_each_sgt_page(page, sgt_iter, pages) {
810 if (page_to_phys(page) & (1 << 17))
811 __set_bit(i, obj->bit_17);
812 else
813 __clear_bit(i, obj->bit_17);
814 i++;
815 }
816}
817
818void intel_ggtt_init_fences(struct i915_ggtt *ggtt)
819{
820 struct drm_i915_private *i915 = ggtt->vm.i915;
821 struct intel_uncore *uncore = ggtt->vm.gt->uncore;
822 int num_fences;
823 int i;
824
825 INIT_LIST_HEAD(&ggtt->fence_list);
826 INIT_LIST_HEAD(&ggtt->userfault_list);
827 intel_wakeref_auto_init(&ggtt->userfault_wakeref, uncore->rpm);
828
829 detect_bit_6_swizzle(ggtt);
830
831 if (!i915_ggtt_has_aperture(ggtt))
832 num_fences = 0;
833 else if (INTEL_GEN(i915) >= 7 &&
834 !(IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)))
835 num_fences = 32;
836 else if (INTEL_GEN(i915) >= 4 ||
837 IS_I945G(i915) || IS_I945GM(i915) ||
838 IS_G33(i915) || IS_PINEVIEW(i915))
839 num_fences = 16;
840 else
841 num_fences = 8;
842
843 if (intel_vgpu_active(i915))
844 num_fences = intel_uncore_read(uncore,
845 vgtif_reg(avail_rs.fence_num));
846 ggtt->fence_regs = kcalloc(num_fences,
847 sizeof(*ggtt->fence_regs),
848 GFP_KERNEL);
849 if (!ggtt->fence_regs)
850 num_fences = 0;
851
852
853 for (i = 0; i < num_fences; i++) {
854 struct i915_fence_reg *fence = &ggtt->fence_regs[i];
855
856 i915_active_init(&fence->active, NULL, NULL);
857 fence->ggtt = ggtt;
858 fence->id = i;
859 list_add_tail(&fence->link, &ggtt->fence_list);
860 }
861 ggtt->num_fences = num_fences;
862
863 intel_ggtt_restore_fences(ggtt);
864}
865
866void intel_ggtt_fini_fences(struct i915_ggtt *ggtt)
867{
868 int i;
869
870 for (i = 0; i < ggtt->num_fences; i++) {
871 struct i915_fence_reg *fence = &ggtt->fence_regs[i];
872
873 i915_active_fini(&fence->active);
874 }
875
876 kfree(ggtt->fence_regs);
877}
878
879void intel_gt_init_swizzling(struct intel_gt *gt)
880{
881 struct drm_i915_private *i915 = gt->i915;
882 struct intel_uncore *uncore = gt->uncore;
883
884 if (INTEL_GEN(i915) < 5 ||
885 i915->ggtt.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
886 return;
887
888 intel_uncore_rmw(uncore, DISP_ARB_CTL, 0, DISP_TILE_SURFACE_SWIZZLING);
889
890 if (IS_GEN(i915, 5))
891 return;
892
893 intel_uncore_rmw(uncore, TILECTL, 0, TILECTL_SWZCTL);
894
895 if (IS_GEN(i915, 6))
896 intel_uncore_write(uncore,
897 ARB_MODE,
898 _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
899 else if (IS_GEN(i915, 7))
900 intel_uncore_write(uncore,
901 ARB_MODE,
902 _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
903 else if (IS_GEN(i915, 8))
904 intel_uncore_write(uncore,
905 GAMTARBMODE,
906 _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
907 else
908 MISSING_CASE(INTEL_GEN(i915));
909}
910