1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#include <drm/drmP.h>
31#include "i915_drv.h"
32#include <drm/i915_drm.h>
33#include "i915_trace.h"
34#include "intel_drv.h"
35
36
37
38
39
40struct pipe_control {
41 struct drm_i915_gem_object *obj;
42 volatile u32 *cpu_page;
43 u32 gtt_offset;
44};
45
46static inline int ring_space(struct intel_ring_buffer *ring)
47{
48 int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
49 if (space < 0)
50 space += ring->size;
51 return space;
52}
53
54static int
55gen2_render_ring_flush(struct intel_ring_buffer *ring,
56 u32 invalidate_domains,
57 u32 flush_domains)
58{
59 u32 cmd;
60 int ret;
61
62 cmd = MI_FLUSH;
63 if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
64 cmd |= MI_NO_WRITE_FLUSH;
65
66 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
67 cmd |= MI_READ_FLUSH;
68
69 ret = intel_ring_begin(ring, 2);
70 if (ret)
71 return ret;
72
73 intel_ring_emit(ring, cmd);
74 intel_ring_emit(ring, MI_NOOP);
75 intel_ring_advance(ring);
76
77 return 0;
78}
79
80static int
81gen4_render_ring_flush(struct intel_ring_buffer *ring,
82 u32 invalidate_domains,
83 u32 flush_domains)
84{
85 struct drm_device *dev = ring->dev;
86 u32 cmd;
87 int ret;
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
118 if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
119 cmd &= ~MI_NO_WRITE_FLUSH;
120 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
121 cmd |= MI_EXE_FLUSH;
122
123 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
124 (IS_G4X(dev) || IS_GEN5(dev)))
125 cmd |= MI_INVALIDATE_ISP;
126
127 ret = intel_ring_begin(ring, 2);
128 if (ret)
129 return ret;
130
131 intel_ring_emit(ring, cmd);
132 intel_ring_emit(ring, MI_NOOP);
133 intel_ring_advance(ring);
134
135 return 0;
136}
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175static int
176intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
177{
178 struct pipe_control *pc = ring->private;
179 u32 scratch_addr = pc->gtt_offset + 128;
180 int ret;
181
182
183 ret = intel_ring_begin(ring, 6);
184 if (ret)
185 return ret;
186
187 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
188 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
189 PIPE_CONTROL_STALL_AT_SCOREBOARD);
190 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
191 intel_ring_emit(ring, 0);
192 intel_ring_emit(ring, 0);
193 intel_ring_emit(ring, MI_NOOP);
194 intel_ring_advance(ring);
195
196 ret = intel_ring_begin(ring, 6);
197 if (ret)
198 return ret;
199
200 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
201 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
202 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
203 intel_ring_emit(ring, 0);
204 intel_ring_emit(ring, 0);
205 intel_ring_emit(ring, MI_NOOP);
206 intel_ring_advance(ring);
207
208 return 0;
209}
210
211static int
212gen6_render_ring_flush(struct intel_ring_buffer *ring,
213 u32 invalidate_domains, u32 flush_domains)
214{
215 u32 flags = 0;
216 struct pipe_control *pc = ring->private;
217 u32 scratch_addr = pc->gtt_offset + 128;
218 int ret;
219
220
221 ret = intel_emit_post_sync_nonzero_flush(ring);
222 if (ret)
223 return ret;
224
225
226
227
228
229 if (flush_domains) {
230 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
231 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
232
233
234
235
236 flags |= PIPE_CONTROL_CS_STALL;
237 }
238 if (invalidate_domains) {
239 flags |= PIPE_CONTROL_TLB_INVALIDATE;
240 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
241 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
242 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
243 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
244 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
245
246
247
248 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
249 }
250
251 ret = intel_ring_begin(ring, 4);
252 if (ret)
253 return ret;
254
255 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
256 intel_ring_emit(ring, flags);
257 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
258 intel_ring_emit(ring, 0);
259 intel_ring_advance(ring);
260
261 return 0;
262}
263
264static int
265gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
266{
267 int ret;
268
269 ret = intel_ring_begin(ring, 4);
270 if (ret)
271 return ret;
272
273 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
274 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
275 PIPE_CONTROL_STALL_AT_SCOREBOARD);
276 intel_ring_emit(ring, 0);
277 intel_ring_emit(ring, 0);
278 intel_ring_advance(ring);
279
280 return 0;
281}
282
283static int
284gen7_render_ring_flush(struct intel_ring_buffer *ring,
285 u32 invalidate_domains, u32 flush_domains)
286{
287 u32 flags = 0;
288 struct pipe_control *pc = ring->private;
289 u32 scratch_addr = pc->gtt_offset + 128;
290 int ret;
291
292
293
294
295
296
297
298
299
300 flags |= PIPE_CONTROL_CS_STALL;
301
302
303
304
305
306 if (flush_domains) {
307 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
308 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
309 }
310 if (invalidate_domains) {
311 flags |= PIPE_CONTROL_TLB_INVALIDATE;
312 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
313 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
314 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
315 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
316 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
317
318
319
320 flags |= PIPE_CONTROL_QW_WRITE;
321 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
322
323
324
325
326 gen7_render_ring_cs_stall_wa(ring);
327 }
328
329 ret = intel_ring_begin(ring, 4);
330 if (ret)
331 return ret;
332
333 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
334 intel_ring_emit(ring, flags);
335 intel_ring_emit(ring, scratch_addr);
336 intel_ring_emit(ring, 0);
337 intel_ring_advance(ring);
338
339 return 0;
340}
341
342static void ring_write_tail(struct intel_ring_buffer *ring,
343 u32 value)
344{
345 drm_i915_private_t *dev_priv = ring->dev->dev_private;
346 I915_WRITE_TAIL(ring, value);
347}
348
349u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
350{
351 drm_i915_private_t *dev_priv = ring->dev->dev_private;
352 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
353 RING_ACTHD(ring->mmio_base) : ACTHD;
354
355 return I915_READ(acthd_reg);
356}
357
358static int init_ring_common(struct intel_ring_buffer *ring)
359{
360 struct drm_device *dev = ring->dev;
361 drm_i915_private_t *dev_priv = dev->dev_private;
362 struct drm_i915_gem_object *obj = ring->obj;
363 int ret = 0;
364 u32 head;
365
366 if (HAS_FORCE_WAKE(dev))
367 gen6_gt_force_wake_get(dev_priv);
368
369
370 I915_WRITE_CTL(ring, 0);
371 I915_WRITE_HEAD(ring, 0);
372 ring->write_tail(ring, 0);
373
374 head = I915_READ_HEAD(ring) & HEAD_ADDR;
375
376
377 if (head != 0) {
378 DRM_DEBUG_KMS("%s head not reset to zero "
379 "ctl %08x head %08x tail %08x start %08x\n",
380 ring->name,
381 I915_READ_CTL(ring),
382 I915_READ_HEAD(ring),
383 I915_READ_TAIL(ring),
384 I915_READ_START(ring));
385
386 I915_WRITE_HEAD(ring, 0);
387
388 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
389 DRM_ERROR("failed to set %s head to zero "
390 "ctl %08x head %08x tail %08x start %08x\n",
391 ring->name,
392 I915_READ_CTL(ring),
393 I915_READ_HEAD(ring),
394 I915_READ_TAIL(ring),
395 I915_READ_START(ring));
396 }
397 }
398
399
400
401
402
403 I915_WRITE_START(ring, obj->gtt_offset);
404 I915_WRITE_CTL(ring,
405 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
406 | RING_VALID);
407
408
409 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
410 I915_READ_START(ring) == obj->gtt_offset &&
411 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
412 DRM_ERROR("%s initialization failed "
413 "ctl %08x head %08x tail %08x start %08x\n",
414 ring->name,
415 I915_READ_CTL(ring),
416 I915_READ_HEAD(ring),
417 I915_READ_TAIL(ring),
418 I915_READ_START(ring));
419 ret = -EIO;
420 goto out;
421 }
422
423 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
424 i915_kernel_lost_context(ring->dev);
425 else {
426 ring->head = I915_READ_HEAD(ring);
427 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
428 ring->space = ring_space(ring);
429 ring->last_retired_head = -1;
430 }
431
432out:
433 if (HAS_FORCE_WAKE(dev))
434 gen6_gt_force_wake_put(dev_priv);
435
436 return ret;
437}
438
439static int
440init_pipe_control(struct intel_ring_buffer *ring)
441{
442 struct pipe_control *pc;
443 struct drm_i915_gem_object *obj;
444 int ret;
445
446 if (ring->private)
447 return 0;
448
449 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
450 if (!pc)
451 return -ENOMEM;
452
453 obj = i915_gem_alloc_object(ring->dev, 4096);
454 if (obj == NULL) {
455 DRM_ERROR("Failed to allocate seqno page\n");
456 ret = -ENOMEM;
457 goto err;
458 }
459
460 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
461
462 ret = i915_gem_object_pin(obj, 4096, true, false);
463 if (ret)
464 goto err_unref;
465
466 pc->gtt_offset = obj->gtt_offset;
467 pc->cpu_page = kmap(sg_page(obj->pages->sgl));
468 if (pc->cpu_page == NULL)
469 goto err_unpin;
470
471 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
472 ring->name, pc->gtt_offset);
473
474 pc->obj = obj;
475 ring->private = pc;
476 return 0;
477
478err_unpin:
479 i915_gem_object_unpin(obj);
480err_unref:
481 drm_gem_object_unreference(&obj->base);
482err:
483 kfree(pc);
484 return ret;
485}
486
487static void
488cleanup_pipe_control(struct intel_ring_buffer *ring)
489{
490 struct pipe_control *pc = ring->private;
491 struct drm_i915_gem_object *obj;
492
493 if (!ring->private)
494 return;
495
496 obj = pc->obj;
497
498 kunmap(sg_page(obj->pages->sgl));
499 i915_gem_object_unpin(obj);
500 drm_gem_object_unreference(&obj->base);
501
502 kfree(pc);
503 ring->private = NULL;
504}
505
506static int init_render_ring(struct intel_ring_buffer *ring)
507{
508 struct drm_device *dev = ring->dev;
509 struct drm_i915_private *dev_priv = dev->dev_private;
510 int ret = init_ring_common(ring);
511
512 if (INTEL_INFO(dev)->gen > 3)
513 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
514
515
516
517
518
519 if (INTEL_INFO(dev)->gen >= 6)
520 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
521
522
523 if (INTEL_INFO(dev)->gen == 6)
524 I915_WRITE(GFX_MODE,
525 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS));
526
527 if (IS_GEN7(dev))
528 I915_WRITE(GFX_MODE_GEN7,
529 _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
530 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
531
532 if (INTEL_INFO(dev)->gen >= 5) {
533 ret = init_pipe_control(ring);
534 if (ret)
535 return ret;
536 }
537
538 if (IS_GEN6(dev)) {
539
540
541
542
543
544 I915_WRITE(CACHE_MODE_0,
545 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
546
547
548
549
550
551 ring->itlb_before_ctx_switch =
552 !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
553 }
554
555 if (INTEL_INFO(dev)->gen >= 6)
556 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
557
558 if (HAS_L3_GPU_CACHE(dev))
559 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
560
561 return ret;
562}
563
564static void render_ring_cleanup(struct intel_ring_buffer *ring)
565{
566 struct drm_device *dev = ring->dev;
567
568 if (!ring->private)
569 return;
570
571 if (HAS_BROKEN_CS_TLB(dev))
572 drm_gem_object_unreference(to_gem_object(ring->private));
573
574 cleanup_pipe_control(ring);
575}
576
577static void
578update_mboxes(struct intel_ring_buffer *ring,
579 u32 mmio_offset)
580{
581 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
582 intel_ring_emit(ring, mmio_offset);
583 intel_ring_emit(ring, ring->outstanding_lazy_request);
584}
585
586
587
588
589
590
591
592
593
594
595static int
596gen6_add_request(struct intel_ring_buffer *ring)
597{
598 u32 mbox1_reg;
599 u32 mbox2_reg;
600 int ret;
601
602 ret = intel_ring_begin(ring, 10);
603 if (ret)
604 return ret;
605
606 mbox1_reg = ring->signal_mbox[0];
607 mbox2_reg = ring->signal_mbox[1];
608
609 update_mboxes(ring, mbox1_reg);
610 update_mboxes(ring, mbox2_reg);
611 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
612 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
613 intel_ring_emit(ring, ring->outstanding_lazy_request);
614 intel_ring_emit(ring, MI_USER_INTERRUPT);
615 intel_ring_advance(ring);
616
617 return 0;
618}
619
620static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
621 u32 seqno)
622{
623 struct drm_i915_private *dev_priv = dev->dev_private;
624 return dev_priv->last_seqno < seqno;
625}
626
627
628
629
630
631
632
633
634static int
635gen6_ring_sync(struct intel_ring_buffer *waiter,
636 struct intel_ring_buffer *signaller,
637 u32 seqno)
638{
639 int ret;
640 u32 dw1 = MI_SEMAPHORE_MBOX |
641 MI_SEMAPHORE_COMPARE |
642 MI_SEMAPHORE_REGISTER;
643
644
645
646
647
648 seqno -= 1;
649
650 WARN_ON(signaller->semaphore_register[waiter->id] ==
651 MI_SEMAPHORE_SYNC_INVALID);
652
653 ret = intel_ring_begin(waiter, 4);
654 if (ret)
655 return ret;
656
657
658 if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
659 intel_ring_emit(waiter,
660 dw1 |
661 signaller->semaphore_register[waiter->id]);
662 intel_ring_emit(waiter, seqno);
663 intel_ring_emit(waiter, 0);
664 intel_ring_emit(waiter, MI_NOOP);
665 } else {
666 intel_ring_emit(waiter, MI_NOOP);
667 intel_ring_emit(waiter, MI_NOOP);
668 intel_ring_emit(waiter, MI_NOOP);
669 intel_ring_emit(waiter, MI_NOOP);
670 }
671 intel_ring_advance(waiter);
672
673 return 0;
674}
675
676#define PIPE_CONTROL_FLUSH(ring__, addr__) \
677do { \
678 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
679 PIPE_CONTROL_DEPTH_STALL); \
680 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
681 intel_ring_emit(ring__, 0); \
682 intel_ring_emit(ring__, 0); \
683} while (0)
684
685static int
686pc_render_add_request(struct intel_ring_buffer *ring)
687{
688 struct pipe_control *pc = ring->private;
689 u32 scratch_addr = pc->gtt_offset + 128;
690 int ret;
691
692
693
694
695
696
697
698
699
700 ret = intel_ring_begin(ring, 32);
701 if (ret)
702 return ret;
703
704 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
705 PIPE_CONTROL_WRITE_FLUSH |
706 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
707 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
708 intel_ring_emit(ring, ring->outstanding_lazy_request);
709 intel_ring_emit(ring, 0);
710 PIPE_CONTROL_FLUSH(ring, scratch_addr);
711 scratch_addr += 128;
712 PIPE_CONTROL_FLUSH(ring, scratch_addr);
713 scratch_addr += 128;
714 PIPE_CONTROL_FLUSH(ring, scratch_addr);
715 scratch_addr += 128;
716 PIPE_CONTROL_FLUSH(ring, scratch_addr);
717 scratch_addr += 128;
718 PIPE_CONTROL_FLUSH(ring, scratch_addr);
719 scratch_addr += 128;
720 PIPE_CONTROL_FLUSH(ring, scratch_addr);
721
722 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
723 PIPE_CONTROL_WRITE_FLUSH |
724 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
725 PIPE_CONTROL_NOTIFY);
726 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
727 intel_ring_emit(ring, ring->outstanding_lazy_request);
728 intel_ring_emit(ring, 0);
729 intel_ring_advance(ring);
730
731 return 0;
732}
733
734static u32
735gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
736{
737
738
739
740 if (!lazy_coherency)
741 intel_ring_get_active_head(ring);
742 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
743}
744
745static u32
746ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
747{
748 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
749}
750
751static void
752ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
753{
754 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
755}
756
757static u32
758pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
759{
760 struct pipe_control *pc = ring->private;
761 return pc->cpu_page[0];
762}
763
764static void
765pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
766{
767 struct pipe_control *pc = ring->private;
768 pc->cpu_page[0] = seqno;
769}
770
771static bool
772gen5_ring_get_irq(struct intel_ring_buffer *ring)
773{
774 struct drm_device *dev = ring->dev;
775 drm_i915_private_t *dev_priv = dev->dev_private;
776 unsigned long flags;
777
778 if (!dev->irq_enabled)
779 return false;
780
781 spin_lock_irqsave(&dev_priv->irq_lock, flags);
782 if (ring->irq_refcount++ == 0) {
783 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
784 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
785 POSTING_READ(GTIMR);
786 }
787 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
788
789 return true;
790}
791
792static void
793gen5_ring_put_irq(struct intel_ring_buffer *ring)
794{
795 struct drm_device *dev = ring->dev;
796 drm_i915_private_t *dev_priv = dev->dev_private;
797 unsigned long flags;
798
799 spin_lock_irqsave(&dev_priv->irq_lock, flags);
800 if (--ring->irq_refcount == 0) {
801 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
802 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
803 POSTING_READ(GTIMR);
804 }
805 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
806}
807
808static bool
809i9xx_ring_get_irq(struct intel_ring_buffer *ring)
810{
811 struct drm_device *dev = ring->dev;
812 drm_i915_private_t *dev_priv = dev->dev_private;
813 unsigned long flags;
814
815 if (!dev->irq_enabled)
816 return false;
817
818 spin_lock_irqsave(&dev_priv->irq_lock, flags);
819 if (ring->irq_refcount++ == 0) {
820 dev_priv->irq_mask &= ~ring->irq_enable_mask;
821 I915_WRITE(IMR, dev_priv->irq_mask);
822 POSTING_READ(IMR);
823 }
824 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
825
826 return true;
827}
828
829static void
830i9xx_ring_put_irq(struct intel_ring_buffer *ring)
831{
832 struct drm_device *dev = ring->dev;
833 drm_i915_private_t *dev_priv = dev->dev_private;
834 unsigned long flags;
835
836 spin_lock_irqsave(&dev_priv->irq_lock, flags);
837 if (--ring->irq_refcount == 0) {
838 dev_priv->irq_mask |= ring->irq_enable_mask;
839 I915_WRITE(IMR, dev_priv->irq_mask);
840 POSTING_READ(IMR);
841 }
842 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
843}
844
845static bool
846i8xx_ring_get_irq(struct intel_ring_buffer *ring)
847{
848 struct drm_device *dev = ring->dev;
849 drm_i915_private_t *dev_priv = dev->dev_private;
850 unsigned long flags;
851
852 if (!dev->irq_enabled)
853 return false;
854
855 spin_lock_irqsave(&dev_priv->irq_lock, flags);
856 if (ring->irq_refcount++ == 0) {
857 dev_priv->irq_mask &= ~ring->irq_enable_mask;
858 I915_WRITE16(IMR, dev_priv->irq_mask);
859 POSTING_READ16(IMR);
860 }
861 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
862
863 return true;
864}
865
866static void
867i8xx_ring_put_irq(struct intel_ring_buffer *ring)
868{
869 struct drm_device *dev = ring->dev;
870 drm_i915_private_t *dev_priv = dev->dev_private;
871 unsigned long flags;
872
873 spin_lock_irqsave(&dev_priv->irq_lock, flags);
874 if (--ring->irq_refcount == 0) {
875 dev_priv->irq_mask |= ring->irq_enable_mask;
876 I915_WRITE16(IMR, dev_priv->irq_mask);
877 POSTING_READ16(IMR);
878 }
879 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
880}
881
882void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
883{
884 struct drm_device *dev = ring->dev;
885 drm_i915_private_t *dev_priv = ring->dev->dev_private;
886 u32 mmio = 0;
887
888
889
890
891 if (IS_GEN7(dev)) {
892 switch (ring->id) {
893 case RCS:
894 mmio = RENDER_HWS_PGA_GEN7;
895 break;
896 case BCS:
897 mmio = BLT_HWS_PGA_GEN7;
898 break;
899 case VCS:
900 mmio = BSD_HWS_PGA_GEN7;
901 break;
902 }
903 } else if (IS_GEN6(ring->dev)) {
904 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
905 } else {
906 mmio = RING_HWS_PGA(ring->mmio_base);
907 }
908
909 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
910 POSTING_READ(mmio);
911}
912
913static int
914bsd_ring_flush(struct intel_ring_buffer *ring,
915 u32 invalidate_domains,
916 u32 flush_domains)
917{
918 int ret;
919
920 ret = intel_ring_begin(ring, 2);
921 if (ret)
922 return ret;
923
924 intel_ring_emit(ring, MI_FLUSH);
925 intel_ring_emit(ring, MI_NOOP);
926 intel_ring_advance(ring);
927 return 0;
928}
929
930static int
931i9xx_add_request(struct intel_ring_buffer *ring)
932{
933 int ret;
934
935 ret = intel_ring_begin(ring, 4);
936 if (ret)
937 return ret;
938
939 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
940 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
941 intel_ring_emit(ring, ring->outstanding_lazy_request);
942 intel_ring_emit(ring, MI_USER_INTERRUPT);
943 intel_ring_advance(ring);
944
945 return 0;
946}
947
948static bool
949gen6_ring_get_irq(struct intel_ring_buffer *ring)
950{
951 struct drm_device *dev = ring->dev;
952 drm_i915_private_t *dev_priv = dev->dev_private;
953 unsigned long flags;
954
955 if (!dev->irq_enabled)
956 return false;
957
958
959
960
961 gen6_gt_force_wake_get(dev_priv);
962
963 spin_lock_irqsave(&dev_priv->irq_lock, flags);
964 if (ring->irq_refcount++ == 0) {
965 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
966 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
967 GEN6_RENDER_L3_PARITY_ERROR));
968 else
969 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
970 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
971 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
972 POSTING_READ(GTIMR);
973 }
974 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
975
976 return true;
977}
978
979static void
980gen6_ring_put_irq(struct intel_ring_buffer *ring)
981{
982 struct drm_device *dev = ring->dev;
983 drm_i915_private_t *dev_priv = dev->dev_private;
984 unsigned long flags;
985
986 spin_lock_irqsave(&dev_priv->irq_lock, flags);
987 if (--ring->irq_refcount == 0) {
988 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
989 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
990 else
991 I915_WRITE_IMR(ring, ~0);
992 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
993 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
994 POSTING_READ(GTIMR);
995 }
996 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
997
998 gen6_gt_force_wake_put(dev_priv);
999}
1000
1001static int
1002i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
1003 u32 offset, u32 length,
1004 unsigned flags)
1005{
1006 int ret;
1007
1008 ret = intel_ring_begin(ring, 2);
1009 if (ret)
1010 return ret;
1011
1012 intel_ring_emit(ring,
1013 MI_BATCH_BUFFER_START |
1014 MI_BATCH_GTT |
1015 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1016 intel_ring_emit(ring, offset);
1017 intel_ring_advance(ring);
1018
1019 return 0;
1020}
1021
1022
1023#define I830_BATCH_LIMIT (256*1024)
1024static int
1025i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
1026 u32 offset, u32 len,
1027 unsigned flags)
1028{
1029 int ret;
1030
1031 if (flags & I915_DISPATCH_PINNED) {
1032 ret = intel_ring_begin(ring, 4);
1033 if (ret)
1034 return ret;
1035
1036 intel_ring_emit(ring, MI_BATCH_BUFFER);
1037 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1038 intel_ring_emit(ring, offset + len - 8);
1039 intel_ring_emit(ring, MI_NOOP);
1040 intel_ring_advance(ring);
1041 } else {
1042 struct drm_i915_gem_object *obj = ring->private;
1043 u32 cs_offset = obj->gtt_offset;
1044
1045 if (len > I830_BATCH_LIMIT)
1046 return -ENOSPC;
1047
1048 ret = intel_ring_begin(ring, 9+3);
1049 if (ret)
1050 return ret;
1051
1052
1053
1054 intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
1055 XY_SRC_COPY_BLT_WRITE_ALPHA |
1056 XY_SRC_COPY_BLT_WRITE_RGB);
1057 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
1058 intel_ring_emit(ring, 0);
1059 intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
1060 intel_ring_emit(ring, cs_offset);
1061 intel_ring_emit(ring, 0);
1062 intel_ring_emit(ring, 4096);
1063 intel_ring_emit(ring, offset);
1064 intel_ring_emit(ring, MI_FLUSH);
1065
1066
1067 intel_ring_emit(ring, MI_BATCH_BUFFER);
1068 intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1069 intel_ring_emit(ring, cs_offset + len - 8);
1070 intel_ring_advance(ring);
1071 }
1072
1073 return 0;
1074}
1075
1076static int
1077i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
1078 u32 offset, u32 len,
1079 unsigned flags)
1080{
1081 int ret;
1082
1083 ret = intel_ring_begin(ring, 2);
1084 if (ret)
1085 return ret;
1086
1087 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1088 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1089 intel_ring_advance(ring);
1090
1091 return 0;
1092}
1093
1094static void cleanup_status_page(struct intel_ring_buffer *ring)
1095{
1096 struct drm_i915_gem_object *obj;
1097
1098 obj = ring->status_page.obj;
1099 if (obj == NULL)
1100 return;
1101
1102 kunmap(sg_page(obj->pages->sgl));
1103 i915_gem_object_unpin(obj);
1104 drm_gem_object_unreference(&obj->base);
1105 ring->status_page.obj = NULL;
1106}
1107
1108static int init_status_page(struct intel_ring_buffer *ring)
1109{
1110 struct drm_device *dev = ring->dev;
1111 struct drm_i915_gem_object *obj;
1112 int ret;
1113
1114 obj = i915_gem_alloc_object(dev, 4096);
1115 if (obj == NULL) {
1116 DRM_ERROR("Failed to allocate status page\n");
1117 ret = -ENOMEM;
1118 goto err;
1119 }
1120
1121 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1122
1123 ret = i915_gem_object_pin(obj, 4096, true, false);
1124 if (ret != 0) {
1125 goto err_unref;
1126 }
1127
1128 ring->status_page.gfx_addr = obj->gtt_offset;
1129 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
1130 if (ring->status_page.page_addr == NULL) {
1131 ret = -ENOMEM;
1132 goto err_unpin;
1133 }
1134 ring->status_page.obj = obj;
1135 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1136
1137 intel_ring_setup_status_page(ring);
1138 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1139 ring->name, ring->status_page.gfx_addr);
1140
1141 return 0;
1142
1143err_unpin:
1144 i915_gem_object_unpin(obj);
1145err_unref:
1146 drm_gem_object_unreference(&obj->base);
1147err:
1148 return ret;
1149}
1150
1151static int init_phys_hws_pga(struct intel_ring_buffer *ring)
1152{
1153 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1154 u32 addr;
1155
1156 if (!dev_priv->status_page_dmah) {
1157 dev_priv->status_page_dmah =
1158 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1159 if (!dev_priv->status_page_dmah)
1160 return -ENOMEM;
1161 }
1162
1163 addr = dev_priv->status_page_dmah->busaddr;
1164 if (INTEL_INFO(ring->dev)->gen >= 4)
1165 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
1166 I915_WRITE(HWS_PGA, addr);
1167
1168 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1169 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1170
1171 return 0;
1172}
1173
1174static int intel_init_ring_buffer(struct drm_device *dev,
1175 struct intel_ring_buffer *ring)
1176{
1177 struct drm_i915_gem_object *obj;
1178 struct drm_i915_private *dev_priv = dev->dev_private;
1179 int ret;
1180
1181 ring->dev = dev;
1182 INIT_LIST_HEAD(&ring->active_list);
1183 INIT_LIST_HEAD(&ring->request_list);
1184 ring->size = 32 * PAGE_SIZE;
1185 memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
1186
1187 init_waitqueue_head(&ring->irq_queue);
1188
1189 if (I915_NEED_GFX_HWS(dev)) {
1190 ret = init_status_page(ring);
1191 if (ret)
1192 return ret;
1193 } else {
1194 BUG_ON(ring->id != RCS);
1195 ret = init_phys_hws_pga(ring);
1196 if (ret)
1197 return ret;
1198 }
1199
1200 obj = NULL;
1201 if (!HAS_LLC(dev))
1202 obj = i915_gem_object_create_stolen(dev, ring->size);
1203 if (obj == NULL)
1204 obj = i915_gem_alloc_object(dev, ring->size);
1205 if (obj == NULL) {
1206 DRM_ERROR("Failed to allocate ringbuffer\n");
1207 ret = -ENOMEM;
1208 goto err_hws;
1209 }
1210
1211 ring->obj = obj;
1212
1213 ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
1214 if (ret)
1215 goto err_unref;
1216
1217 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1218 if (ret)
1219 goto err_unpin;
1220
1221 ring->virtual_start =
1222 ioremap_wc(dev_priv->gtt.mappable_base + obj->gtt_offset,
1223 ring->size);
1224 if (ring->virtual_start == NULL) {
1225 DRM_ERROR("Failed to map ringbuffer.\n");
1226 ret = -EINVAL;
1227 goto err_unpin;
1228 }
1229
1230 ret = ring->init(ring);
1231 if (ret)
1232 goto err_unmap;
1233
1234
1235
1236
1237
1238 ring->effective_size = ring->size;
1239 if (IS_I830(ring->dev) || IS_845G(ring->dev))
1240 ring->effective_size -= 128;
1241
1242 return 0;
1243
1244err_unmap:
1245 iounmap(ring->virtual_start);
1246err_unpin:
1247 i915_gem_object_unpin(obj);
1248err_unref:
1249 drm_gem_object_unreference(&obj->base);
1250 ring->obj = NULL;
1251err_hws:
1252 cleanup_status_page(ring);
1253 return ret;
1254}
1255
1256void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1257{
1258 struct drm_i915_private *dev_priv;
1259 int ret;
1260
1261 if (ring->obj == NULL)
1262 return;
1263
1264
1265 dev_priv = ring->dev->dev_private;
1266 ret = intel_ring_idle(ring);
1267 if (ret)
1268 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1269 ring->name, ret);
1270
1271 I915_WRITE_CTL(ring, 0);
1272
1273 iounmap(ring->virtual_start);
1274
1275 i915_gem_object_unpin(ring->obj);
1276 drm_gem_object_unreference(&ring->obj->base);
1277 ring->obj = NULL;
1278
1279 if (ring->cleanup)
1280 ring->cleanup(ring);
1281
1282 cleanup_status_page(ring);
1283}
1284
1285static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1286{
1287 int ret;
1288
1289 ret = i915_wait_seqno(ring, seqno);
1290 if (!ret)
1291 i915_gem_retire_requests_ring(ring);
1292
1293 return ret;
1294}
1295
1296static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1297{
1298 struct drm_i915_gem_request *request;
1299 u32 seqno = 0;
1300 int ret;
1301
1302 i915_gem_retire_requests_ring(ring);
1303
1304 if (ring->last_retired_head != -1) {
1305 ring->head = ring->last_retired_head;
1306 ring->last_retired_head = -1;
1307 ring->space = ring_space(ring);
1308 if (ring->space >= n)
1309 return 0;
1310 }
1311
1312 list_for_each_entry(request, &ring->request_list, list) {
1313 int space;
1314
1315 if (request->tail == -1)
1316 continue;
1317
1318 space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
1319 if (space < 0)
1320 space += ring->size;
1321 if (space >= n) {
1322 seqno = request->seqno;
1323 break;
1324 }
1325
1326
1327
1328
1329
1330
1331 request->tail = -1;
1332 }
1333
1334 if (seqno == 0)
1335 return -ENOSPC;
1336
1337 ret = intel_ring_wait_seqno(ring, seqno);
1338 if (ret)
1339 return ret;
1340
1341 if (WARN_ON(ring->last_retired_head == -1))
1342 return -ENOSPC;
1343
1344 ring->head = ring->last_retired_head;
1345 ring->last_retired_head = -1;
1346 ring->space = ring_space(ring);
1347 if (WARN_ON(ring->space < n))
1348 return -ENOSPC;
1349
1350 return 0;
1351}
1352
1353static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
1354{
1355 struct drm_device *dev = ring->dev;
1356 struct drm_i915_private *dev_priv = dev->dev_private;
1357 unsigned long end;
1358 int ret;
1359
1360 ret = intel_ring_wait_request(ring, n);
1361 if (ret != -ENOSPC)
1362 return ret;
1363
1364 trace_i915_ring_wait_begin(ring);
1365
1366
1367
1368
1369
1370 end = jiffies + 60 * HZ;
1371
1372 do {
1373 ring->head = I915_READ_HEAD(ring);
1374 ring->space = ring_space(ring);
1375 if (ring->space >= n) {
1376 trace_i915_ring_wait_end(ring);
1377 return 0;
1378 }
1379
1380 if (dev->primary->master) {
1381 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1382 if (master_priv->sarea_priv)
1383 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1384 }
1385
1386 msleep(1);
1387
1388 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1389 dev_priv->mm.interruptible);
1390 if (ret)
1391 return ret;
1392 } while (!time_after(jiffies, end));
1393 trace_i915_ring_wait_end(ring);
1394 return -EBUSY;
1395}
1396
1397static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1398{
1399 uint32_t __iomem *virt;
1400 int rem = ring->size - ring->tail;
1401
1402 if (ring->space < rem) {
1403 int ret = ring_wait_for_space(ring, rem);
1404 if (ret)
1405 return ret;
1406 }
1407
1408 virt = ring->virtual_start + ring->tail;
1409 rem /= 4;
1410 while (rem--)
1411 iowrite32(MI_NOOP, virt++);
1412
1413 ring->tail = 0;
1414 ring->space = ring_space(ring);
1415
1416 return 0;
1417}
1418
1419int intel_ring_idle(struct intel_ring_buffer *ring)
1420{
1421 u32 seqno;
1422 int ret;
1423
1424
1425 if (ring->outstanding_lazy_request) {
1426 ret = i915_add_request(ring, NULL, NULL);
1427 if (ret)
1428 return ret;
1429 }
1430
1431
1432 if (list_empty(&ring->request_list))
1433 return 0;
1434
1435 seqno = list_entry(ring->request_list.prev,
1436 struct drm_i915_gem_request,
1437 list)->seqno;
1438
1439 return i915_wait_seqno(ring, seqno);
1440}
1441
1442static int
1443intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
1444{
1445 if (ring->outstanding_lazy_request)
1446 return 0;
1447
1448 return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_request);
1449}
1450
1451static int __intel_ring_begin(struct intel_ring_buffer *ring,
1452 int bytes)
1453{
1454 int ret;
1455
1456 if (unlikely(ring->tail + bytes > ring->effective_size)) {
1457 ret = intel_wrap_ring_buffer(ring);
1458 if (unlikely(ret))
1459 return ret;
1460 }
1461
1462 if (unlikely(ring->space < bytes)) {
1463 ret = ring_wait_for_space(ring, bytes);
1464 if (unlikely(ret))
1465 return ret;
1466 }
1467
1468 ring->space -= bytes;
1469 return 0;
1470}
1471
1472int intel_ring_begin(struct intel_ring_buffer *ring,
1473 int num_dwords)
1474{
1475 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1476 int ret;
1477
1478 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1479 dev_priv->mm.interruptible);
1480 if (ret)
1481 return ret;
1482
1483
1484 ret = intel_ring_alloc_seqno(ring);
1485 if (ret)
1486 return ret;
1487
1488 return __intel_ring_begin(ring, num_dwords * sizeof(uint32_t));
1489}
1490
1491void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
1492{
1493 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1494
1495 BUG_ON(ring->outstanding_lazy_request);
1496
1497 if (INTEL_INFO(ring->dev)->gen >= 6) {
1498 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
1499 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
1500 }
1501
1502 ring->set_seqno(ring, seqno);
1503}
1504
1505void intel_ring_advance(struct intel_ring_buffer *ring)
1506{
1507 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1508
1509 ring->tail &= ring->size - 1;
1510 if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
1511 return;
1512 ring->write_tail(ring, ring->tail);
1513}
1514
1515
1516static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1517 u32 value)
1518{
1519 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1520
1521
1522
1523
1524
1525
1526 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1527 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1528
1529
1530 I915_WRITE64(GEN6_BSD_RNCID, 0x0);
1531
1532
1533 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1534 GEN6_BSD_SLEEP_INDICATOR) == 0,
1535 50))
1536 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1537
1538
1539 I915_WRITE_TAIL(ring, value);
1540 POSTING_READ(RING_TAIL(ring->mmio_base));
1541
1542
1543
1544
1545 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1546 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1547}
1548
1549static int gen6_ring_flush(struct intel_ring_buffer *ring,
1550 u32 invalidate, u32 flush)
1551{
1552 uint32_t cmd;
1553 int ret;
1554
1555 ret = intel_ring_begin(ring, 4);
1556 if (ret)
1557 return ret;
1558
1559 cmd = MI_FLUSH_DW;
1560
1561
1562
1563
1564
1565
1566 if (invalidate & I915_GEM_GPU_DOMAINS)
1567 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
1568 MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1569 intel_ring_emit(ring, cmd);
1570 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1571 intel_ring_emit(ring, 0);
1572 intel_ring_emit(ring, MI_NOOP);
1573 intel_ring_advance(ring);
1574 return 0;
1575}
1576
1577static int
1578hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1579 u32 offset, u32 len,
1580 unsigned flags)
1581{
1582 int ret;
1583
1584 ret = intel_ring_begin(ring, 2);
1585 if (ret)
1586 return ret;
1587
1588 intel_ring_emit(ring,
1589 MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
1590 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
1591
1592 intel_ring_emit(ring, offset);
1593 intel_ring_advance(ring);
1594
1595 return 0;
1596}
1597
1598static int
1599gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1600 u32 offset, u32 len,
1601 unsigned flags)
1602{
1603 int ret;
1604
1605 ret = intel_ring_begin(ring, 2);
1606 if (ret)
1607 return ret;
1608
1609 intel_ring_emit(ring,
1610 MI_BATCH_BUFFER_START |
1611 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1612
1613 intel_ring_emit(ring, offset);
1614 intel_ring_advance(ring);
1615
1616 return 0;
1617}
1618
1619
1620
1621static int blt_ring_flush(struct intel_ring_buffer *ring,
1622 u32 invalidate, u32 flush)
1623{
1624 uint32_t cmd;
1625 int ret;
1626
1627 ret = intel_ring_begin(ring, 4);
1628 if (ret)
1629 return ret;
1630
1631 cmd = MI_FLUSH_DW;
1632
1633
1634
1635
1636
1637
1638 if (invalidate & I915_GEM_DOMAIN_RENDER)
1639 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
1640 MI_FLUSH_DW_OP_STOREDW;
1641 intel_ring_emit(ring, cmd);
1642 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1643 intel_ring_emit(ring, 0);
1644 intel_ring_emit(ring, MI_NOOP);
1645 intel_ring_advance(ring);
1646 return 0;
1647}
1648
1649int intel_init_render_ring_buffer(struct drm_device *dev)
1650{
1651 drm_i915_private_t *dev_priv = dev->dev_private;
1652 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1653
1654 ring->name = "render ring";
1655 ring->id = RCS;
1656 ring->mmio_base = RENDER_RING_BASE;
1657
1658 if (INTEL_INFO(dev)->gen >= 6) {
1659 ring->add_request = gen6_add_request;
1660 ring->flush = gen7_render_ring_flush;
1661 if (INTEL_INFO(dev)->gen == 6)
1662 ring->flush = gen6_render_ring_flush;
1663 ring->irq_get = gen6_ring_get_irq;
1664 ring->irq_put = gen6_ring_put_irq;
1665 ring->irq_enable_mask = GT_USER_INTERRUPT;
1666 ring->get_seqno = gen6_ring_get_seqno;
1667 ring->set_seqno = ring_set_seqno;
1668 ring->sync_to = gen6_ring_sync;
1669 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
1670 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
1671 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
1672 ring->signal_mbox[0] = GEN6_VRSYNC;
1673 ring->signal_mbox[1] = GEN6_BRSYNC;
1674 } else if (IS_GEN5(dev)) {
1675 ring->add_request = pc_render_add_request;
1676 ring->flush = gen4_render_ring_flush;
1677 ring->get_seqno = pc_render_get_seqno;
1678 ring->set_seqno = pc_render_set_seqno;
1679 ring->irq_get = gen5_ring_get_irq;
1680 ring->irq_put = gen5_ring_put_irq;
1681 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
1682 } else {
1683 ring->add_request = i9xx_add_request;
1684 if (INTEL_INFO(dev)->gen < 4)
1685 ring->flush = gen2_render_ring_flush;
1686 else
1687 ring->flush = gen4_render_ring_flush;
1688 ring->get_seqno = ring_get_seqno;
1689 ring->set_seqno = ring_set_seqno;
1690 if (IS_GEN2(dev)) {
1691 ring->irq_get = i8xx_ring_get_irq;
1692 ring->irq_put = i8xx_ring_put_irq;
1693 } else {
1694 ring->irq_get = i9xx_ring_get_irq;
1695 ring->irq_put = i9xx_ring_put_irq;
1696 }
1697 ring->irq_enable_mask = I915_USER_INTERRUPT;
1698 }
1699 ring->write_tail = ring_write_tail;
1700 if (IS_HASWELL(dev))
1701 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
1702 else if (INTEL_INFO(dev)->gen >= 6)
1703 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1704 else if (INTEL_INFO(dev)->gen >= 4)
1705 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1706 else if (IS_I830(dev) || IS_845G(dev))
1707 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1708 else
1709 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1710 ring->init = init_render_ring;
1711 ring->cleanup = render_ring_cleanup;
1712
1713
1714 if (HAS_BROKEN_CS_TLB(dev)) {
1715 struct drm_i915_gem_object *obj;
1716 int ret;
1717
1718 obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
1719 if (obj == NULL) {
1720 DRM_ERROR("Failed to allocate batch bo\n");
1721 return -ENOMEM;
1722 }
1723
1724 ret = i915_gem_object_pin(obj, 0, true, false);
1725 if (ret != 0) {
1726 drm_gem_object_unreference(&obj->base);
1727 DRM_ERROR("Failed to ping batch bo\n");
1728 return ret;
1729 }
1730
1731 ring->private = obj;
1732 }
1733
1734 return intel_init_ring_buffer(dev, ring);
1735}
1736
1737int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1738{
1739 drm_i915_private_t *dev_priv = dev->dev_private;
1740 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1741 int ret;
1742
1743 ring->name = "render ring";
1744 ring->id = RCS;
1745 ring->mmio_base = RENDER_RING_BASE;
1746
1747 if (INTEL_INFO(dev)->gen >= 6) {
1748
1749 return -ENODEV;
1750 }
1751
1752
1753
1754
1755 ring->add_request = i9xx_add_request;
1756 if (INTEL_INFO(dev)->gen < 4)
1757 ring->flush = gen2_render_ring_flush;
1758 else
1759 ring->flush = gen4_render_ring_flush;
1760 ring->get_seqno = ring_get_seqno;
1761 ring->set_seqno = ring_set_seqno;
1762 if (IS_GEN2(dev)) {
1763 ring->irq_get = i8xx_ring_get_irq;
1764 ring->irq_put = i8xx_ring_put_irq;
1765 } else {
1766 ring->irq_get = i9xx_ring_get_irq;
1767 ring->irq_put = i9xx_ring_put_irq;
1768 }
1769 ring->irq_enable_mask = I915_USER_INTERRUPT;
1770 ring->write_tail = ring_write_tail;
1771 if (INTEL_INFO(dev)->gen >= 4)
1772 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1773 else if (IS_I830(dev) || IS_845G(dev))
1774 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1775 else
1776 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1777 ring->init = init_render_ring;
1778 ring->cleanup = render_ring_cleanup;
1779
1780 ring->dev = dev;
1781 INIT_LIST_HEAD(&ring->active_list);
1782 INIT_LIST_HEAD(&ring->request_list);
1783
1784 ring->size = size;
1785 ring->effective_size = ring->size;
1786 if (IS_I830(ring->dev) || IS_845G(ring->dev))
1787 ring->effective_size -= 128;
1788
1789 ring->virtual_start = ioremap_wc(start, size);
1790 if (ring->virtual_start == NULL) {
1791 DRM_ERROR("can not ioremap virtual address for"
1792 " ring buffer\n");
1793 return -ENOMEM;
1794 }
1795
1796 if (!I915_NEED_GFX_HWS(dev)) {
1797 ret = init_phys_hws_pga(ring);
1798 if (ret)
1799 return ret;
1800 }
1801
1802 return 0;
1803}
1804
1805int intel_init_bsd_ring_buffer(struct drm_device *dev)
1806{
1807 drm_i915_private_t *dev_priv = dev->dev_private;
1808 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1809
1810 ring->name = "bsd ring";
1811 ring->id = VCS;
1812
1813 ring->write_tail = ring_write_tail;
1814 if (IS_GEN6(dev) || IS_GEN7(dev)) {
1815 ring->mmio_base = GEN6_BSD_RING_BASE;
1816
1817 if (IS_GEN6(dev))
1818 ring->write_tail = gen6_bsd_ring_write_tail;
1819 ring->flush = gen6_ring_flush;
1820 ring->add_request = gen6_add_request;
1821 ring->get_seqno = gen6_ring_get_seqno;
1822 ring->set_seqno = ring_set_seqno;
1823 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
1824 ring->irq_get = gen6_ring_get_irq;
1825 ring->irq_put = gen6_ring_put_irq;
1826 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1827 ring->sync_to = gen6_ring_sync;
1828 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
1829 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
1830 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
1831 ring->signal_mbox[0] = GEN6_RVSYNC;
1832 ring->signal_mbox[1] = GEN6_BVSYNC;
1833 } else {
1834 ring->mmio_base = BSD_RING_BASE;
1835 ring->flush = bsd_ring_flush;
1836 ring->add_request = i9xx_add_request;
1837 ring->get_seqno = ring_get_seqno;
1838 ring->set_seqno = ring_set_seqno;
1839 if (IS_GEN5(dev)) {
1840 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1841 ring->irq_get = gen5_ring_get_irq;
1842 ring->irq_put = gen5_ring_put_irq;
1843 } else {
1844 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1845 ring->irq_get = i9xx_ring_get_irq;
1846 ring->irq_put = i9xx_ring_put_irq;
1847 }
1848 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1849 }
1850 ring->init = init_ring_common;
1851
1852 return intel_init_ring_buffer(dev, ring);
1853}
1854
1855int intel_init_blt_ring_buffer(struct drm_device *dev)
1856{
1857 drm_i915_private_t *dev_priv = dev->dev_private;
1858 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1859
1860 ring->name = "blitter ring";
1861 ring->id = BCS;
1862
1863 ring->mmio_base = BLT_RING_BASE;
1864 ring->write_tail = ring_write_tail;
1865 ring->flush = blt_ring_flush;
1866 ring->add_request = gen6_add_request;
1867 ring->get_seqno = gen6_ring_get_seqno;
1868 ring->set_seqno = ring_set_seqno;
1869 ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
1870 ring->irq_get = gen6_ring_get_irq;
1871 ring->irq_put = gen6_ring_put_irq;
1872 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1873 ring->sync_to = gen6_ring_sync;
1874 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
1875 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
1876 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
1877 ring->signal_mbox[0] = GEN6_RBSYNC;
1878 ring->signal_mbox[1] = GEN6_VBSYNC;
1879 ring->init = init_ring_common;
1880
1881 return intel_init_ring_buffer(dev, ring);
1882}
1883
1884int
1885intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
1886{
1887 int ret;
1888
1889 if (!ring->gpu_caches_dirty)
1890 return 0;
1891
1892 ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
1893 if (ret)
1894 return ret;
1895
1896 trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
1897
1898 ring->gpu_caches_dirty = false;
1899 return 0;
1900}
1901
1902int
1903intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
1904{
1905 uint32_t flush_domains;
1906 int ret;
1907
1908 flush_domains = 0;
1909 if (ring->gpu_caches_dirty)
1910 flush_domains = I915_GEM_GPU_DOMAINS;
1911
1912 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
1913 if (ret)
1914 return ret;
1915
1916 trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
1917
1918 ring->gpu_caches_dirty = false;
1919 return 0;
1920}
1921