1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#include <drm/drmP.h>
31#include "i915_drv.h"
32#include <drm/i915_drm.h>
33#include "i915_trace.h"
34#include "intel_drv.h"
35
36
37
38
39
40struct pipe_control {
41 struct drm_i915_gem_object *obj;
42 volatile u32 *cpu_page;
43 u32 gtt_offset;
44};
45
46static inline int ring_space(struct intel_ring_buffer *ring)
47{
48 int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
49 if (space < 0)
50 space += ring->size;
51 return space;
52}
53
54static int
55gen2_render_ring_flush(struct intel_ring_buffer *ring,
56 u32 invalidate_domains,
57 u32 flush_domains)
58{
59 u32 cmd;
60 int ret;
61
62 cmd = MI_FLUSH;
63 if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
64 cmd |= MI_NO_WRITE_FLUSH;
65
66 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
67 cmd |= MI_READ_FLUSH;
68
69 ret = intel_ring_begin(ring, 2);
70 if (ret)
71 return ret;
72
73 intel_ring_emit(ring, cmd);
74 intel_ring_emit(ring, MI_NOOP);
75 intel_ring_advance(ring);
76
77 return 0;
78}
79
80static int
81gen4_render_ring_flush(struct intel_ring_buffer *ring,
82 u32 invalidate_domains,
83 u32 flush_domains)
84{
85 struct drm_device *dev = ring->dev;
86 u32 cmd;
87 int ret;
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
118 if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
119 cmd &= ~MI_NO_WRITE_FLUSH;
120 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
121 cmd |= MI_EXE_FLUSH;
122
123 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
124 (IS_G4X(dev) || IS_GEN5(dev)))
125 cmd |= MI_INVALIDATE_ISP;
126
127 ret = intel_ring_begin(ring, 2);
128 if (ret)
129 return ret;
130
131 intel_ring_emit(ring, cmd);
132 intel_ring_emit(ring, MI_NOOP);
133 intel_ring_advance(ring);
134
135 return 0;
136}
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175static int
176intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
177{
178 struct pipe_control *pc = ring->private;
179 u32 scratch_addr = pc->gtt_offset + 128;
180 int ret;
181
182
183 ret = intel_ring_begin(ring, 6);
184 if (ret)
185 return ret;
186
187 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
188 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
189 PIPE_CONTROL_STALL_AT_SCOREBOARD);
190 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
191 intel_ring_emit(ring, 0);
192 intel_ring_emit(ring, 0);
193 intel_ring_emit(ring, MI_NOOP);
194 intel_ring_advance(ring);
195
196 ret = intel_ring_begin(ring, 6);
197 if (ret)
198 return ret;
199
200 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
201 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
202 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
203 intel_ring_emit(ring, 0);
204 intel_ring_emit(ring, 0);
205 intel_ring_emit(ring, MI_NOOP);
206 intel_ring_advance(ring);
207
208 return 0;
209}
210
211static int
212gen6_render_ring_flush(struct intel_ring_buffer *ring,
213 u32 invalidate_domains, u32 flush_domains)
214{
215 u32 flags = 0;
216 struct pipe_control *pc = ring->private;
217 u32 scratch_addr = pc->gtt_offset + 128;
218 int ret;
219
220
221 ret = intel_emit_post_sync_nonzero_flush(ring);
222 if (ret)
223 return ret;
224
225
226
227
228
229 if (flush_domains) {
230 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
231 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
232
233
234
235
236 flags |= PIPE_CONTROL_CS_STALL;
237 }
238 if (invalidate_domains) {
239 flags |= PIPE_CONTROL_TLB_INVALIDATE;
240 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
241 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
242 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
243 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
244 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
245
246
247
248 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
249 }
250
251 ret = intel_ring_begin(ring, 4);
252 if (ret)
253 return ret;
254
255 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
256 intel_ring_emit(ring, flags);
257 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
258 intel_ring_emit(ring, 0);
259 intel_ring_advance(ring);
260
261 return 0;
262}
263
264static int
265gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
266{
267 int ret;
268
269 ret = intel_ring_begin(ring, 4);
270 if (ret)
271 return ret;
272
273 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
274 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
275 PIPE_CONTROL_STALL_AT_SCOREBOARD);
276 intel_ring_emit(ring, 0);
277 intel_ring_emit(ring, 0);
278 intel_ring_advance(ring);
279
280 return 0;
281}
282
283static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value)
284{
285 int ret;
286
287 if (!ring->fbc_dirty)
288 return 0;
289
290 ret = intel_ring_begin(ring, 4);
291 if (ret)
292 return ret;
293 intel_ring_emit(ring, MI_NOOP);
294
295 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
296 intel_ring_emit(ring, MSG_FBC_REND_STATE);
297 intel_ring_emit(ring, value);
298 intel_ring_advance(ring);
299
300 ring->fbc_dirty = false;
301 return 0;
302}
303
304static int
305gen7_render_ring_flush(struct intel_ring_buffer *ring,
306 u32 invalidate_domains, u32 flush_domains)
307{
308 u32 flags = 0;
309 struct pipe_control *pc = ring->private;
310 u32 scratch_addr = pc->gtt_offset + 128;
311 int ret;
312
313
314
315
316
317
318
319
320
321 flags |= PIPE_CONTROL_CS_STALL;
322
323
324
325
326
327 if (flush_domains) {
328 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
329 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
330 }
331 if (invalidate_domains) {
332 flags |= PIPE_CONTROL_TLB_INVALIDATE;
333 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
334 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
335 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
336 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
337 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
338
339
340
341 flags |= PIPE_CONTROL_QW_WRITE;
342 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
343
344
345
346
347 gen7_render_ring_cs_stall_wa(ring);
348 }
349
350 ret = intel_ring_begin(ring, 4);
351 if (ret)
352 return ret;
353
354 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
355 intel_ring_emit(ring, flags);
356 intel_ring_emit(ring, scratch_addr);
357 intel_ring_emit(ring, 0);
358 intel_ring_advance(ring);
359
360 if (flush_domains)
361 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
362
363 return 0;
364}
365
366static void ring_write_tail(struct intel_ring_buffer *ring,
367 u32 value)
368{
369 drm_i915_private_t *dev_priv = ring->dev->dev_private;
370 I915_WRITE_TAIL(ring, value);
371}
372
373u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
374{
375 drm_i915_private_t *dev_priv = ring->dev->dev_private;
376 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
377 RING_ACTHD(ring->mmio_base) : ACTHD;
378
379 return I915_READ(acthd_reg);
380}
381
382static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
383{
384 struct drm_i915_private *dev_priv = ring->dev->dev_private;
385 u32 addr;
386
387 addr = dev_priv->status_page_dmah->busaddr;
388 if (INTEL_INFO(ring->dev)->gen >= 4)
389 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
390 I915_WRITE(HWS_PGA, addr);
391}
392
393static int init_ring_common(struct intel_ring_buffer *ring)
394{
395 struct drm_device *dev = ring->dev;
396 drm_i915_private_t *dev_priv = dev->dev_private;
397 struct drm_i915_gem_object *obj = ring->obj;
398 int ret = 0;
399 u32 head;
400
401 if (HAS_FORCE_WAKE(dev))
402 gen6_gt_force_wake_get(dev_priv);
403
404 if (I915_NEED_GFX_HWS(dev))
405 intel_ring_setup_status_page(ring);
406 else
407 ring_setup_phys_status_page(ring);
408
409
410 I915_WRITE_CTL(ring, 0);
411 I915_WRITE_HEAD(ring, 0);
412 ring->write_tail(ring, 0);
413
414 head = I915_READ_HEAD(ring) & HEAD_ADDR;
415
416
417 if (head != 0) {
418 DRM_DEBUG_KMS("%s head not reset to zero "
419 "ctl %08x head %08x tail %08x start %08x\n",
420 ring->name,
421 I915_READ_CTL(ring),
422 I915_READ_HEAD(ring),
423 I915_READ_TAIL(ring),
424 I915_READ_START(ring));
425
426 I915_WRITE_HEAD(ring, 0);
427
428 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
429 DRM_ERROR("failed to set %s head to zero "
430 "ctl %08x head %08x tail %08x start %08x\n",
431 ring->name,
432 I915_READ_CTL(ring),
433 I915_READ_HEAD(ring),
434 I915_READ_TAIL(ring),
435 I915_READ_START(ring));
436 }
437 }
438
439
440
441
442
443 I915_WRITE_START(ring, obj->gtt_offset);
444 I915_WRITE_CTL(ring,
445 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
446 | RING_VALID);
447
448
449 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
450 I915_READ_START(ring) == obj->gtt_offset &&
451 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
452 DRM_ERROR("%s initialization failed "
453 "ctl %08x head %08x tail %08x start %08x\n",
454 ring->name,
455 I915_READ_CTL(ring),
456 I915_READ_HEAD(ring),
457 I915_READ_TAIL(ring),
458 I915_READ_START(ring));
459 ret = -EIO;
460 goto out;
461 }
462
463 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
464 i915_kernel_lost_context(ring->dev);
465 else {
466 ring->head = I915_READ_HEAD(ring);
467 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
468 ring->space = ring_space(ring);
469 ring->last_retired_head = -1;
470 }
471
472 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
473
474out:
475 if (HAS_FORCE_WAKE(dev))
476 gen6_gt_force_wake_put(dev_priv);
477
478 return ret;
479}
480
481static int
482init_pipe_control(struct intel_ring_buffer *ring)
483{
484 struct pipe_control *pc;
485 struct drm_i915_gem_object *obj;
486 int ret;
487
488 if (ring->private)
489 return 0;
490
491 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
492 if (!pc)
493 return -ENOMEM;
494
495 obj = i915_gem_alloc_object(ring->dev, 4096);
496 if (obj == NULL) {
497 DRM_ERROR("Failed to allocate seqno page\n");
498 ret = -ENOMEM;
499 goto err;
500 }
501
502 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
503
504 ret = i915_gem_object_pin(obj, 4096, true, false);
505 if (ret)
506 goto err_unref;
507
508 pc->gtt_offset = obj->gtt_offset;
509 pc->cpu_page = kmap(sg_page(obj->pages->sgl));
510 if (pc->cpu_page == NULL) {
511 ret = -ENOMEM;
512 goto err_unpin;
513 }
514
515 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
516 ring->name, pc->gtt_offset);
517
518 pc->obj = obj;
519 ring->private = pc;
520 return 0;
521
522err_unpin:
523 i915_gem_object_unpin(obj);
524err_unref:
525 drm_gem_object_unreference(&obj->base);
526err:
527 kfree(pc);
528 return ret;
529}
530
531static void
532cleanup_pipe_control(struct intel_ring_buffer *ring)
533{
534 struct pipe_control *pc = ring->private;
535 struct drm_i915_gem_object *obj;
536
537 obj = pc->obj;
538
539 kunmap(sg_page(obj->pages->sgl));
540 i915_gem_object_unpin(obj);
541 drm_gem_object_unreference(&obj->base);
542
543 kfree(pc);
544}
545
546static int init_render_ring(struct intel_ring_buffer *ring)
547{
548 struct drm_device *dev = ring->dev;
549 struct drm_i915_private *dev_priv = dev->dev_private;
550 int ret = init_ring_common(ring);
551
552 if (INTEL_INFO(dev)->gen > 3)
553 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
554
555
556
557
558
559
560
561 if (INTEL_INFO(dev)->gen >= 6)
562 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
563
564
565 if (INTEL_INFO(dev)->gen == 6)
566 I915_WRITE(GFX_MODE,
567 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS));
568
569 if (IS_GEN7(dev))
570 I915_WRITE(GFX_MODE_GEN7,
571 _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
572 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
573
574 if (INTEL_INFO(dev)->gen >= 5) {
575 ret = init_pipe_control(ring);
576 if (ret)
577 return ret;
578 }
579
580 if (IS_GEN6(dev)) {
581
582
583
584
585
586 I915_WRITE(CACHE_MODE_0,
587 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
588
589
590
591
592
593 ring->itlb_before_ctx_switch =
594 !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
595 }
596
597 if (INTEL_INFO(dev)->gen >= 6)
598 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
599
600 if (HAS_L3_GPU_CACHE(dev))
601 I915_WRITE_IMR(ring, ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
602
603 return ret;
604}
605
606static void render_ring_cleanup(struct intel_ring_buffer *ring)
607{
608 struct drm_device *dev = ring->dev;
609
610 if (!ring->private)
611 return;
612
613 if (HAS_BROKEN_CS_TLB(dev))
614 drm_gem_object_unreference(to_gem_object(ring->private));
615
616 if (INTEL_INFO(dev)->gen >= 5)
617 cleanup_pipe_control(ring);
618
619 ring->private = NULL;
620}
621
622static void
623update_mboxes(struct intel_ring_buffer *ring,
624 u32 mmio_offset)
625{
626
627
628
629
630
631#define MBOX_UPDATE_DWORDS 4
632 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
633 intel_ring_emit(ring, mmio_offset);
634 intel_ring_emit(ring, ring->outstanding_lazy_request);
635 intel_ring_emit(ring, MI_NOOP);
636}
637
638
639
640
641
642
643
644
645
646
647static int
648gen6_add_request(struct intel_ring_buffer *ring)
649{
650 struct drm_device *dev = ring->dev;
651 struct drm_i915_private *dev_priv = dev->dev_private;
652 struct intel_ring_buffer *useless;
653 int i, ret;
654
655 ret = intel_ring_begin(ring, ((I915_NUM_RINGS-1) *
656 MBOX_UPDATE_DWORDS) +
657 4);
658 if (ret)
659 return ret;
660#undef MBOX_UPDATE_DWORDS
661
662 for_each_ring(useless, dev_priv, i) {
663 u32 mbox_reg = ring->signal_mbox[i];
664 if (mbox_reg != GEN6_NOSYNC)
665 update_mboxes(ring, mbox_reg);
666 }
667
668 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
669 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
670 intel_ring_emit(ring, ring->outstanding_lazy_request);
671 intel_ring_emit(ring, MI_USER_INTERRUPT);
672 intel_ring_advance(ring);
673
674 return 0;
675}
676
677static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
678 u32 seqno)
679{
680 struct drm_i915_private *dev_priv = dev->dev_private;
681 return dev_priv->last_seqno < seqno;
682}
683
684
685
686
687
688
689
690
691static int
692gen6_ring_sync(struct intel_ring_buffer *waiter,
693 struct intel_ring_buffer *signaller,
694 u32 seqno)
695{
696 int ret;
697 u32 dw1 = MI_SEMAPHORE_MBOX |
698 MI_SEMAPHORE_COMPARE |
699 MI_SEMAPHORE_REGISTER;
700
701
702
703
704
705 seqno -= 1;
706
707 WARN_ON(signaller->semaphore_register[waiter->id] ==
708 MI_SEMAPHORE_SYNC_INVALID);
709
710 ret = intel_ring_begin(waiter, 4);
711 if (ret)
712 return ret;
713
714
715 if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
716 intel_ring_emit(waiter,
717 dw1 |
718 signaller->semaphore_register[waiter->id]);
719 intel_ring_emit(waiter, seqno);
720 intel_ring_emit(waiter, 0);
721 intel_ring_emit(waiter, MI_NOOP);
722 } else {
723 intel_ring_emit(waiter, MI_NOOP);
724 intel_ring_emit(waiter, MI_NOOP);
725 intel_ring_emit(waiter, MI_NOOP);
726 intel_ring_emit(waiter, MI_NOOP);
727 }
728 intel_ring_advance(waiter);
729
730 return 0;
731}
732
733#define PIPE_CONTROL_FLUSH(ring__, addr__) \
734do { \
735 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
736 PIPE_CONTROL_DEPTH_STALL); \
737 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
738 intel_ring_emit(ring__, 0); \
739 intel_ring_emit(ring__, 0); \
740} while (0)
741
742static int
743pc_render_add_request(struct intel_ring_buffer *ring)
744{
745 struct pipe_control *pc = ring->private;
746 u32 scratch_addr = pc->gtt_offset + 128;
747 int ret;
748
749
750
751
752
753
754
755
756
757 ret = intel_ring_begin(ring, 32);
758 if (ret)
759 return ret;
760
761 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
762 PIPE_CONTROL_WRITE_FLUSH |
763 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
764 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
765 intel_ring_emit(ring, ring->outstanding_lazy_request);
766 intel_ring_emit(ring, 0);
767 PIPE_CONTROL_FLUSH(ring, scratch_addr);
768 scratch_addr += 128;
769 PIPE_CONTROL_FLUSH(ring, scratch_addr);
770 scratch_addr += 128;
771 PIPE_CONTROL_FLUSH(ring, scratch_addr);
772 scratch_addr += 128;
773 PIPE_CONTROL_FLUSH(ring, scratch_addr);
774 scratch_addr += 128;
775 PIPE_CONTROL_FLUSH(ring, scratch_addr);
776 scratch_addr += 128;
777 PIPE_CONTROL_FLUSH(ring, scratch_addr);
778
779 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
780 PIPE_CONTROL_WRITE_FLUSH |
781 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
782 PIPE_CONTROL_NOTIFY);
783 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
784 intel_ring_emit(ring, ring->outstanding_lazy_request);
785 intel_ring_emit(ring, 0);
786 intel_ring_advance(ring);
787
788 return 0;
789}
790
791static u32
792gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
793{
794
795
796
797 if (!lazy_coherency)
798 intel_ring_get_active_head(ring);
799 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
800}
801
802static u32
803ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
804{
805 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
806}
807
808static void
809ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
810{
811 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
812}
813
814static u32
815pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
816{
817 struct pipe_control *pc = ring->private;
818 return pc->cpu_page[0];
819}
820
821static void
822pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
823{
824 struct pipe_control *pc = ring->private;
825 pc->cpu_page[0] = seqno;
826}
827
828static bool
829gen5_ring_get_irq(struct intel_ring_buffer *ring)
830{
831 struct drm_device *dev = ring->dev;
832 drm_i915_private_t *dev_priv = dev->dev_private;
833 unsigned long flags;
834
835 if (!dev->irq_enabled)
836 return false;
837
838 spin_lock_irqsave(&dev_priv->irq_lock, flags);
839 if (ring->irq_refcount.gt++ == 0) {
840 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
841 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
842 POSTING_READ(GTIMR);
843 }
844 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
845
846 return true;
847}
848
849static void
850gen5_ring_put_irq(struct intel_ring_buffer *ring)
851{
852 struct drm_device *dev = ring->dev;
853 drm_i915_private_t *dev_priv = dev->dev_private;
854 unsigned long flags;
855
856 spin_lock_irqsave(&dev_priv->irq_lock, flags);
857 if (--ring->irq_refcount.gt == 0) {
858 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
859 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
860 POSTING_READ(GTIMR);
861 }
862 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
863}
864
865static bool
866i9xx_ring_get_irq(struct intel_ring_buffer *ring)
867{
868 struct drm_device *dev = ring->dev;
869 drm_i915_private_t *dev_priv = dev->dev_private;
870 unsigned long flags;
871
872 if (!dev->irq_enabled)
873 return false;
874
875 spin_lock_irqsave(&dev_priv->irq_lock, flags);
876 if (ring->irq_refcount.gt++ == 0) {
877 dev_priv->irq_mask &= ~ring->irq_enable_mask;
878 I915_WRITE(IMR, dev_priv->irq_mask);
879 POSTING_READ(IMR);
880 }
881 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
882
883 return true;
884}
885
886static void
887i9xx_ring_put_irq(struct intel_ring_buffer *ring)
888{
889 struct drm_device *dev = ring->dev;
890 drm_i915_private_t *dev_priv = dev->dev_private;
891 unsigned long flags;
892
893 spin_lock_irqsave(&dev_priv->irq_lock, flags);
894 if (--ring->irq_refcount.gt == 0) {
895 dev_priv->irq_mask |= ring->irq_enable_mask;
896 I915_WRITE(IMR, dev_priv->irq_mask);
897 POSTING_READ(IMR);
898 }
899 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
900}
901
902static bool
903i8xx_ring_get_irq(struct intel_ring_buffer *ring)
904{
905 struct drm_device *dev = ring->dev;
906 drm_i915_private_t *dev_priv = dev->dev_private;
907 unsigned long flags;
908
909 if (!dev->irq_enabled)
910 return false;
911
912 spin_lock_irqsave(&dev_priv->irq_lock, flags);
913 if (ring->irq_refcount.gt++ == 0) {
914 dev_priv->irq_mask &= ~ring->irq_enable_mask;
915 I915_WRITE16(IMR, dev_priv->irq_mask);
916 POSTING_READ16(IMR);
917 }
918 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
919
920 return true;
921}
922
923static void
924i8xx_ring_put_irq(struct intel_ring_buffer *ring)
925{
926 struct drm_device *dev = ring->dev;
927 drm_i915_private_t *dev_priv = dev->dev_private;
928 unsigned long flags;
929
930 spin_lock_irqsave(&dev_priv->irq_lock, flags);
931 if (--ring->irq_refcount.gt == 0) {
932 dev_priv->irq_mask |= ring->irq_enable_mask;
933 I915_WRITE16(IMR, dev_priv->irq_mask);
934 POSTING_READ16(IMR);
935 }
936 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
937}
938
939void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
940{
941 struct drm_device *dev = ring->dev;
942 drm_i915_private_t *dev_priv = ring->dev->dev_private;
943 u32 mmio = 0;
944
945
946
947
948 if (IS_GEN7(dev)) {
949 switch (ring->id) {
950 case RCS:
951 mmio = RENDER_HWS_PGA_GEN7;
952 break;
953 case BCS:
954 mmio = BLT_HWS_PGA_GEN7;
955 break;
956 case VCS:
957 mmio = BSD_HWS_PGA_GEN7;
958 break;
959 case VECS:
960 mmio = VEBOX_HWS_PGA_GEN7;
961 break;
962 }
963 } else if (IS_GEN6(ring->dev)) {
964 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
965 } else {
966 mmio = RING_HWS_PGA(ring->mmio_base);
967 }
968
969 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
970 POSTING_READ(mmio);
971
972
973 if (INTEL_INFO(dev)->gen >= 6) {
974 u32 reg = RING_INSTPM(ring->mmio_base);
975 I915_WRITE(reg,
976 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
977 INSTPM_SYNC_FLUSH));
978 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
979 1000))
980 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
981 ring->name);
982 }
983}
984
985static int
986bsd_ring_flush(struct intel_ring_buffer *ring,
987 u32 invalidate_domains,
988 u32 flush_domains)
989{
990 int ret;
991
992 ret = intel_ring_begin(ring, 2);
993 if (ret)
994 return ret;
995
996 intel_ring_emit(ring, MI_FLUSH);
997 intel_ring_emit(ring, MI_NOOP);
998 intel_ring_advance(ring);
999 return 0;
1000}
1001
1002static int
1003i9xx_add_request(struct intel_ring_buffer *ring)
1004{
1005 int ret;
1006
1007 ret = intel_ring_begin(ring, 4);
1008 if (ret)
1009 return ret;
1010
1011 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1012 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1013 intel_ring_emit(ring, ring->outstanding_lazy_request);
1014 intel_ring_emit(ring, MI_USER_INTERRUPT);
1015 intel_ring_advance(ring);
1016
1017 return 0;
1018}
1019
1020static bool
1021gen6_ring_get_irq(struct intel_ring_buffer *ring)
1022{
1023 struct drm_device *dev = ring->dev;
1024 drm_i915_private_t *dev_priv = dev->dev_private;
1025 unsigned long flags;
1026
1027 if (!dev->irq_enabled)
1028 return false;
1029
1030
1031
1032
1033 gen6_gt_force_wake_get(dev_priv);
1034
1035 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1036 if (ring->irq_refcount.gt++ == 0) {
1037 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
1038 I915_WRITE_IMR(ring,
1039 ~(ring->irq_enable_mask |
1040 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1041 else
1042 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1043 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
1044 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1045 POSTING_READ(GTIMR);
1046 }
1047 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1048
1049 return true;
1050}
1051
1052static void
1053gen6_ring_put_irq(struct intel_ring_buffer *ring)
1054{
1055 struct drm_device *dev = ring->dev;
1056 drm_i915_private_t *dev_priv = dev->dev_private;
1057 unsigned long flags;
1058
1059 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1060 if (--ring->irq_refcount.gt == 0) {
1061 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
1062 I915_WRITE_IMR(ring,
1063 ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1064 else
1065 I915_WRITE_IMR(ring, ~0);
1066 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
1067 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1068 POSTING_READ(GTIMR);
1069 }
1070 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1071
1072 gen6_gt_force_wake_put(dev_priv);
1073}
1074
1075static bool
1076hsw_vebox_get_irq(struct intel_ring_buffer *ring)
1077{
1078 struct drm_device *dev = ring->dev;
1079 struct drm_i915_private *dev_priv = dev->dev_private;
1080 unsigned long flags;
1081
1082 if (!dev->irq_enabled)
1083 return false;
1084
1085 spin_lock_irqsave(&dev_priv->rps.lock, flags);
1086 if (ring->irq_refcount.pm++ == 0) {
1087 u32 pm_imr = I915_READ(GEN6_PMIMR);
1088 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1089 I915_WRITE(GEN6_PMIMR, pm_imr & ~ring->irq_enable_mask);
1090 POSTING_READ(GEN6_PMIMR);
1091 }
1092 spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
1093
1094 return true;
1095}
1096
1097static void
1098hsw_vebox_put_irq(struct intel_ring_buffer *ring)
1099{
1100 struct drm_device *dev = ring->dev;
1101 struct drm_i915_private *dev_priv = dev->dev_private;
1102 unsigned long flags;
1103
1104 if (!dev->irq_enabled)
1105 return;
1106
1107 spin_lock_irqsave(&dev_priv->rps.lock, flags);
1108 if (--ring->irq_refcount.pm == 0) {
1109 u32 pm_imr = I915_READ(GEN6_PMIMR);
1110 I915_WRITE_IMR(ring, ~0);
1111 I915_WRITE(GEN6_PMIMR, pm_imr | ring->irq_enable_mask);
1112 POSTING_READ(GEN6_PMIMR);
1113 }
1114 spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
1115}
1116
1117static int
1118i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
1119 u32 offset, u32 length,
1120 unsigned flags)
1121{
1122 int ret;
1123
1124 ret = intel_ring_begin(ring, 2);
1125 if (ret)
1126 return ret;
1127
1128 intel_ring_emit(ring,
1129 MI_BATCH_BUFFER_START |
1130 MI_BATCH_GTT |
1131 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1132 intel_ring_emit(ring, offset);
1133 intel_ring_advance(ring);
1134
1135 return 0;
1136}
1137
1138
1139#define I830_BATCH_LIMIT (256*1024)
1140static int
1141i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
1142 u32 offset, u32 len,
1143 unsigned flags)
1144{
1145 int ret;
1146
1147 if (flags & I915_DISPATCH_PINNED) {
1148 ret = intel_ring_begin(ring, 4);
1149 if (ret)
1150 return ret;
1151
1152 intel_ring_emit(ring, MI_BATCH_BUFFER);
1153 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1154 intel_ring_emit(ring, offset + len - 8);
1155 intel_ring_emit(ring, MI_NOOP);
1156 intel_ring_advance(ring);
1157 } else {
1158 struct drm_i915_gem_object *obj = ring->private;
1159 u32 cs_offset = obj->gtt_offset;
1160
1161 if (len > I830_BATCH_LIMIT)
1162 return -ENOSPC;
1163
1164 ret = intel_ring_begin(ring, 9+3);
1165 if (ret)
1166 return ret;
1167
1168
1169
1170 intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
1171 XY_SRC_COPY_BLT_WRITE_ALPHA |
1172 XY_SRC_COPY_BLT_WRITE_RGB);
1173 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
1174 intel_ring_emit(ring, 0);
1175 intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
1176 intel_ring_emit(ring, cs_offset);
1177 intel_ring_emit(ring, 0);
1178 intel_ring_emit(ring, 4096);
1179 intel_ring_emit(ring, offset);
1180 intel_ring_emit(ring, MI_FLUSH);
1181
1182
1183 intel_ring_emit(ring, MI_BATCH_BUFFER);
1184 intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1185 intel_ring_emit(ring, cs_offset + len - 8);
1186 intel_ring_advance(ring);
1187 }
1188
1189 return 0;
1190}
1191
1192static int
1193i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
1194 u32 offset, u32 len,
1195 unsigned flags)
1196{
1197 int ret;
1198
1199 ret = intel_ring_begin(ring, 2);
1200 if (ret)
1201 return ret;
1202
1203 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1204 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1205 intel_ring_advance(ring);
1206
1207 return 0;
1208}
1209
1210static void cleanup_status_page(struct intel_ring_buffer *ring)
1211{
1212 struct drm_i915_gem_object *obj;
1213
1214 obj = ring->status_page.obj;
1215 if (obj == NULL)
1216 return;
1217
1218 kunmap(sg_page(obj->pages->sgl));
1219 i915_gem_object_unpin(obj);
1220 drm_gem_object_unreference(&obj->base);
1221 ring->status_page.obj = NULL;
1222}
1223
1224static int init_status_page(struct intel_ring_buffer *ring)
1225{
1226 struct drm_device *dev = ring->dev;
1227 struct drm_i915_gem_object *obj;
1228 int ret;
1229
1230 obj = i915_gem_alloc_object(dev, 4096);
1231 if (obj == NULL) {
1232 DRM_ERROR("Failed to allocate status page\n");
1233 ret = -ENOMEM;
1234 goto err;
1235 }
1236
1237 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1238
1239 ret = i915_gem_object_pin(obj, 4096, true, false);
1240 if (ret != 0) {
1241 goto err_unref;
1242 }
1243
1244 ring->status_page.gfx_addr = obj->gtt_offset;
1245 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
1246 if (ring->status_page.page_addr == NULL) {
1247 ret = -ENOMEM;
1248 goto err_unpin;
1249 }
1250 ring->status_page.obj = obj;
1251 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1252
1253 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1254 ring->name, ring->status_page.gfx_addr);
1255
1256 return 0;
1257
1258err_unpin:
1259 i915_gem_object_unpin(obj);
1260err_unref:
1261 drm_gem_object_unreference(&obj->base);
1262err:
1263 return ret;
1264}
1265
1266static int init_phys_status_page(struct intel_ring_buffer *ring)
1267{
1268 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1269
1270 if (!dev_priv->status_page_dmah) {
1271 dev_priv->status_page_dmah =
1272 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1273 if (!dev_priv->status_page_dmah)
1274 return -ENOMEM;
1275 }
1276
1277 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1278 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1279
1280 return 0;
1281}
1282
1283static int intel_init_ring_buffer(struct drm_device *dev,
1284 struct intel_ring_buffer *ring)
1285{
1286 struct drm_i915_gem_object *obj;
1287 struct drm_i915_private *dev_priv = dev->dev_private;
1288 int ret;
1289
1290 ring->dev = dev;
1291 INIT_LIST_HEAD(&ring->active_list);
1292 INIT_LIST_HEAD(&ring->request_list);
1293 ring->size = 32 * PAGE_SIZE;
1294 memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
1295
1296 init_waitqueue_head(&ring->irq_queue);
1297
1298 if (I915_NEED_GFX_HWS(dev)) {
1299 ret = init_status_page(ring);
1300 if (ret)
1301 return ret;
1302 } else {
1303 BUG_ON(ring->id != RCS);
1304 ret = init_phys_status_page(ring);
1305 if (ret)
1306 return ret;
1307 }
1308
1309 obj = NULL;
1310 if (!HAS_LLC(dev))
1311 obj = i915_gem_object_create_stolen(dev, ring->size);
1312 if (obj == NULL)
1313 obj = i915_gem_alloc_object(dev, ring->size);
1314 if (obj == NULL) {
1315 DRM_ERROR("Failed to allocate ringbuffer\n");
1316 ret = -ENOMEM;
1317 goto err_hws;
1318 }
1319
1320 ring->obj = obj;
1321
1322 ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
1323 if (ret)
1324 goto err_unref;
1325
1326 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1327 if (ret)
1328 goto err_unpin;
1329
1330 ring->virtual_start =
1331 ioremap_wc(dev_priv->gtt.mappable_base + obj->gtt_offset,
1332 ring->size);
1333 if (ring->virtual_start == NULL) {
1334 DRM_ERROR("Failed to map ringbuffer.\n");
1335 ret = -EINVAL;
1336 goto err_unpin;
1337 }
1338
1339 ret = ring->init(ring);
1340 if (ret)
1341 goto err_unmap;
1342
1343
1344
1345
1346
1347 ring->effective_size = ring->size;
1348 if (IS_I830(ring->dev) || IS_845G(ring->dev))
1349 ring->effective_size -= 128;
1350
1351 return 0;
1352
1353err_unmap:
1354 iounmap(ring->virtual_start);
1355err_unpin:
1356 i915_gem_object_unpin(obj);
1357err_unref:
1358 drm_gem_object_unreference(&obj->base);
1359 ring->obj = NULL;
1360err_hws:
1361 cleanup_status_page(ring);
1362 return ret;
1363}
1364
1365void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1366{
1367 struct drm_i915_private *dev_priv;
1368 int ret;
1369
1370 if (ring->obj == NULL)
1371 return;
1372
1373
1374 dev_priv = ring->dev->dev_private;
1375 ret = intel_ring_idle(ring);
1376 if (ret)
1377 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1378 ring->name, ret);
1379
1380 I915_WRITE_CTL(ring, 0);
1381
1382 iounmap(ring->virtual_start);
1383
1384 i915_gem_object_unpin(ring->obj);
1385 drm_gem_object_unreference(&ring->obj->base);
1386 ring->obj = NULL;
1387
1388 if (ring->cleanup)
1389 ring->cleanup(ring);
1390
1391 cleanup_status_page(ring);
1392}
1393
1394static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1395{
1396 int ret;
1397
1398 ret = i915_wait_seqno(ring, seqno);
1399 if (!ret)
1400 i915_gem_retire_requests_ring(ring);
1401
1402 return ret;
1403}
1404
1405static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1406{
1407 struct drm_i915_gem_request *request;
1408 u32 seqno = 0;
1409 int ret;
1410
1411 i915_gem_retire_requests_ring(ring);
1412
1413 if (ring->last_retired_head != -1) {
1414 ring->head = ring->last_retired_head;
1415 ring->last_retired_head = -1;
1416 ring->space = ring_space(ring);
1417 if (ring->space >= n)
1418 return 0;
1419 }
1420
1421 list_for_each_entry(request, &ring->request_list, list) {
1422 int space;
1423
1424 if (request->tail == -1)
1425 continue;
1426
1427 space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
1428 if (space < 0)
1429 space += ring->size;
1430 if (space >= n) {
1431 seqno = request->seqno;
1432 break;
1433 }
1434
1435
1436
1437
1438
1439
1440 request->tail = -1;
1441 }
1442
1443 if (seqno == 0)
1444 return -ENOSPC;
1445
1446 ret = intel_ring_wait_seqno(ring, seqno);
1447 if (ret)
1448 return ret;
1449
1450 if (WARN_ON(ring->last_retired_head == -1))
1451 return -ENOSPC;
1452
1453 ring->head = ring->last_retired_head;
1454 ring->last_retired_head = -1;
1455 ring->space = ring_space(ring);
1456 if (WARN_ON(ring->space < n))
1457 return -ENOSPC;
1458
1459 return 0;
1460}
1461
1462static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
1463{
1464 struct drm_device *dev = ring->dev;
1465 struct drm_i915_private *dev_priv = dev->dev_private;
1466 unsigned long end;
1467 int ret;
1468
1469 ret = intel_ring_wait_request(ring, n);
1470 if (ret != -ENOSPC)
1471 return ret;
1472
1473 trace_i915_ring_wait_begin(ring);
1474
1475
1476
1477
1478
1479 end = jiffies + 60 * HZ;
1480
1481 do {
1482 ring->head = I915_READ_HEAD(ring);
1483 ring->space = ring_space(ring);
1484 if (ring->space >= n) {
1485 trace_i915_ring_wait_end(ring);
1486 return 0;
1487 }
1488
1489 if (dev->primary->master) {
1490 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1491 if (master_priv->sarea_priv)
1492 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1493 }
1494
1495 msleep(1);
1496
1497 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1498 dev_priv->mm.interruptible);
1499 if (ret)
1500 return ret;
1501 } while (!time_after(jiffies, end));
1502 trace_i915_ring_wait_end(ring);
1503 return -EBUSY;
1504}
1505
1506static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1507{
1508 uint32_t __iomem *virt;
1509 int rem = ring->size - ring->tail;
1510
1511 if (ring->space < rem) {
1512 int ret = ring_wait_for_space(ring, rem);
1513 if (ret)
1514 return ret;
1515 }
1516
1517 virt = ring->virtual_start + ring->tail;
1518 rem /= 4;
1519 while (rem--)
1520 iowrite32(MI_NOOP, virt++);
1521
1522 ring->tail = 0;
1523 ring->space = ring_space(ring);
1524
1525 return 0;
1526}
1527
1528int intel_ring_idle(struct intel_ring_buffer *ring)
1529{
1530 u32 seqno;
1531 int ret;
1532
1533
1534 if (ring->outstanding_lazy_request) {
1535 ret = i915_add_request(ring, NULL);
1536 if (ret)
1537 return ret;
1538 }
1539
1540
1541 if (list_empty(&ring->request_list))
1542 return 0;
1543
1544 seqno = list_entry(ring->request_list.prev,
1545 struct drm_i915_gem_request,
1546 list)->seqno;
1547
1548 return i915_wait_seqno(ring, seqno);
1549}
1550
1551static int
1552intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
1553{
1554 if (ring->outstanding_lazy_request)
1555 return 0;
1556
1557 return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_request);
1558}
1559
1560static int __intel_ring_begin(struct intel_ring_buffer *ring,
1561 int bytes)
1562{
1563 int ret;
1564
1565 if (unlikely(ring->tail + bytes > ring->effective_size)) {
1566 ret = intel_wrap_ring_buffer(ring);
1567 if (unlikely(ret))
1568 return ret;
1569 }
1570
1571 if (unlikely(ring->space < bytes)) {
1572 ret = ring_wait_for_space(ring, bytes);
1573 if (unlikely(ret))
1574 return ret;
1575 }
1576
1577 ring->space -= bytes;
1578 return 0;
1579}
1580
1581int intel_ring_begin(struct intel_ring_buffer *ring,
1582 int num_dwords)
1583{
1584 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1585 int ret;
1586
1587 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1588 dev_priv->mm.interruptible);
1589 if (ret)
1590 return ret;
1591
1592
1593 ret = intel_ring_alloc_seqno(ring);
1594 if (ret)
1595 return ret;
1596
1597 return __intel_ring_begin(ring, num_dwords * sizeof(uint32_t));
1598}
1599
1600void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
1601{
1602 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1603
1604 BUG_ON(ring->outstanding_lazy_request);
1605
1606 if (INTEL_INFO(ring->dev)->gen >= 6) {
1607 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
1608 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
1609 }
1610
1611 ring->set_seqno(ring, seqno);
1612 ring->hangcheck.seqno = seqno;
1613}
1614
1615void intel_ring_advance(struct intel_ring_buffer *ring)
1616{
1617 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1618
1619 ring->tail &= ring->size - 1;
1620 if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
1621 return;
1622 ring->write_tail(ring, ring->tail);
1623}
1624
1625
1626static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1627 u32 value)
1628{
1629 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1630
1631
1632
1633
1634
1635
1636 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1637 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1638
1639
1640 I915_WRITE64(GEN6_BSD_RNCID, 0x0);
1641
1642
1643 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1644 GEN6_BSD_SLEEP_INDICATOR) == 0,
1645 50))
1646 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1647
1648
1649 I915_WRITE_TAIL(ring, value);
1650 POSTING_READ(RING_TAIL(ring->mmio_base));
1651
1652
1653
1654
1655 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1656 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1657}
1658
1659static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
1660 u32 invalidate, u32 flush)
1661{
1662 uint32_t cmd;
1663 int ret;
1664
1665 ret = intel_ring_begin(ring, 4);
1666 if (ret)
1667 return ret;
1668
1669 cmd = MI_FLUSH_DW;
1670
1671
1672
1673
1674
1675
1676 if (invalidate & I915_GEM_GPU_DOMAINS)
1677 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
1678 MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1679 intel_ring_emit(ring, cmd);
1680 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1681 intel_ring_emit(ring, 0);
1682 intel_ring_emit(ring, MI_NOOP);
1683 intel_ring_advance(ring);
1684 return 0;
1685}
1686
1687static int
1688hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1689 u32 offset, u32 len,
1690 unsigned flags)
1691{
1692 int ret;
1693
1694 ret = intel_ring_begin(ring, 2);
1695 if (ret)
1696 return ret;
1697
1698 intel_ring_emit(ring,
1699 MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
1700 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
1701
1702 intel_ring_emit(ring, offset);
1703 intel_ring_advance(ring);
1704
1705 return 0;
1706}
1707
1708static int
1709gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1710 u32 offset, u32 len,
1711 unsigned flags)
1712{
1713 int ret;
1714
1715 ret = intel_ring_begin(ring, 2);
1716 if (ret)
1717 return ret;
1718
1719 intel_ring_emit(ring,
1720 MI_BATCH_BUFFER_START |
1721 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1722
1723 intel_ring_emit(ring, offset);
1724 intel_ring_advance(ring);
1725
1726 return 0;
1727}
1728
1729
1730
1731static int gen6_ring_flush(struct intel_ring_buffer *ring,
1732 u32 invalidate, u32 flush)
1733{
1734 struct drm_device *dev = ring->dev;
1735 uint32_t cmd;
1736 int ret;
1737
1738 ret = intel_ring_begin(ring, 4);
1739 if (ret)
1740 return ret;
1741
1742 cmd = MI_FLUSH_DW;
1743
1744
1745
1746
1747
1748
1749 if (invalidate & I915_GEM_DOMAIN_RENDER)
1750 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
1751 MI_FLUSH_DW_OP_STOREDW;
1752 intel_ring_emit(ring, cmd);
1753 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1754 intel_ring_emit(ring, 0);
1755 intel_ring_emit(ring, MI_NOOP);
1756 intel_ring_advance(ring);
1757
1758 if (IS_GEN7(dev) && flush)
1759 return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
1760
1761 return 0;
1762}
1763
1764int intel_init_render_ring_buffer(struct drm_device *dev)
1765{
1766 drm_i915_private_t *dev_priv = dev->dev_private;
1767 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1768
1769 ring->name = "render ring";
1770 ring->id = RCS;
1771 ring->mmio_base = RENDER_RING_BASE;
1772
1773 if (INTEL_INFO(dev)->gen >= 6) {
1774 ring->add_request = gen6_add_request;
1775 ring->flush = gen7_render_ring_flush;
1776 if (INTEL_INFO(dev)->gen == 6)
1777 ring->flush = gen6_render_ring_flush;
1778 ring->irq_get = gen6_ring_get_irq;
1779 ring->irq_put = gen6_ring_put_irq;
1780 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
1781 ring->get_seqno = gen6_ring_get_seqno;
1782 ring->set_seqno = ring_set_seqno;
1783 ring->sync_to = gen6_ring_sync;
1784 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID;
1785 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV;
1786 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB;
1787 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE;
1788 ring->signal_mbox[RCS] = GEN6_NOSYNC;
1789 ring->signal_mbox[VCS] = GEN6_VRSYNC;
1790 ring->signal_mbox[BCS] = GEN6_BRSYNC;
1791 ring->signal_mbox[VECS] = GEN6_VERSYNC;
1792 } else if (IS_GEN5(dev)) {
1793 ring->add_request = pc_render_add_request;
1794 ring->flush = gen4_render_ring_flush;
1795 ring->get_seqno = pc_render_get_seqno;
1796 ring->set_seqno = pc_render_set_seqno;
1797 ring->irq_get = gen5_ring_get_irq;
1798 ring->irq_put = gen5_ring_put_irq;
1799 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
1800 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
1801 } else {
1802 ring->add_request = i9xx_add_request;
1803 if (INTEL_INFO(dev)->gen < 4)
1804 ring->flush = gen2_render_ring_flush;
1805 else
1806 ring->flush = gen4_render_ring_flush;
1807 ring->get_seqno = ring_get_seqno;
1808 ring->set_seqno = ring_set_seqno;
1809 if (IS_GEN2(dev)) {
1810 ring->irq_get = i8xx_ring_get_irq;
1811 ring->irq_put = i8xx_ring_put_irq;
1812 } else {
1813 ring->irq_get = i9xx_ring_get_irq;
1814 ring->irq_put = i9xx_ring_put_irq;
1815 }
1816 ring->irq_enable_mask = I915_USER_INTERRUPT;
1817 }
1818 ring->write_tail = ring_write_tail;
1819 if (IS_HASWELL(dev))
1820 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
1821 else if (INTEL_INFO(dev)->gen >= 6)
1822 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1823 else if (INTEL_INFO(dev)->gen >= 4)
1824 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1825 else if (IS_I830(dev) || IS_845G(dev))
1826 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1827 else
1828 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1829 ring->init = init_render_ring;
1830 ring->cleanup = render_ring_cleanup;
1831
1832
1833 if (HAS_BROKEN_CS_TLB(dev)) {
1834 struct drm_i915_gem_object *obj;
1835 int ret;
1836
1837 obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
1838 if (obj == NULL) {
1839 DRM_ERROR("Failed to allocate batch bo\n");
1840 return -ENOMEM;
1841 }
1842
1843 ret = i915_gem_object_pin(obj, 0, true, false);
1844 if (ret != 0) {
1845 drm_gem_object_unreference(&obj->base);
1846 DRM_ERROR("Failed to ping batch bo\n");
1847 return ret;
1848 }
1849
1850 ring->private = obj;
1851 }
1852
1853 return intel_init_ring_buffer(dev, ring);
1854}
1855
1856int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1857{
1858 drm_i915_private_t *dev_priv = dev->dev_private;
1859 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1860 int ret;
1861
1862 ring->name = "render ring";
1863 ring->id = RCS;
1864 ring->mmio_base = RENDER_RING_BASE;
1865
1866 if (INTEL_INFO(dev)->gen >= 6) {
1867
1868 return -ENODEV;
1869 }
1870
1871
1872
1873
1874 ring->add_request = i9xx_add_request;
1875 if (INTEL_INFO(dev)->gen < 4)
1876 ring->flush = gen2_render_ring_flush;
1877 else
1878 ring->flush = gen4_render_ring_flush;
1879 ring->get_seqno = ring_get_seqno;
1880 ring->set_seqno = ring_set_seqno;
1881 if (IS_GEN2(dev)) {
1882 ring->irq_get = i8xx_ring_get_irq;
1883 ring->irq_put = i8xx_ring_put_irq;
1884 } else {
1885 ring->irq_get = i9xx_ring_get_irq;
1886 ring->irq_put = i9xx_ring_put_irq;
1887 }
1888 ring->irq_enable_mask = I915_USER_INTERRUPT;
1889 ring->write_tail = ring_write_tail;
1890 if (INTEL_INFO(dev)->gen >= 4)
1891 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1892 else if (IS_I830(dev) || IS_845G(dev))
1893 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1894 else
1895 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1896 ring->init = init_render_ring;
1897 ring->cleanup = render_ring_cleanup;
1898
1899 ring->dev = dev;
1900 INIT_LIST_HEAD(&ring->active_list);
1901 INIT_LIST_HEAD(&ring->request_list);
1902
1903 ring->size = size;
1904 ring->effective_size = ring->size;
1905 if (IS_I830(ring->dev) || IS_845G(ring->dev))
1906 ring->effective_size -= 128;
1907
1908 ring->virtual_start = ioremap_wc(start, size);
1909 if (ring->virtual_start == NULL) {
1910 DRM_ERROR("can not ioremap virtual address for"
1911 " ring buffer\n");
1912 return -ENOMEM;
1913 }
1914
1915 if (!I915_NEED_GFX_HWS(dev)) {
1916 ret = init_phys_status_page(ring);
1917 if (ret)
1918 return ret;
1919 }
1920
1921 return 0;
1922}
1923
1924int intel_init_bsd_ring_buffer(struct drm_device *dev)
1925{
1926 drm_i915_private_t *dev_priv = dev->dev_private;
1927 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1928
1929 ring->name = "bsd ring";
1930 ring->id = VCS;
1931
1932 ring->write_tail = ring_write_tail;
1933 if (IS_GEN6(dev) || IS_GEN7(dev)) {
1934 ring->mmio_base = GEN6_BSD_RING_BASE;
1935
1936 if (IS_GEN6(dev))
1937 ring->write_tail = gen6_bsd_ring_write_tail;
1938 ring->flush = gen6_bsd_ring_flush;
1939 ring->add_request = gen6_add_request;
1940 ring->get_seqno = gen6_ring_get_seqno;
1941 ring->set_seqno = ring_set_seqno;
1942 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1943 ring->irq_get = gen6_ring_get_irq;
1944 ring->irq_put = gen6_ring_put_irq;
1945 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1946 ring->sync_to = gen6_ring_sync;
1947 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
1948 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
1949 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB;
1950 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE;
1951 ring->signal_mbox[RCS] = GEN6_RVSYNC;
1952 ring->signal_mbox[VCS] = GEN6_NOSYNC;
1953 ring->signal_mbox[BCS] = GEN6_BVSYNC;
1954 ring->signal_mbox[VECS] = GEN6_VEVSYNC;
1955 } else {
1956 ring->mmio_base = BSD_RING_BASE;
1957 ring->flush = bsd_ring_flush;
1958 ring->add_request = i9xx_add_request;
1959 ring->get_seqno = ring_get_seqno;
1960 ring->set_seqno = ring_set_seqno;
1961 if (IS_GEN5(dev)) {
1962 ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
1963 ring->irq_get = gen5_ring_get_irq;
1964 ring->irq_put = gen5_ring_put_irq;
1965 } else {
1966 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1967 ring->irq_get = i9xx_ring_get_irq;
1968 ring->irq_put = i9xx_ring_put_irq;
1969 }
1970 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1971 }
1972 ring->init = init_ring_common;
1973
1974 return intel_init_ring_buffer(dev, ring);
1975}
1976
1977int intel_init_blt_ring_buffer(struct drm_device *dev)
1978{
1979 drm_i915_private_t *dev_priv = dev->dev_private;
1980 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1981
1982 ring->name = "blitter ring";
1983 ring->id = BCS;
1984
1985 ring->mmio_base = BLT_RING_BASE;
1986 ring->write_tail = ring_write_tail;
1987 ring->flush = gen6_ring_flush;
1988 ring->add_request = gen6_add_request;
1989 ring->get_seqno = gen6_ring_get_seqno;
1990 ring->set_seqno = ring_set_seqno;
1991 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
1992 ring->irq_get = gen6_ring_get_irq;
1993 ring->irq_put = gen6_ring_put_irq;
1994 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1995 ring->sync_to = gen6_ring_sync;
1996 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
1997 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
1998 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID;
1999 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE;
2000 ring->signal_mbox[RCS] = GEN6_RBSYNC;
2001 ring->signal_mbox[VCS] = GEN6_VBSYNC;
2002 ring->signal_mbox[BCS] = GEN6_NOSYNC;
2003 ring->signal_mbox[VECS] = GEN6_VEBSYNC;
2004 ring->init = init_ring_common;
2005
2006 return intel_init_ring_buffer(dev, ring);
2007}
2008
2009int intel_init_vebox_ring_buffer(struct drm_device *dev)
2010{
2011 drm_i915_private_t *dev_priv = dev->dev_private;
2012 struct intel_ring_buffer *ring = &dev_priv->ring[VECS];
2013
2014 ring->name = "video enhancement ring";
2015 ring->id = VECS;
2016
2017 ring->mmio_base = VEBOX_RING_BASE;
2018 ring->write_tail = ring_write_tail;
2019 ring->flush = gen6_ring_flush;
2020 ring->add_request = gen6_add_request;
2021 ring->get_seqno = gen6_ring_get_seqno;
2022 ring->set_seqno = ring_set_seqno;
2023 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT |
2024 PM_VEBOX_CS_ERROR_INTERRUPT;
2025 ring->irq_get = hsw_vebox_get_irq;
2026 ring->irq_put = hsw_vebox_put_irq;
2027 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2028 ring->sync_to = gen6_ring_sync;
2029 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
2030 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
2031 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB;
2032 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID;
2033 ring->signal_mbox[RCS] = GEN6_RVESYNC;
2034 ring->signal_mbox[VCS] = GEN6_VVESYNC;
2035 ring->signal_mbox[BCS] = GEN6_BVESYNC;
2036 ring->signal_mbox[VECS] = GEN6_NOSYNC;
2037 ring->init = init_ring_common;
2038
2039 return intel_init_ring_buffer(dev, ring);
2040}
2041
2042int
2043intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
2044{
2045 int ret;
2046
2047 if (!ring->gpu_caches_dirty)
2048 return 0;
2049
2050 ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
2051 if (ret)
2052 return ret;
2053
2054 trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
2055
2056 ring->gpu_caches_dirty = false;
2057 return 0;
2058}
2059
2060int
2061intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
2062{
2063 uint32_t flush_domains;
2064 int ret;
2065
2066 flush_domains = 0;
2067 if (ring->gpu_caches_dirty)
2068 flush_domains = I915_GEM_GPU_DOMAINS;
2069
2070 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2071 if (ret)
2072 return ret;
2073
2074 trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2075
2076 ring->gpu_caches_dirty = false;
2077 return 0;
2078}
2079