1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#include <linux/log2.h>
31#include <drm/drmP.h>
32#include "i915_drv.h"
33#include <drm/i915_drm.h>
34#include "i915_trace.h"
35#include "intel_drv.h"
36
37int __intel_ring_space(int head, int tail, int size)
38{
39 int space = head - tail;
40 if (space <= 0)
41 space += size;
42 return space - I915_RING_FREE_SPACE;
43}
44
45void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
46{
47 if (ringbuf->last_retired_head != -1) {
48 ringbuf->head = ringbuf->last_retired_head;
49 ringbuf->last_retired_head = -1;
50 }
51
52 ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,
53 ringbuf->tail, ringbuf->size);
54}
55
56bool intel_engine_stopped(struct intel_engine_cs *engine)
57{
58 struct drm_i915_private *dev_priv = engine->dev->dev_private;
59 return dev_priv->gpu_error.stop_rings & intel_engine_flag(engine);
60}
61
62static void __intel_ring_advance(struct intel_engine_cs *engine)
63{
64 struct intel_ringbuffer *ringbuf = engine->buffer;
65 ringbuf->tail &= ringbuf->size - 1;
66 if (intel_engine_stopped(engine))
67 return;
68 engine->write_tail(engine, ringbuf->tail);
69}
70
71static int
72gen2_render_ring_flush(struct drm_i915_gem_request *req,
73 u32 invalidate_domains,
74 u32 flush_domains)
75{
76 struct intel_engine_cs *engine = req->engine;
77 u32 cmd;
78 int ret;
79
80 cmd = MI_FLUSH;
81 if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
82 cmd |= MI_NO_WRITE_FLUSH;
83
84 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
85 cmd |= MI_READ_FLUSH;
86
87 ret = intel_ring_begin(req, 2);
88 if (ret)
89 return ret;
90
91 intel_ring_emit(engine, cmd);
92 intel_ring_emit(engine, MI_NOOP);
93 intel_ring_advance(engine);
94
95 return 0;
96}
97
98static int
99gen4_render_ring_flush(struct drm_i915_gem_request *req,
100 u32 invalidate_domains,
101 u32 flush_domains)
102{
103 struct intel_engine_cs *engine = req->engine;
104 struct drm_device *dev = engine->dev;
105 u32 cmd;
106 int ret;
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
137 if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
138 cmd &= ~MI_NO_WRITE_FLUSH;
139 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
140 cmd |= MI_EXE_FLUSH;
141
142 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
143 (IS_G4X(dev) || IS_GEN5(dev)))
144 cmd |= MI_INVALIDATE_ISP;
145
146 ret = intel_ring_begin(req, 2);
147 if (ret)
148 return ret;
149
150 intel_ring_emit(engine, cmd);
151 intel_ring_emit(engine, MI_NOOP);
152 intel_ring_advance(engine);
153
154 return 0;
155}
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194static int
195intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
196{
197 struct intel_engine_cs *engine = req->engine;
198 u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
199 int ret;
200
201 ret = intel_ring_begin(req, 6);
202 if (ret)
203 return ret;
204
205 intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(5));
206 intel_ring_emit(engine, PIPE_CONTROL_CS_STALL |
207 PIPE_CONTROL_STALL_AT_SCOREBOARD);
208 intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
209 intel_ring_emit(engine, 0);
210 intel_ring_emit(engine, 0);
211 intel_ring_emit(engine, MI_NOOP);
212 intel_ring_advance(engine);
213
214 ret = intel_ring_begin(req, 6);
215 if (ret)
216 return ret;
217
218 intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(5));
219 intel_ring_emit(engine, PIPE_CONTROL_QW_WRITE);
220 intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
221 intel_ring_emit(engine, 0);
222 intel_ring_emit(engine, 0);
223 intel_ring_emit(engine, MI_NOOP);
224 intel_ring_advance(engine);
225
226 return 0;
227}
228
229static int
230gen6_render_ring_flush(struct drm_i915_gem_request *req,
231 u32 invalidate_domains, u32 flush_domains)
232{
233 struct intel_engine_cs *engine = req->engine;
234 u32 flags = 0;
235 u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
236 int ret;
237
238
239 ret = intel_emit_post_sync_nonzero_flush(req);
240 if (ret)
241 return ret;
242
243
244
245
246
247 if (flush_domains) {
248 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
249 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
250
251
252
253
254 flags |= PIPE_CONTROL_CS_STALL;
255 }
256 if (invalidate_domains) {
257 flags |= PIPE_CONTROL_TLB_INVALIDATE;
258 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
259 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
260 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
261 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
262 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
263
264
265
266 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
267 }
268
269 ret = intel_ring_begin(req, 4);
270 if (ret)
271 return ret;
272
273 intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
274 intel_ring_emit(engine, flags);
275 intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
276 intel_ring_emit(engine, 0);
277 intel_ring_advance(engine);
278
279 return 0;
280}
281
282static int
283gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
284{
285 struct intel_engine_cs *engine = req->engine;
286 int ret;
287
288 ret = intel_ring_begin(req, 4);
289 if (ret)
290 return ret;
291
292 intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
293 intel_ring_emit(engine, PIPE_CONTROL_CS_STALL |
294 PIPE_CONTROL_STALL_AT_SCOREBOARD);
295 intel_ring_emit(engine, 0);
296 intel_ring_emit(engine, 0);
297 intel_ring_advance(engine);
298
299 return 0;
300}
301
302static int
303gen7_render_ring_flush(struct drm_i915_gem_request *req,
304 u32 invalidate_domains, u32 flush_domains)
305{
306 struct intel_engine_cs *engine = req->engine;
307 u32 flags = 0;
308 u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
309 int ret;
310
311
312
313
314
315
316
317
318
319 flags |= PIPE_CONTROL_CS_STALL;
320
321
322
323
324
325 if (flush_domains) {
326 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
327 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
328 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
329 flags |= PIPE_CONTROL_FLUSH_ENABLE;
330 }
331 if (invalidate_domains) {
332 flags |= PIPE_CONTROL_TLB_INVALIDATE;
333 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
334 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
335 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
336 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
337 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
338 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
339
340
341
342 flags |= PIPE_CONTROL_QW_WRITE;
343 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
344
345 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
346
347
348
349
350 gen7_render_ring_cs_stall_wa(req);
351 }
352
353 ret = intel_ring_begin(req, 4);
354 if (ret)
355 return ret;
356
357 intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
358 intel_ring_emit(engine, flags);
359 intel_ring_emit(engine, scratch_addr);
360 intel_ring_emit(engine, 0);
361 intel_ring_advance(engine);
362
363 return 0;
364}
365
366static int
367gen8_emit_pipe_control(struct drm_i915_gem_request *req,
368 u32 flags, u32 scratch_addr)
369{
370 struct intel_engine_cs *engine = req->engine;
371 int ret;
372
373 ret = intel_ring_begin(req, 6);
374 if (ret)
375 return ret;
376
377 intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6));
378 intel_ring_emit(engine, flags);
379 intel_ring_emit(engine, scratch_addr);
380 intel_ring_emit(engine, 0);
381 intel_ring_emit(engine, 0);
382 intel_ring_emit(engine, 0);
383 intel_ring_advance(engine);
384
385 return 0;
386}
387
388static int
389gen8_render_ring_flush(struct drm_i915_gem_request *req,
390 u32 invalidate_domains, u32 flush_domains)
391{
392 u32 flags = 0;
393 u32 scratch_addr = req->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
394 int ret;
395
396 flags |= PIPE_CONTROL_CS_STALL;
397
398 if (flush_domains) {
399 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
400 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
401 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
402 flags |= PIPE_CONTROL_FLUSH_ENABLE;
403 }
404 if (invalidate_domains) {
405 flags |= PIPE_CONTROL_TLB_INVALIDATE;
406 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
407 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
408 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
409 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
410 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
411 flags |= PIPE_CONTROL_QW_WRITE;
412 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
413
414
415 ret = gen8_emit_pipe_control(req,
416 PIPE_CONTROL_CS_STALL |
417 PIPE_CONTROL_STALL_AT_SCOREBOARD,
418 0);
419 if (ret)
420 return ret;
421 }
422
423 return gen8_emit_pipe_control(req, flags, scratch_addr);
424}
425
426static void ring_write_tail(struct intel_engine_cs *engine,
427 u32 value)
428{
429 struct drm_i915_private *dev_priv = engine->dev->dev_private;
430 I915_WRITE_TAIL(engine, value);
431}
432
433u64 intel_ring_get_active_head(struct intel_engine_cs *engine)
434{
435 struct drm_i915_private *dev_priv = engine->dev->dev_private;
436 u64 acthd;
437
438 if (INTEL_INFO(engine->dev)->gen >= 8)
439 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
440 RING_ACTHD_UDW(engine->mmio_base));
441 else if (INTEL_INFO(engine->dev)->gen >= 4)
442 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
443 else
444 acthd = I915_READ(ACTHD);
445
446 return acthd;
447}
448
449static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
450{
451 struct drm_i915_private *dev_priv = engine->dev->dev_private;
452 u32 addr;
453
454 addr = dev_priv->status_page_dmah->busaddr;
455 if (INTEL_INFO(engine->dev)->gen >= 4)
456 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
457 I915_WRITE(HWS_PGA, addr);
458}
459
460static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
461{
462 struct drm_device *dev = engine->dev;
463 struct drm_i915_private *dev_priv = engine->dev->dev_private;
464 i915_reg_t mmio;
465
466
467
468
469 if (IS_GEN7(dev)) {
470 switch (engine->id) {
471 case RCS:
472 mmio = RENDER_HWS_PGA_GEN7;
473 break;
474 case BCS:
475 mmio = BLT_HWS_PGA_GEN7;
476 break;
477
478
479
480
481 case VCS2:
482 case VCS:
483 mmio = BSD_HWS_PGA_GEN7;
484 break;
485 case VECS:
486 mmio = VEBOX_HWS_PGA_GEN7;
487 break;
488 }
489 } else if (IS_GEN6(engine->dev)) {
490 mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
491 } else {
492
493 mmio = RING_HWS_PGA(engine->mmio_base);
494 }
495
496 I915_WRITE(mmio, (u32)engine->status_page.gfx_addr);
497 POSTING_READ(mmio);
498
499
500
501
502
503
504
505
506 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
507 i915_reg_t reg = RING_INSTPM(engine->mmio_base);
508
509
510 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
511
512 I915_WRITE(reg,
513 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
514 INSTPM_SYNC_FLUSH));
515 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
516 1000))
517 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
518 engine->name);
519 }
520}
521
522static bool stop_ring(struct intel_engine_cs *engine)
523{
524 struct drm_i915_private *dev_priv = to_i915(engine->dev);
525
526 if (!IS_GEN2(engine->dev)) {
527 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
528 if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) {
529 DRM_ERROR("%s : timed out trying to stop ring\n",
530 engine->name);
531
532
533
534
535 if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
536 return false;
537 }
538 }
539
540 I915_WRITE_CTL(engine, 0);
541 I915_WRITE_HEAD(engine, 0);
542 engine->write_tail(engine, 0);
543
544 if (!IS_GEN2(engine->dev)) {
545 (void)I915_READ_CTL(engine);
546 I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
547 }
548
549 return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
550}
551
552void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
553{
554 memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
555}
556
557static int init_ring_common(struct intel_engine_cs *engine)
558{
559 struct drm_device *dev = engine->dev;
560 struct drm_i915_private *dev_priv = dev->dev_private;
561 struct intel_ringbuffer *ringbuf = engine->buffer;
562 struct drm_i915_gem_object *obj = ringbuf->obj;
563 int ret = 0;
564
565 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
566
567 if (!stop_ring(engine)) {
568
569 DRM_DEBUG_KMS("%s head not reset to zero "
570 "ctl %08x head %08x tail %08x start %08x\n",
571 engine->name,
572 I915_READ_CTL(engine),
573 I915_READ_HEAD(engine),
574 I915_READ_TAIL(engine),
575 I915_READ_START(engine));
576
577 if (!stop_ring(engine)) {
578 DRM_ERROR("failed to set %s head to zero "
579 "ctl %08x head %08x tail %08x start %08x\n",
580 engine->name,
581 I915_READ_CTL(engine),
582 I915_READ_HEAD(engine),
583 I915_READ_TAIL(engine),
584 I915_READ_START(engine));
585 ret = -EIO;
586 goto out;
587 }
588 }
589
590 if (I915_NEED_GFX_HWS(dev))
591 intel_ring_setup_status_page(engine);
592 else
593 ring_setup_phys_status_page(engine);
594
595
596 I915_READ_HEAD(engine);
597
598
599
600
601
602 I915_WRITE_START(engine, i915_gem_obj_ggtt_offset(obj));
603
604
605 if (I915_READ_HEAD(engine))
606 DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
607 engine->name, I915_READ_HEAD(engine));
608 I915_WRITE_HEAD(engine, 0);
609 (void)I915_READ_HEAD(engine);
610
611 I915_WRITE_CTL(engine,
612 ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
613 | RING_VALID);
614
615
616 if (wait_for((I915_READ_CTL(engine) & RING_VALID) != 0 &&
617 I915_READ_START(engine) == i915_gem_obj_ggtt_offset(obj) &&
618 (I915_READ_HEAD(engine) & HEAD_ADDR) == 0, 50)) {
619 DRM_ERROR("%s initialization failed "
620 "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
621 engine->name,
622 I915_READ_CTL(engine),
623 I915_READ_CTL(engine) & RING_VALID,
624 I915_READ_HEAD(engine), I915_READ_TAIL(engine),
625 I915_READ_START(engine),
626 (unsigned long)i915_gem_obj_ggtt_offset(obj));
627 ret = -EIO;
628 goto out;
629 }
630
631 ringbuf->last_retired_head = -1;
632 ringbuf->head = I915_READ_HEAD(engine);
633 ringbuf->tail = I915_READ_TAIL(engine) & TAIL_ADDR;
634 intel_ring_update_space(ringbuf);
635
636 intel_engine_init_hangcheck(engine);
637
638out:
639 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
640
641 return ret;
642}
643
644void
645intel_fini_pipe_control(struct intel_engine_cs *engine)
646{
647 struct drm_device *dev = engine->dev;
648
649 if (engine->scratch.obj == NULL)
650 return;
651
652 if (INTEL_INFO(dev)->gen >= 5) {
653 kunmap(sg_page(engine->scratch.obj->pages->sgl));
654 i915_gem_object_ggtt_unpin(engine->scratch.obj);
655 }
656
657 drm_gem_object_unreference(&engine->scratch.obj->base);
658 engine->scratch.obj = NULL;
659}
660
661int
662intel_init_pipe_control(struct intel_engine_cs *engine)
663{
664 int ret;
665
666 WARN_ON(engine->scratch.obj);
667
668 engine->scratch.obj = i915_gem_alloc_object(engine->dev, 4096);
669 if (engine->scratch.obj == NULL) {
670 DRM_ERROR("Failed to allocate seqno page\n");
671 ret = -ENOMEM;
672 goto err;
673 }
674
675 ret = i915_gem_object_set_cache_level(engine->scratch.obj,
676 I915_CACHE_LLC);
677 if (ret)
678 goto err_unref;
679
680 ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0);
681 if (ret)
682 goto err_unref;
683
684 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj);
685 engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl));
686 if (engine->scratch.cpu_page == NULL) {
687 ret = -ENOMEM;
688 goto err_unpin;
689 }
690
691 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
692 engine->name, engine->scratch.gtt_offset);
693 return 0;
694
695err_unpin:
696 i915_gem_object_ggtt_unpin(engine->scratch.obj);
697err_unref:
698 drm_gem_object_unreference(&engine->scratch.obj->base);
699err:
700 return ret;
701}
702
703static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
704{
705 int ret, i;
706 struct intel_engine_cs *engine = req->engine;
707 struct drm_device *dev = engine->dev;
708 struct drm_i915_private *dev_priv = dev->dev_private;
709 struct i915_workarounds *w = &dev_priv->workarounds;
710
711 if (w->count == 0)
712 return 0;
713
714 engine->gpu_caches_dirty = true;
715 ret = intel_ring_flush_all_caches(req);
716 if (ret)
717 return ret;
718
719 ret = intel_ring_begin(req, (w->count * 2 + 2));
720 if (ret)
721 return ret;
722
723 intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(w->count));
724 for (i = 0; i < w->count; i++) {
725 intel_ring_emit_reg(engine, w->reg[i].addr);
726 intel_ring_emit(engine, w->reg[i].value);
727 }
728 intel_ring_emit(engine, MI_NOOP);
729
730 intel_ring_advance(engine);
731
732 engine->gpu_caches_dirty = true;
733 ret = intel_ring_flush_all_caches(req);
734 if (ret)
735 return ret;
736
737 DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
738
739 return 0;
740}
741
742static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
743{
744 int ret;
745
746 ret = intel_ring_workarounds_emit(req);
747 if (ret != 0)
748 return ret;
749
750 ret = i915_gem_render_state_init(req);
751 if (ret)
752 return ret;
753
754 return 0;
755}
756
757static int wa_add(struct drm_i915_private *dev_priv,
758 i915_reg_t addr,
759 const u32 mask, const u32 val)
760{
761 const u32 idx = dev_priv->workarounds.count;
762
763 if (WARN_ON(idx >= I915_MAX_WA_REGS))
764 return -ENOSPC;
765
766 dev_priv->workarounds.reg[idx].addr = addr;
767 dev_priv->workarounds.reg[idx].value = val;
768 dev_priv->workarounds.reg[idx].mask = mask;
769
770 dev_priv->workarounds.count++;
771
772 return 0;
773}
774
775#define WA_REG(addr, mask, val) do { \
776 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
777 if (r) \
778 return r; \
779 } while (0)
780
781#define WA_SET_BIT_MASKED(addr, mask) \
782 WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
783
784#define WA_CLR_BIT_MASKED(addr, mask) \
785 WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
786
787#define WA_SET_FIELD_MASKED(addr, mask, value) \
788 WA_REG(addr, mask, _MASKED_FIELD(mask, value))
789
790#define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
791#define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
792
793#define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
794
795static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
796 i915_reg_t reg)
797{
798 struct drm_i915_private *dev_priv = engine->dev->dev_private;
799 struct i915_workarounds *wa = &dev_priv->workarounds;
800 const uint32_t index = wa->hw_whitelist_count[engine->id];
801
802 if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
803 return -EINVAL;
804
805 WA_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
806 i915_mmio_reg_offset(reg));
807 wa->hw_whitelist_count[engine->id]++;
808
809 return 0;
810}
811
812static int gen8_init_workarounds(struct intel_engine_cs *engine)
813{
814 struct drm_device *dev = engine->dev;
815 struct drm_i915_private *dev_priv = dev->dev_private;
816
817 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
818
819
820 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
821
822
823 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
824 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
825
826
827
828
829
830
831
832 WA_SET_BIT_MASKED(HDC_CHICKEN0,
833 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
834 HDC_FORCE_NON_COHERENT);
835
836
837
838
839
840
841
842
843
844 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
845
846
847 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
848
849
850
851
852
853
854
855
856
857 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
858 GEN6_WIZ_HASHING_MASK,
859 GEN6_WIZ_HASHING_16x4);
860
861 return 0;
862}
863
864static int bdw_init_workarounds(struct intel_engine_cs *engine)
865{
866 int ret;
867 struct drm_device *dev = engine->dev;
868 struct drm_i915_private *dev_priv = dev->dev_private;
869
870 ret = gen8_init_workarounds(engine);
871 if (ret)
872 return ret;
873
874
875 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
876
877
878 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
879 DOP_CLOCK_GATING_DISABLE);
880
881 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
882 GEN8_SAMPLER_POWER_BYPASS_DIS);
883
884 WA_SET_BIT_MASKED(HDC_CHICKEN0,
885
886 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
887
888 (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
889
890 return 0;
891}
892
893static int chv_init_workarounds(struct intel_engine_cs *engine)
894{
895 int ret;
896 struct drm_device *dev = engine->dev;
897 struct drm_i915_private *dev_priv = dev->dev_private;
898
899 ret = gen8_init_workarounds(engine);
900 if (ret)
901 return ret;
902
903
904 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
905
906
907 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
908
909 return 0;
910}
911
912static int gen9_init_workarounds(struct intel_engine_cs *engine)
913{
914 struct drm_device *dev = engine->dev;
915 struct drm_i915_private *dev_priv = dev->dev_private;
916 int ret;
917
918
919 I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
920
921
922 I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
923 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
924
925
926 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
927 ECOCHK_DIS_TLB);
928
929
930
931 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
932 FLOW_CONTROL_ENABLE |
933 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
934
935
936 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
937 GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
938
939
940 if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
941 IS_BXT_REVID(dev, 0, BXT_REVID_A1))
942 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
943 GEN9_DG_MIRROR_FIX_ENABLE);
944
945
946 if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
947 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
948 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
949 GEN9_RHWO_OPTIMIZATION_DISABLE);
950
951
952
953
954
955 }
956
957
958
959 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
960 GEN9_ENABLE_YV12_BUGFIX |
961 GEN9_ENABLE_GPGPU_PREEMPTION);
962
963
964
965 WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
966 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
967
968
969 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
970 GEN9_CCS_TLB_PREFETCH_ENABLE);
971
972
973 if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) ||
974 IS_BXT_REVID(dev, 0, BXT_REVID_A1))
975 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
976 PIXEL_MASK_CAMMING_DISABLE);
977
978
979 WA_SET_BIT_MASKED(HDC_CHICKEN0,
980 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
981 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997 WA_SET_BIT_MASKED(HDC_CHICKEN0,
998 HDC_FORCE_NON_COHERENT);
999
1000
1001 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
1002 BDW_DISABLE_HDC_INVALIDATION);
1003
1004
1005 if (IS_SKYLAKE(dev_priv) ||
1006 IS_KABYLAKE(dev_priv) ||
1007 IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1008 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1009 GEN8_SAMPLER_POWER_BYPASS_DIS);
1010
1011
1012 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
1013
1014
1015 I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
1016 GEN8_LQSC_FLUSH_COHERENT_LINES));
1017
1018
1019 ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
1020 if (ret)
1021 return ret;
1022
1023
1024 ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1025 if (ret)
1026 return ret;
1027
1028
1029 ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
1030 if (ret)
1031 return ret;
1032
1033 return 0;
1034}
1035
1036static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1037{
1038 struct drm_device *dev = engine->dev;
1039 struct drm_i915_private *dev_priv = dev->dev_private;
1040 u8 vals[3] = { 0, 0, 0 };
1041 unsigned int i;
1042
1043 for (i = 0; i < 3; i++) {
1044 u8 ss;
1045
1046
1047
1048
1049
1050 if (!is_power_of_2(dev_priv->info.subslice_7eu[i]))
1051 continue;
1052
1053
1054
1055
1056
1057
1058
1059 ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;
1060 vals[i] = 3 - ss;
1061 }
1062
1063 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
1064 return 0;
1065
1066
1067 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1068 GEN9_IZ_HASHING_MASK(2) |
1069 GEN9_IZ_HASHING_MASK(1) |
1070 GEN9_IZ_HASHING_MASK(0),
1071 GEN9_IZ_HASHING(2, vals[2]) |
1072 GEN9_IZ_HASHING(1, vals[1]) |
1073 GEN9_IZ_HASHING(0, vals[0]));
1074
1075 return 0;
1076}
1077
1078static int skl_init_workarounds(struct intel_engine_cs *engine)
1079{
1080 int ret;
1081 struct drm_device *dev = engine->dev;
1082 struct drm_i915_private *dev_priv = dev->dev_private;
1083
1084 ret = gen9_init_workarounds(engine);
1085 if (ret)
1086 return ret;
1087
1088
1089
1090
1091
1092
1093 if (IS_SKL_REVID(dev, SKL_REVID_E0, REVID_FOREVER)) {
1094 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1095 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1096 }
1097
1098 if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) {
1099
1100 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1101 _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
1102 }
1103
1104
1105
1106
1107 if (IS_SKL_REVID(dev, 0, SKL_REVID_E0))
1108
1109 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1110 GEN8_LQSC_RO_PERF_DIS);
1111
1112
1113 if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) {
1114 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1115 GEN9_GAPS_TSV_CREDIT_DISABLE));
1116 }
1117
1118
1119 if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0))
1120 WA_SET_BIT_MASKED(HIZ_CHICKEN,
1121 BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
1122
1123
1124 if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0))
1125 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1126 HDC_FENCE_DEST_SLM_DISABLE |
1127 HDC_BARRIER_PERFORMANCE_DISABLE);
1128
1129
1130 if (IS_SKL_REVID(dev, 0, SKL_REVID_F0))
1131 WA_SET_BIT_MASKED(
1132 GEN7_HALF_SLICE_CHICKEN1,
1133 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1134
1135
1136 WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1137
1138
1139 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1140 if (ret)
1141 return ret;
1142
1143 return skl_tune_iz_hashing(engine);
1144}
1145
1146static int bxt_init_workarounds(struct intel_engine_cs *engine)
1147{
1148 int ret;
1149 struct drm_device *dev = engine->dev;
1150 struct drm_i915_private *dev_priv = dev->dev_private;
1151
1152 ret = gen9_init_workarounds(engine);
1153 if (ret)
1154 return ret;
1155
1156
1157
1158 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
1159 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1160
1161
1162 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
1163 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1164 ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1165 }
1166
1167
1168 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1169 STALL_DOP_GATING_DISABLE);
1170
1171
1172 if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) {
1173 WA_SET_BIT_MASKED(
1174 GEN7_HALF_SLICE_CHICKEN1,
1175 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1176 }
1177
1178
1179
1180
1181
1182 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
1183 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1184 if (ret)
1185 return ret;
1186
1187 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1188 if (ret)
1189 return ret;
1190 }
1191
1192
1193 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1194 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1195 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1196
1197 return 0;
1198}
1199
1200static int kbl_init_workarounds(struct intel_engine_cs *engine)
1201{
1202 struct drm_i915_private *dev_priv = engine->dev->dev_private;
1203 int ret;
1204
1205 ret = gen9_init_workarounds(engine);
1206 if (ret)
1207 return ret;
1208
1209
1210 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1211 GEN9_GAPS_TSV_CREDIT_DISABLE));
1212
1213
1214 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1215 WA_SET_BIT(GAMT_CHKN_BIT_REG,
1216 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1217
1218
1219 if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
1220 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1221 HDC_FENCE_DEST_SLM_DISABLE);
1222
1223
1224
1225
1226 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
1227
1228 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1229 GEN8_LQSC_RO_PERF_DIS);
1230
1231
1232 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1233 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1234 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1235
1236
1237 WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1238
1239
1240 WA_SET_BIT_MASKED(
1241 GEN7_HALF_SLICE_CHICKEN1,
1242 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1243
1244
1245 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1246 if (ret)
1247 return ret;
1248
1249 return 0;
1250}
1251
1252int init_workarounds_ring(struct intel_engine_cs *engine)
1253{
1254 struct drm_device *dev = engine->dev;
1255 struct drm_i915_private *dev_priv = dev->dev_private;
1256
1257 WARN_ON(engine->id != RCS);
1258
1259 dev_priv->workarounds.count = 0;
1260 dev_priv->workarounds.hw_whitelist_count[RCS] = 0;
1261
1262 if (IS_BROADWELL(dev))
1263 return bdw_init_workarounds(engine);
1264
1265 if (IS_CHERRYVIEW(dev))
1266 return chv_init_workarounds(engine);
1267
1268 if (IS_SKYLAKE(dev))
1269 return skl_init_workarounds(engine);
1270
1271 if (IS_BROXTON(dev))
1272 return bxt_init_workarounds(engine);
1273
1274 if (IS_KABYLAKE(dev_priv))
1275 return kbl_init_workarounds(engine);
1276
1277 return 0;
1278}
1279
1280static int init_render_ring(struct intel_engine_cs *engine)
1281{
1282 struct drm_device *dev = engine->dev;
1283 struct drm_i915_private *dev_priv = dev->dev_private;
1284 int ret = init_ring_common(engine);
1285 if (ret)
1286 return ret;
1287
1288
1289 if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
1290 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
1291
1292
1293
1294
1295
1296
1297
1298 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
1299 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1300
1301
1302
1303 if (INTEL_INFO(dev)->gen == 6)
1304 I915_WRITE(GFX_MODE,
1305 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
1306
1307
1308 if (IS_GEN7(dev))
1309 I915_WRITE(GFX_MODE_GEN7,
1310 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
1311 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
1312
1313 if (IS_GEN6(dev)) {
1314
1315
1316
1317
1318
1319 I915_WRITE(CACHE_MODE_0,
1320 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
1321 }
1322
1323 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
1324 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1325
1326 if (HAS_L3_DPF(dev))
1327 I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev));
1328
1329 return init_workarounds_ring(engine);
1330}
1331
1332static void render_ring_cleanup(struct intel_engine_cs *engine)
1333{
1334 struct drm_device *dev = engine->dev;
1335 struct drm_i915_private *dev_priv = dev->dev_private;
1336
1337 if (dev_priv->semaphore_obj) {
1338 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
1339 drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
1340 dev_priv->semaphore_obj = NULL;
1341 }
1342
1343 intel_fini_pipe_control(engine);
1344}
1345
1346static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
1347 unsigned int num_dwords)
1348{
1349#define MBOX_UPDATE_DWORDS 8
1350 struct intel_engine_cs *signaller = signaller_req->engine;
1351 struct drm_device *dev = signaller->dev;
1352 struct drm_i915_private *dev_priv = dev->dev_private;
1353 struct intel_engine_cs *waiter;
1354 enum intel_engine_id id;
1355 int ret, num_rings;
1356
1357 num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1358 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1359#undef MBOX_UPDATE_DWORDS
1360
1361 ret = intel_ring_begin(signaller_req, num_dwords);
1362 if (ret)
1363 return ret;
1364
1365 for_each_engine_id(waiter, dev_priv, id) {
1366 u32 seqno;
1367 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1368 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1369 continue;
1370
1371 seqno = i915_gem_request_get_seqno(signaller_req);
1372 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
1373 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
1374 PIPE_CONTROL_QW_WRITE |
1375 PIPE_CONTROL_FLUSH_ENABLE);
1376 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
1377 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1378 intel_ring_emit(signaller, seqno);
1379 intel_ring_emit(signaller, 0);
1380 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1381 MI_SEMAPHORE_TARGET(waiter->hw_id));
1382 intel_ring_emit(signaller, 0);
1383 }
1384
1385 return 0;
1386}
1387
1388static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
1389 unsigned int num_dwords)
1390{
1391#define MBOX_UPDATE_DWORDS 6
1392 struct intel_engine_cs *signaller = signaller_req->engine;
1393 struct drm_device *dev = signaller->dev;
1394 struct drm_i915_private *dev_priv = dev->dev_private;
1395 struct intel_engine_cs *waiter;
1396 enum intel_engine_id id;
1397 int ret, num_rings;
1398
1399 num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1400 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1401#undef MBOX_UPDATE_DWORDS
1402
1403 ret = intel_ring_begin(signaller_req, num_dwords);
1404 if (ret)
1405 return ret;
1406
1407 for_each_engine_id(waiter, dev_priv, id) {
1408 u32 seqno;
1409 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1410 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1411 continue;
1412
1413 seqno = i915_gem_request_get_seqno(signaller_req);
1414 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
1415 MI_FLUSH_DW_OP_STOREDW);
1416 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
1417 MI_FLUSH_DW_USE_GTT);
1418 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1419 intel_ring_emit(signaller, seqno);
1420 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1421 MI_SEMAPHORE_TARGET(waiter->hw_id));
1422 intel_ring_emit(signaller, 0);
1423 }
1424
1425 return 0;
1426}
1427
1428static int gen6_signal(struct drm_i915_gem_request *signaller_req,
1429 unsigned int num_dwords)
1430{
1431 struct intel_engine_cs *signaller = signaller_req->engine;
1432 struct drm_device *dev = signaller->dev;
1433 struct drm_i915_private *dev_priv = dev->dev_private;
1434 struct intel_engine_cs *useless;
1435 enum intel_engine_id id;
1436 int ret, num_rings;
1437
1438#define MBOX_UPDATE_DWORDS 3
1439 num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1440 num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
1441#undef MBOX_UPDATE_DWORDS
1442
1443 ret = intel_ring_begin(signaller_req, num_dwords);
1444 if (ret)
1445 return ret;
1446
1447 for_each_engine_id(useless, dev_priv, id) {
1448 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id];
1449
1450 if (i915_mmio_reg_valid(mbox_reg)) {
1451 u32 seqno = i915_gem_request_get_seqno(signaller_req);
1452
1453 intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
1454 intel_ring_emit_reg(signaller, mbox_reg);
1455 intel_ring_emit(signaller, seqno);
1456 }
1457 }
1458
1459
1460 if (num_rings % 2 == 0)
1461 intel_ring_emit(signaller, MI_NOOP);
1462
1463 return 0;
1464}
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474static int
1475gen6_add_request(struct drm_i915_gem_request *req)
1476{
1477 struct intel_engine_cs *engine = req->engine;
1478 int ret;
1479
1480 if (engine->semaphore.signal)
1481 ret = engine->semaphore.signal(req, 4);
1482 else
1483 ret = intel_ring_begin(req, 4);
1484
1485 if (ret)
1486 return ret;
1487
1488 intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1489 intel_ring_emit(engine,
1490 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1491 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1492 intel_ring_emit(engine, MI_USER_INTERRUPT);
1493 __intel_ring_advance(engine);
1494
1495 return 0;
1496}
1497
1498static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
1499 u32 seqno)
1500{
1501 struct drm_i915_private *dev_priv = dev->dev_private;
1502 return dev_priv->last_seqno < seqno;
1503}
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513static int
1514gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
1515 struct intel_engine_cs *signaller,
1516 u32 seqno)
1517{
1518 struct intel_engine_cs *waiter = waiter_req->engine;
1519 struct drm_i915_private *dev_priv = waiter->dev->dev_private;
1520 int ret;
1521
1522 ret = intel_ring_begin(waiter_req, 4);
1523 if (ret)
1524 return ret;
1525
1526 intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
1527 MI_SEMAPHORE_GLOBAL_GTT |
1528 MI_SEMAPHORE_POLL |
1529 MI_SEMAPHORE_SAD_GTE_SDD);
1530 intel_ring_emit(waiter, seqno);
1531 intel_ring_emit(waiter,
1532 lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1533 intel_ring_emit(waiter,
1534 upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1535 intel_ring_advance(waiter);
1536 return 0;
1537}
1538
1539static int
1540gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
1541 struct intel_engine_cs *signaller,
1542 u32 seqno)
1543{
1544 struct intel_engine_cs *waiter = waiter_req->engine;
1545 u32 dw1 = MI_SEMAPHORE_MBOX |
1546 MI_SEMAPHORE_COMPARE |
1547 MI_SEMAPHORE_REGISTER;
1548 u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
1549 int ret;
1550
1551
1552
1553
1554
1555 seqno -= 1;
1556
1557 WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
1558
1559 ret = intel_ring_begin(waiter_req, 4);
1560 if (ret)
1561 return ret;
1562
1563
1564 if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
1565 intel_ring_emit(waiter, dw1 | wait_mbox);
1566 intel_ring_emit(waiter, seqno);
1567 intel_ring_emit(waiter, 0);
1568 intel_ring_emit(waiter, MI_NOOP);
1569 } else {
1570 intel_ring_emit(waiter, MI_NOOP);
1571 intel_ring_emit(waiter, MI_NOOP);
1572 intel_ring_emit(waiter, MI_NOOP);
1573 intel_ring_emit(waiter, MI_NOOP);
1574 }
1575 intel_ring_advance(waiter);
1576
1577 return 0;
1578}
1579
1580#define PIPE_CONTROL_FLUSH(ring__, addr__) \
1581do { \
1582 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
1583 PIPE_CONTROL_DEPTH_STALL); \
1584 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
1585 intel_ring_emit(ring__, 0); \
1586 intel_ring_emit(ring__, 0); \
1587} while (0)
1588
1589static int
1590pc_render_add_request(struct drm_i915_gem_request *req)
1591{
1592 struct intel_engine_cs *engine = req->engine;
1593 u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
1594 int ret;
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604 ret = intel_ring_begin(req, 32);
1605 if (ret)
1606 return ret;
1607
1608 intel_ring_emit(engine,
1609 GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1610 PIPE_CONTROL_WRITE_FLUSH |
1611 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
1612 intel_ring_emit(engine,
1613 engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1614 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1615 intel_ring_emit(engine, 0);
1616 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1617 scratch_addr += 2 * CACHELINE_BYTES;
1618 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1619 scratch_addr += 2 * CACHELINE_BYTES;
1620 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1621 scratch_addr += 2 * CACHELINE_BYTES;
1622 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1623 scratch_addr += 2 * CACHELINE_BYTES;
1624 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1625 scratch_addr += 2 * CACHELINE_BYTES;
1626 PIPE_CONTROL_FLUSH(engine, scratch_addr);
1627
1628 intel_ring_emit(engine,
1629 GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1630 PIPE_CONTROL_WRITE_FLUSH |
1631 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
1632 PIPE_CONTROL_NOTIFY);
1633 intel_ring_emit(engine,
1634 engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1635 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1636 intel_ring_emit(engine, 0);
1637 __intel_ring_advance(engine);
1638
1639 return 0;
1640}
1641
1642static void
1643gen6_seqno_barrier(struct intel_engine_cs *engine)
1644{
1645 struct drm_i915_private *dev_priv = engine->dev->dev_private;
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662 spin_lock_irq(&dev_priv->uncore.lock);
1663 POSTING_READ_FW(RING_ACTHD(engine->mmio_base));
1664 spin_unlock_irq(&dev_priv->uncore.lock);
1665}
1666
1667static u32
1668ring_get_seqno(struct intel_engine_cs *engine)
1669{
1670 return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
1671}
1672
1673static void
1674ring_set_seqno(struct intel_engine_cs *engine, u32 seqno)
1675{
1676 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
1677}
1678
1679static u32
1680pc_render_get_seqno(struct intel_engine_cs *engine)
1681{
1682 return engine->scratch.cpu_page[0];
1683}
1684
1685static void
1686pc_render_set_seqno(struct intel_engine_cs *engine, u32 seqno)
1687{
1688 engine->scratch.cpu_page[0] = seqno;
1689}
1690
1691static bool
1692gen5_ring_get_irq(struct intel_engine_cs *engine)
1693{
1694 struct drm_device *dev = engine->dev;
1695 struct drm_i915_private *dev_priv = dev->dev_private;
1696 unsigned long flags;
1697
1698 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1699 return false;
1700
1701 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1702 if (engine->irq_refcount++ == 0)
1703 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1704 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1705
1706 return true;
1707}
1708
1709static void
1710gen5_ring_put_irq(struct intel_engine_cs *engine)
1711{
1712 struct drm_device *dev = engine->dev;
1713 struct drm_i915_private *dev_priv = dev->dev_private;
1714 unsigned long flags;
1715
1716 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1717 if (--engine->irq_refcount == 0)
1718 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1719 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1720}
1721
1722static bool
1723i9xx_ring_get_irq(struct intel_engine_cs *engine)
1724{
1725 struct drm_device *dev = engine->dev;
1726 struct drm_i915_private *dev_priv = dev->dev_private;
1727 unsigned long flags;
1728
1729 if (!intel_irqs_enabled(dev_priv))
1730 return false;
1731
1732 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1733 if (engine->irq_refcount++ == 0) {
1734 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1735 I915_WRITE(IMR, dev_priv->irq_mask);
1736 POSTING_READ(IMR);
1737 }
1738 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1739
1740 return true;
1741}
1742
1743static void
1744i9xx_ring_put_irq(struct intel_engine_cs *engine)
1745{
1746 struct drm_device *dev = engine->dev;
1747 struct drm_i915_private *dev_priv = dev->dev_private;
1748 unsigned long flags;
1749
1750 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1751 if (--engine->irq_refcount == 0) {
1752 dev_priv->irq_mask |= engine->irq_enable_mask;
1753 I915_WRITE(IMR, dev_priv->irq_mask);
1754 POSTING_READ(IMR);
1755 }
1756 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1757}
1758
1759static bool
1760i8xx_ring_get_irq(struct intel_engine_cs *engine)
1761{
1762 struct drm_device *dev = engine->dev;
1763 struct drm_i915_private *dev_priv = dev->dev_private;
1764 unsigned long flags;
1765
1766 if (!intel_irqs_enabled(dev_priv))
1767 return false;
1768
1769 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1770 if (engine->irq_refcount++ == 0) {
1771 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1772 I915_WRITE16(IMR, dev_priv->irq_mask);
1773 POSTING_READ16(IMR);
1774 }
1775 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1776
1777 return true;
1778}
1779
1780static void
1781i8xx_ring_put_irq(struct intel_engine_cs *engine)
1782{
1783 struct drm_device *dev = engine->dev;
1784 struct drm_i915_private *dev_priv = dev->dev_private;
1785 unsigned long flags;
1786
1787 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1788 if (--engine->irq_refcount == 0) {
1789 dev_priv->irq_mask |= engine->irq_enable_mask;
1790 I915_WRITE16(IMR, dev_priv->irq_mask);
1791 POSTING_READ16(IMR);
1792 }
1793 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1794}
1795
1796static int
1797bsd_ring_flush(struct drm_i915_gem_request *req,
1798 u32 invalidate_domains,
1799 u32 flush_domains)
1800{
1801 struct intel_engine_cs *engine = req->engine;
1802 int ret;
1803
1804 ret = intel_ring_begin(req, 2);
1805 if (ret)
1806 return ret;
1807
1808 intel_ring_emit(engine, MI_FLUSH);
1809 intel_ring_emit(engine, MI_NOOP);
1810 intel_ring_advance(engine);
1811 return 0;
1812}
1813
1814static int
1815i9xx_add_request(struct drm_i915_gem_request *req)
1816{
1817 struct intel_engine_cs *engine = req->engine;
1818 int ret;
1819
1820 ret = intel_ring_begin(req, 4);
1821 if (ret)
1822 return ret;
1823
1824 intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1825 intel_ring_emit(engine,
1826 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1827 intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1828 intel_ring_emit(engine, MI_USER_INTERRUPT);
1829 __intel_ring_advance(engine);
1830
1831 return 0;
1832}
1833
1834static bool
1835gen6_ring_get_irq(struct intel_engine_cs *engine)
1836{
1837 struct drm_device *dev = engine->dev;
1838 struct drm_i915_private *dev_priv = dev->dev_private;
1839 unsigned long flags;
1840
1841 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1842 return false;
1843
1844 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1845 if (engine->irq_refcount++ == 0) {
1846 if (HAS_L3_DPF(dev) && engine->id == RCS)
1847 I915_WRITE_IMR(engine,
1848 ~(engine->irq_enable_mask |
1849 GT_PARITY_ERROR(dev)));
1850 else
1851 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1852 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1853 }
1854 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1855
1856 return true;
1857}
1858
1859static void
1860gen6_ring_put_irq(struct intel_engine_cs *engine)
1861{
1862 struct drm_device *dev = engine->dev;
1863 struct drm_i915_private *dev_priv = dev->dev_private;
1864 unsigned long flags;
1865
1866 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1867 if (--engine->irq_refcount == 0) {
1868 if (HAS_L3_DPF(dev) && engine->id == RCS)
1869 I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev));
1870 else
1871 I915_WRITE_IMR(engine, ~0);
1872 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1873 }
1874 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1875}
1876
1877static bool
1878hsw_vebox_get_irq(struct intel_engine_cs *engine)
1879{
1880 struct drm_device *dev = engine->dev;
1881 struct drm_i915_private *dev_priv = dev->dev_private;
1882 unsigned long flags;
1883
1884 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1885 return false;
1886
1887 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1888 if (engine->irq_refcount++ == 0) {
1889 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1890 gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
1891 }
1892 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1893
1894 return true;
1895}
1896
1897static void
1898hsw_vebox_put_irq(struct intel_engine_cs *engine)
1899{
1900 struct drm_device *dev = engine->dev;
1901 struct drm_i915_private *dev_priv = dev->dev_private;
1902 unsigned long flags;
1903
1904 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1905 if (--engine->irq_refcount == 0) {
1906 I915_WRITE_IMR(engine, ~0);
1907 gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
1908 }
1909 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1910}
1911
1912static bool
1913gen8_ring_get_irq(struct intel_engine_cs *engine)
1914{
1915 struct drm_device *dev = engine->dev;
1916 struct drm_i915_private *dev_priv = dev->dev_private;
1917 unsigned long flags;
1918
1919 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1920 return false;
1921
1922 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1923 if (engine->irq_refcount++ == 0) {
1924 if (HAS_L3_DPF(dev) && engine->id == RCS) {
1925 I915_WRITE_IMR(engine,
1926 ~(engine->irq_enable_mask |
1927 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1928 } else {
1929 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1930 }
1931 POSTING_READ(RING_IMR(engine->mmio_base));
1932 }
1933 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1934
1935 return true;
1936}
1937
1938static void
1939gen8_ring_put_irq(struct intel_engine_cs *engine)
1940{
1941 struct drm_device *dev = engine->dev;
1942 struct drm_i915_private *dev_priv = dev->dev_private;
1943 unsigned long flags;
1944
1945 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1946 if (--engine->irq_refcount == 0) {
1947 if (HAS_L3_DPF(dev) && engine->id == RCS) {
1948 I915_WRITE_IMR(engine,
1949 ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1950 } else {
1951 I915_WRITE_IMR(engine, ~0);
1952 }
1953 POSTING_READ(RING_IMR(engine->mmio_base));
1954 }
1955 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1956}
1957
1958static int
1959i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
1960 u64 offset, u32 length,
1961 unsigned dispatch_flags)
1962{
1963 struct intel_engine_cs *engine = req->engine;
1964 int ret;
1965
1966 ret = intel_ring_begin(req, 2);
1967 if (ret)
1968 return ret;
1969
1970 intel_ring_emit(engine,
1971 MI_BATCH_BUFFER_START |
1972 MI_BATCH_GTT |
1973 (dispatch_flags & I915_DISPATCH_SECURE ?
1974 0 : MI_BATCH_NON_SECURE_I965));
1975 intel_ring_emit(engine, offset);
1976 intel_ring_advance(engine);
1977
1978 return 0;
1979}
1980
1981
1982#define I830_BATCH_LIMIT (256*1024)
1983#define I830_TLB_ENTRIES (2)
1984#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1985static int
1986i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
1987 u64 offset, u32 len,
1988 unsigned dispatch_flags)
1989{
1990 struct intel_engine_cs *engine = req->engine;
1991 u32 cs_offset = engine->scratch.gtt_offset;
1992 int ret;
1993
1994 ret = intel_ring_begin(req, 6);
1995 if (ret)
1996 return ret;
1997
1998
1999 intel_ring_emit(engine, COLOR_BLT_CMD | BLT_WRITE_RGBA);
2000 intel_ring_emit(engine, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
2001 intel_ring_emit(engine, I830_TLB_ENTRIES << 16 | 4);
2002 intel_ring_emit(engine, cs_offset);
2003 intel_ring_emit(engine, 0xdeadbeef);
2004 intel_ring_emit(engine, MI_NOOP);
2005 intel_ring_advance(engine);
2006
2007 if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
2008 if (len > I830_BATCH_LIMIT)
2009 return -ENOSPC;
2010
2011 ret = intel_ring_begin(req, 6 + 2);
2012 if (ret)
2013 return ret;
2014
2015
2016
2017
2018
2019 intel_ring_emit(engine, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
2020 intel_ring_emit(engine,
2021 BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
2022 intel_ring_emit(engine, DIV_ROUND_UP(len, 4096) << 16 | 4096);
2023 intel_ring_emit(engine, cs_offset);
2024 intel_ring_emit(engine, 4096);
2025 intel_ring_emit(engine, offset);
2026
2027 intel_ring_emit(engine, MI_FLUSH);
2028 intel_ring_emit(engine, MI_NOOP);
2029 intel_ring_advance(engine);
2030
2031
2032 offset = cs_offset;
2033 }
2034
2035 ret = intel_ring_begin(req, 2);
2036 if (ret)
2037 return ret;
2038
2039 intel_ring_emit(engine, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
2040 intel_ring_emit(engine, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
2041 0 : MI_BATCH_NON_SECURE));
2042 intel_ring_advance(engine);
2043
2044 return 0;
2045}
2046
2047static int
2048i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
2049 u64 offset, u32 len,
2050 unsigned dispatch_flags)
2051{
2052 struct intel_engine_cs *engine = req->engine;
2053 int ret;
2054
2055 ret = intel_ring_begin(req, 2);
2056 if (ret)
2057 return ret;
2058
2059 intel_ring_emit(engine, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
2060 intel_ring_emit(engine, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
2061 0 : MI_BATCH_NON_SECURE));
2062 intel_ring_advance(engine);
2063
2064 return 0;
2065}
2066
2067static void cleanup_phys_status_page(struct intel_engine_cs *engine)
2068{
2069 struct drm_i915_private *dev_priv = to_i915(engine->dev);
2070
2071 if (!dev_priv->status_page_dmah)
2072 return;
2073
2074 drm_pci_free(engine->dev, dev_priv->status_page_dmah);
2075 engine->status_page.page_addr = NULL;
2076}
2077
2078static void cleanup_status_page(struct intel_engine_cs *engine)
2079{
2080 struct drm_i915_gem_object *obj;
2081
2082 obj = engine->status_page.obj;
2083 if (obj == NULL)
2084 return;
2085
2086 kunmap(sg_page(obj->pages->sgl));
2087 i915_gem_object_ggtt_unpin(obj);
2088 drm_gem_object_unreference(&obj->base);
2089 engine->status_page.obj = NULL;
2090}
2091
2092static int init_status_page(struct intel_engine_cs *engine)
2093{
2094 struct drm_i915_gem_object *obj = engine->status_page.obj;
2095
2096 if (obj == NULL) {
2097 unsigned flags;
2098 int ret;
2099
2100 obj = i915_gem_alloc_object(engine->dev, 4096);
2101 if (obj == NULL) {
2102 DRM_ERROR("Failed to allocate status page\n");
2103 return -ENOMEM;
2104 }
2105
2106 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2107 if (ret)
2108 goto err_unref;
2109
2110 flags = 0;
2111 if (!HAS_LLC(engine->dev))
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122 flags |= PIN_MAPPABLE;
2123 ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
2124 if (ret) {
2125err_unref:
2126 drm_gem_object_unreference(&obj->base);
2127 return ret;
2128 }
2129
2130 engine->status_page.obj = obj;
2131 }
2132
2133 engine->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
2134 engine->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
2135 memset(engine->status_page.page_addr, 0, PAGE_SIZE);
2136
2137 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
2138 engine->name, engine->status_page.gfx_addr);
2139
2140 return 0;
2141}
2142
2143static int init_phys_status_page(struct intel_engine_cs *engine)
2144{
2145 struct drm_i915_private *dev_priv = engine->dev->dev_private;
2146
2147 if (!dev_priv->status_page_dmah) {
2148 dev_priv->status_page_dmah =
2149 drm_pci_alloc(engine->dev, PAGE_SIZE, PAGE_SIZE);
2150 if (!dev_priv->status_page_dmah)
2151 return -ENOMEM;
2152 }
2153
2154 engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
2155 memset(engine->status_page.page_addr, 0, PAGE_SIZE);
2156
2157 return 0;
2158}
2159
2160void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2161{
2162 if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
2163 i915_gem_object_unpin_map(ringbuf->obj);
2164 else
2165 iounmap(ringbuf->virtual_start);
2166 ringbuf->virtual_start = NULL;
2167 ringbuf->vma = NULL;
2168 i915_gem_object_ggtt_unpin(ringbuf->obj);
2169}
2170
2171int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
2172 struct intel_ringbuffer *ringbuf)
2173{
2174 struct drm_i915_private *dev_priv = to_i915(dev);
2175 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2176 struct drm_i915_gem_object *obj = ringbuf->obj;
2177
2178 unsigned flags = PIN_OFFSET_BIAS | 4096;
2179 void *addr;
2180 int ret;
2181
2182 if (HAS_LLC(dev_priv) && !obj->stolen) {
2183 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, flags);
2184 if (ret)
2185 return ret;
2186
2187 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2188 if (ret)
2189 goto err_unpin;
2190
2191 addr = i915_gem_object_pin_map(obj);
2192 if (IS_ERR(addr)) {
2193 ret = PTR_ERR(addr);
2194 goto err_unpin;
2195 }
2196 } else {
2197 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE,
2198 flags | PIN_MAPPABLE);
2199 if (ret)
2200 return ret;
2201
2202 ret = i915_gem_object_set_to_gtt_domain(obj, true);
2203 if (ret)
2204 goto err_unpin;
2205
2206
2207 assert_rpm_wakelock_held(dev_priv);
2208
2209 addr = ioremap_wc(ggtt->mappable_base +
2210 i915_gem_obj_ggtt_offset(obj), ringbuf->size);
2211 if (addr == NULL) {
2212 ret = -ENOMEM;
2213 goto err_unpin;
2214 }
2215 }
2216
2217 ringbuf->virtual_start = addr;
2218 ringbuf->vma = i915_gem_obj_to_ggtt(obj);
2219 return 0;
2220
2221err_unpin:
2222 i915_gem_object_ggtt_unpin(obj);
2223 return ret;
2224}
2225
2226static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2227{
2228 drm_gem_object_unreference(&ringbuf->obj->base);
2229 ringbuf->obj = NULL;
2230}
2231
2232static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
2233 struct intel_ringbuffer *ringbuf)
2234{
2235 struct drm_i915_gem_object *obj;
2236
2237 obj = NULL;
2238 if (!HAS_LLC(dev))
2239 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
2240 if (obj == NULL)
2241 obj = i915_gem_alloc_object(dev, ringbuf->size);
2242 if (obj == NULL)
2243 return -ENOMEM;
2244
2245
2246 obj->gt_ro = 1;
2247
2248 ringbuf->obj = obj;
2249
2250 return 0;
2251}
2252
2253struct intel_ringbuffer *
2254intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
2255{
2256 struct intel_ringbuffer *ring;
2257 int ret;
2258
2259 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
2260 if (ring == NULL) {
2261 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
2262 engine->name);
2263 return ERR_PTR(-ENOMEM);
2264 }
2265
2266 ring->engine = engine;
2267 list_add(&ring->link, &engine->buffers);
2268
2269 ring->size = size;
2270
2271
2272
2273
2274 ring->effective_size = size;
2275 if (IS_I830(engine->dev) || IS_845G(engine->dev))
2276 ring->effective_size -= 2 * CACHELINE_BYTES;
2277
2278 ring->last_retired_head = -1;
2279 intel_ring_update_space(ring);
2280
2281 ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
2282 if (ret) {
2283 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
2284 engine->name, ret);
2285 list_del(&ring->link);
2286 kfree(ring);
2287 return ERR_PTR(ret);
2288 }
2289
2290 return ring;
2291}
2292
2293void
2294intel_ringbuffer_free(struct intel_ringbuffer *ring)
2295{
2296 intel_destroy_ringbuffer_obj(ring);
2297 list_del(&ring->link);
2298 kfree(ring);
2299}
2300
2301static int intel_init_ring_buffer(struct drm_device *dev,
2302 struct intel_engine_cs *engine)
2303{
2304 struct intel_ringbuffer *ringbuf;
2305 int ret;
2306
2307 WARN_ON(engine->buffer);
2308
2309 engine->dev = dev;
2310 INIT_LIST_HEAD(&engine->active_list);
2311 INIT_LIST_HEAD(&engine->request_list);
2312 INIT_LIST_HEAD(&engine->execlist_queue);
2313 INIT_LIST_HEAD(&engine->buffers);
2314 i915_gem_batch_pool_init(dev, &engine->batch_pool);
2315 memset(engine->semaphore.sync_seqno, 0,
2316 sizeof(engine->semaphore.sync_seqno));
2317
2318 init_waitqueue_head(&engine->irq_queue);
2319
2320 ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE);
2321 if (IS_ERR(ringbuf)) {
2322 ret = PTR_ERR(ringbuf);
2323 goto error;
2324 }
2325 engine->buffer = ringbuf;
2326
2327 if (I915_NEED_GFX_HWS(dev)) {
2328 ret = init_status_page(engine);
2329 if (ret)
2330 goto error;
2331 } else {
2332 WARN_ON(engine->id != RCS);
2333 ret = init_phys_status_page(engine);
2334 if (ret)
2335 goto error;
2336 }
2337
2338 ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
2339 if (ret) {
2340 DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
2341 engine->name, ret);
2342 intel_destroy_ringbuffer_obj(ringbuf);
2343 goto error;
2344 }
2345
2346 ret = i915_cmd_parser_init_ring(engine);
2347 if (ret)
2348 goto error;
2349
2350 return 0;
2351
2352error:
2353 intel_cleanup_engine(engine);
2354 return ret;
2355}
2356
2357void intel_cleanup_engine(struct intel_engine_cs *engine)
2358{
2359 struct drm_i915_private *dev_priv;
2360
2361 if (!intel_engine_initialized(engine))
2362 return;
2363
2364 dev_priv = to_i915(engine->dev);
2365
2366 if (engine->buffer) {
2367 intel_stop_engine(engine);
2368 WARN_ON(!IS_GEN2(engine->dev) && (I915_READ_MODE(engine) & MODE_IDLE) == 0);
2369
2370 intel_unpin_ringbuffer_obj(engine->buffer);
2371 intel_ringbuffer_free(engine->buffer);
2372 engine->buffer = NULL;
2373 }
2374
2375 if (engine->cleanup)
2376 engine->cleanup(engine);
2377
2378 if (I915_NEED_GFX_HWS(engine->dev)) {
2379 cleanup_status_page(engine);
2380 } else {
2381 WARN_ON(engine->id != RCS);
2382 cleanup_phys_status_page(engine);
2383 }
2384
2385 i915_cmd_parser_fini_ring(engine);
2386 i915_gem_batch_pool_fini(&engine->batch_pool);
2387 engine->dev = NULL;
2388}
2389
2390int intel_engine_idle(struct intel_engine_cs *engine)
2391{
2392 struct drm_i915_gem_request *req;
2393
2394
2395 if (list_empty(&engine->request_list))
2396 return 0;
2397
2398 req = list_entry(engine->request_list.prev,
2399 struct drm_i915_gem_request,
2400 list);
2401
2402
2403 return __i915_wait_request(req,
2404 req->i915->mm.interruptible,
2405 NULL, NULL);
2406}
2407
2408int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
2409{
2410 request->ringbuf = request->engine->buffer;
2411 return 0;
2412}
2413
2414int intel_ring_reserve_space(struct drm_i915_gem_request *request)
2415{
2416
2417
2418
2419
2420
2421
2422
2423
2424 intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
2425
2426 return intel_ring_begin(request, 0);
2427}
2428
2429void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
2430{
2431 GEM_BUG_ON(ringbuf->reserved_size);
2432 ringbuf->reserved_size = size;
2433}
2434
2435void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
2436{
2437 GEM_BUG_ON(!ringbuf->reserved_size);
2438 ringbuf->reserved_size = 0;
2439}
2440
2441void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
2442{
2443 GEM_BUG_ON(!ringbuf->reserved_size);
2444 ringbuf->reserved_size = 0;
2445}
2446
2447void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
2448{
2449 GEM_BUG_ON(ringbuf->reserved_size);
2450}
2451
2452static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
2453{
2454 struct intel_ringbuffer *ringbuf = req->ringbuf;
2455 struct intel_engine_cs *engine = req->engine;
2456 struct drm_i915_gem_request *target;
2457
2458 intel_ring_update_space(ringbuf);
2459 if (ringbuf->space >= bytes)
2460 return 0;
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471 GEM_BUG_ON(!ringbuf->reserved_size);
2472
2473 list_for_each_entry(target, &engine->request_list, list) {
2474 unsigned space;
2475
2476
2477
2478
2479
2480
2481 if (target->ringbuf != ringbuf)
2482 continue;
2483
2484
2485 space = __intel_ring_space(target->postfix, ringbuf->tail,
2486 ringbuf->size);
2487 if (space >= bytes)
2488 break;
2489 }
2490
2491 if (WARN_ON(&target->list == &engine->request_list))
2492 return -ENOSPC;
2493
2494 return i915_wait_request(target);
2495}
2496
2497int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
2498{
2499 struct intel_ringbuffer *ringbuf = req->ringbuf;
2500 int remain_actual = ringbuf->size - ringbuf->tail;
2501 int remain_usable = ringbuf->effective_size - ringbuf->tail;
2502 int bytes = num_dwords * sizeof(u32);
2503 int total_bytes, wait_bytes;
2504 bool need_wrap = false;
2505
2506 total_bytes = bytes + ringbuf->reserved_size;
2507
2508 if (unlikely(bytes > remain_usable)) {
2509
2510
2511
2512
2513 wait_bytes = remain_actual + total_bytes;
2514 need_wrap = true;
2515 } else if (unlikely(total_bytes > remain_usable)) {
2516
2517
2518
2519
2520
2521
2522 wait_bytes = remain_actual + ringbuf->reserved_size;
2523 } else {
2524
2525 wait_bytes = total_bytes;
2526 }
2527
2528 if (wait_bytes > ringbuf->space) {
2529 int ret = wait_for_space(req, wait_bytes);
2530 if (unlikely(ret))
2531 return ret;
2532
2533 intel_ring_update_space(ringbuf);
2534 if (unlikely(ringbuf->space < wait_bytes))
2535 return -EAGAIN;
2536 }
2537
2538 if (unlikely(need_wrap)) {
2539 GEM_BUG_ON(remain_actual > ringbuf->space);
2540 GEM_BUG_ON(ringbuf->tail + remain_actual > ringbuf->size);
2541
2542
2543 memset(ringbuf->virtual_start + ringbuf->tail,
2544 0, remain_actual);
2545 ringbuf->tail = 0;
2546 ringbuf->space -= remain_actual;
2547 }
2548
2549 ringbuf->space -= bytes;
2550 GEM_BUG_ON(ringbuf->space < 0);
2551 return 0;
2552}
2553
2554
2555int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
2556{
2557 struct intel_engine_cs *engine = req->engine;
2558 int num_dwords = (engine->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
2559 int ret;
2560
2561 if (num_dwords == 0)
2562 return 0;
2563
2564 num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
2565 ret = intel_ring_begin(req, num_dwords);
2566 if (ret)
2567 return ret;
2568
2569 while (num_dwords--)
2570 intel_ring_emit(engine, MI_NOOP);
2571
2572 intel_ring_advance(engine);
2573
2574 return 0;
2575}
2576
2577void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
2578{
2579 struct drm_i915_private *dev_priv = to_i915(engine->dev);
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589 if (INTEL_INFO(dev_priv)->gen == 6 || INTEL_INFO(dev_priv)->gen == 7) {
2590 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
2591 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
2592 if (HAS_VEBOX(dev_priv))
2593 I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
2594 }
2595 if (dev_priv->semaphore_obj) {
2596 struct drm_i915_gem_object *obj = dev_priv->semaphore_obj;
2597 struct page *page = i915_gem_object_get_dirty_page(obj, 0);
2598 void *semaphores = kmap(page);
2599 memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
2600 0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
2601 kunmap(page);
2602 }
2603 memset(engine->semaphore.sync_seqno, 0,
2604 sizeof(engine->semaphore.sync_seqno));
2605
2606 engine->set_seqno(engine, seqno);
2607 engine->last_submitted_seqno = seqno;
2608
2609 engine->hangcheck.seqno = seqno;
2610}
2611
2612static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
2613 u32 value)
2614{
2615 struct drm_i915_private *dev_priv = engine->dev->dev_private;
2616
2617
2618
2619
2620
2621
2622 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2623 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2624
2625
2626 I915_WRITE64(GEN6_BSD_RNCID, 0x0);
2627
2628
2629 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
2630 GEN6_BSD_SLEEP_INDICATOR) == 0,
2631 50))
2632 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2633
2634
2635 I915_WRITE_TAIL(engine, value);
2636 POSTING_READ(RING_TAIL(engine->mmio_base));
2637
2638
2639
2640
2641 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2642 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2643}
2644
2645static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
2646 u32 invalidate, u32 flush)
2647{
2648 struct intel_engine_cs *engine = req->engine;
2649 uint32_t cmd;
2650 int ret;
2651
2652 ret = intel_ring_begin(req, 4);
2653 if (ret)
2654 return ret;
2655
2656 cmd = MI_FLUSH_DW;
2657 if (INTEL_INFO(engine->dev)->gen >= 8)
2658 cmd += 1;
2659
2660
2661
2662
2663
2664
2665 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2666
2667
2668
2669
2670
2671
2672
2673 if (invalidate & I915_GEM_GPU_DOMAINS)
2674 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
2675
2676 intel_ring_emit(engine, cmd);
2677 intel_ring_emit(engine,
2678 I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2679 if (INTEL_INFO(engine->dev)->gen >= 8) {
2680 intel_ring_emit(engine, 0);
2681 intel_ring_emit(engine, 0);
2682 } else {
2683 intel_ring_emit(engine, 0);
2684 intel_ring_emit(engine, MI_NOOP);
2685 }
2686 intel_ring_advance(engine);
2687 return 0;
2688}
2689
2690static int
2691gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2692 u64 offset, u32 len,
2693 unsigned dispatch_flags)
2694{
2695 struct intel_engine_cs *engine = req->engine;
2696 bool ppgtt = USES_PPGTT(engine->dev) &&
2697 !(dispatch_flags & I915_DISPATCH_SECURE);
2698 int ret;
2699
2700 ret = intel_ring_begin(req, 4);
2701 if (ret)
2702 return ret;
2703
2704
2705 intel_ring_emit(engine, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
2706 (dispatch_flags & I915_DISPATCH_RS ?
2707 MI_BATCH_RESOURCE_STREAMER : 0));
2708 intel_ring_emit(engine, lower_32_bits(offset));
2709 intel_ring_emit(engine, upper_32_bits(offset));
2710 intel_ring_emit(engine, MI_NOOP);
2711 intel_ring_advance(engine);
2712
2713 return 0;
2714}
2715
2716static int
2717hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2718 u64 offset, u32 len,
2719 unsigned dispatch_flags)
2720{
2721 struct intel_engine_cs *engine = req->engine;
2722 int ret;
2723
2724 ret = intel_ring_begin(req, 2);
2725 if (ret)
2726 return ret;
2727
2728 intel_ring_emit(engine,
2729 MI_BATCH_BUFFER_START |
2730 (dispatch_flags & I915_DISPATCH_SECURE ?
2731 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
2732 (dispatch_flags & I915_DISPATCH_RS ?
2733 MI_BATCH_RESOURCE_STREAMER : 0));
2734
2735 intel_ring_emit(engine, offset);
2736 intel_ring_advance(engine);
2737
2738 return 0;
2739}
2740
2741static int
2742gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2743 u64 offset, u32 len,
2744 unsigned dispatch_flags)
2745{
2746 struct intel_engine_cs *engine = req->engine;
2747 int ret;
2748
2749 ret = intel_ring_begin(req, 2);
2750 if (ret)
2751 return ret;
2752
2753 intel_ring_emit(engine,
2754 MI_BATCH_BUFFER_START |
2755 (dispatch_flags & I915_DISPATCH_SECURE ?
2756 0 : MI_BATCH_NON_SECURE_I965));
2757
2758 intel_ring_emit(engine, offset);
2759 intel_ring_advance(engine);
2760
2761 return 0;
2762}
2763
2764
2765
2766static int gen6_ring_flush(struct drm_i915_gem_request *req,
2767 u32 invalidate, u32 flush)
2768{
2769 struct intel_engine_cs *engine = req->engine;
2770 struct drm_device *dev = engine->dev;
2771 uint32_t cmd;
2772 int ret;
2773
2774 ret = intel_ring_begin(req, 4);
2775 if (ret)
2776 return ret;
2777
2778 cmd = MI_FLUSH_DW;
2779 if (INTEL_INFO(dev)->gen >= 8)
2780 cmd += 1;
2781
2782
2783
2784
2785
2786
2787 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2788
2789
2790
2791
2792
2793
2794
2795 if (invalidate & I915_GEM_DOMAIN_RENDER)
2796 cmd |= MI_INVALIDATE_TLB;
2797 intel_ring_emit(engine, cmd);
2798 intel_ring_emit(engine,
2799 I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2800 if (INTEL_INFO(dev)->gen >= 8) {
2801 intel_ring_emit(engine, 0);
2802 intel_ring_emit(engine, 0);
2803 } else {
2804 intel_ring_emit(engine, 0);
2805 intel_ring_emit(engine, MI_NOOP);
2806 }
2807 intel_ring_advance(engine);
2808
2809 return 0;
2810}
2811
2812int intel_init_render_ring_buffer(struct drm_device *dev)
2813{
2814 struct drm_i915_private *dev_priv = dev->dev_private;
2815 struct intel_engine_cs *engine = &dev_priv->engine[RCS];
2816 struct drm_i915_gem_object *obj;
2817 int ret;
2818
2819 engine->name = "render ring";
2820 engine->id = RCS;
2821 engine->exec_id = I915_EXEC_RENDER;
2822 engine->hw_id = 0;
2823 engine->mmio_base = RENDER_RING_BASE;
2824
2825 if (INTEL_INFO(dev)->gen >= 8) {
2826 if (i915_semaphore_is_enabled(dev)) {
2827 obj = i915_gem_alloc_object(dev, 4096);
2828 if (obj == NULL) {
2829 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2830 i915.semaphores = 0;
2831 } else {
2832 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2833 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2834 if (ret != 0) {
2835 drm_gem_object_unreference(&obj->base);
2836 DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2837 i915.semaphores = 0;
2838 } else
2839 dev_priv->semaphore_obj = obj;
2840 }
2841 }
2842
2843 engine->init_context = intel_rcs_ctx_init;
2844 engine->add_request = gen6_add_request;
2845 engine->flush = gen8_render_ring_flush;
2846 engine->irq_get = gen8_ring_get_irq;
2847 engine->irq_put = gen8_ring_put_irq;
2848 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2849 engine->irq_seqno_barrier = gen6_seqno_barrier;
2850 engine->get_seqno = ring_get_seqno;
2851 engine->set_seqno = ring_set_seqno;
2852 if (i915_semaphore_is_enabled(dev)) {
2853 WARN_ON(!dev_priv->semaphore_obj);
2854 engine->semaphore.sync_to = gen8_ring_sync;
2855 engine->semaphore.signal = gen8_rcs_signal;
2856 GEN8_RING_SEMAPHORE_INIT(engine);
2857 }
2858 } else if (INTEL_INFO(dev)->gen >= 6) {
2859 engine->init_context = intel_rcs_ctx_init;
2860 engine->add_request = gen6_add_request;
2861 engine->flush = gen7_render_ring_flush;
2862 if (INTEL_INFO(dev)->gen == 6)
2863 engine->flush = gen6_render_ring_flush;
2864 engine->irq_get = gen6_ring_get_irq;
2865 engine->irq_put = gen6_ring_put_irq;
2866 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2867 engine->irq_seqno_barrier = gen6_seqno_barrier;
2868 engine->get_seqno = ring_get_seqno;
2869 engine->set_seqno = ring_set_seqno;
2870 if (i915_semaphore_is_enabled(dev)) {
2871 engine->semaphore.sync_to = gen6_ring_sync;
2872 engine->semaphore.signal = gen6_signal;
2873
2874
2875
2876
2877
2878
2879
2880 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2881 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
2882 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
2883 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
2884 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2885 engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2886 engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
2887 engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
2888 engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
2889 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2890 }
2891 } else if (IS_GEN5(dev)) {
2892 engine->add_request = pc_render_add_request;
2893 engine->flush = gen4_render_ring_flush;
2894 engine->get_seqno = pc_render_get_seqno;
2895 engine->set_seqno = pc_render_set_seqno;
2896 engine->irq_get = gen5_ring_get_irq;
2897 engine->irq_put = gen5_ring_put_irq;
2898 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
2899 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
2900 } else {
2901 engine->add_request = i9xx_add_request;
2902 if (INTEL_INFO(dev)->gen < 4)
2903 engine->flush = gen2_render_ring_flush;
2904 else
2905 engine->flush = gen4_render_ring_flush;
2906 engine->get_seqno = ring_get_seqno;
2907 engine->set_seqno = ring_set_seqno;
2908 if (IS_GEN2(dev)) {
2909 engine->irq_get = i8xx_ring_get_irq;
2910 engine->irq_put = i8xx_ring_put_irq;
2911 } else {
2912 engine->irq_get = i9xx_ring_get_irq;
2913 engine->irq_put = i9xx_ring_put_irq;
2914 }
2915 engine->irq_enable_mask = I915_USER_INTERRUPT;
2916 }
2917 engine->write_tail = ring_write_tail;
2918
2919 if (IS_HASWELL(dev))
2920 engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
2921 else if (IS_GEN8(dev))
2922 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2923 else if (INTEL_INFO(dev)->gen >= 6)
2924 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2925 else if (INTEL_INFO(dev)->gen >= 4)
2926 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
2927 else if (IS_I830(dev) || IS_845G(dev))
2928 engine->dispatch_execbuffer = i830_dispatch_execbuffer;
2929 else
2930 engine->dispatch_execbuffer = i915_dispatch_execbuffer;
2931 engine->init_hw = init_render_ring;
2932 engine->cleanup = render_ring_cleanup;
2933
2934
2935 if (HAS_BROKEN_CS_TLB(dev)) {
2936 obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
2937 if (obj == NULL) {
2938 DRM_ERROR("Failed to allocate batch bo\n");
2939 return -ENOMEM;
2940 }
2941
2942 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
2943 if (ret != 0) {
2944 drm_gem_object_unreference(&obj->base);
2945 DRM_ERROR("Failed to ping batch bo\n");
2946 return ret;
2947 }
2948
2949 engine->scratch.obj = obj;
2950 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2951 }
2952
2953 ret = intel_init_ring_buffer(dev, engine);
2954 if (ret)
2955 return ret;
2956
2957 if (INTEL_INFO(dev)->gen >= 5) {
2958 ret = intel_init_pipe_control(engine);
2959 if (ret)
2960 return ret;
2961 }
2962
2963 return 0;
2964}
2965
2966int intel_init_bsd_ring_buffer(struct drm_device *dev)
2967{
2968 struct drm_i915_private *dev_priv = dev->dev_private;
2969 struct intel_engine_cs *engine = &dev_priv->engine[VCS];
2970
2971 engine->name = "bsd ring";
2972 engine->id = VCS;
2973 engine->exec_id = I915_EXEC_BSD;
2974 engine->hw_id = 1;
2975
2976 engine->write_tail = ring_write_tail;
2977 if (INTEL_INFO(dev)->gen >= 6) {
2978 engine->mmio_base = GEN6_BSD_RING_BASE;
2979
2980 if (IS_GEN6(dev))
2981 engine->write_tail = gen6_bsd_ring_write_tail;
2982 engine->flush = gen6_bsd_ring_flush;
2983 engine->add_request = gen6_add_request;
2984 engine->irq_seqno_barrier = gen6_seqno_barrier;
2985 engine->get_seqno = ring_get_seqno;
2986 engine->set_seqno = ring_set_seqno;
2987 if (INTEL_INFO(dev)->gen >= 8) {
2988 engine->irq_enable_mask =
2989 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2990 engine->irq_get = gen8_ring_get_irq;
2991 engine->irq_put = gen8_ring_put_irq;
2992 engine->dispatch_execbuffer =
2993 gen8_ring_dispatch_execbuffer;
2994 if (i915_semaphore_is_enabled(dev)) {
2995 engine->semaphore.sync_to = gen8_ring_sync;
2996 engine->semaphore.signal = gen8_xcs_signal;
2997 GEN8_RING_SEMAPHORE_INIT(engine);
2998 }
2999 } else {
3000 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
3001 engine->irq_get = gen6_ring_get_irq;
3002 engine->irq_put = gen6_ring_put_irq;
3003 engine->dispatch_execbuffer =
3004 gen6_ring_dispatch_execbuffer;
3005 if (i915_semaphore_is_enabled(dev)) {
3006 engine->semaphore.sync_to = gen6_ring_sync;
3007 engine->semaphore.signal = gen6_signal;
3008 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
3009 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
3010 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
3011 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
3012 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3013 engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
3014 engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
3015 engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
3016 engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
3017 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3018 }
3019 }
3020 } else {
3021 engine->mmio_base = BSD_RING_BASE;
3022 engine->flush = bsd_ring_flush;
3023 engine->add_request = i9xx_add_request;
3024 engine->get_seqno = ring_get_seqno;
3025 engine->set_seqno = ring_set_seqno;
3026 if (IS_GEN5(dev)) {
3027 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3028 engine->irq_get = gen5_ring_get_irq;
3029 engine->irq_put = gen5_ring_put_irq;
3030 } else {
3031 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
3032 engine->irq_get = i9xx_ring_get_irq;
3033 engine->irq_put = i9xx_ring_put_irq;
3034 }
3035 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
3036 }
3037 engine->init_hw = init_ring_common;
3038
3039 return intel_init_ring_buffer(dev, engine);
3040}
3041
3042
3043
3044
3045int intel_init_bsd2_ring_buffer(struct drm_device *dev)
3046{
3047 struct drm_i915_private *dev_priv = dev->dev_private;
3048 struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
3049
3050 engine->name = "bsd2 ring";
3051 engine->id = VCS2;
3052 engine->exec_id = I915_EXEC_BSD;
3053 engine->hw_id = 4;
3054
3055 engine->write_tail = ring_write_tail;
3056 engine->mmio_base = GEN8_BSD2_RING_BASE;
3057 engine->flush = gen6_bsd_ring_flush;
3058 engine->add_request = gen6_add_request;
3059 engine->irq_seqno_barrier = gen6_seqno_barrier;
3060 engine->get_seqno = ring_get_seqno;
3061 engine->set_seqno = ring_set_seqno;
3062 engine->irq_enable_mask =
3063 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
3064 engine->irq_get = gen8_ring_get_irq;
3065 engine->irq_put = gen8_ring_put_irq;
3066 engine->dispatch_execbuffer =
3067 gen8_ring_dispatch_execbuffer;
3068 if (i915_semaphore_is_enabled(dev)) {
3069 engine->semaphore.sync_to = gen8_ring_sync;
3070 engine->semaphore.signal = gen8_xcs_signal;
3071 GEN8_RING_SEMAPHORE_INIT(engine);
3072 }
3073 engine->init_hw = init_ring_common;
3074
3075 return intel_init_ring_buffer(dev, engine);
3076}
3077
3078int intel_init_blt_ring_buffer(struct drm_device *dev)
3079{
3080 struct drm_i915_private *dev_priv = dev->dev_private;
3081 struct intel_engine_cs *engine = &dev_priv->engine[BCS];
3082
3083 engine->name = "blitter ring";
3084 engine->id = BCS;
3085 engine->exec_id = I915_EXEC_BLT;
3086 engine->hw_id = 2;
3087
3088 engine->mmio_base = BLT_RING_BASE;
3089 engine->write_tail = ring_write_tail;
3090 engine->flush = gen6_ring_flush;
3091 engine->add_request = gen6_add_request;
3092 engine->irq_seqno_barrier = gen6_seqno_barrier;
3093 engine->get_seqno = ring_get_seqno;
3094 engine->set_seqno = ring_set_seqno;
3095 if (INTEL_INFO(dev)->gen >= 8) {
3096 engine->irq_enable_mask =
3097 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
3098 engine->irq_get = gen8_ring_get_irq;
3099 engine->irq_put = gen8_ring_put_irq;
3100 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3101 if (i915_semaphore_is_enabled(dev)) {
3102 engine->semaphore.sync_to = gen8_ring_sync;
3103 engine->semaphore.signal = gen8_xcs_signal;
3104 GEN8_RING_SEMAPHORE_INIT(engine);
3105 }
3106 } else {
3107 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
3108 engine->irq_get = gen6_ring_get_irq;
3109 engine->irq_put = gen6_ring_put_irq;
3110 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3111 if (i915_semaphore_is_enabled(dev)) {
3112 engine->semaphore.signal = gen6_signal;
3113 engine->semaphore.sync_to = gen6_ring_sync;
3114
3115
3116
3117
3118
3119
3120
3121 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
3122 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
3123 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
3124 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
3125 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3126 engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
3127 engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
3128 engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
3129 engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
3130 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3131 }
3132 }
3133 engine->init_hw = init_ring_common;
3134
3135 return intel_init_ring_buffer(dev, engine);
3136}
3137
3138int intel_init_vebox_ring_buffer(struct drm_device *dev)
3139{
3140 struct drm_i915_private *dev_priv = dev->dev_private;
3141 struct intel_engine_cs *engine = &dev_priv->engine[VECS];
3142
3143 engine->name = "video enhancement ring";
3144 engine->id = VECS;
3145 engine->exec_id = I915_EXEC_VEBOX;
3146 engine->hw_id = 3;
3147
3148 engine->mmio_base = VEBOX_RING_BASE;
3149 engine->write_tail = ring_write_tail;
3150 engine->flush = gen6_ring_flush;
3151 engine->add_request = gen6_add_request;
3152 engine->irq_seqno_barrier = gen6_seqno_barrier;
3153 engine->get_seqno = ring_get_seqno;
3154 engine->set_seqno = ring_set_seqno;
3155
3156 if (INTEL_INFO(dev)->gen >= 8) {
3157 engine->irq_enable_mask =
3158 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
3159 engine->irq_get = gen8_ring_get_irq;
3160 engine->irq_put = gen8_ring_put_irq;
3161 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3162 if (i915_semaphore_is_enabled(dev)) {
3163 engine->semaphore.sync_to = gen8_ring_sync;
3164 engine->semaphore.signal = gen8_xcs_signal;
3165 GEN8_RING_SEMAPHORE_INIT(engine);
3166 }
3167 } else {
3168 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
3169 engine->irq_get = hsw_vebox_get_irq;
3170 engine->irq_put = hsw_vebox_put_irq;
3171 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3172 if (i915_semaphore_is_enabled(dev)) {
3173 engine->semaphore.sync_to = gen6_ring_sync;
3174 engine->semaphore.signal = gen6_signal;
3175 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
3176 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
3177 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
3178 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
3179 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3180 engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
3181 engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
3182 engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
3183 engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
3184 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3185 }
3186 }
3187 engine->init_hw = init_ring_common;
3188
3189 return intel_init_ring_buffer(dev, engine);
3190}
3191
3192int
3193intel_ring_flush_all_caches(struct drm_i915_gem_request *req)
3194{
3195 struct intel_engine_cs *engine = req->engine;
3196 int ret;
3197
3198 if (!engine->gpu_caches_dirty)
3199 return 0;
3200
3201 ret = engine->flush(req, 0, I915_GEM_GPU_DOMAINS);
3202 if (ret)
3203 return ret;
3204
3205 trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS);
3206
3207 engine->gpu_caches_dirty = false;
3208 return 0;
3209}
3210
3211int
3212intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
3213{
3214 struct intel_engine_cs *engine = req->engine;
3215 uint32_t flush_domains;
3216 int ret;
3217
3218 flush_domains = 0;
3219 if (engine->gpu_caches_dirty)
3220 flush_domains = I915_GEM_GPU_DOMAINS;
3221
3222 ret = engine->flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3223 if (ret)
3224 return ret;
3225
3226 trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3227
3228 engine->gpu_caches_dirty = false;
3229 return 0;
3230}
3231
3232void
3233intel_stop_engine(struct intel_engine_cs *engine)
3234{
3235 int ret;
3236
3237 if (!intel_engine_initialized(engine))
3238 return;
3239
3240 ret = intel_engine_idle(engine);
3241 if (ret)
3242 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
3243 engine->name, ret);
3244
3245 stop_ring(engine);
3246}
3247