1
2
3
4
5
6#include "gen2_engine_cs.h"
7#include "gen6_engine_cs.h"
8#include "gen6_ppgtt.h"
9#include "gen7_renderclear.h"
10#include "i915_drv.h"
11#include "i915_mitigations.h"
12#include "intel_breadcrumbs.h"
13#include "intel_context.h"
14#include "intel_gt.h"
15#include "intel_gt_irq.h"
16#include "intel_reset.h"
17#include "intel_ring.h"
18#include "shmem_utils.h"
19#include "intel_engine_heartbeat.h"
20
21
22
23
24#define LEGACY_REQUEST_SIZE 200
25
26static void set_hwstam(struct intel_engine_cs *engine, u32 mask)
27{
28
29
30
31
32 if (engine->class == RENDER_CLASS) {
33 if (GRAPHICS_VER(engine->i915) >= 6)
34 mask &= ~BIT(0);
35 else
36 mask &= ~I915_USER_INTERRUPT;
37 }
38
39 intel_engine_set_hwsp_writemask(engine, mask);
40}
41
42static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys)
43{
44 u32 addr;
45
46 addr = lower_32_bits(phys);
47 if (GRAPHICS_VER(engine->i915) >= 4)
48 addr |= (phys >> 28) & 0xf0;
49
50 intel_uncore_write(engine->uncore, HWS_PGA, addr);
51}
52
53static struct page *status_page(struct intel_engine_cs *engine)
54{
55 struct drm_i915_gem_object *obj = engine->status_page.vma->obj;
56
57 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
58 return sg_page(obj->mm.pages->sgl);
59}
60
61static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
62{
63 set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine))));
64 set_hwstam(engine, ~0u);
65}
66
67static void set_hwsp(struct intel_engine_cs *engine, u32 offset)
68{
69 i915_reg_t hwsp;
70
71
72
73
74
75 if (GRAPHICS_VER(engine->i915) == 7) {
76 switch (engine->id) {
77
78
79
80
81 default:
82 GEM_BUG_ON(engine->id);
83 fallthrough;
84 case RCS0:
85 hwsp = RENDER_HWS_PGA_GEN7;
86 break;
87 case BCS0:
88 hwsp = BLT_HWS_PGA_GEN7;
89 break;
90 case VCS0:
91 hwsp = BSD_HWS_PGA_GEN7;
92 break;
93 case VECS0:
94 hwsp = VEBOX_HWS_PGA_GEN7;
95 break;
96 }
97 } else if (GRAPHICS_VER(engine->i915) == 6) {
98 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base);
99 } else {
100 hwsp = RING_HWS_PGA(engine->mmio_base);
101 }
102
103 intel_uncore_write_fw(engine->uncore, hwsp, offset);
104 intel_uncore_posting_read_fw(engine->uncore, hwsp);
105}
106
107static void flush_cs_tlb(struct intel_engine_cs *engine)
108{
109 if (!IS_GRAPHICS_VER(engine->i915, 6, 7))
110 return;
111
112
113 GEM_DEBUG_WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
114
115 ENGINE_WRITE_FW(engine, RING_INSTPM,
116 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
117 INSTPM_SYNC_FLUSH));
118 if (__intel_wait_for_register_fw(engine->uncore,
119 RING_INSTPM(engine->mmio_base),
120 INSTPM_SYNC_FLUSH, 0,
121 2000, 0, NULL))
122 ENGINE_TRACE(engine,
123 "wait for SyncFlush to complete for TLB invalidation timed out\n");
124}
125
126static void ring_setup_status_page(struct intel_engine_cs *engine)
127{
128 set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma));
129 set_hwstam(engine, ~0u);
130
131 flush_cs_tlb(engine);
132}
133
134static struct i915_address_space *vm_alias(struct i915_address_space *vm)
135{
136 if (i915_is_ggtt(vm))
137 vm = &i915_vm_to_ggtt(vm)->alias->vm;
138
139 return vm;
140}
141
142static u32 pp_dir(struct i915_address_space *vm)
143{
144 return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir;
145}
146
147static void set_pp_dir(struct intel_engine_cs *engine)
148{
149 struct i915_address_space *vm = vm_alias(engine->gt->vm);
150
151 if (!vm)
152 return;
153
154 ENGINE_WRITE_FW(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G);
155 ENGINE_WRITE_FW(engine, RING_PP_DIR_BASE, pp_dir(vm));
156
157 if (GRAPHICS_VER(engine->i915) >= 7) {
158 ENGINE_WRITE_FW(engine,
159 RING_MODE_GEN7,
160 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
161 }
162}
163
164static bool stop_ring(struct intel_engine_cs *engine)
165{
166
167 ENGINE_WRITE_FW(engine, RING_HEAD, ENGINE_READ_FW(engine, RING_TAIL));
168 ENGINE_POSTING_READ(engine, RING_HEAD);
169
170
171 ENGINE_WRITE_FW(engine, RING_CTL, 0);
172 ENGINE_POSTING_READ(engine, RING_CTL);
173
174
175 ENGINE_WRITE_FW(engine, RING_HEAD, 0);
176 ENGINE_WRITE_FW(engine, RING_TAIL, 0);
177
178 return (ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) == 0;
179}
180
181static int xcs_resume(struct intel_engine_cs *engine)
182{
183 struct intel_ring *ring = engine->legacy.ring;
184
185 ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",
186 ring->head, ring->tail);
187
188
189
190
191
192 intel_synchronize_hardirq(engine->i915);
193 if (!stop_ring(engine))
194 goto err;
195
196 if (HWS_NEEDS_PHYSICAL(engine->i915))
197 ring_setup_phys_status_page(engine);
198 else
199 ring_setup_status_page(engine);
200
201 intel_breadcrumbs_reset(engine->breadcrumbs);
202
203
204 ENGINE_POSTING_READ(engine, RING_HEAD);
205
206
207
208
209
210
211
212 ENGINE_WRITE_FW(engine, RING_START, i915_ggtt_offset(ring->vma));
213
214
215 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
216 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));
217 intel_ring_update_space(ring);
218
219 set_pp_dir(engine);
220
221
222 ENGINE_WRITE_FW(engine, RING_HEAD, ring->head);
223 ENGINE_WRITE_FW(engine, RING_TAIL, ring->head);
224 ENGINE_POSTING_READ(engine, RING_TAIL);
225
226 ENGINE_WRITE_FW(engine, RING_CTL,
227 RING_CTL_SIZE(ring->size) | RING_VALID);
228
229
230 if (__intel_wait_for_register_fw(engine->uncore,
231 RING_CTL(engine->mmio_base),
232 RING_VALID, RING_VALID,
233 5000, 0, NULL))
234 goto err;
235
236 if (GRAPHICS_VER(engine->i915) > 2)
237 ENGINE_WRITE_FW(engine,
238 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
239
240
241 if (ring->tail != ring->head) {
242 ENGINE_WRITE_FW(engine, RING_TAIL, ring->tail);
243 ENGINE_POSTING_READ(engine, RING_TAIL);
244 }
245
246
247 intel_engine_signal_breadcrumbs(engine);
248 return 0;
249
250err:
251 drm_err(&engine->i915->drm,
252 "%s initialization failed; "
253 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
254 engine->name,
255 ENGINE_READ(engine, RING_CTL),
256 ENGINE_READ(engine, RING_CTL) & RING_VALID,
257 ENGINE_READ(engine, RING_HEAD), ring->head,
258 ENGINE_READ(engine, RING_TAIL), ring->tail,
259 ENGINE_READ(engine, RING_START),
260 i915_ggtt_offset(ring->vma));
261 return -EIO;
262}
263
264static void sanitize_hwsp(struct intel_engine_cs *engine)
265{
266 struct intel_timeline *tl;
267
268 list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
269 intel_timeline_reset_seqno(tl);
270}
271
272static void xcs_sanitize(struct intel_engine_cs *engine)
273{
274
275
276
277
278
279
280
281
282
283 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
284 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
285
286
287
288
289
290
291 sanitize_hwsp(engine);
292
293
294 clflush_cache_range(engine->status_page.addr, PAGE_SIZE);
295}
296
297static void reset_prepare(struct intel_engine_cs *engine)
298{
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314 ENGINE_TRACE(engine, "\n");
315 intel_engine_stop_cs(engine);
316
317 if (!stop_ring(engine)) {
318
319 ENGINE_TRACE(engine,
320 "HEAD not reset to zero, "
321 "{ CTL:%08x, HEAD:%08x, TAIL:%08x, START:%08x }\n",
322 ENGINE_READ_FW(engine, RING_CTL),
323 ENGINE_READ_FW(engine, RING_HEAD),
324 ENGINE_READ_FW(engine, RING_TAIL),
325 ENGINE_READ_FW(engine, RING_START));
326 if (!stop_ring(engine)) {
327 drm_err(&engine->i915->drm,
328 "failed to set %s head to zero "
329 "ctl %08x head %08x tail %08x start %08x\n",
330 engine->name,
331 ENGINE_READ_FW(engine, RING_CTL),
332 ENGINE_READ_FW(engine, RING_HEAD),
333 ENGINE_READ_FW(engine, RING_TAIL),
334 ENGINE_READ_FW(engine, RING_START));
335 }
336 }
337}
338
339static void reset_rewind(struct intel_engine_cs *engine, bool stalled)
340{
341 struct i915_request *pos, *rq;
342 unsigned long flags;
343 u32 head;
344
345 rq = NULL;
346 spin_lock_irqsave(&engine->sched_engine->lock, flags);
347 rcu_read_lock();
348 list_for_each_entry(pos, &engine->sched_engine->requests, sched.link) {
349 if (!__i915_request_is_complete(pos)) {
350 rq = pos;
351 break;
352 }
353 }
354 rcu_read_unlock();
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378 if (rq) {
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394 __i915_request_reset(rq, stalled);
395
396 GEM_BUG_ON(rq->ring != engine->legacy.ring);
397 head = rq->head;
398 } else {
399 head = engine->legacy.ring->tail;
400 }
401 engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head);
402
403 spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
404}
405
406static void reset_finish(struct intel_engine_cs *engine)
407{
408}
409
410static void reset_cancel(struct intel_engine_cs *engine)
411{
412 struct i915_request *request;
413 unsigned long flags;
414
415 spin_lock_irqsave(&engine->sched_engine->lock, flags);
416
417
418 list_for_each_entry(request, &engine->sched_engine->requests, sched.link)
419 i915_request_put(i915_request_mark_eio(request));
420 intel_engine_signal_breadcrumbs(engine);
421
422
423
424 spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
425}
426
427static void i9xx_submit_request(struct i915_request *request)
428{
429 i915_request_submit(request);
430 wmb();
431
432 ENGINE_WRITE(request->engine, RING_TAIL,
433 intel_ring_set_tail(request->ring, request->tail));
434}
435
436static void __ring_context_fini(struct intel_context *ce)
437{
438 i915_vma_put(ce->state);
439}
440
441static void ring_context_destroy(struct kref *ref)
442{
443 struct intel_context *ce = container_of(ref, typeof(*ce), ref);
444
445 GEM_BUG_ON(intel_context_is_pinned(ce));
446
447 if (ce->state)
448 __ring_context_fini(ce);
449
450 intel_context_fini(ce);
451 intel_context_free(ce);
452}
453
454static int ring_context_init_default_state(struct intel_context *ce,
455 struct i915_gem_ww_ctx *ww)
456{
457 struct drm_i915_gem_object *obj = ce->state->obj;
458 void *vaddr;
459
460 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
461 if (IS_ERR(vaddr))
462 return PTR_ERR(vaddr);
463
464 shmem_read(ce->engine->default_state, 0,
465 vaddr, ce->engine->context_size);
466
467 i915_gem_object_flush_map(obj);
468 __i915_gem_object_release_map(obj);
469
470 __set_bit(CONTEXT_VALID_BIT, &ce->flags);
471 return 0;
472}
473
474static int ring_context_pre_pin(struct intel_context *ce,
475 struct i915_gem_ww_ctx *ww,
476 void **unused)
477{
478 struct i915_address_space *vm;
479 int err = 0;
480
481 if (ce->engine->default_state &&
482 !test_bit(CONTEXT_VALID_BIT, &ce->flags)) {
483 err = ring_context_init_default_state(ce, ww);
484 if (err)
485 return err;
486 }
487
488 vm = vm_alias(ce->vm);
489 if (vm)
490 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww);
491
492 return err;
493}
494
495static void __context_unpin_ppgtt(struct intel_context *ce)
496{
497 struct i915_address_space *vm;
498
499 vm = vm_alias(ce->vm);
500 if (vm)
501 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm));
502}
503
504static void ring_context_unpin(struct intel_context *ce)
505{
506}
507
508static void ring_context_post_unpin(struct intel_context *ce)
509{
510 __context_unpin_ppgtt(ce);
511}
512
513static struct i915_vma *
514alloc_context_vma(struct intel_engine_cs *engine)
515{
516 struct drm_i915_private *i915 = engine->i915;
517 struct drm_i915_gem_object *obj;
518 struct i915_vma *vma;
519 int err;
520
521 obj = i915_gem_object_create_shmem(i915, engine->context_size);
522 if (IS_ERR(obj))
523 return ERR_CAST(obj);
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540 if (IS_IVYBRIDGE(i915))
541 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
542
543 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
544 if (IS_ERR(vma)) {
545 err = PTR_ERR(vma);
546 goto err_obj;
547 }
548
549 return vma;
550
551err_obj:
552 i915_gem_object_put(obj);
553 return ERR_PTR(err);
554}
555
556static int ring_context_alloc(struct intel_context *ce)
557{
558 struct intel_engine_cs *engine = ce->engine;
559
560
561 GEM_BUG_ON(!engine->legacy.ring);
562 ce->ring = engine->legacy.ring;
563 ce->timeline = intel_timeline_get(engine->legacy.timeline);
564
565 GEM_BUG_ON(ce->state);
566 if (engine->context_size) {
567 struct i915_vma *vma;
568
569 vma = alloc_context_vma(engine);
570 if (IS_ERR(vma))
571 return PTR_ERR(vma);
572
573 ce->state = vma;
574 }
575
576 return 0;
577}
578
579static int ring_context_pin(struct intel_context *ce, void *unused)
580{
581 return 0;
582}
583
584static void ring_context_reset(struct intel_context *ce)
585{
586 intel_ring_reset(ce->ring, ce->ring->emit);
587 clear_bit(CONTEXT_VALID_BIT, &ce->flags);
588}
589
590static void ring_context_ban(struct intel_context *ce,
591 struct i915_request *rq)
592{
593 struct intel_engine_cs *engine;
594
595 if (!rq || !i915_request_is_active(rq))
596 return;
597
598 engine = rq->engine;
599 lockdep_assert_held(&engine->sched_engine->lock);
600 list_for_each_entry_continue(rq, &engine->sched_engine->requests,
601 sched.link)
602 if (rq->context == ce) {
603 i915_request_set_error_once(rq, -EIO);
604 __i915_request_skip(rq);
605 }
606}
607
608static void ring_context_cancel_request(struct intel_context *ce,
609 struct i915_request *rq)
610{
611 struct intel_engine_cs *engine = NULL;
612
613 i915_request_active_engine(rq, &engine);
614
615 if (engine && intel_engine_pulse(engine))
616 intel_gt_handle_error(engine->gt, engine->mask, 0,
617 "request cancellation by %s",
618 current->comm);
619}
620
621static const struct intel_context_ops ring_context_ops = {
622 .alloc = ring_context_alloc,
623
624 .cancel_request = ring_context_cancel_request,
625
626 .ban = ring_context_ban,
627
628 .pre_pin = ring_context_pre_pin,
629 .pin = ring_context_pin,
630 .unpin = ring_context_unpin,
631 .post_unpin = ring_context_post_unpin,
632
633 .enter = intel_context_enter_engine,
634 .exit = intel_context_exit_engine,
635
636 .reset = ring_context_reset,
637 .destroy = ring_context_destroy,
638};
639
640static int load_pd_dir(struct i915_request *rq,
641 struct i915_address_space *vm,
642 u32 valid)
643{
644 const struct intel_engine_cs * const engine = rq->engine;
645 u32 *cs;
646
647 cs = intel_ring_begin(rq, 12);
648 if (IS_ERR(cs))
649 return PTR_ERR(cs);
650
651 *cs++ = MI_LOAD_REGISTER_IMM(1);
652 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
653 *cs++ = valid;
654
655 *cs++ = MI_LOAD_REGISTER_IMM(1);
656 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
657 *cs++ = pp_dir(vm);
658
659
660 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
661 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
662 *cs++ = intel_gt_scratch_offset(engine->gt,
663 INTEL_GT_SCRATCH_FIELD_DEFAULT);
664
665 *cs++ = MI_LOAD_REGISTER_IMM(1);
666 *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));
667 *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE);
668
669 intel_ring_advance(rq, cs);
670
671 return rq->engine->emit_flush(rq, EMIT_FLUSH);
672}
673
674static int mi_set_context(struct i915_request *rq,
675 struct intel_context *ce,
676 u32 flags)
677{
678 struct intel_engine_cs *engine = rq->engine;
679 struct drm_i915_private *i915 = engine->i915;
680 enum intel_engine_id id;
681 const int num_engines =
682 IS_HASWELL(i915) ? engine->gt->info.num_engines - 1 : 0;
683 bool force_restore = false;
684 int len;
685 u32 *cs;
686
687 len = 4;
688 if (GRAPHICS_VER(i915) == 7)
689 len += 2 + (num_engines ? 4 * num_engines + 6 : 0);
690 else if (GRAPHICS_VER(i915) == 5)
691 len += 2;
692 if (flags & MI_FORCE_RESTORE) {
693 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT);
694 flags &= ~MI_FORCE_RESTORE;
695 force_restore = true;
696 len += 2;
697 }
698
699 cs = intel_ring_begin(rq, len);
700 if (IS_ERR(cs))
701 return PTR_ERR(cs);
702
703
704 if (GRAPHICS_VER(i915) == 7) {
705 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
706 if (num_engines) {
707 struct intel_engine_cs *signaller;
708
709 *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
710 for_each_engine(signaller, engine->gt, id) {
711 if (signaller == engine)
712 continue;
713
714 *cs++ = i915_mmio_reg_offset(
715 RING_PSMI_CTL(signaller->mmio_base));
716 *cs++ = _MASKED_BIT_ENABLE(
717 GEN6_PSMI_SLEEP_MSG_DISABLE);
718 }
719 }
720 } else if (GRAPHICS_VER(i915) == 5) {
721
722
723
724
725
726
727 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN;
728 }
729
730 if (force_restore) {
731
732
733
734
735
736
737
738
739
740
741
742
743 *cs++ = MI_SET_CONTEXT;
744 *cs++ = i915_ggtt_offset(engine->kernel_context->state) |
745 MI_MM_SPACE_GTT |
746 MI_RESTORE_INHIBIT;
747 }
748
749 *cs++ = MI_NOOP;
750 *cs++ = MI_SET_CONTEXT;
751 *cs++ = i915_ggtt_offset(ce->state) | flags;
752
753
754
755
756 *cs++ = MI_NOOP;
757
758 if (GRAPHICS_VER(i915) == 7) {
759 if (num_engines) {
760 struct intel_engine_cs *signaller;
761 i915_reg_t last_reg = {};
762
763 *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
764 for_each_engine(signaller, engine->gt, id) {
765 if (signaller == engine)
766 continue;
767
768 last_reg = RING_PSMI_CTL(signaller->mmio_base);
769 *cs++ = i915_mmio_reg_offset(last_reg);
770 *cs++ = _MASKED_BIT_DISABLE(
771 GEN6_PSMI_SLEEP_MSG_DISABLE);
772 }
773
774
775 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
776 *cs++ = i915_mmio_reg_offset(last_reg);
777 *cs++ = intel_gt_scratch_offset(engine->gt,
778 INTEL_GT_SCRATCH_FIELD_DEFAULT);
779 *cs++ = MI_NOOP;
780 }
781 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
782 } else if (GRAPHICS_VER(i915) == 5) {
783 *cs++ = MI_SUSPEND_FLUSH;
784 }
785
786 intel_ring_advance(rq, cs);
787
788 return 0;
789}
790
791static int remap_l3_slice(struct i915_request *rq, int slice)
792{
793#define L3LOG_DW (GEN7_L3LOG_SIZE / sizeof(u32))
794 u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice];
795 int i;
796
797 if (!remap_info)
798 return 0;
799
800 cs = intel_ring_begin(rq, L3LOG_DW * 2 + 2);
801 if (IS_ERR(cs))
802 return PTR_ERR(cs);
803
804
805
806
807
808
809 *cs++ = MI_LOAD_REGISTER_IMM(L3LOG_DW);
810 for (i = 0; i < L3LOG_DW; i++) {
811 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
812 *cs++ = remap_info[i];
813 }
814 *cs++ = MI_NOOP;
815 intel_ring_advance(rq, cs);
816
817 return 0;
818#undef L3LOG_DW
819}
820
821static int remap_l3(struct i915_request *rq)
822{
823 struct i915_gem_context *ctx = i915_request_gem_context(rq);
824 int i, err;
825
826 if (!ctx || !ctx->remap_slice)
827 return 0;
828
829 for (i = 0; i < MAX_L3_SLICES; i++) {
830 if (!(ctx->remap_slice & BIT(i)))
831 continue;
832
833 err = remap_l3_slice(rq, i);
834 if (err)
835 return err;
836 }
837
838 ctx->remap_slice = 0;
839 return 0;
840}
841
842static int switch_mm(struct i915_request *rq, struct i915_address_space *vm)
843{
844 int ret;
845
846 if (!vm)
847 return 0;
848
849 ret = rq->engine->emit_flush(rq, EMIT_FLUSH);
850 if (ret)
851 return ret;
852
853
854
855
856
857
858
859
860
861 ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G);
862 if (ret)
863 return ret;
864
865 return rq->engine->emit_flush(rq, EMIT_INVALIDATE);
866}
867
868static int clear_residuals(struct i915_request *rq)
869{
870 struct intel_engine_cs *engine = rq->engine;
871 int ret;
872
873 ret = switch_mm(rq, vm_alias(engine->kernel_context->vm));
874 if (ret)
875 return ret;
876
877 if (engine->kernel_context->state) {
878 ret = mi_set_context(rq,
879 engine->kernel_context,
880 MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT);
881 if (ret)
882 return ret;
883 }
884
885 ret = engine->emit_bb_start(rq,
886 engine->wa_ctx.vma->node.start, 0,
887 0);
888 if (ret)
889 return ret;
890
891 ret = engine->emit_flush(rq, EMIT_FLUSH);
892 if (ret)
893 return ret;
894
895
896 return engine->emit_flush(rq, EMIT_INVALIDATE);
897}
898
899static int switch_context(struct i915_request *rq)
900{
901 struct intel_engine_cs *engine = rq->engine;
902 struct intel_context *ce = rq->context;
903 void **residuals = NULL;
904 int ret;
905
906 GEM_BUG_ON(HAS_EXECLISTS(engine->i915));
907
908 if (engine->wa_ctx.vma && ce != engine->kernel_context) {
909 if (engine->wa_ctx.vma->private != ce &&
910 i915_mitigate_clear_residuals()) {
911 ret = clear_residuals(rq);
912 if (ret)
913 return ret;
914
915 residuals = &engine->wa_ctx.vma->private;
916 }
917 }
918
919 ret = switch_mm(rq, vm_alias(ce->vm));
920 if (ret)
921 return ret;
922
923 if (ce->state) {
924 u32 flags;
925
926 GEM_BUG_ON(engine->id != RCS0);
927
928
929 BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN);
930 BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN);
931
932 flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT;
933 if (test_bit(CONTEXT_VALID_BIT, &ce->flags))
934 flags |= MI_RESTORE_EXT_STATE_EN;
935 else
936 flags |= MI_RESTORE_INHIBIT;
937
938 ret = mi_set_context(rq, ce, flags);
939 if (ret)
940 return ret;
941 }
942
943 ret = remap_l3(rq);
944 if (ret)
945 return ret;
946
947
948
949
950
951
952
953
954
955
956 if (residuals) {
957 intel_context_put(*residuals);
958 *residuals = intel_context_get(ce);
959 }
960
961 return 0;
962}
963
964static int ring_request_alloc(struct i915_request *request)
965{
966 int ret;
967
968 GEM_BUG_ON(!intel_context_is_pinned(request->context));
969 GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb);
970
971
972
973
974
975
976 request->reserved_space += LEGACY_REQUEST_SIZE;
977
978
979 ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
980 if (ret)
981 return ret;
982
983 ret = switch_context(request);
984 if (ret)
985 return ret;
986
987 request->reserved_space -= LEGACY_REQUEST_SIZE;
988 return 0;
989}
990
991static void gen6_bsd_submit_request(struct i915_request *request)
992{
993 struct intel_uncore *uncore = request->engine->uncore;
994
995 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
996
997
998
999
1000
1001
1002 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
1003 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1004
1005
1006 intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0);
1007
1008
1009 if (__intel_wait_for_register_fw(uncore,
1010 GEN6_BSD_SLEEP_PSMI_CONTROL,
1011 GEN6_BSD_SLEEP_INDICATOR,
1012 0,
1013 1000, 0, NULL))
1014 drm_err(&uncore->i915->drm,
1015 "timed out waiting for the BSD ring to wake up\n");
1016
1017
1018 i9xx_submit_request(request);
1019
1020
1021
1022
1023 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
1024 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1025
1026 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1027}
1028
1029static void i9xx_set_default_submission(struct intel_engine_cs *engine)
1030{
1031 engine->submit_request = i9xx_submit_request;
1032}
1033
1034static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
1035{
1036 engine->submit_request = gen6_bsd_submit_request;
1037}
1038
1039static void ring_release(struct intel_engine_cs *engine)
1040{
1041 struct drm_i915_private *dev_priv = engine->i915;
1042
1043 drm_WARN_ON(&dev_priv->drm, GRAPHICS_VER(dev_priv) > 2 &&
1044 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
1045
1046 intel_engine_cleanup_common(engine);
1047
1048 if (engine->wa_ctx.vma) {
1049 intel_context_put(engine->wa_ctx.vma->private);
1050 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
1051 }
1052
1053 intel_ring_unpin(engine->legacy.ring);
1054 intel_ring_put(engine->legacy.ring);
1055
1056 intel_timeline_unpin(engine->legacy.timeline);
1057 intel_timeline_put(engine->legacy.timeline);
1058}
1059
1060static void irq_handler(struct intel_engine_cs *engine, u16 iir)
1061{
1062 intel_engine_signal_breadcrumbs(engine);
1063}
1064
1065static void setup_irq(struct intel_engine_cs *engine)
1066{
1067 struct drm_i915_private *i915 = engine->i915;
1068
1069 intel_engine_set_irq_handler(engine, irq_handler);
1070
1071 if (GRAPHICS_VER(i915) >= 6) {
1072 engine->irq_enable = gen6_irq_enable;
1073 engine->irq_disable = gen6_irq_disable;
1074 } else if (GRAPHICS_VER(i915) >= 5) {
1075 engine->irq_enable = gen5_irq_enable;
1076 engine->irq_disable = gen5_irq_disable;
1077 } else if (GRAPHICS_VER(i915) >= 3) {
1078 engine->irq_enable = gen3_irq_enable;
1079 engine->irq_disable = gen3_irq_disable;
1080 } else {
1081 engine->irq_enable = gen2_irq_enable;
1082 engine->irq_disable = gen2_irq_disable;
1083 }
1084}
1085
1086static void add_to_engine(struct i915_request *rq)
1087{
1088 lockdep_assert_held(&rq->engine->sched_engine->lock);
1089 list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests);
1090}
1091
1092static void remove_from_engine(struct i915_request *rq)
1093{
1094 spin_lock_irq(&rq->engine->sched_engine->lock);
1095 list_del_init(&rq->sched.link);
1096
1097
1098 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
1099
1100 spin_unlock_irq(&rq->engine->sched_engine->lock);
1101
1102 i915_request_notify_execute_cb_imm(rq);
1103}
1104
1105static void setup_common(struct intel_engine_cs *engine)
1106{
1107 struct drm_i915_private *i915 = engine->i915;
1108
1109
1110 GEM_BUG_ON(GRAPHICS_VER(i915) >= 8);
1111
1112 setup_irq(engine);
1113
1114 engine->resume = xcs_resume;
1115 engine->sanitize = xcs_sanitize;
1116
1117 engine->reset.prepare = reset_prepare;
1118 engine->reset.rewind = reset_rewind;
1119 engine->reset.cancel = reset_cancel;
1120 engine->reset.finish = reset_finish;
1121
1122 engine->add_active_request = add_to_engine;
1123 engine->remove_active_request = remove_from_engine;
1124
1125 engine->cops = &ring_context_ops;
1126 engine->request_alloc = ring_request_alloc;
1127
1128
1129
1130
1131
1132
1133 engine->emit_fini_breadcrumb = gen3_emit_breadcrumb;
1134 if (GRAPHICS_VER(i915) == 5)
1135 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb;
1136
1137 engine->set_default_submission = i9xx_set_default_submission;
1138
1139 if (GRAPHICS_VER(i915) >= 6)
1140 engine->emit_bb_start = gen6_emit_bb_start;
1141 else if (GRAPHICS_VER(i915) >= 4)
1142 engine->emit_bb_start = gen4_emit_bb_start;
1143 else if (IS_I830(i915) || IS_I845G(i915))
1144 engine->emit_bb_start = i830_emit_bb_start;
1145 else
1146 engine->emit_bb_start = gen3_emit_bb_start;
1147}
1148
1149static void setup_rcs(struct intel_engine_cs *engine)
1150{
1151 struct drm_i915_private *i915 = engine->i915;
1152
1153 if (HAS_L3_DPF(i915))
1154 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1155
1156 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
1157
1158 if (GRAPHICS_VER(i915) >= 7) {
1159 engine->emit_flush = gen7_emit_flush_rcs;
1160 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs;
1161 } else if (GRAPHICS_VER(i915) == 6) {
1162 engine->emit_flush = gen6_emit_flush_rcs;
1163 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs;
1164 } else if (GRAPHICS_VER(i915) == 5) {
1165 engine->emit_flush = gen4_emit_flush_rcs;
1166 } else {
1167 if (GRAPHICS_VER(i915) < 4)
1168 engine->emit_flush = gen2_emit_flush;
1169 else
1170 engine->emit_flush = gen4_emit_flush_rcs;
1171 engine->irq_enable_mask = I915_USER_INTERRUPT;
1172 }
1173
1174 if (IS_HASWELL(i915))
1175 engine->emit_bb_start = hsw_emit_bb_start;
1176}
1177
1178static void setup_vcs(struct intel_engine_cs *engine)
1179{
1180 struct drm_i915_private *i915 = engine->i915;
1181
1182 if (GRAPHICS_VER(i915) >= 6) {
1183
1184 if (GRAPHICS_VER(i915) == 6)
1185 engine->set_default_submission = gen6_bsd_set_default_submission;
1186 engine->emit_flush = gen6_emit_flush_vcs;
1187 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1188
1189 if (GRAPHICS_VER(i915) == 6)
1190 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;
1191 else
1192 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1193 } else {
1194 engine->emit_flush = gen4_emit_flush_vcs;
1195 if (GRAPHICS_VER(i915) == 5)
1196 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
1197 else
1198 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1199 }
1200}
1201
1202static void setup_bcs(struct intel_engine_cs *engine)
1203{
1204 struct drm_i915_private *i915 = engine->i915;
1205
1206 engine->emit_flush = gen6_emit_flush_xcs;
1207 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
1208
1209 if (GRAPHICS_VER(i915) == 6)
1210 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;
1211 else
1212 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1213}
1214
1215static void setup_vecs(struct intel_engine_cs *engine)
1216{
1217 struct drm_i915_private *i915 = engine->i915;
1218
1219 GEM_BUG_ON(GRAPHICS_VER(i915) < 7);
1220
1221 engine->emit_flush = gen6_emit_flush_xcs;
1222 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
1223 engine->irq_enable = hsw_irq_enable_vecs;
1224 engine->irq_disable = hsw_irq_disable_vecs;
1225
1226 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1227}
1228
1229static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine,
1230 struct i915_vma * const vma)
1231{
1232 return gen7_setup_clear_gpr_bb(engine, vma);
1233}
1234
1235static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine,
1236 struct i915_gem_ww_ctx *ww,
1237 struct i915_vma *vma)
1238{
1239 int err;
1240
1241 err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER | PIN_HIGH);
1242 if (err)
1243 return err;
1244
1245 err = i915_vma_sync(vma);
1246 if (err)
1247 goto err_unpin;
1248
1249 err = gen7_ctx_switch_bb_setup(engine, vma);
1250 if (err)
1251 goto err_unpin;
1252
1253 engine->wa_ctx.vma = vma;
1254 return 0;
1255
1256err_unpin:
1257 i915_vma_unpin(vma);
1258 return err;
1259}
1260
1261static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine)
1262{
1263 struct drm_i915_gem_object *obj;
1264 struct i915_vma *vma;
1265 int size, err;
1266
1267 if (GRAPHICS_VER(engine->i915) != 7 || engine->class != RENDER_CLASS)
1268 return 0;
1269
1270 err = gen7_ctx_switch_bb_setup(engine, NULL );
1271 if (err < 0)
1272 return ERR_PTR(err);
1273 if (!err)
1274 return NULL;
1275
1276 size = ALIGN(err, PAGE_SIZE);
1277
1278 obj = i915_gem_object_create_internal(engine->i915, size);
1279 if (IS_ERR(obj))
1280 return ERR_CAST(obj);
1281
1282 vma = i915_vma_instance(obj, engine->gt->vm, NULL);
1283 if (IS_ERR(vma)) {
1284 i915_gem_object_put(obj);
1285 return ERR_CAST(vma);
1286 }
1287
1288 vma->private = intel_context_create(engine);
1289 if (IS_ERR(vma->private)) {
1290 err = PTR_ERR(vma->private);
1291 vma->private = NULL;
1292 i915_gem_object_put(obj);
1293 return ERR_PTR(err);
1294 }
1295
1296 return vma;
1297}
1298
1299int intel_ring_submission_setup(struct intel_engine_cs *engine)
1300{
1301 struct i915_gem_ww_ctx ww;
1302 struct intel_timeline *timeline;
1303 struct intel_ring *ring;
1304 struct i915_vma *gen7_wa_vma;
1305 int err;
1306
1307 setup_common(engine);
1308
1309 switch (engine->class) {
1310 case RENDER_CLASS:
1311 setup_rcs(engine);
1312 break;
1313 case VIDEO_DECODE_CLASS:
1314 setup_vcs(engine);
1315 break;
1316 case COPY_ENGINE_CLASS:
1317 setup_bcs(engine);
1318 break;
1319 case VIDEO_ENHANCEMENT_CLASS:
1320 setup_vecs(engine);
1321 break;
1322 default:
1323 MISSING_CASE(engine->class);
1324 return -ENODEV;
1325 }
1326
1327 timeline = intel_timeline_create_from_engine(engine,
1328 I915_GEM_HWS_SEQNO_ADDR);
1329 if (IS_ERR(timeline)) {
1330 err = PTR_ERR(timeline);
1331 goto err;
1332 }
1333 GEM_BUG_ON(timeline->has_initial_breadcrumb);
1334
1335 ring = intel_engine_create_ring(engine, SZ_16K);
1336 if (IS_ERR(ring)) {
1337 err = PTR_ERR(ring);
1338 goto err_timeline;
1339 }
1340
1341 GEM_BUG_ON(engine->legacy.ring);
1342 engine->legacy.ring = ring;
1343 engine->legacy.timeline = timeline;
1344
1345 gen7_wa_vma = gen7_ctx_vma(engine);
1346 if (IS_ERR(gen7_wa_vma)) {
1347 err = PTR_ERR(gen7_wa_vma);
1348 goto err_ring;
1349 }
1350
1351 i915_gem_ww_ctx_init(&ww, false);
1352
1353retry:
1354 err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww);
1355 if (!err && gen7_wa_vma)
1356 err = i915_gem_object_lock(gen7_wa_vma->obj, &ww);
1357 if (!err && engine->legacy.ring->vma->obj)
1358 err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww);
1359 if (!err)
1360 err = intel_timeline_pin(timeline, &ww);
1361 if (!err) {
1362 err = intel_ring_pin(ring, &ww);
1363 if (err)
1364 intel_timeline_unpin(timeline);
1365 }
1366 if (err)
1367 goto out;
1368
1369 GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
1370
1371 if (gen7_wa_vma) {
1372 err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma);
1373 if (err) {
1374 intel_ring_unpin(ring);
1375 intel_timeline_unpin(timeline);
1376 }
1377 }
1378
1379out:
1380 if (err == -EDEADLK) {
1381 err = i915_gem_ww_ctx_backoff(&ww);
1382 if (!err)
1383 goto retry;
1384 }
1385 i915_gem_ww_ctx_fini(&ww);
1386 if (err)
1387 goto err_gen7_put;
1388
1389
1390 engine->release = ring_release;
1391
1392 return 0;
1393
1394err_gen7_put:
1395 if (gen7_wa_vma) {
1396 intel_context_put(gen7_wa_vma->private);
1397 i915_gem_object_put(gen7_wa_vma->obj);
1398 }
1399err_ring:
1400 intel_ring_put(ring);
1401err_timeline:
1402 intel_timeline_put(timeline);
1403err:
1404 intel_engine_cleanup_common(engine);
1405 return err;
1406}
1407
1408#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1409#include "selftest_ring_submission.c"
1410#endif
1411