1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134#include <linux/interrupt.h>
135
136#include <drm/drmP.h>
137#include <drm/i915_drm.h>
138#include "i915_drv.h"
139#include "i915_gem_render_state.h"
140#include "intel_lrc_reg.h"
141#include "intel_mocs.h"
142
143#define RING_EXECLIST_QFULL (1 << 0x2)
144#define RING_EXECLIST1_VALID (1 << 0x3)
145#define RING_EXECLIST0_VALID (1 << 0x4)
146#define RING_EXECLIST_ACTIVE_STATUS (3 << 0xE)
147#define RING_EXECLIST1_ACTIVE (1 << 0x11)
148#define RING_EXECLIST0_ACTIVE (1 << 0x12)
149
150#define GEN8_CTX_STATUS_IDLE_ACTIVE (1 << 0)
151#define GEN8_CTX_STATUS_PREEMPTED (1 << 1)
152#define GEN8_CTX_STATUS_ELEMENT_SWITCH (1 << 2)
153#define GEN8_CTX_STATUS_ACTIVE_IDLE (1 << 3)
154#define GEN8_CTX_STATUS_COMPLETE (1 << 4)
155#define GEN8_CTX_STATUS_LITE_RESTORE (1 << 15)
156
157#define GEN8_CTX_STATUS_COMPLETED_MASK \
158 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
159
160
161#define EXECLISTS_REQUEST_SIZE 64
162#define WA_TAIL_DWORDS 2
163#define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS)
164
165static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
166 struct intel_engine_cs *engine);
167static void execlists_init_reg_state(u32 *reg_state,
168 struct i915_gem_context *ctx,
169 struct intel_engine_cs *engine,
170 struct intel_ring *ring);
171
172static inline struct i915_priolist *to_priolist(struct rb_node *rb)
173{
174 return rb_entry(rb, struct i915_priolist, node);
175}
176
177static inline int rq_prio(const struct i915_request *rq)
178{
179 return rq->priotree.priority;
180}
181
182static inline bool need_preempt(const struct intel_engine_cs *engine,
183 const struct i915_request *last,
184 int prio)
185{
186 return engine->i915->preempt_context && prio > max(rq_prio(last), 0);
187}
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220static void
221intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
222 struct intel_engine_cs *engine)
223{
224 struct intel_context *ce = &ctx->engine[engine->id];
225 u64 desc;
226
227 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (BIT(GEN8_CTX_ID_WIDTH)));
228 BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > (BIT(GEN11_SW_CTX_ID_WIDTH)));
229
230 desc = ctx->desc_template;
231 GEM_BUG_ON(desc & GENMASK_ULL(63, 12));
232
233 desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE;
234
235 GEM_BUG_ON(desc & GENMASK_ULL(63, 32));
236
237 if (INTEL_GEN(ctx->i915) >= 11) {
238 GEM_BUG_ON(ctx->hw_id >= BIT(GEN11_SW_CTX_ID_WIDTH));
239 desc |= (u64)ctx->hw_id << GEN11_SW_CTX_ID_SHIFT;
240
241
242 desc |= (u64)engine->instance << GEN11_ENGINE_INSTANCE_SHIFT;
243
244
245
246
247 desc |= (u64)engine->class << GEN11_ENGINE_CLASS_SHIFT;
248
249 } else {
250 GEM_BUG_ON(ctx->hw_id >= BIT(GEN8_CTX_ID_WIDTH));
251 desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;
252 }
253
254 ce->lrc_desc = desc;
255}
256
257static struct i915_priolist *
258lookup_priolist(struct intel_engine_cs *engine,
259 struct i915_priotree *pt,
260 int prio)
261{
262 struct intel_engine_execlists * const execlists = &engine->execlists;
263 struct i915_priolist *p;
264 struct rb_node **parent, *rb;
265 bool first = true;
266
267 if (unlikely(execlists->no_priolist))
268 prio = I915_PRIORITY_NORMAL;
269
270find_priolist:
271
272 rb = NULL;
273 parent = &execlists->queue.rb_node;
274 while (*parent) {
275 rb = *parent;
276 p = to_priolist(rb);
277 if (prio > p->priority) {
278 parent = &rb->rb_left;
279 } else if (prio < p->priority) {
280 parent = &rb->rb_right;
281 first = false;
282 } else {
283 return p;
284 }
285 }
286
287 if (prio == I915_PRIORITY_NORMAL) {
288 p = &execlists->default_priolist;
289 } else {
290 p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC);
291
292 if (unlikely(!p)) {
293 prio = I915_PRIORITY_NORMAL;
294
295
296
297
298
299
300
301
302
303 execlists->no_priolist = true;
304 goto find_priolist;
305 }
306 }
307
308 p->priority = prio;
309 INIT_LIST_HEAD(&p->requests);
310 rb_link_node(&p->node, rb, parent);
311 rb_insert_color(&p->node, &execlists->queue);
312
313 if (first)
314 execlists->first = &p->node;
315
316 return p;
317}
318
319static void unwind_wa_tail(struct i915_request *rq)
320{
321 rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES);
322 assert_ring_tail_valid(rq->ring, rq->tail);
323}
324
325static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
326{
327 struct i915_request *rq, *rn;
328 struct i915_priolist *uninitialized_var(p);
329 int last_prio = I915_PRIORITY_INVALID;
330
331 lockdep_assert_held(&engine->timeline->lock);
332
333 list_for_each_entry_safe_reverse(rq, rn,
334 &engine->timeline->requests,
335 link) {
336 if (i915_request_completed(rq))
337 return;
338
339 __i915_request_unsubmit(rq);
340 unwind_wa_tail(rq);
341
342 GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
343 if (rq_prio(rq) != last_prio) {
344 last_prio = rq_prio(rq);
345 p = lookup_priolist(engine, &rq->priotree, last_prio);
346 }
347
348 list_add(&rq->priotree.link, &p->requests);
349 }
350}
351
352void
353execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
354{
355 struct intel_engine_cs *engine =
356 container_of(execlists, typeof(*engine), execlists);
357
358 spin_lock_irq(&engine->timeline->lock);
359 __unwind_incomplete_requests(engine);
360 spin_unlock_irq(&engine->timeline->lock);
361}
362
363static inline void
364execlists_context_status_change(struct i915_request *rq, unsigned long status)
365{
366
367
368
369
370 if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
371 return;
372
373 atomic_notifier_call_chain(&rq->engine->context_status_notifier,
374 status, rq);
375}
376
377static inline void
378execlists_context_schedule_in(struct i915_request *rq)
379{
380 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
381 intel_engine_context_in(rq->engine);
382}
383
384static inline void
385execlists_context_schedule_out(struct i915_request *rq)
386{
387 intel_engine_context_out(rq->engine);
388 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
389}
390
391static void
392execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
393{
394 ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
395 ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
396 ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
397 ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
398}
399
400static u64 execlists_update_context(struct i915_request *rq)
401{
402 struct intel_context *ce = &rq->ctx->engine[rq->engine->id];
403 struct i915_hw_ppgtt *ppgtt =
404 rq->ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt;
405 u32 *reg_state = ce->lrc_reg_state;
406
407 reg_state[CTX_RING_TAIL+1] = intel_ring_set_tail(rq->ring, rq->tail);
408
409
410
411
412
413
414 if (ppgtt && !i915_vm_is_48bit(&ppgtt->base))
415 execlists_update_context_pdps(ppgtt, reg_state);
416
417 return ce->lrc_desc;
418}
419
420static inline void write_desc(struct intel_engine_execlists *execlists, u64 desc, u32 port)
421{
422 if (execlists->ctrl_reg) {
423 writel(lower_32_bits(desc), execlists->submit_reg + port * 2);
424 writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);
425 } else {
426 writel(upper_32_bits(desc), execlists->submit_reg);
427 writel(lower_32_bits(desc), execlists->submit_reg);
428 }
429}
430
431static void execlists_submit_ports(struct intel_engine_cs *engine)
432{
433 struct intel_engine_execlists *execlists = &engine->execlists;
434 struct execlist_port *port = execlists->port;
435 unsigned int n;
436
437
438
439
440
441
442
443 for (n = execlists_num_ports(execlists); n--; ) {
444 struct i915_request *rq;
445 unsigned int count;
446 u64 desc;
447
448 rq = port_unpack(&port[n], &count);
449 if (rq) {
450 GEM_BUG_ON(count > !n);
451 if (!count++)
452 execlists_context_schedule_in(rq);
453 port_set(&port[n], port_pack(rq, count));
454 desc = execlists_update_context(rq);
455 GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
456
457 GEM_TRACE("%s in[%d]: ctx=%d.%d, seqno=%x, prio=%d\n",
458 engine->name, n,
459 port[n].context_id, count,
460 rq->global_seqno,
461 rq_prio(rq));
462 } else {
463 GEM_BUG_ON(!n);
464 desc = 0;
465 }
466
467 write_desc(execlists, desc, n);
468 }
469
470
471 if (execlists->ctrl_reg)
472 writel(EL_CTRL_LOAD, execlists->ctrl_reg);
473
474 execlists_clear_active(execlists, EXECLISTS_ACTIVE_HWACK);
475}
476
477static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
478{
479 return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
480 i915_gem_context_force_single_submission(ctx));
481}
482
483static bool can_merge_ctx(const struct i915_gem_context *prev,
484 const struct i915_gem_context *next)
485{
486 if (prev != next)
487 return false;
488
489 if (ctx_single_port_submission(prev))
490 return false;
491
492 return true;
493}
494
495static void port_assign(struct execlist_port *port, struct i915_request *rq)
496{
497 GEM_BUG_ON(rq == port_request(port));
498
499 if (port_isset(port))
500 i915_request_put(port_request(port));
501
502 port_set(port, port_pack(i915_request_get(rq), port_count(port)));
503}
504
505static void inject_preempt_context(struct intel_engine_cs *engine)
506{
507 struct intel_engine_execlists *execlists = &engine->execlists;
508 struct intel_context *ce =
509 &engine->i915->preempt_context->engine[engine->id];
510 unsigned int n;
511
512 GEM_BUG_ON(execlists->preempt_complete_status !=
513 upper_32_bits(ce->lrc_desc));
514 GEM_BUG_ON((ce->lrc_reg_state[CTX_CONTEXT_CONTROL + 1] &
515 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
516 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT)) !=
517 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
518 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT));
519
520
521
522
523
524 GEM_TRACE("%s\n", engine->name);
525 for (n = execlists_num_ports(execlists); --n; )
526 write_desc(execlists, 0, n);
527
528 write_desc(execlists, ce->lrc_desc, n);
529
530
531 if (execlists->ctrl_reg)
532 writel(EL_CTRL_LOAD, execlists->ctrl_reg);
533
534 execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK);
535 execlists_set_active(&engine->execlists, EXECLISTS_ACTIVE_PREEMPT);
536}
537
538static void execlists_dequeue(struct intel_engine_cs *engine)
539{
540 struct intel_engine_execlists * const execlists = &engine->execlists;
541 struct execlist_port *port = execlists->port;
542 const struct execlist_port * const last_port =
543 &execlists->port[execlists->port_mask];
544 struct i915_request *last = port_request(port);
545 struct rb_node *rb;
546 bool submit = false;
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569 spin_lock_irq(&engine->timeline->lock);
570 rb = execlists->first;
571 GEM_BUG_ON(rb_first(&execlists->queue) != rb);
572
573 if (last) {
574
575
576
577
578
579
580 GEM_BUG_ON(!execlists_is_active(execlists,
581 EXECLISTS_ACTIVE_USER));
582 GEM_BUG_ON(!port_count(&port[0]));
583 if (port_count(&port[0]) > 1)
584 goto unlock;
585
586
587
588
589
590
591
592
593 if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
594 goto unlock;
595
596 if (need_preempt(engine, last, execlists->queue_priority)) {
597 inject_preempt_context(engine);
598 goto unlock;
599 }
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622 if (port_count(&port[1]))
623 goto unlock;
624
625
626
627
628
629
630
631
632
633 last->tail = last->wa_tail;
634 }
635
636 while (rb) {
637 struct i915_priolist *p = to_priolist(rb);
638 struct i915_request *rq, *rn;
639
640 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
641
642
643
644
645
646
647
648
649
650
651
652 if (last && !can_merge_ctx(rq->ctx, last->ctx)) {
653
654
655
656
657
658 if (port == last_port) {
659 __list_del_many(&p->requests,
660 &rq->priotree.link);
661 goto done;
662 }
663
664
665
666
667
668
669
670
671 if (ctx_single_port_submission(last->ctx) ||
672 ctx_single_port_submission(rq->ctx)) {
673 __list_del_many(&p->requests,
674 &rq->priotree.link);
675 goto done;
676 }
677
678 GEM_BUG_ON(last->ctx == rq->ctx);
679
680 if (submit)
681 port_assign(port, last);
682 port++;
683
684 GEM_BUG_ON(port_isset(port));
685 }
686
687 INIT_LIST_HEAD(&rq->priotree.link);
688 __i915_request_submit(rq);
689 trace_i915_request_in(rq, port_index(port, execlists));
690 last = rq;
691 submit = true;
692 }
693
694 rb = rb_next(rb);
695 rb_erase(&p->node, &execlists->queue);
696 INIT_LIST_HEAD(&p->requests);
697 if (p->priority != I915_PRIORITY_NORMAL)
698 kmem_cache_free(engine->i915->priorities, p);
699 }
700done:
701 execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
702 execlists->first = rb;
703 if (submit)
704 port_assign(port, last);
705
706
707 GEM_BUG_ON(execlists->first && !port_isset(execlists->port));
708
709unlock:
710 spin_unlock_irq(&engine->timeline->lock);
711
712 if (submit) {
713 execlists_set_active(execlists, EXECLISTS_ACTIVE_USER);
714 execlists_submit_ports(engine);
715 }
716
717 GEM_BUG_ON(port_isset(execlists->port) &&
718 !execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
719}
720
721void
722execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
723{
724 struct execlist_port *port = execlists->port;
725 unsigned int num_ports = execlists_num_ports(execlists);
726
727 while (num_ports-- && port_isset(port)) {
728 struct i915_request *rq = port_request(port);
729
730 GEM_BUG_ON(!execlists->active);
731 intel_engine_context_out(rq->engine);
732
733 execlists_context_status_change(rq,
734 i915_request_completed(rq) ?
735 INTEL_CONTEXT_SCHEDULE_OUT :
736 INTEL_CONTEXT_SCHEDULE_PREEMPTED);
737
738 i915_request_put(rq);
739
740 memset(port, 0, sizeof(*port));
741 port++;
742 }
743
744 execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
745}
746
747static void execlists_cancel_requests(struct intel_engine_cs *engine)
748{
749 struct intel_engine_execlists * const execlists = &engine->execlists;
750 struct i915_request *rq, *rn;
751 struct rb_node *rb;
752 unsigned long flags;
753
754 GEM_TRACE("%s\n", engine->name);
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770 local_irq_save(flags);
771
772
773 execlists_cancel_port_requests(execlists);
774
775 spin_lock(&engine->timeline->lock);
776
777
778 list_for_each_entry(rq, &engine->timeline->requests, link) {
779 GEM_BUG_ON(!rq->global_seqno);
780 if (!i915_request_completed(rq))
781 dma_fence_set_error(&rq->fence, -EIO);
782 }
783
784
785 rb = execlists->first;
786 while (rb) {
787 struct i915_priolist *p = to_priolist(rb);
788
789 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
790 INIT_LIST_HEAD(&rq->priotree.link);
791
792 dma_fence_set_error(&rq->fence, -EIO);
793 __i915_request_submit(rq);
794 }
795
796 rb = rb_next(rb);
797 rb_erase(&p->node, &execlists->queue);
798 INIT_LIST_HEAD(&p->requests);
799 if (p->priority != I915_PRIORITY_NORMAL)
800 kmem_cache_free(engine->i915->priorities, p);
801 }
802
803
804
805 execlists->queue_priority = INT_MIN;
806 execlists->queue = RB_ROOT;
807 execlists->first = NULL;
808 GEM_BUG_ON(port_isset(execlists->port));
809
810 spin_unlock(&engine->timeline->lock);
811
812
813
814
815
816
817
818 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
819
820
821 execlists->active = 0;
822
823 local_irq_restore(flags);
824}
825
826
827
828
829
830static void execlists_submission_tasklet(unsigned long data)
831{
832 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
833 struct intel_engine_execlists * const execlists = &engine->execlists;
834 struct execlist_port * const port = execlists->port;
835 struct drm_i915_private *dev_priv = engine->i915;
836 bool fw = false;
837
838
839
840
841
842
843
844
845
846 GEM_BUG_ON(!dev_priv->gt.awake);
847
848
849
850
851
852
853 while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) {
854
855 const u32 *buf =
856 &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
857 unsigned int head, tail;
858
859 if (unlikely(execlists->csb_use_mmio)) {
860 buf = (u32 * __force)
861 (dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));
862 execlists->csb_head = -1;
863 }
864
865
866 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
867 smp_mb__after_atomic();
868
869 if (unlikely(execlists->csb_head == -1)) {
870 if (!fw) {
871 intel_uncore_forcewake_get(dev_priv,
872 execlists->fw_domains);
873 fw = true;
874 }
875
876 head = readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
877 tail = GEN8_CSB_WRITE_PTR(head);
878 head = GEN8_CSB_READ_PTR(head);
879 execlists->csb_head = head;
880 } else {
881 const int write_idx =
882 intel_hws_csb_write_index(dev_priv) -
883 I915_HWS_CSB_BUF0_INDEX;
884
885 head = execlists->csb_head;
886 tail = READ_ONCE(buf[write_idx]);
887 rmb();
888 }
889 GEM_TRACE("%s cs-irq head=%d [%d%s], tail=%d [%d%s]\n",
890 engine->name,
891 head, GEN8_CSB_READ_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?",
892 tail, GEN8_CSB_WRITE_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?");
893
894 while (head != tail) {
895 struct i915_request *rq;
896 unsigned int status;
897 unsigned int count;
898
899 if (++head == GEN8_CSB_ENTRIES)
900 head = 0;
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919 status = READ_ONCE(buf[2 * head]);
920 GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
921 engine->name, head,
922 status, buf[2*head + 1],
923 execlists->active);
924
925 if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
926 GEN8_CTX_STATUS_PREEMPTED))
927 execlists_set_active(execlists,
928 EXECLISTS_ACTIVE_HWACK);
929 if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
930 execlists_clear_active(execlists,
931 EXECLISTS_ACTIVE_HWACK);
932
933 if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
934 continue;
935
936
937 GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);
938
939 if (status & GEN8_CTX_STATUS_COMPLETE &&
940 buf[2*head + 1] == execlists->preempt_complete_status) {
941 GEM_TRACE("%s preempt-idle\n", engine->name);
942
943 execlists_cancel_port_requests(execlists);
944 execlists_unwind_incomplete_requests(execlists);
945
946 GEM_BUG_ON(!execlists_is_active(execlists,
947 EXECLISTS_ACTIVE_PREEMPT));
948 execlists_clear_active(execlists,
949 EXECLISTS_ACTIVE_PREEMPT);
950 continue;
951 }
952
953 if (status & GEN8_CTX_STATUS_PREEMPTED &&
954 execlists_is_active(execlists,
955 EXECLISTS_ACTIVE_PREEMPT))
956 continue;
957
958 GEM_BUG_ON(!execlists_is_active(execlists,
959 EXECLISTS_ACTIVE_USER));
960
961 rq = port_unpack(port, &count);
962 GEM_TRACE("%s out[0]: ctx=%d.%d, seqno=%x, prio=%d\n",
963 engine->name,
964 port->context_id, count,
965 rq ? rq->global_seqno : 0,
966 rq ? rq_prio(rq) : 0);
967
968
969 GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
970
971 GEM_BUG_ON(count == 0);
972 if (--count == 0) {
973 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
974 GEM_BUG_ON(port_isset(&port[1]) &&
975 !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
976 GEM_BUG_ON(!i915_request_completed(rq));
977 execlists_context_schedule_out(rq);
978 trace_i915_request_out(rq);
979 i915_request_put(rq);
980
981 GEM_TRACE("%s completed ctx=%d\n",
982 engine->name, port->context_id);
983
984 execlists_port_complete(execlists, port);
985 } else {
986 port_set(port, port_pack(rq, count));
987 }
988
989
990 GEM_BUG_ON(port_count(port) == 0 &&
991 !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
992 if (port_count(port) == 0)
993 execlists_clear_active(execlists,
994 EXECLISTS_ACTIVE_USER);
995 }
996
997 if (head != execlists->csb_head) {
998 execlists->csb_head = head;
999 writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8),
1000 dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
1001 }
1002 }
1003
1004 if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT))
1005 execlists_dequeue(engine);
1006
1007 if (fw)
1008 intel_uncore_forcewake_put(dev_priv, execlists->fw_domains);
1009
1010
1011 GEM_BUG_ON(execlists_is_active(&engine->execlists,
1012 EXECLISTS_ACTIVE_USER) ==
1013 !port_isset(engine->execlists.port));
1014}
1015
1016static void queue_request(struct intel_engine_cs *engine,
1017 struct i915_priotree *pt,
1018 int prio)
1019{
1020 list_add_tail(&pt->link, &lookup_priolist(engine, pt, prio)->requests);
1021}
1022
1023static void submit_queue(struct intel_engine_cs *engine, int prio)
1024{
1025 if (prio > engine->execlists.queue_priority) {
1026 engine->execlists.queue_priority = prio;
1027 tasklet_hi_schedule(&engine->execlists.tasklet);
1028 }
1029}
1030
1031static void execlists_submit_request(struct i915_request *request)
1032{
1033 struct intel_engine_cs *engine = request->engine;
1034 unsigned long flags;
1035
1036
1037 spin_lock_irqsave(&engine->timeline->lock, flags);
1038
1039 queue_request(engine, &request->priotree, rq_prio(request));
1040 submit_queue(engine, rq_prio(request));
1041
1042 GEM_BUG_ON(!engine->execlists.first);
1043 GEM_BUG_ON(list_empty(&request->priotree.link));
1044
1045 spin_unlock_irqrestore(&engine->timeline->lock, flags);
1046}
1047
1048static struct i915_request *pt_to_request(struct i915_priotree *pt)
1049{
1050 return container_of(pt, struct i915_request, priotree);
1051}
1052
1053static struct intel_engine_cs *
1054pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
1055{
1056 struct intel_engine_cs *engine = pt_to_request(pt)->engine;
1057
1058 GEM_BUG_ON(!locked);
1059
1060 if (engine != locked) {
1061 spin_unlock(&locked->timeline->lock);
1062 spin_lock(&engine->timeline->lock);
1063 }
1064
1065 return engine;
1066}
1067
1068static void execlists_schedule(struct i915_request *request, int prio)
1069{
1070 struct intel_engine_cs *engine;
1071 struct i915_dependency *dep, *p;
1072 struct i915_dependency stack;
1073 LIST_HEAD(dfs);
1074
1075 GEM_BUG_ON(prio == I915_PRIORITY_INVALID);
1076
1077 if (i915_request_completed(request))
1078 return;
1079
1080 if (prio <= READ_ONCE(request->priotree.priority))
1081 return;
1082
1083
1084 lockdep_assert_held(&request->i915->drm.struct_mutex);
1085
1086 stack.signaler = &request->priotree;
1087 list_add(&stack.dfs_link, &dfs);
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107 list_for_each_entry(dep, &dfs, dfs_link) {
1108 struct i915_priotree *pt = dep->signaler;
1109
1110
1111
1112
1113
1114
1115
1116 list_for_each_entry(p, &pt->signalers_list, signal_link) {
1117 GEM_BUG_ON(p == dep);
1118
1119 if (i915_priotree_signaled(p->signaler))
1120 continue;
1121
1122 GEM_BUG_ON(p->signaler->priority < pt->priority);
1123 if (prio > READ_ONCE(p->signaler->priority))
1124 list_move_tail(&p->dfs_link, &dfs);
1125 }
1126 }
1127
1128
1129
1130
1131
1132
1133
1134 if (request->priotree.priority == I915_PRIORITY_INVALID) {
1135 GEM_BUG_ON(!list_empty(&request->priotree.link));
1136 request->priotree.priority = prio;
1137 if (stack.dfs_link.next == stack.dfs_link.prev)
1138 return;
1139 __list_del_entry(&stack.dfs_link);
1140 }
1141
1142 engine = request->engine;
1143 spin_lock_irq(&engine->timeline->lock);
1144
1145
1146 list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
1147 struct i915_priotree *pt = dep->signaler;
1148
1149 INIT_LIST_HEAD(&dep->dfs_link);
1150
1151 engine = pt_lock_engine(pt, engine);
1152
1153 if (prio <= pt->priority)
1154 continue;
1155
1156 pt->priority = prio;
1157 if (!list_empty(&pt->link)) {
1158 __list_del_entry(&pt->link);
1159 queue_request(engine, pt, prio);
1160 }
1161 submit_queue(engine, prio);
1162 }
1163
1164 spin_unlock_irq(&engine->timeline->lock);
1165}
1166
1167static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma)
1168{
1169 unsigned int flags;
1170 int err;
1171
1172
1173
1174
1175
1176
1177 if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
1178 err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
1179 if (err)
1180 return err;
1181 }
1182
1183 flags = PIN_GLOBAL | PIN_HIGH;
1184 if (ctx->ggtt_offset_bias)
1185 flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;
1186
1187 return i915_vma_pin(vma, 0, GEN8_LR_CONTEXT_ALIGN, flags);
1188}
1189
1190static struct intel_ring *
1191execlists_context_pin(struct intel_engine_cs *engine,
1192 struct i915_gem_context *ctx)
1193{
1194 struct intel_context *ce = &ctx->engine[engine->id];
1195 void *vaddr;
1196 int ret;
1197
1198 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1199
1200 if (likely(ce->pin_count++))
1201 goto out;
1202 GEM_BUG_ON(!ce->pin_count);
1203
1204 ret = execlists_context_deferred_alloc(ctx, engine);
1205 if (ret)
1206 goto err;
1207 GEM_BUG_ON(!ce->state);
1208
1209 ret = __context_pin(ctx, ce->state);
1210 if (ret)
1211 goto err;
1212
1213 vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
1214 if (IS_ERR(vaddr)) {
1215 ret = PTR_ERR(vaddr);
1216 goto unpin_vma;
1217 }
1218
1219 ret = intel_ring_pin(ce->ring, ctx->i915, ctx->ggtt_offset_bias);
1220 if (ret)
1221 goto unpin_map;
1222
1223 intel_lr_context_descriptor_update(ctx, engine);
1224
1225 ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
1226 ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
1227 i915_ggtt_offset(ce->ring->vma);
1228
1229 ce->state->obj->pin_global++;
1230 i915_gem_context_get(ctx);
1231out:
1232 return ce->ring;
1233
1234unpin_map:
1235 i915_gem_object_unpin_map(ce->state->obj);
1236unpin_vma:
1237 __i915_vma_unpin(ce->state);
1238err:
1239 ce->pin_count = 0;
1240 return ERR_PTR(ret);
1241}
1242
1243static void execlists_context_unpin(struct intel_engine_cs *engine,
1244 struct i915_gem_context *ctx)
1245{
1246 struct intel_context *ce = &ctx->engine[engine->id];
1247
1248 lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1249 GEM_BUG_ON(ce->pin_count == 0);
1250
1251 if (--ce->pin_count)
1252 return;
1253
1254 intel_ring_unpin(ce->ring);
1255
1256 ce->state->obj->pin_global--;
1257 i915_gem_object_unpin_map(ce->state->obj);
1258 i915_vma_unpin(ce->state);
1259
1260 i915_gem_context_put(ctx);
1261}
1262
1263static int execlists_request_alloc(struct i915_request *request)
1264{
1265 struct intel_engine_cs *engine = request->engine;
1266 struct intel_context *ce = &request->ctx->engine[engine->id];
1267 int ret;
1268
1269 GEM_BUG_ON(!ce->pin_count);
1270
1271
1272
1273
1274
1275 request->reserved_space += EXECLISTS_REQUEST_SIZE;
1276
1277 ret = intel_ring_wait_for_space(request->ring, request->reserved_space);
1278 if (ret)
1279 return ret;
1280
1281
1282
1283
1284
1285
1286
1287
1288 request->reserved_space -= EXECLISTS_REQUEST_SIZE;
1289 return 0;
1290}
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308static u32 *
1309gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch)
1310{
1311 *batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
1312 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
1313 *batch++ = i915_ggtt_offset(engine->scratch) + 256;
1314 *batch++ = 0;
1315
1316 *batch++ = MI_LOAD_REGISTER_IMM(1);
1317 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
1318 *batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES;
1319
1320 batch = gen8_emit_pipe_control(batch,
1321 PIPE_CONTROL_CS_STALL |
1322 PIPE_CONTROL_DC_FLUSH_ENABLE,
1323 0);
1324
1325 *batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
1326 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
1327 *batch++ = i915_ggtt_offset(engine->scratch) + 256;
1328 *batch++ = 0;
1329
1330 return batch;
1331}
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
1349{
1350
1351 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1352
1353
1354 if (IS_BROADWELL(engine->i915))
1355 batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1356
1357
1358
1359 batch = gen8_emit_pipe_control(batch,
1360 PIPE_CONTROL_FLUSH_L3 |
1361 PIPE_CONTROL_GLOBAL_GTT_IVB |
1362 PIPE_CONTROL_CS_STALL |
1363 PIPE_CONTROL_QW_WRITE,
1364 i915_ggtt_offset(engine->scratch) +
1365 2 * CACHELINE_BYTES);
1366
1367 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1368
1369
1370 while ((unsigned long)batch % CACHELINE_BYTES)
1371 *batch++ = MI_NOOP;
1372
1373
1374
1375
1376
1377
1378
1379 return batch;
1380}
1381
1382static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
1383{
1384 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1385
1386
1387 batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1388
1389
1390 *batch++ = MI_LOAD_REGISTER_IMM(1);
1391 *batch++ = i915_mmio_reg_offset(COMMON_SLICE_CHICKEN2);
1392 *batch++ = _MASKED_BIT_DISABLE(
1393 GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE);
1394 *batch++ = MI_NOOP;
1395
1396
1397
1398 if (IS_KBL_REVID(engine->i915, 0, KBL_REVID_A0)) {
1399 batch = gen8_emit_pipe_control(batch,
1400 PIPE_CONTROL_FLUSH_L3 |
1401 PIPE_CONTROL_GLOBAL_GTT_IVB |
1402 PIPE_CONTROL_CS_STALL |
1403 PIPE_CONTROL_QW_WRITE,
1404 i915_ggtt_offset(engine->scratch)
1405 + 2 * CACHELINE_BYTES);
1406 }
1407
1408
1409 if (HAS_POOLED_EU(engine->i915)) {
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423 *batch++ = GEN9_MEDIA_POOL_STATE;
1424 *batch++ = GEN9_MEDIA_POOL_ENABLE;
1425 *batch++ = 0x00777000;
1426 *batch++ = 0;
1427 *batch++ = 0;
1428 *batch++ = 0;
1429 }
1430
1431 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1432
1433
1434 while ((unsigned long)batch % CACHELINE_BYTES)
1435 *batch++ = MI_NOOP;
1436
1437 return batch;
1438}
1439
1440static u32 *
1441gen10_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
1442{
1443 int i;
1444
1445
1446
1447
1448
1449
1450
1451 batch = gen8_emit_pipe_control(batch,
1452 PIPE_CONTROL_CS_STALL,
1453 0);
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464 for (i = 0; i < 10; i++)
1465 *batch++ = MI_NOOP;
1466
1467
1468 while ((unsigned long)batch % CACHELINE_BYTES)
1469 *batch++ = MI_NOOP;
1470
1471 return batch;
1472}
1473
1474#define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE)
1475
1476static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
1477{
1478 struct drm_i915_gem_object *obj;
1479 struct i915_vma *vma;
1480 int err;
1481
1482 obj = i915_gem_object_create(engine->i915, CTX_WA_BB_OBJ_SIZE);
1483 if (IS_ERR(obj))
1484 return PTR_ERR(obj);
1485
1486 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
1487 if (IS_ERR(vma)) {
1488 err = PTR_ERR(vma);
1489 goto err;
1490 }
1491
1492 err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH);
1493 if (err)
1494 goto err;
1495
1496 engine->wa_ctx.vma = vma;
1497 return 0;
1498
1499err:
1500 i915_gem_object_put(obj);
1501 return err;
1502}
1503
1504static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine)
1505{
1506 i915_vma_unpin_and_release(&engine->wa_ctx.vma);
1507}
1508
1509typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);
1510
1511static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1512{
1513 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
1514 struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx,
1515 &wa_ctx->per_ctx };
1516 wa_bb_func_t wa_bb_fn[2];
1517 struct page *page;
1518 void *batch, *batch_ptr;
1519 unsigned int i;
1520 int ret;
1521
1522 if (GEM_WARN_ON(engine->id != RCS))
1523 return -EINVAL;
1524
1525 switch (INTEL_GEN(engine->i915)) {
1526 case 10:
1527 wa_bb_fn[0] = gen10_init_indirectctx_bb;
1528 wa_bb_fn[1] = NULL;
1529 break;
1530 case 9:
1531 wa_bb_fn[0] = gen9_init_indirectctx_bb;
1532 wa_bb_fn[1] = NULL;
1533 break;
1534 case 8:
1535 wa_bb_fn[0] = gen8_init_indirectctx_bb;
1536 wa_bb_fn[1] = NULL;
1537 break;
1538 default:
1539 MISSING_CASE(INTEL_GEN(engine->i915));
1540 return 0;
1541 }
1542
1543 ret = lrc_setup_wa_ctx(engine);
1544 if (ret) {
1545 DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret);
1546 return ret;
1547 }
1548
1549 page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0);
1550 batch = batch_ptr = kmap_atomic(page);
1551
1552
1553
1554
1555
1556
1557 for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) {
1558 wa_bb[i]->offset = batch_ptr - batch;
1559 if (GEM_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset,
1560 CACHELINE_BYTES))) {
1561 ret = -EINVAL;
1562 break;
1563 }
1564 if (wa_bb_fn[i])
1565 batch_ptr = wa_bb_fn[i](engine, batch_ptr);
1566 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset);
1567 }
1568
1569 BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE);
1570
1571 kunmap_atomic(batch);
1572 if (ret)
1573 lrc_destroy_wa_ctx(engine);
1574
1575 return ret;
1576}
1577
1578static u8 gtiir[] = {
1579 [RCS] = 0,
1580 [BCS] = 0,
1581 [VCS] = 1,
1582 [VCS2] = 1,
1583 [VECS] = 3,
1584};
1585
1586static void enable_execlists(struct intel_engine_cs *engine)
1587{
1588 struct drm_i915_private *dev_priv = engine->i915;
1589
1590 I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600 if (INTEL_GEN(dev_priv) >= 11)
1601 I915_WRITE(RING_MODE_GEN7(engine),
1602 _MASKED_BIT_DISABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
1603 else
1604 I915_WRITE(RING_MODE_GEN7(engine),
1605 _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
1606
1607 I915_WRITE(RING_HWS_PGA(engine->mmio_base),
1608 engine->status_page.ggtt_offset);
1609 POSTING_READ(RING_HWS_PGA(engine->mmio_base));
1610
1611
1612 engine->execlists.csb_head = -1;
1613}
1614
1615static int gen8_init_common_ring(struct intel_engine_cs *engine)
1616{
1617 struct intel_engine_execlists * const execlists = &engine->execlists;
1618 int ret;
1619
1620 ret = intel_mocs_init_engine(engine);
1621 if (ret)
1622 return ret;
1623
1624 intel_engine_reset_breadcrumbs(engine);
1625 intel_engine_init_hangcheck(engine);
1626
1627 enable_execlists(engine);
1628
1629
1630 if (execlists->first)
1631 tasklet_schedule(&execlists->tasklet);
1632
1633 return 0;
1634}
1635
1636static int gen8_init_render_ring(struct intel_engine_cs *engine)
1637{
1638 struct drm_i915_private *dev_priv = engine->i915;
1639 int ret;
1640
1641 ret = gen8_init_common_ring(engine);
1642 if (ret)
1643 return ret;
1644
1645
1646
1647
1648
1649
1650
1651 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1652
1653 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1654
1655 return init_workarounds_ring(engine);
1656}
1657
1658static int gen9_init_render_ring(struct intel_engine_cs *engine)
1659{
1660 int ret;
1661
1662 ret = gen8_init_common_ring(engine);
1663 if (ret)
1664 return ret;
1665
1666 return init_workarounds_ring(engine);
1667}
1668
1669static void reset_irq(struct intel_engine_cs *engine)
1670{
1671 struct drm_i915_private *dev_priv = engine->i915;
1672 int i;
1673
1674 GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir));
1675
1676
1677
1678
1679
1680
1681
1682
1683 for (i = 0; i < 2; i++) {
1684 I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
1685 GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
1686 POSTING_READ(GEN8_GT_IIR(gtiir[engine->id]));
1687 }
1688 GEM_BUG_ON(I915_READ(GEN8_GT_IIR(gtiir[engine->id])) &
1689 (GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift));
1690
1691 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
1692}
1693
1694static void reset_common_ring(struct intel_engine_cs *engine,
1695 struct i915_request *request)
1696{
1697 struct intel_engine_execlists * const execlists = &engine->execlists;
1698 struct intel_context *ce;
1699 unsigned long flags;
1700
1701 GEM_TRACE("%s seqno=%x\n",
1702 engine->name, request ? request->global_seqno : 0);
1703
1704
1705 local_irq_save(flags);
1706
1707 reset_irq(engine);
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718 execlists_cancel_port_requests(execlists);
1719
1720
1721 spin_lock(&engine->timeline->lock);
1722 __unwind_incomplete_requests(engine);
1723 spin_unlock(&engine->timeline->lock);
1724
1725
1726 execlists->active = 0;
1727
1728 local_irq_restore(flags);
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741 if (!request || request->fence.error != -EIO)
1742 return;
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752 ce = &request->ctx->engine[engine->id];
1753 execlists_init_reg_state(ce->lrc_reg_state,
1754 request->ctx, engine, ce->ring);
1755
1756
1757 ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
1758 i915_ggtt_offset(ce->ring->vma);
1759 ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix;
1760
1761 request->ring->head = request->postfix;
1762 intel_ring_update_space(request->ring);
1763
1764
1765 unwind_wa_tail(request);
1766}
1767
1768static int intel_logical_ring_emit_pdps(struct i915_request *rq)
1769{
1770 struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt;
1771 struct intel_engine_cs *engine = rq->engine;
1772 const int num_lri_cmds = GEN8_3LVL_PDPES * 2;
1773 u32 *cs;
1774 int i;
1775
1776 cs = intel_ring_begin(rq, num_lri_cmds * 2 + 2);
1777 if (IS_ERR(cs))
1778 return PTR_ERR(cs);
1779
1780 *cs++ = MI_LOAD_REGISTER_IMM(num_lri_cmds);
1781 for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
1782 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
1783
1784 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i));
1785 *cs++ = upper_32_bits(pd_daddr);
1786 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i));
1787 *cs++ = lower_32_bits(pd_daddr);
1788 }
1789
1790 *cs++ = MI_NOOP;
1791 intel_ring_advance(rq, cs);
1792
1793 return 0;
1794}
1795
1796static int gen8_emit_bb_start(struct i915_request *rq,
1797 u64 offset, u32 len,
1798 const unsigned int flags)
1799{
1800 u32 *cs;
1801 int ret;
1802
1803
1804
1805
1806
1807
1808
1809 if (rq->ctx->ppgtt &&
1810 (intel_engine_flag(rq->engine) & rq->ctx->ppgtt->pd_dirty_rings) &&
1811 !i915_vm_is_48bit(&rq->ctx->ppgtt->base) &&
1812 !intel_vgpu_active(rq->i915)) {
1813 ret = intel_logical_ring_emit_pdps(rq);
1814 if (ret)
1815 return ret;
1816
1817 rq->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(rq->engine);
1818 }
1819
1820 cs = intel_ring_begin(rq, 4);
1821 if (IS_ERR(cs))
1822 return PTR_ERR(cs);
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1842
1843
1844 *cs++ = MI_BATCH_BUFFER_START_GEN8 |
1845 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)) |
1846 (flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
1847 *cs++ = lower_32_bits(offset);
1848 *cs++ = upper_32_bits(offset);
1849 intel_ring_advance(rq, cs);
1850
1851 return 0;
1852}
1853
1854static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
1855{
1856 struct drm_i915_private *dev_priv = engine->i915;
1857 I915_WRITE_IMR(engine,
1858 ~(engine->irq_enable_mask | engine->irq_keep_mask));
1859 POSTING_READ_FW(RING_IMR(engine->mmio_base));
1860}
1861
1862static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
1863{
1864 struct drm_i915_private *dev_priv = engine->i915;
1865 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1866}
1867
1868static int gen8_emit_flush(struct i915_request *request, u32 mode)
1869{
1870 u32 cmd, *cs;
1871
1872 cs = intel_ring_begin(request, 4);
1873 if (IS_ERR(cs))
1874 return PTR_ERR(cs);
1875
1876 cmd = MI_FLUSH_DW + 1;
1877
1878
1879
1880
1881
1882
1883 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1884
1885 if (mode & EMIT_INVALIDATE) {
1886 cmd |= MI_INVALIDATE_TLB;
1887 if (request->engine->id == VCS)
1888 cmd |= MI_INVALIDATE_BSD;
1889 }
1890
1891 *cs++ = cmd;
1892 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
1893 *cs++ = 0;
1894 *cs++ = 0;
1895 intel_ring_advance(request, cs);
1896
1897 return 0;
1898}
1899
1900static int gen8_emit_flush_render(struct i915_request *request,
1901 u32 mode)
1902{
1903 struct intel_engine_cs *engine = request->engine;
1904 u32 scratch_addr =
1905 i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
1906 bool vf_flush_wa = false, dc_flush_wa = false;
1907 u32 *cs, flags = 0;
1908 int len;
1909
1910 flags |= PIPE_CONTROL_CS_STALL;
1911
1912 if (mode & EMIT_FLUSH) {
1913 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
1914 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
1915 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
1916 flags |= PIPE_CONTROL_FLUSH_ENABLE;
1917 }
1918
1919 if (mode & EMIT_INVALIDATE) {
1920 flags |= PIPE_CONTROL_TLB_INVALIDATE;
1921 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
1922 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1923 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
1924 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
1925 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
1926 flags |= PIPE_CONTROL_QW_WRITE;
1927 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
1928
1929
1930
1931
1932
1933 if (IS_GEN9(request->i915))
1934 vf_flush_wa = true;
1935
1936
1937 if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0))
1938 dc_flush_wa = true;
1939 }
1940
1941 len = 6;
1942
1943 if (vf_flush_wa)
1944 len += 6;
1945
1946 if (dc_flush_wa)
1947 len += 12;
1948
1949 cs = intel_ring_begin(request, len);
1950 if (IS_ERR(cs))
1951 return PTR_ERR(cs);
1952
1953 if (vf_flush_wa)
1954 cs = gen8_emit_pipe_control(cs, 0, 0);
1955
1956 if (dc_flush_wa)
1957 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,
1958 0);
1959
1960 cs = gen8_emit_pipe_control(cs, flags, scratch_addr);
1961
1962 if (dc_flush_wa)
1963 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);
1964
1965 intel_ring_advance(request, cs);
1966
1967 return 0;
1968}
1969
1970
1971
1972
1973
1974
1975static void gen8_emit_wa_tail(struct i915_request *request, u32 *cs)
1976{
1977
1978 *cs++ = MI_ARB_CHECK;
1979 *cs++ = MI_NOOP;
1980 request->wa_tail = intel_ring_offset(request, cs);
1981}
1982
1983static void gen8_emit_breadcrumb(struct i915_request *request, u32 *cs)
1984{
1985
1986 BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
1987
1988 cs = gen8_emit_ggtt_write(cs, request->global_seqno,
1989 intel_hws_seqno_address(request->engine));
1990 *cs++ = MI_USER_INTERRUPT;
1991 *cs++ = MI_NOOP;
1992 request->tail = intel_ring_offset(request, cs);
1993 assert_ring_tail_valid(request->ring, request->tail);
1994
1995 gen8_emit_wa_tail(request, cs);
1996}
1997static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
1998
1999static void gen8_emit_breadcrumb_rcs(struct i915_request *request, u32 *cs)
2000{
2001
2002 BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
2003
2004 cs = gen8_emit_ggtt_write_rcs(cs, request->global_seqno,
2005 intel_hws_seqno_address(request->engine));
2006 *cs++ = MI_USER_INTERRUPT;
2007 *cs++ = MI_NOOP;
2008 request->tail = intel_ring_offset(request, cs);
2009 assert_ring_tail_valid(request->ring, request->tail);
2010
2011 gen8_emit_wa_tail(request, cs);
2012}
2013static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS;
2014
2015static int gen8_init_rcs_context(struct i915_request *rq)
2016{
2017 int ret;
2018
2019 ret = intel_ring_workarounds_emit(rq);
2020 if (ret)
2021 return ret;
2022
2023 ret = intel_rcs_context_init_mocs(rq);
2024
2025
2026
2027
2028 if (ret)
2029 DRM_ERROR("MOCS failed to program: expect performance issues.\n");
2030
2031 return i915_gem_render_state_emit(rq);
2032}
2033
2034
2035
2036
2037
2038void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
2039{
2040 struct drm_i915_private *dev_priv;
2041
2042
2043
2044
2045
2046 if (WARN_ON(test_bit(TASKLET_STATE_SCHED,
2047 &engine->execlists.tasklet.state)))
2048 tasklet_kill(&engine->execlists.tasklet);
2049
2050 dev_priv = engine->i915;
2051
2052 if (engine->buffer) {
2053 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
2054 }
2055
2056 if (engine->cleanup)
2057 engine->cleanup(engine);
2058
2059 intel_engine_cleanup_common(engine);
2060
2061 lrc_destroy_wa_ctx(engine);
2062
2063 engine->i915 = NULL;
2064 dev_priv->engine[engine->id] = NULL;
2065 kfree(engine);
2066}
2067
2068static void execlists_set_default_submission(struct intel_engine_cs *engine)
2069{
2070 engine->submit_request = execlists_submit_request;
2071 engine->cancel_requests = execlists_cancel_requests;
2072 engine->schedule = execlists_schedule;
2073 engine->execlists.tasklet.func = execlists_submission_tasklet;
2074
2075 engine->park = NULL;
2076 engine->unpark = NULL;
2077
2078 engine->flags |= I915_ENGINE_SUPPORTS_STATS;
2079
2080 engine->i915->caps.scheduler =
2081 I915_SCHEDULER_CAP_ENABLED |
2082 I915_SCHEDULER_CAP_PRIORITY;
2083 if (engine->i915->preempt_context)
2084 engine->i915->caps.scheduler |= I915_SCHEDULER_CAP_PREEMPTION;
2085}
2086
2087static void
2088logical_ring_default_vfuncs(struct intel_engine_cs *engine)
2089{
2090
2091 engine->init_hw = gen8_init_common_ring;
2092 engine->reset_hw = reset_common_ring;
2093
2094 engine->context_pin = execlists_context_pin;
2095 engine->context_unpin = execlists_context_unpin;
2096
2097 engine->request_alloc = execlists_request_alloc;
2098
2099 engine->emit_flush = gen8_emit_flush;
2100 engine->emit_breadcrumb = gen8_emit_breadcrumb;
2101 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
2102
2103 engine->set_default_submission = execlists_set_default_submission;
2104
2105 if (INTEL_GEN(engine->i915) < 11) {
2106 engine->irq_enable = gen8_logical_ring_enable_irq;
2107 engine->irq_disable = gen8_logical_ring_disable_irq;
2108 } else {
2109
2110
2111
2112
2113
2114
2115 }
2116 engine->emit_bb_start = gen8_emit_bb_start;
2117}
2118
2119static inline void
2120logical_ring_default_irqs(struct intel_engine_cs *engine)
2121{
2122 unsigned shift = engine->irq_shift;
2123 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
2124 engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
2125}
2126
2127static void
2128logical_ring_setup(struct intel_engine_cs *engine)
2129{
2130 struct drm_i915_private *dev_priv = engine->i915;
2131 enum forcewake_domains fw_domains;
2132
2133 intel_engine_setup_common(engine);
2134
2135
2136 engine->buffer = NULL;
2137
2138 fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
2139 RING_ELSP(engine),
2140 FW_REG_WRITE);
2141
2142 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
2143 RING_CONTEXT_STATUS_PTR(engine),
2144 FW_REG_READ | FW_REG_WRITE);
2145
2146 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
2147 RING_CONTEXT_STATUS_BUF_BASE(engine),
2148 FW_REG_READ);
2149
2150 engine->execlists.fw_domains = fw_domains;
2151
2152 tasklet_init(&engine->execlists.tasklet,
2153 execlists_submission_tasklet, (unsigned long)engine);
2154
2155 logical_ring_default_vfuncs(engine);
2156 logical_ring_default_irqs(engine);
2157}
2158
2159static int logical_ring_init(struct intel_engine_cs *engine)
2160{
2161 int ret;
2162
2163 ret = intel_engine_init_common(engine);
2164 if (ret)
2165 goto error;
2166
2167 if (HAS_LOGICAL_RING_ELSQ(engine->i915)) {
2168 engine->execlists.submit_reg = engine->i915->regs +
2169 i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(engine));
2170 engine->execlists.ctrl_reg = engine->i915->regs +
2171 i915_mmio_reg_offset(RING_EXECLIST_CONTROL(engine));
2172 } else {
2173 engine->execlists.submit_reg = engine->i915->regs +
2174 i915_mmio_reg_offset(RING_ELSP(engine));
2175 }
2176
2177 engine->execlists.preempt_complete_status = ~0u;
2178 if (engine->i915->preempt_context)
2179 engine->execlists.preempt_complete_status =
2180 upper_32_bits(engine->i915->preempt_context->engine[engine->id].lrc_desc);
2181
2182 return 0;
2183
2184error:
2185 intel_logical_ring_cleanup(engine);
2186 return ret;
2187}
2188
2189int logical_render_ring_init(struct intel_engine_cs *engine)
2190{
2191 struct drm_i915_private *dev_priv = engine->i915;
2192 int ret;
2193
2194 logical_ring_setup(engine);
2195
2196 if (HAS_L3_DPF(dev_priv))
2197 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2198
2199
2200 if (INTEL_GEN(dev_priv) >= 9)
2201 engine->init_hw = gen9_init_render_ring;
2202 else
2203 engine->init_hw = gen8_init_render_ring;
2204 engine->init_context = gen8_init_rcs_context;
2205 engine->emit_flush = gen8_emit_flush_render;
2206 engine->emit_breadcrumb = gen8_emit_breadcrumb_rcs;
2207 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_rcs_sz;
2208
2209 ret = intel_engine_create_scratch(engine, PAGE_SIZE);
2210 if (ret)
2211 return ret;
2212
2213 ret = intel_init_workaround_bb(engine);
2214 if (ret) {
2215
2216
2217
2218
2219
2220 DRM_ERROR("WA batch buffer initialization failed: %d\n",
2221 ret);
2222 }
2223
2224 return logical_ring_init(engine);
2225}
2226
2227int logical_xcs_ring_init(struct intel_engine_cs *engine)
2228{
2229 logical_ring_setup(engine);
2230
2231 return logical_ring_init(engine);
2232}
2233
2234static u32
2235make_rpcs(struct drm_i915_private *dev_priv)
2236{
2237 u32 rpcs = 0;
2238
2239
2240
2241
2242
2243 if (INTEL_GEN(dev_priv) < 9)
2244 return 0;
2245
2246
2247
2248
2249
2250
2251
2252 if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) {
2253 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
2254 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) <<
2255 GEN8_RPCS_S_CNT_SHIFT;
2256 rpcs |= GEN8_RPCS_ENABLE;
2257 }
2258
2259 if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) {
2260 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
2261 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask[0]) <<
2262 GEN8_RPCS_SS_CNT_SHIFT;
2263 rpcs |= GEN8_RPCS_ENABLE;
2264 }
2265
2266 if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) {
2267 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
2268 GEN8_RPCS_EU_MIN_SHIFT;
2269 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
2270 GEN8_RPCS_EU_MAX_SHIFT;
2271 rpcs |= GEN8_RPCS_ENABLE;
2272 }
2273
2274 return rpcs;
2275}
2276
2277static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
2278{
2279 u32 indirect_ctx_offset;
2280
2281 switch (INTEL_GEN(engine->i915)) {
2282 default:
2283 MISSING_CASE(INTEL_GEN(engine->i915));
2284
2285 case 11:
2286 indirect_ctx_offset =
2287 GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
2288 break;
2289 case 10:
2290 indirect_ctx_offset =
2291 GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
2292 break;
2293 case 9:
2294 indirect_ctx_offset =
2295 GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
2296 break;
2297 case 8:
2298 indirect_ctx_offset =
2299 GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
2300 break;
2301 }
2302
2303 return indirect_ctx_offset;
2304}
2305
2306static void execlists_init_reg_state(u32 *regs,
2307 struct i915_gem_context *ctx,
2308 struct intel_engine_cs *engine,
2309 struct intel_ring *ring)
2310{
2311 struct drm_i915_private *dev_priv = engine->i915;
2312 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt;
2313 u32 base = engine->mmio_base;
2314 bool rcs = engine->id == RCS;
2315
2316
2317
2318
2319
2320
2321
2322
2323 regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) |
2324 MI_LRI_FORCE_POSTED;
2325
2326 CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine),
2327 _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
2328 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT) |
2329 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
2330 (HAS_RESOURCE_STREAMER(dev_priv) ?
2331 CTX_CTRL_RS_CTX_ENABLE : 0)));
2332 CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0);
2333 CTX_REG(regs, CTX_RING_TAIL, RING_TAIL(base), 0);
2334 CTX_REG(regs, CTX_RING_BUFFER_START, RING_START(base), 0);
2335 CTX_REG(regs, CTX_RING_BUFFER_CONTROL, RING_CTL(base),
2336 RING_CTL_SIZE(ring->size) | RING_VALID);
2337 CTX_REG(regs, CTX_BB_HEAD_U, RING_BBADDR_UDW(base), 0);
2338 CTX_REG(regs, CTX_BB_HEAD_L, RING_BBADDR(base), 0);
2339 CTX_REG(regs, CTX_BB_STATE, RING_BBSTATE(base), RING_BB_PPGTT);
2340 CTX_REG(regs, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(base), 0);
2341 CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0);
2342 CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0);
2343 if (rcs) {
2344 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
2345
2346 CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0);
2347 CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET,
2348 RING_INDIRECT_CTX_OFFSET(base), 0);
2349 if (wa_ctx->indirect_ctx.size) {
2350 u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2351
2352 regs[CTX_RCS_INDIRECT_CTX + 1] =
2353 (ggtt_offset + wa_ctx->indirect_ctx.offset) |
2354 (wa_ctx->indirect_ctx.size / CACHELINE_BYTES);
2355
2356 regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] =
2357 intel_lr_indirect_ctx_offset(engine) << 6;
2358 }
2359
2360 CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0);
2361 if (wa_ctx->per_ctx.size) {
2362 u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2363
2364 regs[CTX_BB_PER_CTX_PTR + 1] =
2365 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01;
2366 }
2367 }
2368
2369 regs[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED;
2370
2371 CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0);
2372
2373 CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3), 0);
2374 CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3), 0);
2375 CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2), 0);
2376 CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2), 0);
2377 CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1), 0);
2378 CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1), 0);
2379 CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0);
2380 CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0);
2381
2382 if (ppgtt && i915_vm_is_48bit(&ppgtt->base)) {
2383
2384
2385
2386
2387 ASSIGN_CTX_PML4(ppgtt, regs);
2388 }
2389
2390 if (rcs) {
2391 regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
2392 CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
2393 make_rpcs(dev_priv));
2394
2395 i915_oa_init_reg_state(engine, ctx, regs);
2396 }
2397}
2398
2399static int
2400populate_lr_context(struct i915_gem_context *ctx,
2401 struct drm_i915_gem_object *ctx_obj,
2402 struct intel_engine_cs *engine,
2403 struct intel_ring *ring)
2404{
2405 void *vaddr;
2406 u32 *regs;
2407 int ret;
2408
2409 ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
2410 if (ret) {
2411 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
2412 return ret;
2413 }
2414
2415 vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
2416 if (IS_ERR(vaddr)) {
2417 ret = PTR_ERR(vaddr);
2418 DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
2419 return ret;
2420 }
2421 ctx_obj->mm.dirty = true;
2422
2423 if (engine->default_state) {
2424
2425
2426
2427
2428
2429 const unsigned long start = LRC_HEADER_PAGES * PAGE_SIZE;
2430 void *defaults;
2431
2432 defaults = i915_gem_object_pin_map(engine->default_state,
2433 I915_MAP_WB);
2434 if (IS_ERR(defaults))
2435 return PTR_ERR(defaults);
2436
2437 memcpy(vaddr + start, defaults + start, engine->context_size);
2438 i915_gem_object_unpin_map(engine->default_state);
2439 }
2440
2441
2442
2443 regs = vaddr + LRC_STATE_PN * PAGE_SIZE;
2444 execlists_init_reg_state(regs, ctx, engine, ring);
2445 if (!engine->default_state)
2446 regs[CTX_CONTEXT_CONTROL + 1] |=
2447 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
2448 if (ctx == ctx->i915->preempt_context && INTEL_GEN(engine->i915) < 11)
2449 regs[CTX_CONTEXT_CONTROL + 1] |=
2450 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
2451 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT);
2452
2453 i915_gem_object_unpin_map(ctx_obj);
2454
2455 return 0;
2456}
2457
2458static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
2459 struct intel_engine_cs *engine)
2460{
2461 struct drm_i915_gem_object *ctx_obj;
2462 struct intel_context *ce = &ctx->engine[engine->id];
2463 struct i915_vma *vma;
2464 uint32_t context_size;
2465 struct intel_ring *ring;
2466 int ret;
2467
2468 if (ce->state)
2469 return 0;
2470
2471 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);
2472
2473
2474
2475
2476
2477 context_size += LRC_HEADER_PAGES * PAGE_SIZE;
2478
2479 ctx_obj = i915_gem_object_create(ctx->i915, context_size);
2480 if (IS_ERR(ctx_obj)) {
2481 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
2482 return PTR_ERR(ctx_obj);
2483 }
2484
2485 vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.base, NULL);
2486 if (IS_ERR(vma)) {
2487 ret = PTR_ERR(vma);
2488 goto error_deref_obj;
2489 }
2490
2491 ring = intel_engine_create_ring(engine, ctx->ring_size);
2492 if (IS_ERR(ring)) {
2493 ret = PTR_ERR(ring);
2494 goto error_deref_obj;
2495 }
2496
2497 ret = populate_lr_context(ctx, ctx_obj, engine, ring);
2498 if (ret) {
2499 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
2500 goto error_ring_free;
2501 }
2502
2503 ce->ring = ring;
2504 ce->state = vma;
2505
2506 return 0;
2507
2508error_ring_free:
2509 intel_ring_free(ring);
2510error_deref_obj:
2511 i915_gem_object_put(ctx_obj);
2512 return ret;
2513}
2514
2515void intel_lr_context_resume(struct drm_i915_private *dev_priv)
2516{
2517 struct intel_engine_cs *engine;
2518 struct i915_gem_context *ctx;
2519 enum intel_engine_id id;
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531 list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
2532 for_each_engine(engine, dev_priv, id) {
2533 struct intel_context *ce = &ctx->engine[engine->id];
2534 u32 *reg;
2535
2536 if (!ce->state)
2537 continue;
2538
2539 reg = i915_gem_object_pin_map(ce->state->obj,
2540 I915_MAP_WB);
2541 if (WARN_ON(IS_ERR(reg)))
2542 continue;
2543
2544 reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg);
2545 reg[CTX_RING_HEAD+1] = 0;
2546 reg[CTX_RING_TAIL+1] = 0;
2547
2548 ce->state->obj->mm.dirty = true;
2549 i915_gem_object_unpin_map(ce->state->obj);
2550
2551 intel_ring_reset(ce->ring, 0);
2552 }
2553 }
2554}
2555