1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135#include <drm/drmP.h>
136#include <drm/i915_drm.h>
137#include "i915_drv.h"
138
139#define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
140#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
141#define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
142
143#define RING_EXECLIST_QFULL (1 << 0x2)
144#define RING_EXECLIST1_VALID (1 << 0x3)
145#define RING_EXECLIST0_VALID (1 << 0x4)
146#define RING_EXECLIST_ACTIVE_STATUS (3 << 0xE)
147#define RING_EXECLIST1_ACTIVE (1 << 0x11)
148#define RING_EXECLIST0_ACTIVE (1 << 0x12)
149
150#define GEN8_CTX_STATUS_IDLE_ACTIVE (1 << 0)
151#define GEN8_CTX_STATUS_PREEMPTED (1 << 1)
152#define GEN8_CTX_STATUS_ELEMENT_SWITCH (1 << 2)
153#define GEN8_CTX_STATUS_ACTIVE_IDLE (1 << 3)
154#define GEN8_CTX_STATUS_COMPLETE (1 << 4)
155#define GEN8_CTX_STATUS_LITE_RESTORE (1 << 15)
156
157#define CTX_LRI_HEADER_0 0x01
158#define CTX_CONTEXT_CONTROL 0x02
159#define CTX_RING_HEAD 0x04
160#define CTX_RING_TAIL 0x06
161#define CTX_RING_BUFFER_START 0x08
162#define CTX_RING_BUFFER_CONTROL 0x0a
163#define CTX_BB_HEAD_U 0x0c
164#define CTX_BB_HEAD_L 0x0e
165#define CTX_BB_STATE 0x10
166#define CTX_SECOND_BB_HEAD_U 0x12
167#define CTX_SECOND_BB_HEAD_L 0x14
168#define CTX_SECOND_BB_STATE 0x16
169#define CTX_BB_PER_CTX_PTR 0x18
170#define CTX_RCS_INDIRECT_CTX 0x1a
171#define CTX_RCS_INDIRECT_CTX_OFFSET 0x1c
172#define CTX_LRI_HEADER_1 0x21
173#define CTX_CTX_TIMESTAMP 0x22
174#define CTX_PDP3_UDW 0x24
175#define CTX_PDP3_LDW 0x26
176#define CTX_PDP2_UDW 0x28
177#define CTX_PDP2_LDW 0x2a
178#define CTX_PDP1_UDW 0x2c
179#define CTX_PDP1_LDW 0x2e
180#define CTX_PDP0_UDW 0x30
181#define CTX_PDP0_LDW 0x32
182#define CTX_LRI_HEADER_2 0x41
183#define CTX_R_PWR_CLK_STATE 0x42
184#define CTX_GPGPU_CSR_BASE_ADDRESS 0x44
185
186#define GEN8_CTX_VALID (1<<0)
187#define GEN8_CTX_FORCE_PD_RESTORE (1<<1)
188#define GEN8_CTX_FORCE_RESTORE (1<<2)
189#define GEN8_CTX_L3LLC_COHERENT (1<<5)
190#define GEN8_CTX_PRIVILEGE (1<<8)
191enum {
192 ADVANCED_CONTEXT = 0,
193 LEGACY_CONTEXT,
194 ADVANCED_AD_CONTEXT,
195 LEGACY_64B_CONTEXT
196};
197#define GEN8_CTX_MODE_SHIFT 3
198enum {
199 FAULT_AND_HANG = 0,
200 FAULT_AND_HALT,
201 FAULT_AND_STREAM,
202 FAULT_AND_CONTINUE
203};
204#define GEN8_CTX_ID_SHIFT 32
205
206static int intel_lr_context_pin(struct intel_engine_cs *ring,
207 struct intel_context *ctx);
208
209
210
211
212
213
214
215
216
217
218
219int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
220{
221 WARN_ON(i915.enable_ppgtt == -1);
222
223 if (INTEL_INFO(dev)->gen >= 9)
224 return 1;
225
226 if (enable_execlists == 0)
227 return 0;
228
229 if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) &&
230 i915.use_mmio_flip >= 0)
231 return 1;
232
233 return 0;
234}
235
236
237
238
239
240
241
242
243
244
245
246
247
248u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj)
249{
250 u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj);
251
252
253
254 return lrca >> 12;
255}
256
257static uint64_t execlists_ctx_descriptor(struct intel_engine_cs *ring,
258 struct drm_i915_gem_object *ctx_obj)
259{
260 struct drm_device *dev = ring->dev;
261 uint64_t desc;
262 uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj);
263
264 WARN_ON(lrca & 0xFFFFFFFF00000FFFULL);
265
266 desc = GEN8_CTX_VALID;
267 desc |= LEGACY_CONTEXT << GEN8_CTX_MODE_SHIFT;
268 desc |= GEN8_CTX_L3LLC_COHERENT;
269 desc |= GEN8_CTX_PRIVILEGE;
270 desc |= lrca;
271 desc |= (u64)intel_execlists_ctx_id(ctx_obj) << GEN8_CTX_ID_SHIFT;
272
273
274
275
276
277
278 if (IS_GEN9(dev) &&
279 INTEL_REVID(dev) <= SKL_REVID_B0 &&
280 (ring->id == BCS || ring->id == VCS ||
281 ring->id == VECS || ring->id == VCS2))
282 desc |= GEN8_CTX_FORCE_RESTORE;
283
284 return desc;
285}
286
287static void execlists_elsp_write(struct intel_engine_cs *ring,
288 struct drm_i915_gem_object *ctx_obj0,
289 struct drm_i915_gem_object *ctx_obj1)
290{
291 struct drm_device *dev = ring->dev;
292 struct drm_i915_private *dev_priv = dev->dev_private;
293 uint64_t temp = 0;
294 uint32_t desc[4];
295
296
297 if (ctx_obj1)
298 temp = execlists_ctx_descriptor(ring, ctx_obj1);
299 else
300 temp = 0;
301 desc[1] = (u32)(temp >> 32);
302 desc[0] = (u32)temp;
303
304 temp = execlists_ctx_descriptor(ring, ctx_obj0);
305 desc[3] = (u32)(temp >> 32);
306 desc[2] = (u32)temp;
307
308 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
309 I915_WRITE(RING_ELSP(ring), desc[1]);
310 I915_WRITE(RING_ELSP(ring), desc[0]);
311 I915_WRITE(RING_ELSP(ring), desc[3]);
312
313
314 I915_WRITE(RING_ELSP(ring), desc[2]);
315
316
317 POSTING_READ(RING_EXECLIST_STATUS(ring));
318 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
319}
320
321static int execlists_update_context(struct drm_i915_gem_object *ctx_obj,
322 struct drm_i915_gem_object *ring_obj,
323 u32 tail)
324{
325 struct page *page;
326 uint32_t *reg_state;
327
328 page = i915_gem_object_get_page(ctx_obj, 1);
329 reg_state = kmap_atomic(page);
330
331 reg_state[CTX_RING_TAIL+1] = tail;
332 reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj);
333
334 kunmap_atomic(reg_state);
335
336 return 0;
337}
338
339static void execlists_submit_contexts(struct intel_engine_cs *ring,
340 struct intel_context *to0, u32 tail0,
341 struct intel_context *to1, u32 tail1)
342{
343 struct drm_i915_gem_object *ctx_obj0 = to0->engine[ring->id].state;
344 struct intel_ringbuffer *ringbuf0 = to0->engine[ring->id].ringbuf;
345 struct drm_i915_gem_object *ctx_obj1 = NULL;
346 struct intel_ringbuffer *ringbuf1 = NULL;
347
348 BUG_ON(!ctx_obj0);
349 WARN_ON(!i915_gem_obj_is_pinned(ctx_obj0));
350 WARN_ON(!i915_gem_obj_is_pinned(ringbuf0->obj));
351
352 execlists_update_context(ctx_obj0, ringbuf0->obj, tail0);
353
354 if (to1) {
355 ringbuf1 = to1->engine[ring->id].ringbuf;
356 ctx_obj1 = to1->engine[ring->id].state;
357 BUG_ON(!ctx_obj1);
358 WARN_ON(!i915_gem_obj_is_pinned(ctx_obj1));
359 WARN_ON(!i915_gem_obj_is_pinned(ringbuf1->obj));
360
361 execlists_update_context(ctx_obj1, ringbuf1->obj, tail1);
362 }
363
364 execlists_elsp_write(ring, ctx_obj0, ctx_obj1);
365}
366
367static void execlists_context_unqueue(struct intel_engine_cs *ring)
368{
369 struct drm_i915_gem_request *req0 = NULL, *req1 = NULL;
370 struct drm_i915_gem_request *cursor = NULL, *tmp = NULL;
371
372 assert_spin_locked(&ring->execlist_lock);
373
374 if (list_empty(&ring->execlist_queue))
375 return;
376
377
378 list_for_each_entry_safe(cursor, tmp, &ring->execlist_queue,
379 execlist_link) {
380 if (!req0) {
381 req0 = cursor;
382 } else if (req0->ctx == cursor->ctx) {
383
384
385 cursor->elsp_submitted = req0->elsp_submitted;
386 list_del(&req0->execlist_link);
387 list_add_tail(&req0->execlist_link,
388 &ring->execlist_retired_req_list);
389 req0 = cursor;
390 } else {
391 req1 = cursor;
392 break;
393 }
394 }
395
396 if (IS_GEN8(ring->dev) || IS_GEN9(ring->dev)) {
397
398
399
400
401 if (req0 && req0->elsp_submitted) {
402
403
404
405
406
407
408 struct intel_ringbuffer *ringbuf;
409
410 ringbuf = req0->ctx->engine[ring->id].ringbuf;
411 req0->tail += 8;
412 req0->tail &= ringbuf->size - 1;
413 }
414 }
415
416 WARN_ON(req1 && req1->elsp_submitted);
417
418 execlists_submit_contexts(ring, req0->ctx, req0->tail,
419 req1 ? req1->ctx : NULL,
420 req1 ? req1->tail : 0);
421
422 req0->elsp_submitted++;
423 if (req1)
424 req1->elsp_submitted++;
425}
426
427static bool execlists_check_remove_request(struct intel_engine_cs *ring,
428 u32 request_id)
429{
430 struct drm_i915_gem_request *head_req;
431
432 assert_spin_locked(&ring->execlist_lock);
433
434 head_req = list_first_entry_or_null(&ring->execlist_queue,
435 struct drm_i915_gem_request,
436 execlist_link);
437
438 if (head_req != NULL) {
439 struct drm_i915_gem_object *ctx_obj =
440 head_req->ctx->engine[ring->id].state;
441 if (intel_execlists_ctx_id(ctx_obj) == request_id) {
442 WARN(head_req->elsp_submitted == 0,
443 "Never submitted head request\n");
444
445 if (--head_req->elsp_submitted <= 0) {
446 list_del(&head_req->execlist_link);
447 list_add_tail(&head_req->execlist_link,
448 &ring->execlist_retired_req_list);
449 return true;
450 }
451 }
452 }
453
454 return false;
455}
456
457
458
459
460
461
462
463
464void intel_lrc_irq_handler(struct intel_engine_cs *ring)
465{
466 struct drm_i915_private *dev_priv = ring->dev->dev_private;
467 u32 status_pointer;
468 u8 read_pointer;
469 u8 write_pointer;
470 u32 status;
471 u32 status_id;
472 u32 submit_contexts = 0;
473
474 status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring));
475
476 read_pointer = ring->next_context_status_buffer;
477 write_pointer = status_pointer & 0x07;
478 if (read_pointer > write_pointer)
479 write_pointer += 6;
480
481 spin_lock(&ring->execlist_lock);
482
483 while (read_pointer < write_pointer) {
484 read_pointer++;
485 status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
486 (read_pointer % 6) * 8);
487 status_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
488 (read_pointer % 6) * 8 + 4);
489
490 if (status & GEN8_CTX_STATUS_PREEMPTED) {
491 if (status & GEN8_CTX_STATUS_LITE_RESTORE) {
492 if (execlists_check_remove_request(ring, status_id))
493 WARN(1, "Lite Restored request removed from queue\n");
494 } else
495 WARN(1, "Preemption without Lite Restore\n");
496 }
497
498 if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) ||
499 (status & GEN8_CTX_STATUS_ELEMENT_SWITCH)) {
500 if (execlists_check_remove_request(ring, status_id))
501 submit_contexts++;
502 }
503 }
504
505 if (submit_contexts != 0)
506 execlists_context_unqueue(ring);
507
508 spin_unlock(&ring->execlist_lock);
509
510 WARN(submit_contexts > 2, "More than two context complete events?\n");
511 ring->next_context_status_buffer = write_pointer % 6;
512
513 I915_WRITE(RING_CONTEXT_STATUS_PTR(ring),
514 ((u32)ring->next_context_status_buffer & 0x07) << 8);
515}
516
517static int execlists_context_queue(struct intel_engine_cs *ring,
518 struct intel_context *to,
519 u32 tail,
520 struct drm_i915_gem_request *request)
521{
522 struct drm_i915_gem_request *cursor;
523 struct drm_i915_private *dev_priv = ring->dev->dev_private;
524 unsigned long flags;
525 int num_elements = 0;
526
527 if (to != ring->default_context)
528 intel_lr_context_pin(ring, to);
529
530 if (!request) {
531
532
533
534
535 request = kzalloc(sizeof(*request), GFP_KERNEL);
536 if (request == NULL)
537 return -ENOMEM;
538 request->ring = ring;
539 request->ctx = to;
540 kref_init(&request->ref);
541 request->uniq = dev_priv->request_uniq++;
542 i915_gem_context_reference(request->ctx);
543 } else {
544 i915_gem_request_reference(request);
545 WARN_ON(to != request->ctx);
546 }
547 request->tail = tail;
548
549 intel_runtime_pm_get(dev_priv);
550
551 spin_lock_irqsave(&ring->execlist_lock, flags);
552
553 list_for_each_entry(cursor, &ring->execlist_queue, execlist_link)
554 if (++num_elements > 2)
555 break;
556
557 if (num_elements > 2) {
558 struct drm_i915_gem_request *tail_req;
559
560 tail_req = list_last_entry(&ring->execlist_queue,
561 struct drm_i915_gem_request,
562 execlist_link);
563
564 if (to == tail_req->ctx) {
565 WARN(tail_req->elsp_submitted != 0,
566 "More than 2 already-submitted reqs queued\n");
567 list_del(&tail_req->execlist_link);
568 list_add_tail(&tail_req->execlist_link,
569 &ring->execlist_retired_req_list);
570 }
571 }
572
573 list_add_tail(&request->execlist_link, &ring->execlist_queue);
574 if (num_elements == 0)
575 execlists_context_unqueue(ring);
576
577 spin_unlock_irqrestore(&ring->execlist_lock, flags);
578
579 return 0;
580}
581
582static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf,
583 struct intel_context *ctx)
584{
585 struct intel_engine_cs *ring = ringbuf->ring;
586 uint32_t flush_domains;
587 int ret;
588
589 flush_domains = 0;
590 if (ring->gpu_caches_dirty)
591 flush_domains = I915_GEM_GPU_DOMAINS;
592
593 ret = ring->emit_flush(ringbuf, ctx,
594 I915_GEM_GPU_DOMAINS, flush_domains);
595 if (ret)
596 return ret;
597
598 ring->gpu_caches_dirty = false;
599 return 0;
600}
601
602static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf,
603 struct intel_context *ctx,
604 struct list_head *vmas)
605{
606 struct intel_engine_cs *ring = ringbuf->ring;
607 struct i915_vma *vma;
608 uint32_t flush_domains = 0;
609 bool flush_chipset = false;
610 int ret;
611
612 list_for_each_entry(vma, vmas, exec_list) {
613 struct drm_i915_gem_object *obj = vma->obj;
614
615 ret = i915_gem_object_sync(obj, ring);
616 if (ret)
617 return ret;
618
619 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
620 flush_chipset |= i915_gem_clflush_object(obj, false);
621
622 flush_domains |= obj->base.write_domain;
623 }
624
625 if (flush_domains & I915_GEM_DOMAIN_GTT)
626 wmb();
627
628
629
630
631 return logical_ring_invalidate_all_caches(ringbuf, ctx);
632}
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651int intel_execlists_submission(struct drm_device *dev, struct drm_file *file,
652 struct intel_engine_cs *ring,
653 struct intel_context *ctx,
654 struct drm_i915_gem_execbuffer2 *args,
655 struct list_head *vmas,
656 struct drm_i915_gem_object *batch_obj,
657 u64 exec_start, u32 dispatch_flags)
658{
659 struct drm_i915_private *dev_priv = dev->dev_private;
660 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
661 int instp_mode;
662 u32 instp_mask;
663 int ret;
664
665 instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
666 instp_mask = I915_EXEC_CONSTANTS_MASK;
667 switch (instp_mode) {
668 case I915_EXEC_CONSTANTS_REL_GENERAL:
669 case I915_EXEC_CONSTANTS_ABSOLUTE:
670 case I915_EXEC_CONSTANTS_REL_SURFACE:
671 if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
672 DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
673 return -EINVAL;
674 }
675
676 if (instp_mode != dev_priv->relative_constants_mode) {
677 if (instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
678 DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
679 return -EINVAL;
680 }
681
682
683 instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
684 }
685 break;
686 default:
687 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
688 return -EINVAL;
689 }
690
691 if (args->num_cliprects != 0) {
692 DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
693 return -EINVAL;
694 } else {
695 if (args->DR4 == 0xffffffff) {
696 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
697 args->DR4 = 0;
698 }
699
700 if (args->DR1 || args->DR4 || args->cliprects_ptr) {
701 DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
702 return -EINVAL;
703 }
704 }
705
706 if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
707 DRM_DEBUG("sol reset is gen7 only\n");
708 return -EINVAL;
709 }
710
711 ret = execlists_move_to_gpu(ringbuf, ctx, vmas);
712 if (ret)
713 return ret;
714
715 if (ring == &dev_priv->ring[RCS] &&
716 instp_mode != dev_priv->relative_constants_mode) {
717 ret = intel_logical_ring_begin(ringbuf, ctx, 4);
718 if (ret)
719 return ret;
720
721 intel_logical_ring_emit(ringbuf, MI_NOOP);
722 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
723 intel_logical_ring_emit(ringbuf, INSTPM);
724 intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);
725 intel_logical_ring_advance(ringbuf);
726
727 dev_priv->relative_constants_mode = instp_mode;
728 }
729
730 ret = ring->emit_bb_start(ringbuf, ctx, exec_start, dispatch_flags);
731 if (ret)
732 return ret;
733
734 trace_i915_gem_ring_dispatch(intel_ring_get_request(ring), dispatch_flags);
735
736 i915_gem_execbuffer_move_to_active(vmas, ring);
737 i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
738
739 return 0;
740}
741
742void intel_execlists_retire_requests(struct intel_engine_cs *ring)
743{
744 struct drm_i915_gem_request *req, *tmp;
745 struct drm_i915_private *dev_priv = ring->dev->dev_private;
746 unsigned long flags;
747 struct list_head retired_list;
748
749 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
750 if (list_empty(&ring->execlist_retired_req_list))
751 return;
752
753 INIT_LIST_HEAD(&retired_list);
754 spin_lock_irqsave(&ring->execlist_lock, flags);
755 list_replace_init(&ring->execlist_retired_req_list, &retired_list);
756 spin_unlock_irqrestore(&ring->execlist_lock, flags);
757
758 list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) {
759 struct intel_context *ctx = req->ctx;
760 struct drm_i915_gem_object *ctx_obj =
761 ctx->engine[ring->id].state;
762
763 if (ctx_obj && (ctx != ring->default_context))
764 intel_lr_context_unpin(ring, ctx);
765 intel_runtime_pm_put(dev_priv);
766 list_del(&req->execlist_link);
767 i915_gem_request_unreference(req);
768 }
769}
770
771void intel_logical_ring_stop(struct intel_engine_cs *ring)
772{
773 struct drm_i915_private *dev_priv = ring->dev->dev_private;
774 int ret;
775
776 if (!intel_ring_initialized(ring))
777 return;
778
779 ret = intel_ring_idle(ring);
780 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
781 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
782 ring->name, ret);
783
784
785 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
786 if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
787 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
788 return;
789 }
790 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
791}
792
793int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf,
794 struct intel_context *ctx)
795{
796 struct intel_engine_cs *ring = ringbuf->ring;
797 int ret;
798
799 if (!ring->gpu_caches_dirty)
800 return 0;
801
802 ret = ring->emit_flush(ringbuf, ctx, 0, I915_GEM_GPU_DOMAINS);
803 if (ret)
804 return ret;
805
806 ring->gpu_caches_dirty = false;
807 return 0;
808}
809
810
811
812
813
814
815
816
817
818
819static void
820intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf,
821 struct intel_context *ctx,
822 struct drm_i915_gem_request *request)
823{
824 struct intel_engine_cs *ring = ringbuf->ring;
825
826 intel_logical_ring_advance(ringbuf);
827
828 if (intel_ring_stopped(ring))
829 return;
830
831 execlists_context_queue(ring, ctx, ringbuf->tail, request);
832}
833
834static int intel_lr_context_pin(struct intel_engine_cs *ring,
835 struct intel_context *ctx)
836{
837 struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
838 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
839 int ret = 0;
840
841 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
842 if (ctx->engine[ring->id].pin_count++ == 0) {
843 ret = i915_gem_obj_ggtt_pin(ctx_obj,
844 GEN8_LR_CONTEXT_ALIGN, 0);
845 if (ret)
846 goto reset_pin_count;
847
848 ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);
849 if (ret)
850 goto unpin_ctx_obj;
851 }
852
853 return ret;
854
855unpin_ctx_obj:
856 i915_gem_object_ggtt_unpin(ctx_obj);
857reset_pin_count:
858 ctx->engine[ring->id].pin_count = 0;
859
860 return ret;
861}
862
863void intel_lr_context_unpin(struct intel_engine_cs *ring,
864 struct intel_context *ctx)
865{
866 struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
867 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
868
869 if (ctx_obj) {
870 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
871 if (--ctx->engine[ring->id].pin_count == 0) {
872 intel_unpin_ringbuffer_obj(ringbuf);
873 i915_gem_object_ggtt_unpin(ctx_obj);
874 }
875 }
876}
877
878static int logical_ring_alloc_request(struct intel_engine_cs *ring,
879 struct intel_context *ctx)
880{
881 struct drm_i915_gem_request *request;
882 struct drm_i915_private *dev_private = ring->dev->dev_private;
883 int ret;
884
885 if (ring->outstanding_lazy_request)
886 return 0;
887
888 request = kzalloc(sizeof(*request), GFP_KERNEL);
889 if (request == NULL)
890 return -ENOMEM;
891
892 if (ctx != ring->default_context) {
893 ret = intel_lr_context_pin(ring, ctx);
894 if (ret) {
895 kfree(request);
896 return ret;
897 }
898 }
899
900 kref_init(&request->ref);
901 request->ring = ring;
902 request->uniq = dev_private->request_uniq++;
903
904 ret = i915_gem_get_seqno(ring->dev, &request->seqno);
905 if (ret) {
906 intel_lr_context_unpin(ring, ctx);
907 kfree(request);
908 return ret;
909 }
910
911 request->ctx = ctx;
912 i915_gem_context_reference(request->ctx);
913 request->ringbuf = ctx->engine[ring->id].ringbuf;
914
915 ring->outstanding_lazy_request = request;
916 return 0;
917}
918
919static int logical_ring_wait_request(struct intel_ringbuffer *ringbuf,
920 int bytes)
921{
922 struct intel_engine_cs *ring = ringbuf->ring;
923 struct drm_i915_gem_request *request;
924 int ret;
925
926 if (intel_ring_space(ringbuf) >= bytes)
927 return 0;
928
929 list_for_each_entry(request, &ring->request_list, list) {
930
931
932
933
934
935 struct intel_context *ctx = request->ctx;
936 if (ctx->engine[ring->id].ringbuf != ringbuf)
937 continue;
938
939
940 if (__intel_ring_space(request->tail, ringbuf->tail,
941 ringbuf->size) >= bytes) {
942 break;
943 }
944 }
945
946 if (&request->list == &ring->request_list)
947 return -ENOSPC;
948
949 ret = i915_wait_request(request);
950 if (ret)
951 return ret;
952
953 i915_gem_retire_requests_ring(ring);
954
955 return intel_ring_space(ringbuf) >= bytes ? 0 : -ENOSPC;
956}
957
958static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf,
959 struct intel_context *ctx,
960 int bytes)
961{
962 struct intel_engine_cs *ring = ringbuf->ring;
963 struct drm_device *dev = ring->dev;
964 struct drm_i915_private *dev_priv = dev->dev_private;
965 unsigned long end;
966 int ret;
967
968 ret = logical_ring_wait_request(ringbuf, bytes);
969 if (ret != -ENOSPC)
970 return ret;
971
972
973 intel_logical_ring_advance_and_submit(ringbuf, ctx, NULL);
974
975
976
977
978
979
980 end = jiffies + 60 * HZ;
981
982 ret = 0;
983 do {
984 if (intel_ring_space(ringbuf) >= bytes)
985 break;
986
987 msleep(1);
988
989 if (dev_priv->mm.interruptible && signal_pending(current)) {
990 ret = -ERESTARTSYS;
991 break;
992 }
993
994 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
995 dev_priv->mm.interruptible);
996 if (ret)
997 break;
998
999 if (time_after(jiffies, end)) {
1000 ret = -EBUSY;
1001 break;
1002 }
1003 } while (1);
1004
1005 return ret;
1006}
1007
1008static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf,
1009 struct intel_context *ctx)
1010{
1011 uint32_t __iomem *virt;
1012 int rem = ringbuf->size - ringbuf->tail;
1013
1014 if (ringbuf->space < rem) {
1015 int ret = logical_ring_wait_for_space(ringbuf, ctx, rem);
1016
1017 if (ret)
1018 return ret;
1019 }
1020
1021 virt = ringbuf->virtual_start + ringbuf->tail;
1022 rem /= 4;
1023 while (rem--)
1024 iowrite32(MI_NOOP, virt++);
1025
1026 ringbuf->tail = 0;
1027 intel_ring_update_space(ringbuf);
1028
1029 return 0;
1030}
1031
1032static int logical_ring_prepare(struct intel_ringbuffer *ringbuf,
1033 struct intel_context *ctx, int bytes)
1034{
1035 int ret;
1036
1037 if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
1038 ret = logical_ring_wrap_buffer(ringbuf, ctx);
1039 if (unlikely(ret))
1040 return ret;
1041 }
1042
1043 if (unlikely(ringbuf->space < bytes)) {
1044 ret = logical_ring_wait_for_space(ringbuf, ctx, bytes);
1045 if (unlikely(ret))
1046 return ret;
1047 }
1048
1049 return 0;
1050}
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf,
1066 struct intel_context *ctx, int num_dwords)
1067{
1068 struct intel_engine_cs *ring = ringbuf->ring;
1069 struct drm_device *dev = ring->dev;
1070 struct drm_i915_private *dev_priv = dev->dev_private;
1071 int ret;
1072
1073 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1074 dev_priv->mm.interruptible);
1075 if (ret)
1076 return ret;
1077
1078 ret = logical_ring_prepare(ringbuf, ctx, num_dwords * sizeof(uint32_t));
1079 if (ret)
1080 return ret;
1081
1082
1083 ret = logical_ring_alloc_request(ring, ctx);
1084 if (ret)
1085 return ret;
1086
1087 ringbuf->space -= num_dwords * sizeof(uint32_t);
1088 return 0;
1089}
1090
1091static int intel_logical_ring_workarounds_emit(struct intel_engine_cs *ring,
1092 struct intel_context *ctx)
1093{
1094 int ret, i;
1095 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
1096 struct drm_device *dev = ring->dev;
1097 struct drm_i915_private *dev_priv = dev->dev_private;
1098 struct i915_workarounds *w = &dev_priv->workarounds;
1099
1100 if (WARN_ON_ONCE(w->count == 0))
1101 return 0;
1102
1103 ring->gpu_caches_dirty = true;
1104 ret = logical_ring_flush_all_caches(ringbuf, ctx);
1105 if (ret)
1106 return ret;
1107
1108 ret = intel_logical_ring_begin(ringbuf, ctx, w->count * 2 + 2);
1109 if (ret)
1110 return ret;
1111
1112 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count));
1113 for (i = 0; i < w->count; i++) {
1114 intel_logical_ring_emit(ringbuf, w->reg[i].addr);
1115 intel_logical_ring_emit(ringbuf, w->reg[i].value);
1116 }
1117 intel_logical_ring_emit(ringbuf, MI_NOOP);
1118
1119 intel_logical_ring_advance(ringbuf);
1120
1121 ring->gpu_caches_dirty = true;
1122 ret = logical_ring_flush_all_caches(ringbuf, ctx);
1123 if (ret)
1124 return ret;
1125
1126 return 0;
1127}
1128
1129static int gen8_init_common_ring(struct intel_engine_cs *ring)
1130{
1131 struct drm_device *dev = ring->dev;
1132 struct drm_i915_private *dev_priv = dev->dev_private;
1133
1134 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
1135 I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
1136
1137 if (ring->status_page.obj) {
1138 I915_WRITE(RING_HWS_PGA(ring->mmio_base),
1139 (u32)ring->status_page.gfx_addr);
1140 POSTING_READ(RING_HWS_PGA(ring->mmio_base));
1141 }
1142
1143 I915_WRITE(RING_MODE_GEN7(ring),
1144 _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
1145 _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
1146 POSTING_READ(RING_MODE_GEN7(ring));
1147 ring->next_context_status_buffer = 0;
1148 DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
1149
1150 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
1151
1152 return 0;
1153}
1154
1155static int gen8_init_render_ring(struct intel_engine_cs *ring)
1156{
1157 struct drm_device *dev = ring->dev;
1158 struct drm_i915_private *dev_priv = dev->dev_private;
1159 int ret;
1160
1161 ret = gen8_init_common_ring(ring);
1162 if (ret)
1163 return ret;
1164
1165
1166
1167
1168
1169
1170
1171 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1172
1173 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1174
1175 return init_workarounds_ring(ring);
1176}
1177
1178static int gen9_init_render_ring(struct intel_engine_cs *ring)
1179{
1180 int ret;
1181
1182 ret = gen8_init_common_ring(ring);
1183 if (ret)
1184 return ret;
1185
1186 return init_workarounds_ring(ring);
1187}
1188
1189static int gen8_emit_bb_start(struct intel_ringbuffer *ringbuf,
1190 struct intel_context *ctx,
1191 u64 offset, unsigned dispatch_flags)
1192{
1193 bool ppgtt = !(dispatch_flags & I915_DISPATCH_SECURE);
1194 int ret;
1195
1196 ret = intel_logical_ring_begin(ringbuf, ctx, 4);
1197 if (ret)
1198 return ret;
1199
1200
1201 intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
1202 intel_logical_ring_emit(ringbuf, lower_32_bits(offset));
1203 intel_logical_ring_emit(ringbuf, upper_32_bits(offset));
1204 intel_logical_ring_emit(ringbuf, MI_NOOP);
1205 intel_logical_ring_advance(ringbuf);
1206
1207 return 0;
1208}
1209
1210static bool gen8_logical_ring_get_irq(struct intel_engine_cs *ring)
1211{
1212 struct drm_device *dev = ring->dev;
1213 struct drm_i915_private *dev_priv = dev->dev_private;
1214 unsigned long flags;
1215
1216 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1217 return false;
1218
1219 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1220 if (ring->irq_refcount++ == 0) {
1221 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
1222 POSTING_READ(RING_IMR(ring->mmio_base));
1223 }
1224 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1225
1226 return true;
1227}
1228
1229static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring)
1230{
1231 struct drm_device *dev = ring->dev;
1232 struct drm_i915_private *dev_priv = dev->dev_private;
1233 unsigned long flags;
1234
1235 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1236 if (--ring->irq_refcount == 0) {
1237 I915_WRITE_IMR(ring, ~ring->irq_keep_mask);
1238 POSTING_READ(RING_IMR(ring->mmio_base));
1239 }
1240 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1241}
1242
1243static int gen8_emit_flush(struct intel_ringbuffer *ringbuf,
1244 struct intel_context *ctx,
1245 u32 invalidate_domains,
1246 u32 unused)
1247{
1248 struct intel_engine_cs *ring = ringbuf->ring;
1249 struct drm_device *dev = ring->dev;
1250 struct drm_i915_private *dev_priv = dev->dev_private;
1251 uint32_t cmd;
1252 int ret;
1253
1254 ret = intel_logical_ring_begin(ringbuf, ctx, 4);
1255 if (ret)
1256 return ret;
1257
1258 cmd = MI_FLUSH_DW + 1;
1259
1260
1261
1262
1263
1264
1265 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1266
1267 if (invalidate_domains & I915_GEM_GPU_DOMAINS) {
1268 cmd |= MI_INVALIDATE_TLB;
1269 if (ring == &dev_priv->ring[VCS])
1270 cmd |= MI_INVALIDATE_BSD;
1271 }
1272
1273 intel_logical_ring_emit(ringbuf, cmd);
1274 intel_logical_ring_emit(ringbuf,
1275 I915_GEM_HWS_SCRATCH_ADDR |
1276 MI_FLUSH_DW_USE_GTT);
1277 intel_logical_ring_emit(ringbuf, 0);
1278 intel_logical_ring_emit(ringbuf, 0);
1279 intel_logical_ring_advance(ringbuf);
1280
1281 return 0;
1282}
1283
1284static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf,
1285 struct intel_context *ctx,
1286 u32 invalidate_domains,
1287 u32 flush_domains)
1288{
1289 struct intel_engine_cs *ring = ringbuf->ring;
1290 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
1291 u32 flags = 0;
1292 int ret;
1293
1294 flags |= PIPE_CONTROL_CS_STALL;
1295
1296 if (flush_domains) {
1297 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
1298 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
1299 }
1300
1301 if (invalidate_domains) {
1302 flags |= PIPE_CONTROL_TLB_INVALIDATE;
1303 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
1304 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1305 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
1306 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
1307 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
1308 flags |= PIPE_CONTROL_QW_WRITE;
1309 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
1310 }
1311
1312 ret = intel_logical_ring_begin(ringbuf, ctx, 6);
1313 if (ret)
1314 return ret;
1315
1316 intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
1317 intel_logical_ring_emit(ringbuf, flags);
1318 intel_logical_ring_emit(ringbuf, scratch_addr);
1319 intel_logical_ring_emit(ringbuf, 0);
1320 intel_logical_ring_emit(ringbuf, 0);
1321 intel_logical_ring_emit(ringbuf, 0);
1322 intel_logical_ring_advance(ringbuf);
1323
1324 return 0;
1325}
1326
1327static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1328{
1329 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1330}
1331
1332static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1333{
1334 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
1335}
1336
1337static int gen8_emit_request(struct intel_ringbuffer *ringbuf,
1338 struct drm_i915_gem_request *request)
1339{
1340 struct intel_engine_cs *ring = ringbuf->ring;
1341 u32 cmd;
1342 int ret;
1343
1344
1345
1346
1347
1348
1349 ret = intel_logical_ring_begin(ringbuf, request->ctx, 8);
1350 if (ret)
1351 return ret;
1352
1353 cmd = MI_STORE_DWORD_IMM_GEN4;
1354 cmd |= MI_GLOBAL_GTT;
1355
1356 intel_logical_ring_emit(ringbuf, cmd);
1357 intel_logical_ring_emit(ringbuf,
1358 (ring->status_page.gfx_addr +
1359 (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)));
1360 intel_logical_ring_emit(ringbuf, 0);
1361 intel_logical_ring_emit(ringbuf,
1362 i915_gem_request_get_seqno(ring->outstanding_lazy_request));
1363 intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
1364 intel_logical_ring_emit(ringbuf, MI_NOOP);
1365 intel_logical_ring_advance_and_submit(ringbuf, request->ctx, request);
1366
1367
1368
1369
1370
1371 intel_logical_ring_emit(ringbuf, MI_NOOP);
1372 intel_logical_ring_emit(ringbuf, MI_NOOP);
1373 intel_logical_ring_advance(ringbuf);
1374
1375 return 0;
1376}
1377
1378static int intel_lr_context_render_state_init(struct intel_engine_cs *ring,
1379 struct intel_context *ctx)
1380{
1381 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
1382 struct render_state so;
1383 struct drm_i915_file_private *file_priv = ctx->file_priv;
1384 struct drm_file *file = file_priv ? file_priv->file : NULL;
1385 int ret;
1386
1387 ret = i915_gem_render_state_prepare(ring, &so);
1388 if (ret)
1389 return ret;
1390
1391 if (so.rodata == NULL)
1392 return 0;
1393
1394 ret = ring->emit_bb_start(ringbuf,
1395 ctx,
1396 so.ggtt_offset,
1397 I915_DISPATCH_SECURE);
1398 if (ret)
1399 goto out;
1400
1401 i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring);
1402
1403 ret = __i915_add_request(ring, file, so.obj);
1404
1405
1406out:
1407 i915_gem_render_state_fini(&so);
1408 return ret;
1409}
1410
1411static int gen8_init_rcs_context(struct intel_engine_cs *ring,
1412 struct intel_context *ctx)
1413{
1414 int ret;
1415
1416 ret = intel_logical_ring_workarounds_emit(ring, ctx);
1417 if (ret)
1418 return ret;
1419
1420 return intel_lr_context_render_state_init(ring, ctx);
1421}
1422
1423
1424
1425
1426
1427
1428
1429void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
1430{
1431 struct drm_i915_private *dev_priv;
1432
1433 if (!intel_ring_initialized(ring))
1434 return;
1435
1436 dev_priv = ring->dev->dev_private;
1437
1438 intel_logical_ring_stop(ring);
1439 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
1440 i915_gem_request_assign(&ring->outstanding_lazy_request, NULL);
1441
1442 if (ring->cleanup)
1443 ring->cleanup(ring);
1444
1445 i915_cmd_parser_fini_ring(ring);
1446
1447 if (ring->status_page.obj) {
1448 kunmap(sg_page(ring->status_page.obj->pages->sgl));
1449 ring->status_page.obj = NULL;
1450 }
1451}
1452
1453static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
1454{
1455 int ret;
1456
1457
1458 ring->buffer = NULL;
1459
1460 ring->dev = dev;
1461 INIT_LIST_HEAD(&ring->active_list);
1462 INIT_LIST_HEAD(&ring->request_list);
1463 init_waitqueue_head(&ring->irq_queue);
1464
1465 INIT_LIST_HEAD(&ring->execlist_queue);
1466 INIT_LIST_HEAD(&ring->execlist_retired_req_list);
1467 spin_lock_init(&ring->execlist_lock);
1468
1469 ret = i915_cmd_parser_init_ring(ring);
1470 if (ret)
1471 return ret;
1472
1473 ret = intel_lr_context_deferred_create(ring->default_context, ring);
1474
1475 return ret;
1476}
1477
1478static int logical_render_ring_init(struct drm_device *dev)
1479{
1480 struct drm_i915_private *dev_priv = dev->dev_private;
1481 struct intel_engine_cs *ring = &dev_priv->ring[RCS];
1482 int ret;
1483
1484 ring->name = "render ring";
1485 ring->id = RCS;
1486 ring->mmio_base = RENDER_RING_BASE;
1487 ring->irq_enable_mask =
1488 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
1489 ring->irq_keep_mask =
1490 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
1491 if (HAS_L3_DPF(dev))
1492 ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1493
1494 if (INTEL_INFO(dev)->gen >= 9)
1495 ring->init_hw = gen9_init_render_ring;
1496 else
1497 ring->init_hw = gen8_init_render_ring;
1498 ring->init_context = gen8_init_rcs_context;
1499 ring->cleanup = intel_fini_pipe_control;
1500 ring->get_seqno = gen8_get_seqno;
1501 ring->set_seqno = gen8_set_seqno;
1502 ring->emit_request = gen8_emit_request;
1503 ring->emit_flush = gen8_emit_flush_render;
1504 ring->irq_get = gen8_logical_ring_get_irq;
1505 ring->irq_put = gen8_logical_ring_put_irq;
1506 ring->emit_bb_start = gen8_emit_bb_start;
1507
1508 ring->dev = dev;
1509 ret = logical_ring_init(dev, ring);
1510 if (ret)
1511 return ret;
1512
1513 return intel_init_pipe_control(ring);
1514}
1515
1516static int logical_bsd_ring_init(struct drm_device *dev)
1517{
1518 struct drm_i915_private *dev_priv = dev->dev_private;
1519 struct intel_engine_cs *ring = &dev_priv->ring[VCS];
1520
1521 ring->name = "bsd ring";
1522 ring->id = VCS;
1523 ring->mmio_base = GEN6_BSD_RING_BASE;
1524 ring->irq_enable_mask =
1525 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
1526 ring->irq_keep_mask =
1527 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
1528
1529 ring->init_hw = gen8_init_common_ring;
1530 ring->get_seqno = gen8_get_seqno;
1531 ring->set_seqno = gen8_set_seqno;
1532 ring->emit_request = gen8_emit_request;
1533 ring->emit_flush = gen8_emit_flush;
1534 ring->irq_get = gen8_logical_ring_get_irq;
1535 ring->irq_put = gen8_logical_ring_put_irq;
1536 ring->emit_bb_start = gen8_emit_bb_start;
1537
1538 return logical_ring_init(dev, ring);
1539}
1540
1541static int logical_bsd2_ring_init(struct drm_device *dev)
1542{
1543 struct drm_i915_private *dev_priv = dev->dev_private;
1544 struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
1545
1546 ring->name = "bds2 ring";
1547 ring->id = VCS2;
1548 ring->mmio_base = GEN8_BSD2_RING_BASE;
1549 ring->irq_enable_mask =
1550 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
1551 ring->irq_keep_mask =
1552 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
1553
1554 ring->init_hw = gen8_init_common_ring;
1555 ring->get_seqno = gen8_get_seqno;
1556 ring->set_seqno = gen8_set_seqno;
1557 ring->emit_request = gen8_emit_request;
1558 ring->emit_flush = gen8_emit_flush;
1559 ring->irq_get = gen8_logical_ring_get_irq;
1560 ring->irq_put = gen8_logical_ring_put_irq;
1561 ring->emit_bb_start = gen8_emit_bb_start;
1562
1563 return logical_ring_init(dev, ring);
1564}
1565
1566static int logical_blt_ring_init(struct drm_device *dev)
1567{
1568 struct drm_i915_private *dev_priv = dev->dev_private;
1569 struct intel_engine_cs *ring = &dev_priv->ring[BCS];
1570
1571 ring->name = "blitter ring";
1572 ring->id = BCS;
1573 ring->mmio_base = BLT_RING_BASE;
1574 ring->irq_enable_mask =
1575 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
1576 ring->irq_keep_mask =
1577 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
1578
1579 ring->init_hw = gen8_init_common_ring;
1580 ring->get_seqno = gen8_get_seqno;
1581 ring->set_seqno = gen8_set_seqno;
1582 ring->emit_request = gen8_emit_request;
1583 ring->emit_flush = gen8_emit_flush;
1584 ring->irq_get = gen8_logical_ring_get_irq;
1585 ring->irq_put = gen8_logical_ring_put_irq;
1586 ring->emit_bb_start = gen8_emit_bb_start;
1587
1588 return logical_ring_init(dev, ring);
1589}
1590
1591static int logical_vebox_ring_init(struct drm_device *dev)
1592{
1593 struct drm_i915_private *dev_priv = dev->dev_private;
1594 struct intel_engine_cs *ring = &dev_priv->ring[VECS];
1595
1596 ring->name = "video enhancement ring";
1597 ring->id = VECS;
1598 ring->mmio_base = VEBOX_RING_BASE;
1599 ring->irq_enable_mask =
1600 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
1601 ring->irq_keep_mask =
1602 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
1603
1604 ring->init_hw = gen8_init_common_ring;
1605 ring->get_seqno = gen8_get_seqno;
1606 ring->set_seqno = gen8_set_seqno;
1607 ring->emit_request = gen8_emit_request;
1608 ring->emit_flush = gen8_emit_flush;
1609 ring->irq_get = gen8_logical_ring_get_irq;
1610 ring->irq_put = gen8_logical_ring_put_irq;
1611 ring->emit_bb_start = gen8_emit_bb_start;
1612
1613 return logical_ring_init(dev, ring);
1614}
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626int intel_logical_rings_init(struct drm_device *dev)
1627{
1628 struct drm_i915_private *dev_priv = dev->dev_private;
1629 int ret;
1630
1631 ret = logical_render_ring_init(dev);
1632 if (ret)
1633 return ret;
1634
1635 if (HAS_BSD(dev)) {
1636 ret = logical_bsd_ring_init(dev);
1637 if (ret)
1638 goto cleanup_render_ring;
1639 }
1640
1641 if (HAS_BLT(dev)) {
1642 ret = logical_blt_ring_init(dev);
1643 if (ret)
1644 goto cleanup_bsd_ring;
1645 }
1646
1647 if (HAS_VEBOX(dev)) {
1648 ret = logical_vebox_ring_init(dev);
1649 if (ret)
1650 goto cleanup_blt_ring;
1651 }
1652
1653 if (HAS_BSD2(dev)) {
1654 ret = logical_bsd2_ring_init(dev);
1655 if (ret)
1656 goto cleanup_vebox_ring;
1657 }
1658
1659 ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
1660 if (ret)
1661 goto cleanup_bsd2_ring;
1662
1663 return 0;
1664
1665cleanup_bsd2_ring:
1666 intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
1667cleanup_vebox_ring:
1668 intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
1669cleanup_blt_ring:
1670 intel_logical_ring_cleanup(&dev_priv->ring[BCS]);
1671cleanup_bsd_ring:
1672 intel_logical_ring_cleanup(&dev_priv->ring[VCS]);
1673cleanup_render_ring:
1674 intel_logical_ring_cleanup(&dev_priv->ring[RCS]);
1675
1676 return ret;
1677}
1678
1679static u32
1680make_rpcs(struct drm_device *dev)
1681{
1682 u32 rpcs = 0;
1683
1684
1685
1686
1687
1688 if (INTEL_INFO(dev)->gen < 9)
1689 return 0;
1690
1691
1692
1693
1694
1695
1696
1697 if (INTEL_INFO(dev)->has_slice_pg) {
1698 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
1699 rpcs |= INTEL_INFO(dev)->slice_total <<
1700 GEN8_RPCS_S_CNT_SHIFT;
1701 rpcs |= GEN8_RPCS_ENABLE;
1702 }
1703
1704 if (INTEL_INFO(dev)->has_subslice_pg) {
1705 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
1706 rpcs |= INTEL_INFO(dev)->subslice_per_slice <<
1707 GEN8_RPCS_SS_CNT_SHIFT;
1708 rpcs |= GEN8_RPCS_ENABLE;
1709 }
1710
1711 if (INTEL_INFO(dev)->has_eu_pg) {
1712 rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
1713 GEN8_RPCS_EU_MIN_SHIFT;
1714 rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
1715 GEN8_RPCS_EU_MAX_SHIFT;
1716 rpcs |= GEN8_RPCS_ENABLE;
1717 }
1718
1719 return rpcs;
1720}
1721
1722static int
1723populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
1724 struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
1725{
1726 struct drm_device *dev = ring->dev;
1727 struct drm_i915_private *dev_priv = dev->dev_private;
1728 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
1729 struct page *page;
1730 uint32_t *reg_state;
1731 int ret;
1732
1733 if (!ppgtt)
1734 ppgtt = dev_priv->mm.aliasing_ppgtt;
1735
1736 ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
1737 if (ret) {
1738 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
1739 return ret;
1740 }
1741
1742 ret = i915_gem_object_get_pages(ctx_obj);
1743 if (ret) {
1744 DRM_DEBUG_DRIVER("Could not get object pages\n");
1745 return ret;
1746 }
1747
1748 i915_gem_object_pin_pages(ctx_obj);
1749
1750
1751
1752 page = i915_gem_object_get_page(ctx_obj, 1);
1753 reg_state = kmap_atomic(page);
1754
1755
1756
1757
1758
1759
1760 if (ring->id == RCS)
1761 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
1762 else
1763 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
1764 reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
1765 reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
1766 reg_state[CTX_CONTEXT_CONTROL+1] =
1767 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
1768 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
1769 reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
1770 reg_state[CTX_RING_HEAD+1] = 0;
1771 reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
1772 reg_state[CTX_RING_TAIL+1] = 0;
1773 reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
1774
1775
1776
1777 reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
1778 reg_state[CTX_RING_BUFFER_CONTROL+1] =
1779 ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
1780 reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
1781 reg_state[CTX_BB_HEAD_U+1] = 0;
1782 reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
1783 reg_state[CTX_BB_HEAD_L+1] = 0;
1784 reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
1785 reg_state[CTX_BB_STATE+1] = (1<<5);
1786 reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
1787 reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
1788 reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
1789 reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
1790 reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
1791 reg_state[CTX_SECOND_BB_STATE+1] = 0;
1792 if (ring->id == RCS) {
1793
1794
1795
1796 reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
1797 reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
1798 reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
1799 reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
1800 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
1801 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
1802 }
1803 reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
1804 reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
1805 reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
1806 reg_state[CTX_CTX_TIMESTAMP+1] = 0;
1807 reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
1808 reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
1809 reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
1810 reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
1811 reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
1812 reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
1813 reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
1814 reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
1815 reg_state[CTX_PDP3_UDW+1] = upper_32_bits(ppgtt->pdp.page_directory[3]->daddr);
1816 reg_state[CTX_PDP3_LDW+1] = lower_32_bits(ppgtt->pdp.page_directory[3]->daddr);
1817 reg_state[CTX_PDP2_UDW+1] = upper_32_bits(ppgtt->pdp.page_directory[2]->daddr);
1818 reg_state[CTX_PDP2_LDW+1] = lower_32_bits(ppgtt->pdp.page_directory[2]->daddr);
1819 reg_state[CTX_PDP1_UDW+1] = upper_32_bits(ppgtt->pdp.page_directory[1]->daddr);
1820 reg_state[CTX_PDP1_LDW+1] = lower_32_bits(ppgtt->pdp.page_directory[1]->daddr);
1821 reg_state[CTX_PDP0_UDW+1] = upper_32_bits(ppgtt->pdp.page_directory[0]->daddr);
1822 reg_state[CTX_PDP0_LDW+1] = lower_32_bits(ppgtt->pdp.page_directory[0]->daddr);
1823 if (ring->id == RCS) {
1824 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
1825 reg_state[CTX_R_PWR_CLK_STATE] = GEN8_R_PWR_CLK_STATE;
1826 reg_state[CTX_R_PWR_CLK_STATE+1] = make_rpcs(dev);
1827 }
1828
1829 kunmap_atomic(reg_state);
1830
1831 ctx_obj->dirty = 1;
1832 set_page_dirty(page);
1833 i915_gem_object_unpin_pages(ctx_obj);
1834
1835 return 0;
1836}
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846void intel_lr_context_free(struct intel_context *ctx)
1847{
1848 int i;
1849
1850 for (i = 0; i < I915_NUM_RINGS; i++) {
1851 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
1852
1853 if (ctx_obj) {
1854 struct intel_ringbuffer *ringbuf =
1855 ctx->engine[i].ringbuf;
1856 struct intel_engine_cs *ring = ringbuf->ring;
1857
1858 if (ctx == ring->default_context) {
1859 intel_unpin_ringbuffer_obj(ringbuf);
1860 i915_gem_object_ggtt_unpin(ctx_obj);
1861 }
1862 WARN_ON(ctx->engine[ring->id].pin_count);
1863 intel_destroy_ringbuffer_obj(ringbuf);
1864 kfree(ringbuf);
1865 drm_gem_object_unreference(&ctx_obj->base);
1866 }
1867 }
1868}
1869
1870static uint32_t get_lr_context_size(struct intel_engine_cs *ring)
1871{
1872 int ret = 0;
1873
1874 WARN_ON(INTEL_INFO(ring->dev)->gen < 8);
1875
1876 switch (ring->id) {
1877 case RCS:
1878 if (INTEL_INFO(ring->dev)->gen >= 9)
1879 ret = GEN9_LR_CONTEXT_RENDER_SIZE;
1880 else
1881 ret = GEN8_LR_CONTEXT_RENDER_SIZE;
1882 break;
1883 case VCS:
1884 case BCS:
1885 case VECS:
1886 case VCS2:
1887 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
1888 break;
1889 }
1890
1891 return ret;
1892}
1893
1894static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring,
1895 struct drm_i915_gem_object *default_ctx_obj)
1896{
1897 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1898
1899
1900
1901 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj);
1902 ring->status_page.page_addr =
1903 kmap(sg_page(default_ctx_obj->pages->sgl));
1904 ring->status_page.obj = default_ctx_obj;
1905
1906 I915_WRITE(RING_HWS_PGA(ring->mmio_base),
1907 (u32)ring->status_page.gfx_addr);
1908 POSTING_READ(RING_HWS_PGA(ring->mmio_base));
1909}
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924int intel_lr_context_deferred_create(struct intel_context *ctx,
1925 struct intel_engine_cs *ring)
1926{
1927 const bool is_global_default_ctx = (ctx == ring->default_context);
1928 struct drm_device *dev = ring->dev;
1929 struct drm_i915_gem_object *ctx_obj;
1930 uint32_t context_size;
1931 struct intel_ringbuffer *ringbuf;
1932 int ret;
1933
1934 WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
1935 WARN_ON(ctx->engine[ring->id].state);
1936
1937 context_size = round_up(get_lr_context_size(ring), 4096);
1938
1939 ctx_obj = i915_gem_alloc_context_obj(dev, context_size);
1940 if (IS_ERR(ctx_obj)) {
1941 ret = PTR_ERR(ctx_obj);
1942 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed: %d\n", ret);
1943 return ret;
1944 }
1945
1946 if (is_global_default_ctx) {
1947 ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0);
1948 if (ret) {
1949 DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n",
1950 ret);
1951 drm_gem_object_unreference(&ctx_obj->base);
1952 return ret;
1953 }
1954 }
1955
1956 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
1957 if (!ringbuf) {
1958 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
1959 ring->name);
1960 ret = -ENOMEM;
1961 goto error_unpin_ctx;
1962 }
1963
1964 ringbuf->ring = ring;
1965
1966 ringbuf->size = 32 * PAGE_SIZE;
1967 ringbuf->effective_size = ringbuf->size;
1968 ringbuf->head = 0;
1969 ringbuf->tail = 0;
1970 ringbuf->last_retired_head = -1;
1971 intel_ring_update_space(ringbuf);
1972
1973 if (ringbuf->obj == NULL) {
1974 ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
1975 if (ret) {
1976 DRM_DEBUG_DRIVER(
1977 "Failed to allocate ringbuffer obj %s: %d\n",
1978 ring->name, ret);
1979 goto error_free_rbuf;
1980 }
1981
1982 if (is_global_default_ctx) {
1983 ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
1984 if (ret) {
1985 DRM_ERROR(
1986 "Failed to pin and map ringbuffer %s: %d\n",
1987 ring->name, ret);
1988 goto error_destroy_rbuf;
1989 }
1990 }
1991
1992 }
1993
1994 ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
1995 if (ret) {
1996 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
1997 goto error;
1998 }
1999
2000 ctx->engine[ring->id].ringbuf = ringbuf;
2001 ctx->engine[ring->id].state = ctx_obj;
2002
2003 if (ctx == ring->default_context)
2004 lrc_setup_hardware_status_page(ring, ctx_obj);
2005 else if (ring->id == RCS && !ctx->rcs_initialized) {
2006 if (ring->init_context) {
2007 ret = ring->init_context(ring, ctx);
2008 if (ret) {
2009 DRM_ERROR("ring init context: %d\n", ret);
2010 ctx->engine[ring->id].ringbuf = NULL;
2011 ctx->engine[ring->id].state = NULL;
2012 goto error;
2013 }
2014 }
2015
2016 ctx->rcs_initialized = true;
2017 }
2018
2019 return 0;
2020
2021error:
2022 if (is_global_default_ctx)
2023 intel_unpin_ringbuffer_obj(ringbuf);
2024error_destroy_rbuf:
2025 intel_destroy_ringbuffer_obj(ringbuf);
2026error_free_rbuf:
2027 kfree(ringbuf);
2028error_unpin_ctx:
2029 if (is_global_default_ctx)
2030 i915_gem_object_ggtt_unpin(ctx_obj);
2031 drm_gem_object_unreference(&ctx_obj->base);
2032 return ret;
2033}
2034
2035void intel_lr_context_reset(struct drm_device *dev,
2036 struct intel_context *ctx)
2037{
2038 struct drm_i915_private *dev_priv = dev->dev_private;
2039 struct intel_engine_cs *ring;
2040 int i;
2041
2042 for_each_ring(ring, dev_priv, i) {
2043 struct drm_i915_gem_object *ctx_obj =
2044 ctx->engine[ring->id].state;
2045 struct intel_ringbuffer *ringbuf =
2046 ctx->engine[ring->id].ringbuf;
2047 uint32_t *reg_state;
2048 struct page *page;
2049
2050 if (!ctx_obj)
2051 continue;
2052
2053 if (i915_gem_object_get_pages(ctx_obj)) {
2054 WARN(1, "Failed get_pages for context obj\n");
2055 continue;
2056 }
2057 page = i915_gem_object_get_page(ctx_obj, 1);
2058 reg_state = kmap_atomic(page);
2059
2060 reg_state[CTX_RING_HEAD+1] = 0;
2061 reg_state[CTX_RING_TAIL+1] = 0;
2062
2063 kunmap_atomic(reg_state);
2064
2065 ringbuf->head = 0;
2066 ringbuf->tail = 0;
2067 }
2068}
2069