1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <drm/drmP.h>
30#include <drm/i915_drm.h>
31
32#include "i915_drv.h"
33#include "intel_drv.h"
34#include "i915_trace.h"
35
36static bool ggtt_is_idle(struct drm_i915_private *dev_priv)
37{
38 struct i915_ggtt *ggtt = &dev_priv->ggtt;
39 struct intel_engine_cs *engine;
40 enum intel_engine_id id;
41
42 for_each_engine(engine, dev_priv, id) {
43 struct intel_timeline *tl;
44
45 tl = &ggtt->base.timeline.engine[engine->id];
46 if (i915_gem_active_isset(&tl->last_request))
47 return false;
48 }
49
50 return true;
51}
52
53static int ggtt_flush(struct drm_i915_private *i915)
54{
55 int err;
56
57
58
59
60
61
62
63 err = i915_gem_switch_to_kernel_context(i915);
64 if (err)
65 return err;
66
67 err = i915_gem_wait_for_idle(i915,
68 I915_WAIT_INTERRUPTIBLE |
69 I915_WAIT_LOCKED);
70 if (err)
71 return err;
72
73 return 0;
74}
75
76static bool
77mark_free(struct drm_mm_scan *scan,
78 struct i915_vma *vma,
79 unsigned int flags,
80 struct list_head *unwind)
81{
82 if (i915_vma_is_pinned(vma))
83 return false;
84
85 if (flags & PIN_NONFAULT && !list_empty(&vma->obj->userfault_link))
86 return false;
87
88 list_add(&vma->evict_link, unwind);
89 return drm_mm_scan_add_block(scan, &vma->node);
90}
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115int
116i915_gem_evict_something(struct i915_address_space *vm,
117 u64 min_size, u64 alignment,
118 unsigned cache_level,
119 u64 start, u64 end,
120 unsigned flags)
121{
122 struct drm_i915_private *dev_priv = vm->i915;
123 struct drm_mm_scan scan;
124 struct list_head eviction_list;
125 struct list_head *phases[] = {
126 &vm->inactive_list,
127 &vm->active_list,
128 NULL,
129 }, **phase;
130 struct i915_vma *vma, *next;
131 struct drm_mm_node *node;
132 enum drm_mm_insert_mode mode;
133 int ret;
134
135 lockdep_assert_held(&vm->i915->drm.struct_mutex);
136 trace_i915_gem_evict(vm, min_size, alignment, flags);
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151 mode = DRM_MM_INSERT_BEST;
152 if (flags & PIN_HIGH)
153 mode = DRM_MM_INSERT_HIGH;
154 if (flags & PIN_MAPPABLE)
155 mode = DRM_MM_INSERT_LOW;
156 drm_mm_scan_init_with_range(&scan, &vm->mm,
157 min_size, alignment, cache_level,
158 start, end, mode);
159
160
161
162
163
164
165 if (!(flags & PIN_NONBLOCK))
166 i915_gem_retire_requests(dev_priv);
167 else
168 phases[1] = NULL;
169
170search_again:
171 INIT_LIST_HEAD(&eviction_list);
172 phase = phases;
173 do {
174 list_for_each_entry(vma, *phase, vm_link)
175 if (mark_free(&scan, vma, flags, &eviction_list))
176 goto found;
177 } while (*++phase);
178
179
180 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
181 ret = drm_mm_scan_remove_block(&scan, &vma->node);
182 BUG_ON(ret);
183 }
184
185
186
187
188
189
190 if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK)
191 return -ENOSPC;
192
193 if (ggtt_is_idle(dev_priv)) {
194
195
196
197
198 return intel_has_pending_fb_unpin(dev_priv) ? -EAGAIN : -ENOSPC;
199 }
200
201 ret = ggtt_flush(dev_priv);
202 if (ret)
203 return ret;
204
205 goto search_again;
206
207found:
208
209
210
211
212
213
214 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
215 if (drm_mm_scan_remove_block(&scan, &vma->node))
216 __i915_vma_pin(vma);
217 else
218 list_del(&vma->evict_link);
219 }
220
221
222 ret = 0;
223 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
224 __i915_vma_unpin(vma);
225 if (ret == 0)
226 ret = i915_vma_unbind(vma);
227 }
228
229 while (ret == 0 && (node = drm_mm_scan_color_evict(&scan))) {
230 vma = container_of(node, struct i915_vma, node);
231 ret = i915_vma_unbind(vma);
232 }
233
234 return ret;
235}
236
237
238
239
240
241
242
243
244
245
246
247
248int i915_gem_evict_for_node(struct i915_address_space *vm,
249 struct drm_mm_node *target,
250 unsigned int flags)
251{
252 LIST_HEAD(eviction_list);
253 struct drm_mm_node *node;
254 u64 start = target->start;
255 u64 end = start + target->size;
256 struct i915_vma *vma, *next;
257 bool check_color;
258 int ret = 0;
259
260 lockdep_assert_held(&vm->i915->drm.struct_mutex);
261 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
262 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
263
264 trace_i915_gem_evict_node(vm, target, flags);
265
266
267
268
269
270
271 if (!(flags & PIN_NONBLOCK))
272 i915_gem_retire_requests(vm->i915);
273
274 check_color = vm->mm.color_adjust;
275 if (check_color) {
276
277 if (start)
278 start -= I915_GTT_PAGE_SIZE;
279
280
281 end += I915_GTT_PAGE_SIZE;
282 }
283 GEM_BUG_ON(start >= end);
284
285 drm_mm_for_each_node_in_range(node, &vm->mm, start, end) {
286
287 if (node->color == I915_COLOR_UNEVICTABLE) {
288 ret = -ENOSPC;
289 break;
290 }
291
292 GEM_BUG_ON(!node->allocated);
293 vma = container_of(node, typeof(*vma), node);
294
295
296
297
298
299
300
301 if (check_color) {
302 if (node->start + node->size == target->start) {
303 if (node->color == target->color)
304 continue;
305 }
306 if (node->start == target->start + target->size) {
307 if (node->color == target->color)
308 continue;
309 }
310 }
311
312 if (flags & PIN_NONBLOCK &&
313 (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))) {
314 ret = -ENOSPC;
315 break;
316 }
317
318
319 if (i915_vma_is_pinned(vma)) {
320 ret = -ENOSPC;
321 if (vma->exec_entry &&
322 vma->exec_entry->flags & EXEC_OBJECT_PINNED)
323 ret = -EINVAL;
324 break;
325 }
326
327
328
329
330
331
332
333
334
335 __i915_vma_pin(vma);
336 list_add(&vma->evict_link, &eviction_list);
337 }
338
339 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
340 __i915_vma_unpin(vma);
341 if (ret == 0)
342 ret = i915_vma_unbind(vma);
343 }
344
345 return ret;
346}
347
348
349
350
351
352
353
354
355
356
357
358
359
360int i915_gem_evict_vm(struct i915_address_space *vm)
361{
362 struct list_head *phases[] = {
363 &vm->inactive_list,
364 &vm->active_list,
365 NULL
366 }, **phase;
367 struct list_head eviction_list;
368 struct i915_vma *vma, *next;
369 int ret;
370
371 lockdep_assert_held(&vm->i915->drm.struct_mutex);
372 trace_i915_gem_evict_vm(vm);
373
374
375
376
377
378
379 if (i915_is_ggtt(vm)) {
380 ret = ggtt_flush(vm->i915);
381 if (ret)
382 return ret;
383 }
384
385 INIT_LIST_HEAD(&eviction_list);
386 phase = phases;
387 do {
388 list_for_each_entry(vma, *phase, vm_link) {
389 if (i915_vma_is_pinned(vma))
390 continue;
391
392 __i915_vma_pin(vma);
393 list_add(&vma->evict_link, &eviction_list);
394 }
395 } while (*++phase);
396
397 ret = 0;
398 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
399 __i915_vma_unpin(vma);
400 if (ret == 0)
401 ret = i915_vma_unbind(vma);
402 }
403 return ret;
404}
405
406#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
407#include "selftests/i915_gem_evict.c"
408#endif
409