1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#ifndef __I915_VMA_H__
26#define __I915_VMA_H__
27
28#include <linux/io-mapping.h>
29#include <linux/rbtree.h>
30
31#include <drm/drm_mm.h>
32
33#include "i915_gem_gtt.h"
34#include "i915_gem_fence_reg.h"
35#include "gem/i915_gem_object.h"
36
37#include "i915_active.h"
38#include "i915_request.h"
39
40enum i915_cache_level;
41
42
43
44
45
46
47
48
49
50
51
52struct i915_vma {
53 struct drm_mm_node node;
54 struct drm_i915_gem_object *obj;
55 struct i915_address_space *vm;
56 const struct i915_vma_ops *ops;
57 struct i915_fence_reg *fence;
58 struct reservation_object *resv;
59 struct sg_table *pages;
60 void __iomem *iomap;
61 void *private;
62 u64 size;
63 u64 display_alignment;
64 struct i915_page_sizes page_sizes;
65
66 u32 fence_size;
67 u32 fence_alignment;
68
69
70
71
72
73
74 atomic_t open_count;
75 unsigned long flags;
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99#define I915_VMA_PIN_MASK 0xff
100#define I915_VMA_PIN_OVERFLOW BIT(8)
101
102
103#define I915_VMA_GLOBAL_BIND BIT(9)
104#define I915_VMA_LOCAL_BIND BIT(10)
105#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
106
107#define I915_VMA_GGTT BIT(11)
108#define I915_VMA_CAN_FENCE BIT(12)
109#define I915_VMA_USERFAULT_BIT 13
110#define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT)
111#define I915_VMA_GGTT_WRITE BIT(14)
112
113 struct i915_active active;
114 struct i915_active_request last_fence;
115
116
117
118
119
120
121
122
123 struct i915_ggtt_view ggtt_view;
124
125
126 struct list_head vm_link;
127
128 struct list_head obj_link;
129 struct rb_node obj_node;
130 struct hlist_node obj_hash;
131
132
133 struct list_head exec_link;
134 struct list_head reloc_link;
135
136
137 struct list_head evict_link;
138
139 struct list_head closed_link;
140
141
142
143
144 unsigned int *exec_flags;
145 struct hlist_node exec_node;
146 u32 exec_handle;
147};
148
149struct i915_vma *
150i915_vma_instance(struct drm_i915_gem_object *obj,
151 struct i915_address_space *vm,
152 const struct i915_ggtt_view *view);
153
154void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags);
155#define I915_VMA_RELEASE_MAP BIT(0)
156
157static inline bool i915_vma_is_active(const struct i915_vma *vma)
158{
159 return !i915_active_is_idle(&vma->active);
160}
161
162int __must_check i915_vma_move_to_active(struct i915_vma *vma,
163 struct i915_request *rq,
164 unsigned int flags);
165
166static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
167{
168 return vma->flags & I915_VMA_GGTT;
169}
170
171static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma)
172{
173 return vma->flags & I915_VMA_GGTT_WRITE;
174}
175
176static inline void i915_vma_set_ggtt_write(struct i915_vma *vma)
177{
178 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
179 vma->flags |= I915_VMA_GGTT_WRITE;
180}
181
182static inline void i915_vma_unset_ggtt_write(struct i915_vma *vma)
183{
184 vma->flags &= ~I915_VMA_GGTT_WRITE;
185}
186
187void i915_vma_flush_writes(struct i915_vma *vma);
188
189static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
190{
191 return vma->flags & I915_VMA_CAN_FENCE;
192}
193
194static inline bool i915_vma_set_userfault(struct i915_vma *vma)
195{
196 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
197 return __test_and_set_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
198}
199
200static inline void i915_vma_unset_userfault(struct i915_vma *vma)
201{
202 return __clear_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
203}
204
205static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
206{
207 return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
208}
209
210static inline bool i915_vma_is_closed(const struct i915_vma *vma)
211{
212 return !list_empty(&vma->closed_link);
213}
214
215static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
216{
217 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
218 GEM_BUG_ON(!vma->node.allocated);
219 GEM_BUG_ON(upper_32_bits(vma->node.start));
220 GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
221 return lower_32_bits(vma->node.start);
222}
223
224static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma)
225{
226 return i915_vm_to_ggtt(vma->vm)->pin_bias;
227}
228
229static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
230{
231 i915_gem_object_get(vma->obj);
232 return vma;
233}
234
235static inline void i915_vma_put(struct i915_vma *vma)
236{
237 i915_gem_object_put(vma->obj);
238}
239
240static __always_inline ptrdiff_t ptrdiff(const void *a, const void *b)
241{
242 return a - b;
243}
244
245static inline long
246i915_vma_compare(struct i915_vma *vma,
247 struct i915_address_space *vm,
248 const struct i915_ggtt_view *view)
249{
250 ptrdiff_t cmp;
251
252 GEM_BUG_ON(view && !i915_is_ggtt(vm));
253
254 cmp = ptrdiff(vma->vm, vm);
255 if (cmp)
256 return cmp;
257
258 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0);
259 cmp = vma->ggtt_view.type;
260 if (!view)
261 return cmp;
262
263 cmp -= view->type;
264 if (cmp)
265 return cmp;
266
267 assert_i915_gem_gtt_types();
268
269
270
271
272
273
274
275
276
277
278
279 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL);
280 BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED);
281 BUILD_BUG_ON(I915_GGTT_VIEW_ROTATED >= I915_GGTT_VIEW_REMAPPED);
282 BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
283 offsetof(typeof(*view), partial));
284 BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
285 offsetof(typeof(*view), remapped));
286 return memcmp(&vma->ggtt_view.partial, &view->partial, view->type);
287}
288
289int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
290 u32 flags);
291bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level);
292bool i915_vma_misplaced(const struct i915_vma *vma,
293 u64 size, u64 alignment, u64 flags);
294void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
295void i915_vma_revoke_mmap(struct i915_vma *vma);
296int __must_check i915_vma_unbind(struct i915_vma *vma);
297void i915_vma_unlink_ctx(struct i915_vma *vma);
298void i915_vma_close(struct i915_vma *vma);
299void i915_vma_reopen(struct i915_vma *vma);
300void i915_vma_destroy(struct i915_vma *vma);
301
302#define assert_vma_held(vma) reservation_object_assert_held((vma)->resv)
303
304static inline void i915_vma_lock(struct i915_vma *vma)
305{
306 reservation_object_lock(vma->resv, NULL);
307}
308
309static inline void i915_vma_unlock(struct i915_vma *vma)
310{
311 reservation_object_unlock(vma->resv);
312}
313
314int __i915_vma_do_pin(struct i915_vma *vma,
315 u64 size, u64 alignment, u64 flags);
316static inline int __must_check
317i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
318{
319 BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
320 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
321 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
322
323
324
325
326 if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) {
327 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
328 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
329 return 0;
330 }
331
332 return __i915_vma_do_pin(vma, size, alignment, flags);
333}
334
335static inline int i915_vma_pin_count(const struct i915_vma *vma)
336{
337 return vma->flags & I915_VMA_PIN_MASK;
338}
339
340static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
341{
342 return i915_vma_pin_count(vma);
343}
344
345static inline void __i915_vma_pin(struct i915_vma *vma)
346{
347 vma->flags++;
348 GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
349}
350
351static inline void __i915_vma_unpin(struct i915_vma *vma)
352{
353 vma->flags--;
354}
355
356static inline void i915_vma_unpin(struct i915_vma *vma)
357{
358 GEM_BUG_ON(!i915_vma_is_pinned(vma));
359 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
360 __i915_vma_unpin(vma);
361}
362
363static inline bool i915_vma_is_bound(const struct i915_vma *vma,
364 unsigned int where)
365{
366 return vma->flags & where;
367}
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
383#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
384
385
386
387
388
389
390
391
392
393
394void i915_vma_unpin_iomap(struct i915_vma *vma);
395
396static inline struct page *i915_vma_first_page(struct i915_vma *vma)
397{
398 GEM_BUG_ON(!vma->pages);
399 return sg_page(vma->pages->sgl);
400}
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417int i915_vma_pin_fence(struct i915_vma *vma);
418int __must_check i915_vma_put_fence(struct i915_vma *vma);
419
420static inline void __i915_vma_unpin_fence(struct i915_vma *vma)
421{
422 GEM_BUG_ON(vma->fence->pin_count <= 0);
423 vma->fence->pin_count--;
424}
425
426
427
428
429
430
431
432
433
434static inline void
435i915_vma_unpin_fence(struct i915_vma *vma)
436{
437
438 if (vma->fence)
439 __i915_vma_unpin_fence(vma);
440}
441
442void i915_vma_parked(struct drm_i915_private *i915);
443
444#define for_each_until(cond) if (cond) break; else
445
446
447
448
449
450
451
452
453
454
455#define for_each_ggtt_vma(V, OBJ) \
456 list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \
457 for_each_until(!i915_vma_is_ggtt(V))
458
459struct i915_vma *i915_vma_alloc(void);
460void i915_vma_free(struct i915_vma *vma);
461
462#endif
463