1
2
3
4
5
6
7#include "i915_drv.h"
8#include "i915_gem_object.h"
9#include "i915_scatterlist.h"
10#include "i915_gem_lmem.h"
11#include "i915_gem_mman.h"
12
13void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
14 struct sg_table *pages,
15 unsigned int sg_page_sizes)
16{
17 struct drm_i915_private *i915 = to_i915(obj->base.dev);
18 unsigned long supported = INTEL_INFO(i915)->page_sizes;
19 int i;
20
21 lockdep_assert_held(&obj->mm.lock);
22
23 if (i915_gem_object_is_volatile(obj))
24 obj->mm.madv = I915_MADV_DONTNEED;
25
26
27 if (obj->cache_dirty) {
28 obj->write_domain = 0;
29 if (i915_gem_object_has_struct_page(obj))
30 drm_clflush_sg(pages);
31 obj->cache_dirty = false;
32 }
33
34 obj->mm.get_page.sg_pos = pages->sgl;
35 obj->mm.get_page.sg_idx = 0;
36
37 obj->mm.pages = pages;
38
39 if (i915_gem_object_is_tiled(obj) &&
40 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
41 GEM_BUG_ON(obj->mm.quirked);
42 __i915_gem_object_pin_pages(obj);
43 obj->mm.quirked = true;
44 }
45
46 GEM_BUG_ON(!sg_page_sizes);
47 obj->mm.page_sizes.phys = sg_page_sizes;
48
49
50
51
52
53
54
55
56
57 obj->mm.page_sizes.sg = 0;
58 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
59 if (obj->mm.page_sizes.phys & ~0u << i)
60 obj->mm.page_sizes.sg |= BIT(i);
61 }
62 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
63
64 if (i915_gem_object_is_shrinkable(obj)) {
65 struct list_head *list;
66 unsigned long flags;
67
68 spin_lock_irqsave(&i915->mm.obj_lock, flags);
69
70 i915->mm.shrink_count++;
71 i915->mm.shrink_memory += obj->base.size;
72
73 if (obj->mm.madv != I915_MADV_WILLNEED)
74 list = &i915->mm.purge_list;
75 else
76 list = &i915->mm.shrink_list;
77 list_add_tail(&obj->mm.link, list);
78
79 atomic_set(&obj->mm.shrink_pin, 0);
80 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
81 }
82}
83
84int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
85{
86 struct drm_i915_private *i915 = to_i915(obj->base.dev);
87 int err;
88
89 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
90 drm_dbg(&i915->drm,
91 "Attempting to obtain a purgeable object\n");
92 return -EFAULT;
93 }
94
95 err = obj->ops->get_pages(obj);
96 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
97
98 return err;
99}
100
101
102
103
104
105
106
107
108int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
109{
110 int err;
111
112 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
113 if (err)
114 return err;
115
116 if (unlikely(!i915_gem_object_has_pages(obj))) {
117 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
118
119 err = ____i915_gem_object_get_pages(obj);
120 if (err)
121 goto unlock;
122
123 smp_mb__before_atomic();
124 }
125 atomic_inc(&obj->mm.pages_pin_count);
126
127unlock:
128 mutex_unlock(&obj->mm.lock);
129 return err;
130}
131
132
133void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
134{
135 drm_gem_free_mmap_offset(&obj->base);
136 if (obj->ops->truncate)
137 obj->ops->truncate(obj);
138}
139
140
141void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
142{
143 lockdep_assert_held(&obj->mm.lock);
144 GEM_BUG_ON(i915_gem_object_has_pages(obj));
145
146 if (obj->ops->writeback)
147 obj->ops->writeback(obj);
148}
149
150static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
151{
152 struct radix_tree_iter iter;
153 void __rcu **slot;
154
155 rcu_read_lock();
156 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
157 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
158 rcu_read_unlock();
159}
160
161static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
162{
163 if (is_vmalloc_addr(ptr))
164 vunmap(ptr);
165 else
166 kunmap(kmap_to_page(ptr));
167}
168
169struct sg_table *
170__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
171{
172 struct sg_table *pages;
173
174 pages = fetch_and_zero(&obj->mm.pages);
175 if (IS_ERR_OR_NULL(pages))
176 return pages;
177
178 if (i915_gem_object_is_volatile(obj))
179 obj->mm.madv = I915_MADV_WILLNEED;
180
181 i915_gem_object_make_unshrinkable(obj);
182
183 if (obj->mm.mapping) {
184 unmap_object(obj, page_mask_bits(obj->mm.mapping));
185 obj->mm.mapping = NULL;
186 }
187
188 __i915_gem_object_reset_page_iter(obj);
189 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
190
191 return pages;
192}
193
194int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
195{
196 struct sg_table *pages;
197 int err;
198
199 if (i915_gem_object_has_pinned_pages(obj))
200 return -EBUSY;
201
202
203 mutex_lock(&obj->mm.lock);
204 if (unlikely(atomic_read(&obj->mm.pages_pin_count))) {
205 err = -EBUSY;
206 goto unlock;
207 }
208
209 i915_gem_object_release_mmap_offset(obj);
210
211
212
213
214
215
216 pages = __i915_gem_object_unset_pages(obj);
217
218
219
220
221
222
223
224 if (!pages && !i915_gem_object_needs_async_cancel(obj))
225 pages = ERR_PTR(-EINVAL);
226
227 if (!IS_ERR(pages))
228 obj->ops->put_pages(obj, pages);
229
230 err = 0;
231unlock:
232 mutex_unlock(&obj->mm.lock);
233
234 return err;
235}
236
237static inline pte_t iomap_pte(resource_size_t base,
238 dma_addr_t offset,
239 pgprot_t prot)
240{
241 return pte_mkspecial(pfn_pte((base + offset) >> PAGE_SHIFT, prot));
242}
243
244
245static void *i915_gem_object_map(struct drm_i915_gem_object *obj,
246 enum i915_map_type type)
247{
248 unsigned long n_pte = obj->base.size >> PAGE_SHIFT;
249 struct sg_table *sgt = obj->mm.pages;
250 pte_t *stack[32], **mem;
251 struct vm_struct *area;
252 pgprot_t pgprot;
253
254 if (!i915_gem_object_has_struct_page(obj) && type != I915_MAP_WC)
255 return NULL;
256
257
258 if (n_pte == 1 && type == I915_MAP_WB)
259 return kmap(sg_page(sgt->sgl));
260
261 mem = stack;
262 if (n_pte > ARRAY_SIZE(stack)) {
263
264 mem = kvmalloc_array(n_pte, sizeof(*mem), GFP_KERNEL);
265 if (!mem)
266 return NULL;
267 }
268
269 area = alloc_vm_area(obj->base.size, mem);
270 if (!area) {
271 if (mem != stack)
272 kvfree(mem);
273 return NULL;
274 }
275
276 switch (type) {
277 default:
278 MISSING_CASE(type);
279 fallthrough;
280 case I915_MAP_WB:
281 pgprot = PAGE_KERNEL;
282 break;
283 case I915_MAP_WC:
284 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
285 break;
286 }
287
288 if (i915_gem_object_has_struct_page(obj)) {
289 struct sgt_iter iter;
290 struct page *page;
291 pte_t **ptes = mem;
292
293 for_each_sgt_page(page, iter, sgt)
294 **ptes++ = mk_pte(page, pgprot);
295 } else {
296 resource_size_t iomap;
297 struct sgt_iter iter;
298 pte_t **ptes = mem;
299 dma_addr_t addr;
300
301 iomap = obj->mm.region->iomap.base;
302 iomap -= obj->mm.region->region.start;
303
304 for_each_sgt_daddr(addr, iter, sgt)
305 **ptes++ = iomap_pte(iomap, addr, pgprot);
306 }
307
308 if (mem != stack)
309 kvfree(mem);
310
311 return area->addr;
312}
313
314
315void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
316 enum i915_map_type type)
317{
318 enum i915_map_type has_type;
319 unsigned int flags;
320 bool pinned;
321 void *ptr;
322 int err;
323
324 flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_HAS_IOMEM;
325 if (!i915_gem_object_type_has(obj, flags))
326 return ERR_PTR(-ENXIO);
327
328 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
329 if (err)
330 return ERR_PTR(err);
331
332 pinned = !(type & I915_MAP_OVERRIDE);
333 type &= ~I915_MAP_OVERRIDE;
334
335 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
336 if (unlikely(!i915_gem_object_has_pages(obj))) {
337 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
338
339 err = ____i915_gem_object_get_pages(obj);
340 if (err)
341 goto err_unlock;
342
343 smp_mb__before_atomic();
344 }
345 atomic_inc(&obj->mm.pages_pin_count);
346 pinned = false;
347 }
348 GEM_BUG_ON(!i915_gem_object_has_pages(obj));
349
350 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
351 if (ptr && has_type != type) {
352 if (pinned) {
353 err = -EBUSY;
354 goto err_unpin;
355 }
356
357 unmap_object(obj, ptr);
358
359 ptr = obj->mm.mapping = NULL;
360 }
361
362 if (!ptr) {
363 ptr = i915_gem_object_map(obj, type);
364 if (!ptr) {
365 err = -ENOMEM;
366 goto err_unpin;
367 }
368
369 obj->mm.mapping = page_pack_bits(ptr, type);
370 }
371
372out_unlock:
373 mutex_unlock(&obj->mm.lock);
374 return ptr;
375
376err_unpin:
377 atomic_dec(&obj->mm.pages_pin_count);
378err_unlock:
379 ptr = ERR_PTR(err);
380 goto out_unlock;
381}
382
383void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
384 unsigned long offset,
385 unsigned long size)
386{
387 enum i915_map_type has_type;
388 void *ptr;
389
390 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
391 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
392 offset, size, obj->base.size));
393
394 wmb();
395 obj->mm.dirty = true;
396
397 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
398 return;
399
400 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
401 if (has_type == I915_MAP_WC)
402 return;
403
404 drm_clflush_virt_range(ptr + offset, size);
405 if (size == obj->base.size) {
406 obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
407 obj->cache_dirty = false;
408 }
409}
410
411void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
412{
413 GEM_BUG_ON(!obj->mm.mapping);
414
415
416
417
418
419
420
421 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
422
423 i915_gem_object_unpin_map(obj);
424}
425
426struct scatterlist *
427i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
428 unsigned int n,
429 unsigned int *offset)
430{
431 struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
432 struct scatterlist *sg;
433 unsigned int idx, count;
434
435 might_sleep();
436 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
437 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
438
439
440
441
442
443
444
445
446
447
448 if (n < READ_ONCE(iter->sg_idx))
449 goto lookup;
450
451 mutex_lock(&iter->lock);
452
453
454
455
456
457
458 sg = iter->sg_pos;
459 idx = iter->sg_idx;
460 count = __sg_page_count(sg);
461
462 while (idx + count <= n) {
463 void *entry;
464 unsigned long i;
465 int ret;
466
467
468
469
470
471
472
473
474
475 ret = radix_tree_insert(&iter->radix, idx, sg);
476 if (ret && ret != -EEXIST)
477 goto scan;
478
479 entry = xa_mk_value(idx);
480 for (i = 1; i < count; i++) {
481 ret = radix_tree_insert(&iter->radix, idx + i, entry);
482 if (ret && ret != -EEXIST)
483 goto scan;
484 }
485
486 idx += count;
487 sg = ____sg_next(sg);
488 count = __sg_page_count(sg);
489 }
490
491scan:
492 iter->sg_pos = sg;
493 iter->sg_idx = idx;
494
495 mutex_unlock(&iter->lock);
496
497 if (unlikely(n < idx))
498 goto lookup;
499
500
501
502
503 while (idx + count <= n) {
504 idx += count;
505 sg = ____sg_next(sg);
506 count = __sg_page_count(sg);
507 }
508
509 *offset = n - idx;
510 return sg;
511
512lookup:
513 rcu_read_lock();
514
515 sg = radix_tree_lookup(&iter->radix, n);
516 GEM_BUG_ON(!sg);
517
518
519
520
521
522
523
524 *offset = 0;
525 if (unlikely(xa_is_value(sg))) {
526 unsigned long base = xa_to_value(sg);
527
528 sg = radix_tree_lookup(&iter->radix, base);
529 GEM_BUG_ON(!sg);
530
531 *offset = n - base;
532 }
533
534 rcu_read_unlock();
535
536 return sg;
537}
538
539struct page *
540i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
541{
542 struct scatterlist *sg;
543 unsigned int offset;
544
545 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
546
547 sg = i915_gem_object_get_sg(obj, n, &offset);
548 return nth_page(sg_page(sg), offset);
549}
550
551
552struct page *
553i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
554 unsigned int n)
555{
556 struct page *page;
557
558 page = i915_gem_object_get_page(obj, n);
559 if (!obj->mm.dirty)
560 set_page_dirty(page);
561
562 return page;
563}
564
565dma_addr_t
566i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
567 unsigned long n,
568 unsigned int *len)
569{
570 struct scatterlist *sg;
571 unsigned int offset;
572
573 sg = i915_gem_object_get_sg(obj, n, &offset);
574
575 if (len)
576 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
577
578 return sg_dma_address(sg) + (offset << PAGE_SHIFT);
579}
580
581dma_addr_t
582i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
583 unsigned long n)
584{
585 return i915_gem_object_get_dma_address_len(obj, n, NULL);
586}
587