1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <drm/drmP.h>
26#include <drm/i915_drm.h>
27#include "i915_drv.h"
28
29#define QUIET (__GFP_NORETRY | __GFP_NOWARN)
30
31
32#define IO_TLB_SEGPAGES (IO_TLB_SEGSIZE << IO_TLB_SHIFT >> PAGE_SHIFT)
33
34static void internal_free_pages(struct sg_table *st)
35{
36 struct scatterlist *sg;
37
38 for (sg = st->sgl; sg; sg = __sg_next(sg)) {
39 if (sg_page(sg))
40 __free_pages(sg_page(sg), get_order(sg->length));
41 }
42
43 sg_free_table(st);
44 kfree(st);
45}
46
47static struct sg_table *
48i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
49{
50 struct drm_i915_private *i915 = to_i915(obj->base.dev);
51 struct sg_table *st;
52 struct scatterlist *sg;
53 unsigned int npages;
54 int max_order;
55 gfp_t gfp;
56
57 max_order = MAX_ORDER;
58#ifdef CONFIG_SWIOTLB
59 if (swiotlb_nr_tbl()) {
60 unsigned int max_segment;
61
62 max_segment = swiotlb_max_segment();
63 if (max_segment) {
64 max_segment = max_t(unsigned int, max_segment,
65 PAGE_SIZE) >> PAGE_SHIFT;
66 max_order = min(max_order, ilog2(max_segment));
67 }
68 }
69#endif
70
71 gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE;
72 if (IS_I965GM(i915) || IS_I965G(i915)) {
73
74 gfp &= ~__GFP_HIGHMEM;
75 gfp |= __GFP_DMA32;
76 }
77
78create_st:
79 st = kmalloc(sizeof(*st), GFP_KERNEL);
80 if (!st)
81 return ERR_PTR(-ENOMEM);
82
83 npages = obj->base.size / PAGE_SIZE;
84 if (sg_alloc_table(st, npages, GFP_KERNEL)) {
85 kfree(st);
86 return ERR_PTR(-ENOMEM);
87 }
88
89 sg = st->sgl;
90 st->nents = 0;
91
92 do {
93 int order = min(fls(npages) - 1, max_order);
94 struct page *page;
95
96 do {
97 page = alloc_pages(gfp | (order ? QUIET : 0), order);
98 if (page)
99 break;
100 if (!order--)
101 goto err;
102
103
104 max_order = order;
105 } while (1);
106
107 sg_set_page(sg, page, PAGE_SIZE << order, 0);
108 st->nents++;
109
110 npages -= 1 << order;
111 if (!npages) {
112 sg_mark_end(sg);
113 break;
114 }
115
116 sg = __sg_next(sg);
117 } while (1);
118
119 if (i915_gem_gtt_prepare_pages(obj, st)) {
120
121 if (get_order(st->sgl->length)) {
122 internal_free_pages(st);
123 max_order = 0;
124 goto create_st;
125 }
126 goto err;
127 }
128
129
130
131
132
133
134 obj->mm.madv = I915_MADV_DONTNEED;
135 return st;
136
137err:
138 sg_set_page(sg, NULL, 0, 0);
139 sg_mark_end(sg);
140 internal_free_pages(st);
141 return ERR_PTR(-ENOMEM);
142}
143
144static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj,
145 struct sg_table *pages)
146{
147 i915_gem_gtt_finish_pages(obj, pages);
148 internal_free_pages(pages);
149
150 obj->mm.dirty = false;
151 obj->mm.madv = I915_MADV_WILLNEED;
152}
153
154static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
155 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
156 I915_GEM_OBJECT_IS_SHRINKABLE,
157 .get_pages = i915_gem_object_get_pages_internal,
158 .put_pages = i915_gem_object_put_pages_internal,
159};
160
161
162
163
164
165
166
167
168
169
170
171
172struct drm_i915_gem_object *
173i915_gem_object_create_internal(struct drm_i915_private *i915,
174 phys_addr_t size)
175{
176 struct drm_i915_gem_object *obj;
177
178 GEM_BUG_ON(!size);
179 GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));
180
181 if (overflows_type(size, obj->base.size))
182 return ERR_PTR(-E2BIG);
183
184 obj = i915_gem_object_alloc(i915);
185 if (!obj)
186 return ERR_PTR(-ENOMEM);
187
188 drm_gem_private_object_init(&i915->drm, &obj->base, size);
189 i915_gem_object_init(obj, &i915_gem_object_internal_ops);
190
191 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
192 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
193 obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
194 obj->cache_coherent = i915_gem_object_is_coherent(obj);
195 obj->cache_dirty = !obj->cache_coherent;
196
197 return obj;
198}
199