1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31#include <linux/uaccess.h>
32
33#include "amdgpu.h"
34#include "amdgpu_trace.h"
35
36#define AMDGPU_BO_LIST_MAX_PRIORITY 32u
37#define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1)
38
39static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu)
40{
41 struct amdgpu_bo_list *list = container_of(rcu, struct amdgpu_bo_list,
42 rhead);
43
44 kvfree(list);
45}
46
47static void amdgpu_bo_list_free(struct kref *ref)
48{
49 struct amdgpu_bo_list *list = container_of(ref, struct amdgpu_bo_list,
50 refcount);
51 struct amdgpu_bo_list_entry *e;
52
53 amdgpu_bo_list_for_each_entry(e, list) {
54 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
55
56 amdgpu_bo_unref(&bo);
57 }
58
59 call_rcu(&list->rhead, amdgpu_bo_list_free_rcu);
60}
61
62int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
63 struct drm_amdgpu_bo_list_entry *info,
64 unsigned num_entries, struct amdgpu_bo_list **result)
65{
66 unsigned last_entry = 0, first_userptr = num_entries;
67 struct amdgpu_bo_list_entry *array;
68 struct amdgpu_bo_list *list;
69 uint64_t total_size = 0;
70 size_t size;
71 unsigned i;
72 int r;
73
74 if (num_entries > (SIZE_MAX - sizeof(struct amdgpu_bo_list))
75 / sizeof(struct amdgpu_bo_list_entry))
76 return -EINVAL;
77
78 size = sizeof(struct amdgpu_bo_list);
79 size += num_entries * sizeof(struct amdgpu_bo_list_entry);
80 list = kvmalloc(size, GFP_KERNEL);
81 if (!list)
82 return -ENOMEM;
83
84 kref_init(&list->refcount);
85 list->gds_obj = NULL;
86 list->gws_obj = NULL;
87 list->oa_obj = NULL;
88
89 array = amdgpu_bo_list_array_entry(list, 0);
90 memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry));
91
92 for (i = 0; i < num_entries; ++i) {
93 struct amdgpu_bo_list_entry *entry;
94 struct drm_gem_object *gobj;
95 struct amdgpu_bo *bo;
96 struct mm_struct *usermm;
97
98 gobj = drm_gem_object_lookup(filp, info[i].bo_handle);
99 if (!gobj) {
100 r = -ENOENT;
101 goto error_free;
102 }
103
104 bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
105 drm_gem_object_put_unlocked(gobj);
106
107 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
108 if (usermm) {
109 if (usermm != current->mm) {
110 amdgpu_bo_unref(&bo);
111 r = -EPERM;
112 goto error_free;
113 }
114 entry = &array[--first_userptr];
115 } else {
116 entry = &array[last_entry++];
117 }
118
119 entry->priority = min(info[i].bo_priority,
120 AMDGPU_BO_LIST_MAX_PRIORITY);
121 entry->tv.bo = &bo->tbo;
122
123 if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GDS)
124 list->gds_obj = bo;
125 if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GWS)
126 list->gws_obj = bo;
127 if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_OA)
128 list->oa_obj = bo;
129
130 total_size += amdgpu_bo_size(bo);
131 trace_amdgpu_bo_list_set(list, bo);
132 }
133
134 list->first_userptr = first_userptr;
135 list->num_entries = num_entries;
136
137 trace_amdgpu_cs_bo_status(list->num_entries, total_size);
138
139 *result = list;
140 return 0;
141
142error_free:
143 while (i--) {
144 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(array[i].tv.bo);
145
146 amdgpu_bo_unref(&bo);
147 }
148 kvfree(list);
149 return r;
150
151}
152
153static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id)
154{
155 struct amdgpu_bo_list *list;
156
157 mutex_lock(&fpriv->bo_list_lock);
158 list = idr_remove(&fpriv->bo_list_handles, id);
159 mutex_unlock(&fpriv->bo_list_lock);
160 if (list)
161 kref_put(&list->refcount, amdgpu_bo_list_free);
162}
163
164int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id,
165 struct amdgpu_bo_list **result)
166{
167 rcu_read_lock();
168 *result = idr_find(&fpriv->bo_list_handles, id);
169
170 if (*result && kref_get_unless_zero(&(*result)->refcount)) {
171 rcu_read_unlock();
172 return 0;
173 }
174
175 rcu_read_unlock();
176 return -ENOENT;
177}
178
179void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
180 struct list_head *validated)
181{
182
183
184
185
186 struct list_head bucket[AMDGPU_BO_LIST_NUM_BUCKETS];
187 struct amdgpu_bo_list_entry *e;
188 unsigned i;
189
190 for (i = 0; i < AMDGPU_BO_LIST_NUM_BUCKETS; i++)
191 INIT_LIST_HEAD(&bucket[i]);
192
193
194
195
196
197
198 amdgpu_bo_list_for_each_entry(e, list) {
199 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
200 unsigned priority = e->priority;
201
202 if (!bo->parent)
203 list_add_tail(&e->tv.head, &bucket[priority]);
204
205 e->user_pages = NULL;
206 }
207
208
209 for (i = 0; i < AMDGPU_BO_LIST_NUM_BUCKETS; i++)
210 list_splice(&bucket[i], validated);
211}
212
213void amdgpu_bo_list_put(struct amdgpu_bo_list *list)
214{
215 kref_put(&list->refcount, amdgpu_bo_list_free);
216}
217
218int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
219 struct drm_amdgpu_bo_list_entry **info_param)
220{
221 const void __user *uptr = u64_to_user_ptr(in->bo_info_ptr);
222 const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry);
223 struct drm_amdgpu_bo_list_entry *info;
224 int r;
225
226 info = kvmalloc_array(in->bo_number, info_size, GFP_KERNEL);
227 if (!info)
228 return -ENOMEM;
229
230
231 r = -EFAULT;
232 if (likely(info_size == in->bo_info_size)) {
233 unsigned long bytes = in->bo_number *
234 in->bo_info_size;
235
236 if (copy_from_user(info, uptr, bytes))
237 goto error_free;
238
239 } else {
240 unsigned long bytes = min(in->bo_info_size, info_size);
241 unsigned i;
242
243 memset(info, 0, in->bo_number * info_size);
244 for (i = 0; i < in->bo_number; ++i) {
245 if (copy_from_user(&info[i], uptr, bytes))
246 goto error_free;
247
248 uptr += in->bo_info_size;
249 }
250 }
251
252 *info_param = info;
253 return 0;
254
255error_free:
256 kvfree(info);
257 return r;
258}
259
260int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
261 struct drm_file *filp)
262{
263 struct amdgpu_device *adev = dev->dev_private;
264 struct amdgpu_fpriv *fpriv = filp->driver_priv;
265 union drm_amdgpu_bo_list *args = data;
266 uint32_t handle = args->in.list_handle;
267 struct drm_amdgpu_bo_list_entry *info = NULL;
268 struct amdgpu_bo_list *list, *old;
269 int r;
270
271 r = amdgpu_bo_create_list_entry_array(&args->in, &info);
272 if (r)
273 goto error_free;
274
275 switch (args->in.operation) {
276 case AMDGPU_BO_LIST_OP_CREATE:
277 r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
278 &list);
279 if (r)
280 goto error_free;
281
282 mutex_lock(&fpriv->bo_list_lock);
283 r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL);
284 mutex_unlock(&fpriv->bo_list_lock);
285 if (r < 0) {
286 amdgpu_bo_list_put(list);
287 return r;
288 }
289
290 handle = r;
291 break;
292
293 case AMDGPU_BO_LIST_OP_DESTROY:
294 amdgpu_bo_list_destroy(fpriv, handle);
295 handle = 0;
296 break;
297
298 case AMDGPU_BO_LIST_OP_UPDATE:
299 r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
300 &list);
301 if (r)
302 goto error_free;
303
304 mutex_lock(&fpriv->bo_list_lock);
305 old = idr_replace(&fpriv->bo_list_handles, list, handle);
306 mutex_unlock(&fpriv->bo_list_lock);
307
308 if (IS_ERR(old)) {
309 amdgpu_bo_list_put(list);
310 r = PTR_ERR(old);
311 goto error_free;
312 }
313
314 amdgpu_bo_list_put(old);
315 break;
316
317 default:
318 r = -EINVAL;
319 goto error_free;
320 }
321
322 memset(args, 0, sizeof(*args));
323 args->out.list_handle = handle;
324 kvfree(info);
325
326 return 0;
327
328error_free:
329 if (info)
330 kvfree(info);
331 return r;
332}
333