1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#include <linux/reservation.h>
36#include <linux/export.h>
37
38
39
40
41
42
43
44
45
46
47
48
49DEFINE_WW_CLASS(reservation_ww_class);
50EXPORT_SYMBOL(reservation_ww_class);
51
52struct lock_class_key reservation_seqcount_class;
53EXPORT_SYMBOL(reservation_seqcount_class);
54
55const char reservation_seqcount_string[] = "reservation_seqcount";
56EXPORT_SYMBOL(reservation_seqcount_string);
57
58
59
60
61
62
63
64
65
66
67
68
69int reservation_object_reserve_shared(struct reservation_object *obj)
70{
71 struct reservation_object_list *fobj, *old;
72 u32 max;
73
74 old = reservation_object_get_list(obj);
75
76 if (old && old->shared_max) {
77 if (old->shared_count < old->shared_max) {
78
79 kfree(obj->staged);
80 obj->staged = NULL;
81 return 0;
82 } else
83 max = old->shared_max * 2;
84 } else
85 max = 4;
86
87
88
89
90
91 fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]),
92 GFP_KERNEL);
93 if (!fobj)
94 return -ENOMEM;
95
96 obj->staged = fobj;
97 fobj->shared_max = max;
98 return 0;
99}
100EXPORT_SYMBOL(reservation_object_reserve_shared);
101
102static void
103reservation_object_add_shared_inplace(struct reservation_object *obj,
104 struct reservation_object_list *fobj,
105 struct dma_fence *fence)
106{
107 u32 i;
108
109 dma_fence_get(fence);
110
111 preempt_disable();
112 write_seqcount_begin(&obj->seq);
113
114 for (i = 0; i < fobj->shared_count; ++i) {
115 struct dma_fence *old_fence;
116
117 old_fence = rcu_dereference_protected(fobj->shared[i],
118 reservation_object_held(obj));
119
120 if (old_fence->context == fence->context) {
121
122 RCU_INIT_POINTER(fobj->shared[i], fence);
123 write_seqcount_end(&obj->seq);
124 preempt_enable();
125
126 dma_fence_put(old_fence);
127 return;
128 }
129 }
130
131
132
133
134
135 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
136 fobj->shared_count++;
137
138 write_seqcount_end(&obj->seq);
139 preempt_enable();
140}
141
142static void
143reservation_object_add_shared_replace(struct reservation_object *obj,
144 struct reservation_object_list *old,
145 struct reservation_object_list *fobj,
146 struct dma_fence *fence)
147{
148 unsigned i;
149 struct dma_fence *old_fence = NULL;
150
151 dma_fence_get(fence);
152
153 if (!old) {
154 RCU_INIT_POINTER(fobj->shared[0], fence);
155 fobj->shared_count = 1;
156 goto done;
157 }
158
159
160
161
162
163
164
165 fobj->shared_count = old->shared_count;
166
167 for (i = 0; i < old->shared_count; ++i) {
168 struct dma_fence *check;
169
170 check = rcu_dereference_protected(old->shared[i],
171 reservation_object_held(obj));
172
173 if (!old_fence && check->context == fence->context) {
174 old_fence = check;
175 RCU_INIT_POINTER(fobj->shared[i], fence);
176 } else
177 RCU_INIT_POINTER(fobj->shared[i], check);
178 }
179 if (!old_fence) {
180 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
181 fobj->shared_count++;
182 }
183
184done:
185 preempt_disable();
186 write_seqcount_begin(&obj->seq);
187
188
189
190
191 RCU_INIT_POINTER(obj->fence, fobj);
192 write_seqcount_end(&obj->seq);
193 preempt_enable();
194
195 if (old)
196 kfree_rcu(old, rcu);
197
198 dma_fence_put(old_fence);
199}
200
201
202
203
204
205
206
207
208
209void reservation_object_add_shared_fence(struct reservation_object *obj,
210 struct dma_fence *fence)
211{
212 struct reservation_object_list *old, *fobj = obj->staged;
213
214 old = reservation_object_get_list(obj);
215 obj->staged = NULL;
216
217 if (!fobj) {
218 BUG_ON(old->shared_count >= old->shared_max);
219 reservation_object_add_shared_inplace(obj, old, fence);
220 } else
221 reservation_object_add_shared_replace(obj, old, fobj, fence);
222}
223EXPORT_SYMBOL(reservation_object_add_shared_fence);
224
225
226
227
228
229
230
231
232void reservation_object_add_excl_fence(struct reservation_object *obj,
233 struct dma_fence *fence)
234{
235 struct dma_fence *old_fence = reservation_object_get_excl(obj);
236 struct reservation_object_list *old;
237 u32 i = 0;
238
239 old = reservation_object_get_list(obj);
240 if (old)
241 i = old->shared_count;
242
243 if (fence)
244 dma_fence_get(fence);
245
246 preempt_disable();
247 write_seqcount_begin(&obj->seq);
248
249 RCU_INIT_POINTER(obj->fence_excl, fence);
250 if (old)
251 old->shared_count = 0;
252 write_seqcount_end(&obj->seq);
253 preempt_enable();
254
255
256 while (i--)
257 dma_fence_put(rcu_dereference_protected(old->shared[i],
258 reservation_object_held(obj)));
259
260 dma_fence_put(old_fence);
261}
262EXPORT_SYMBOL(reservation_object_add_excl_fence);
263
264
265
266
267
268
269
270
271
272int reservation_object_copy_fences(struct reservation_object *dst,
273 struct reservation_object *src)
274{
275 struct reservation_object_list *src_list, *dst_list;
276 struct dma_fence *old, *new;
277 size_t size;
278 unsigned i;
279
280 src_list = reservation_object_get_list(src);
281
282 if (src_list) {
283 size = offsetof(typeof(*src_list),
284 shared[src_list->shared_count]);
285 dst_list = kmalloc(size, GFP_KERNEL);
286 if (!dst_list)
287 return -ENOMEM;
288
289 dst_list->shared_count = src_list->shared_count;
290 dst_list->shared_max = src_list->shared_count;
291 for (i = 0; i < src_list->shared_count; ++i)
292 dst_list->shared[i] =
293 dma_fence_get(src_list->shared[i]);
294 } else {
295 dst_list = NULL;
296 }
297
298 kfree(dst->staged);
299 dst->staged = NULL;
300
301 src_list = reservation_object_get_list(dst);
302
303 old = reservation_object_get_excl(dst);
304 new = reservation_object_get_excl(src);
305
306 dma_fence_get(new);
307
308 preempt_disable();
309 write_seqcount_begin(&dst->seq);
310
311 RCU_INIT_POINTER(dst->fence_excl, new);
312 RCU_INIT_POINTER(dst->fence, dst_list);
313 write_seqcount_end(&dst->seq);
314 preempt_enable();
315
316 if (src_list)
317 kfree_rcu(src_list, rcu);
318 dma_fence_put(old);
319
320 return 0;
321}
322EXPORT_SYMBOL(reservation_object_copy_fences);
323
324
325
326
327
328
329
330
331
332
333
334
335
336int reservation_object_get_fences_rcu(struct reservation_object *obj,
337 struct dma_fence **pfence_excl,
338 unsigned *pshared_count,
339 struct dma_fence ***pshared)
340{
341 struct dma_fence **shared = NULL;
342 struct dma_fence *fence_excl;
343 unsigned int shared_count;
344 int ret = 1;
345
346 do {
347 struct reservation_object_list *fobj;
348 unsigned seq;
349 unsigned int i;
350
351 shared_count = i = 0;
352
353 rcu_read_lock();
354 seq = read_seqcount_begin(&obj->seq);
355
356 fence_excl = rcu_dereference(obj->fence_excl);
357 if (fence_excl && !dma_fence_get_rcu(fence_excl))
358 goto unlock;
359
360 fobj = rcu_dereference(obj->fence);
361 if (fobj) {
362 struct dma_fence **nshared;
363 size_t sz = sizeof(*shared) * fobj->shared_max;
364
365 nshared = krealloc(shared, sz,
366 GFP_NOWAIT | __GFP_NOWARN);
367 if (!nshared) {
368 rcu_read_unlock();
369 nshared = krealloc(shared, sz, GFP_KERNEL);
370 if (nshared) {
371 shared = nshared;
372 continue;
373 }
374
375 ret = -ENOMEM;
376 break;
377 }
378 shared = nshared;
379 shared_count = fobj->shared_count;
380
381 for (i = 0; i < shared_count; ++i) {
382 shared[i] = rcu_dereference(fobj->shared[i]);
383 if (!dma_fence_get_rcu(shared[i]))
384 break;
385 }
386 }
387
388 if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) {
389 while (i--)
390 dma_fence_put(shared[i]);
391 dma_fence_put(fence_excl);
392 goto unlock;
393 }
394
395 ret = 0;
396unlock:
397 rcu_read_unlock();
398 } while (ret);
399
400 if (!shared_count) {
401 kfree(shared);
402 shared = NULL;
403 }
404
405 *pshared_count = shared_count;
406 *pshared = shared;
407 *pfence_excl = fence_excl;
408
409 return ret;
410}
411EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu);
412
413
414
415
416
417
418
419
420
421
422
423
424
425long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
426 bool wait_all, bool intr,
427 unsigned long timeout)
428{
429 struct dma_fence *fence;
430 unsigned seq, shared_count, i = 0;
431 long ret = timeout ? timeout : 1;
432
433retry:
434 shared_count = 0;
435 seq = read_seqcount_begin(&obj->seq);
436 rcu_read_lock();
437
438 fence = rcu_dereference(obj->fence_excl);
439 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
440 if (!dma_fence_get_rcu(fence))
441 goto unlock_retry;
442
443 if (dma_fence_is_signaled(fence)) {
444 dma_fence_put(fence);
445 fence = NULL;
446 }
447
448 } else {
449 fence = NULL;
450 }
451
452 if (!fence && wait_all) {
453 struct reservation_object_list *fobj =
454 rcu_dereference(obj->fence);
455
456 if (fobj)
457 shared_count = fobj->shared_count;
458
459 for (i = 0; i < shared_count; ++i) {
460 struct dma_fence *lfence = rcu_dereference(fobj->shared[i]);
461
462 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
463 &lfence->flags))
464 continue;
465
466 if (!dma_fence_get_rcu(lfence))
467 goto unlock_retry;
468
469 if (dma_fence_is_signaled(lfence)) {
470 dma_fence_put(lfence);
471 continue;
472 }
473
474 fence = lfence;
475 break;
476 }
477 }
478
479 rcu_read_unlock();
480 if (fence) {
481 if (read_seqcount_retry(&obj->seq, seq)) {
482 dma_fence_put(fence);
483 goto retry;
484 }
485
486 ret = dma_fence_wait_timeout(fence, intr, ret);
487 dma_fence_put(fence);
488 if (ret > 0 && wait_all && (i + 1 < shared_count))
489 goto retry;
490 }
491 return ret;
492
493unlock_retry:
494 rcu_read_unlock();
495 goto retry;
496}
497EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu);
498
499
500static inline int
501reservation_object_test_signaled_single(struct dma_fence *passed_fence)
502{
503 struct dma_fence *fence, *lfence = passed_fence;
504 int ret = 1;
505
506 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) {
507 fence = dma_fence_get_rcu(lfence);
508 if (!fence)
509 return -1;
510
511 ret = !!dma_fence_is_signaled(fence);
512 dma_fence_put(fence);
513 }
514 return ret;
515}
516
517
518
519
520
521
522
523
524
525
526
527bool reservation_object_test_signaled_rcu(struct reservation_object *obj,
528 bool test_all)
529{
530 unsigned seq, shared_count;
531 int ret;
532
533 rcu_read_lock();
534retry:
535 ret = true;
536 shared_count = 0;
537 seq = read_seqcount_begin(&obj->seq);
538
539 if (test_all) {
540 unsigned i;
541
542 struct reservation_object_list *fobj =
543 rcu_dereference(obj->fence);
544
545 if (fobj)
546 shared_count = fobj->shared_count;
547
548 for (i = 0; i < shared_count; ++i) {
549 struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
550
551 ret = reservation_object_test_signaled_single(fence);
552 if (ret < 0)
553 goto retry;
554 else if (!ret)
555 break;
556 }
557
558 if (read_seqcount_retry(&obj->seq, seq))
559 goto retry;
560 }
561
562 if (!shared_count) {
563 struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl);
564
565 if (fence_excl) {
566 ret = reservation_object_test_signaled_single(
567 fence_excl);
568 if (ret < 0)
569 goto retry;
570
571 if (read_seqcount_retry(&obj->seq, seq))
572 goto retry;
573 }
574 }
575
576 rcu_read_unlock();
577 return ret;
578}
579EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu);
580