1
2
3
4
5
6
7
8
9
10#include <linux/dma-fence-chain.h>
11
12static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
13
14
15
16
17
18
19
20
21static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
22{
23 struct dma_fence *prev;
24
25 rcu_read_lock();
26 prev = dma_fence_get_rcu_safe(&chain->prev);
27 rcu_read_unlock();
28 return prev;
29}
30
31
32
33
34
35
36
37
38
39struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
40{
41 struct dma_fence_chain *chain, *prev_chain;
42 struct dma_fence *prev, *replacement, *tmp;
43
44 chain = to_dma_fence_chain(fence);
45 if (!chain) {
46 dma_fence_put(fence);
47 return NULL;
48 }
49
50 while ((prev = dma_fence_chain_get_prev(chain))) {
51
52 prev_chain = to_dma_fence_chain(prev);
53 if (prev_chain) {
54 if (!dma_fence_is_signaled(prev_chain->fence))
55 break;
56
57 replacement = dma_fence_chain_get_prev(prev_chain);
58 } else {
59 if (!dma_fence_is_signaled(prev))
60 break;
61
62 replacement = NULL;
63 }
64
65 tmp = cmpxchg((void **)&chain->prev, (void *)prev, (void *)replacement);
66 if (tmp == prev)
67 dma_fence_put(tmp);
68 else
69 dma_fence_put(replacement);
70 dma_fence_put(prev);
71 }
72
73 dma_fence_put(fence);
74 return prev;
75}
76EXPORT_SYMBOL(dma_fence_chain_walk);
77
78
79
80
81
82
83
84
85
86
87
88
89int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
90{
91 struct dma_fence_chain *chain;
92
93 if (!seqno)
94 return 0;
95
96 chain = to_dma_fence_chain(*pfence);
97 if (!chain || chain->base.seqno < seqno)
98 return -EINVAL;
99
100 dma_fence_chain_for_each(*pfence, &chain->base) {
101 if ((*pfence)->context != chain->base.context ||
102 to_dma_fence_chain(*pfence)->prev_seqno < seqno)
103 break;
104 }
105 dma_fence_put(&chain->base);
106
107 return 0;
108}
109EXPORT_SYMBOL(dma_fence_chain_find_seqno);
110
111static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
112{
113 return "dma_fence_chain";
114}
115
116static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
117{
118 return "unbound";
119}
120
121static void dma_fence_chain_irq_work(struct irq_work *work)
122{
123 struct dma_fence_chain *chain;
124
125 chain = container_of(work, typeof(*chain), work);
126
127
128 if (!dma_fence_chain_enable_signaling(&chain->base))
129
130 dma_fence_signal(&chain->base);
131 dma_fence_put(&chain->base);
132}
133
134static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
135{
136 struct dma_fence_chain *chain;
137
138 chain = container_of(cb, typeof(*chain), cb);
139 irq_work_queue(&chain->work);
140 dma_fence_put(f);
141}
142
143static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
144{
145 struct dma_fence_chain *head = to_dma_fence_chain(fence);
146
147 dma_fence_get(&head->base);
148 dma_fence_chain_for_each(fence, &head->base) {
149 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
150 struct dma_fence *f = chain ? chain->fence : fence;
151
152 dma_fence_get(f);
153 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
154 dma_fence_put(fence);
155 return true;
156 }
157 dma_fence_put(f);
158 }
159 dma_fence_put(&head->base);
160 return false;
161}
162
163static bool dma_fence_chain_signaled(struct dma_fence *fence)
164{
165 dma_fence_chain_for_each(fence, fence) {
166 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
167 struct dma_fence *f = chain ? chain->fence : fence;
168
169 if (!dma_fence_is_signaled(f)) {
170 dma_fence_put(fence);
171 return false;
172 }
173 }
174
175 return true;
176}
177
178static void dma_fence_chain_release(struct dma_fence *fence)
179{
180 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
181
182 dma_fence_put(rcu_dereference_protected(chain->prev, true));
183 dma_fence_put(chain->fence);
184 dma_fence_free(fence);
185}
186
187const struct dma_fence_ops dma_fence_chain_ops = {
188 .use_64bit_seqno = true,
189 .get_driver_name = dma_fence_chain_get_driver_name,
190 .get_timeline_name = dma_fence_chain_get_timeline_name,
191 .enable_signaling = dma_fence_chain_enable_signaling,
192 .signaled = dma_fence_chain_signaled,
193 .release = dma_fence_chain_release,
194};
195EXPORT_SYMBOL(dma_fence_chain_ops);
196
197
198
199
200
201
202
203
204
205
206void dma_fence_chain_init(struct dma_fence_chain *chain,
207 struct dma_fence *prev,
208 struct dma_fence *fence,
209 uint64_t seqno)
210{
211 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
212 uint64_t context;
213
214 spin_lock_init(&chain->lock);
215 rcu_assign_pointer(chain->prev, prev);
216 chain->fence = fence;
217 chain->prev_seqno = 0;
218 init_irq_work(&chain->work, dma_fence_chain_irq_work);
219
220
221 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
222 context = prev->context;
223 chain->prev_seqno = prev->seqno;
224 } else {
225 context = dma_fence_context_alloc(1);
226
227 if (prev_chain)
228 seqno = max(prev->seqno, seqno);
229 }
230
231 dma_fence_init(&chain->base, &dma_fence_chain_ops,
232 &chain->lock, context, seqno);
233}
234EXPORT_SYMBOL(dma_fence_chain_init);
235