1
2
3
4
5
6
7#include <linux/dma-fence-array.h>
8#include <linux/dma-fence-chain.h>
9#include <linux/jiffies.h>
10
11#include "gt/intel_engine.h"
12
13#include "dma_resv_utils.h"
14#include "i915_gem_ioctls.h"
15#include "i915_gem_object.h"
16
17static long
18i915_gem_object_wait_fence(struct dma_fence *fence,
19 unsigned int flags,
20 long timeout)
21{
22 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
23
24 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
25 return timeout;
26
27 if (dma_fence_is_i915(fence))
28 return i915_request_wait(to_request(fence), flags, timeout);
29
30 return dma_fence_wait_timeout(fence,
31 flags & I915_WAIT_INTERRUPTIBLE,
32 timeout);
33}
34
35static long
36i915_gem_object_wait_reservation(struct dma_resv *resv,
37 unsigned int flags,
38 long timeout)
39{
40 struct dma_fence *excl;
41 bool prune_fences = false;
42
43 if (flags & I915_WAIT_ALL) {
44 struct dma_fence **shared;
45 unsigned int count, i;
46 int ret;
47
48 ret = dma_resv_get_fences(resv, &excl, &count, &shared);
49 if (ret)
50 return ret;
51
52 for (i = 0; i < count; i++) {
53 timeout = i915_gem_object_wait_fence(shared[i],
54 flags, timeout);
55 if (timeout < 0)
56 break;
57
58 dma_fence_put(shared[i]);
59 }
60
61 for (; i < count; i++)
62 dma_fence_put(shared[i]);
63 kfree(shared);
64
65
66
67
68
69
70
71
72
73
74 prune_fences = count && timeout >= 0;
75 } else {
76 excl = dma_resv_get_excl_unlocked(resv);
77 }
78
79 if (excl && timeout >= 0)
80 timeout = i915_gem_object_wait_fence(excl, flags, timeout);
81
82 dma_fence_put(excl);
83
84
85
86
87
88 if (prune_fences)
89 dma_resv_prune(resv);
90
91 return timeout;
92}
93
94static void fence_set_priority(struct dma_fence *fence,
95 const struct i915_sched_attr *attr)
96{
97 struct i915_request *rq;
98 struct intel_engine_cs *engine;
99
100 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
101 return;
102
103 rq = to_request(fence);
104 engine = rq->engine;
105
106 rcu_read_lock();
107 if (engine->schedule)
108 engine->schedule(rq, attr);
109 rcu_read_unlock();
110}
111
112static inline bool __dma_fence_is_chain(const struct dma_fence *fence)
113{
114 return fence->ops == &dma_fence_chain_ops;
115}
116
117void i915_gem_fence_wait_priority(struct dma_fence *fence,
118 const struct i915_sched_attr *attr)
119{
120 if (dma_fence_is_signaled(fence))
121 return;
122
123 local_bh_disable();
124
125
126 if (dma_fence_is_array(fence)) {
127 struct dma_fence_array *array = to_dma_fence_array(fence);
128 int i;
129
130 for (i = 0; i < array->num_fences; i++)
131 fence_set_priority(array->fences[i], attr);
132 } else if (__dma_fence_is_chain(fence)) {
133 struct dma_fence *iter;
134
135
136 dma_fence_chain_for_each(iter, fence) {
137 fence_set_priority(to_dma_fence_chain(iter)->fence,
138 attr);
139 break;
140 }
141 dma_fence_put(iter);
142 } else {
143 fence_set_priority(fence, attr);
144 }
145
146 local_bh_enable();
147}
148
149int
150i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
151 unsigned int flags,
152 const struct i915_sched_attr *attr)
153{
154 struct dma_fence *excl;
155
156 if (flags & I915_WAIT_ALL) {
157 struct dma_fence **shared;
158 unsigned int count, i;
159 int ret;
160
161 ret = dma_resv_get_fences(obj->base.resv, &excl, &count,
162 &shared);
163 if (ret)
164 return ret;
165
166 for (i = 0; i < count; i++) {
167 i915_gem_fence_wait_priority(shared[i], attr);
168 dma_fence_put(shared[i]);
169 }
170
171 kfree(shared);
172 } else {
173 excl = dma_resv_get_excl_unlocked(obj->base.resv);
174 }
175
176 if (excl) {
177 i915_gem_fence_wait_priority(excl, attr);
178 dma_fence_put(excl);
179 }
180 return 0;
181}
182
183
184
185
186
187
188
189int
190i915_gem_object_wait(struct drm_i915_gem_object *obj,
191 unsigned int flags,
192 long timeout)
193{
194 might_sleep();
195 GEM_BUG_ON(timeout < 0);
196
197 timeout = i915_gem_object_wait_reservation(obj->base.resv,
198 flags, timeout);
199 return timeout < 0 ? timeout : 0;
200}
201
202static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
203{
204
205 if (NSEC_PER_SEC % HZ &&
206 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
207 return MAX_JIFFY_OFFSET;
208
209 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
210}
211
212static unsigned long to_wait_timeout(s64 timeout_ns)
213{
214 if (timeout_ns < 0)
215 return MAX_SCHEDULE_TIMEOUT;
216
217 if (timeout_ns == 0)
218 return 0;
219
220 return nsecs_to_jiffies_timeout(timeout_ns);
221}
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247int
248i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
249{
250 struct drm_i915_gem_wait *args = data;
251 struct drm_i915_gem_object *obj;
252 ktime_t start;
253 long ret;
254
255 if (args->flags != 0)
256 return -EINVAL;
257
258 obj = i915_gem_object_lookup(file, args->bo_handle);
259 if (!obj)
260 return -ENOENT;
261
262 start = ktime_get();
263
264 ret = i915_gem_object_wait(obj,
265 I915_WAIT_INTERRUPTIBLE |
266 I915_WAIT_PRIORITY |
267 I915_WAIT_ALL,
268 to_wait_timeout(args->timeout_ns));
269
270 if (args->timeout_ns > 0) {
271 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
272 if (args->timeout_ns < 0)
273 args->timeout_ns = 0;
274
275
276
277
278
279
280
281
282 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
283 args->timeout_ns = 0;
284
285
286 if (ret == -ETIME && args->timeout_ns)
287 ret = -EAGAIN;
288 }
289
290 i915_gem_object_put(obj);
291 return ret;
292}
293