1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/prime_numbers.h>
26
27#include "../i915_selftest.h"
28#include "i915_random.h"
29
30static int cpu_set(struct drm_i915_gem_object *obj,
31 unsigned long offset,
32 u32 v)
33{
34 unsigned int needs_clflush;
35 struct page *page;
36 void *map;
37 u32 *cpu;
38 int err;
39
40 err = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
41 if (err)
42 return err;
43
44 page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
45 map = kmap_atomic(page);
46 cpu = map + offset_in_page(offset);
47
48 if (needs_clflush & CLFLUSH_BEFORE)
49 drm_clflush_virt_range(cpu, sizeof(*cpu));
50
51 *cpu = v;
52
53 if (needs_clflush & CLFLUSH_AFTER)
54 drm_clflush_virt_range(cpu, sizeof(*cpu));
55
56 kunmap_atomic(map);
57 i915_gem_obj_finish_shmem_access(obj);
58
59 return 0;
60}
61
62static int cpu_get(struct drm_i915_gem_object *obj,
63 unsigned long offset,
64 u32 *v)
65{
66 unsigned int needs_clflush;
67 struct page *page;
68 void *map;
69 u32 *cpu;
70 int err;
71
72 err = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
73 if (err)
74 return err;
75
76 page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
77 map = kmap_atomic(page);
78 cpu = map + offset_in_page(offset);
79
80 if (needs_clflush & CLFLUSH_BEFORE)
81 drm_clflush_virt_range(cpu, sizeof(*cpu));
82
83 *v = *cpu;
84
85 kunmap_atomic(map);
86 i915_gem_obj_finish_shmem_access(obj);
87
88 return 0;
89}
90
91static int gtt_set(struct drm_i915_gem_object *obj,
92 unsigned long offset,
93 u32 v)
94{
95 struct i915_vma *vma;
96 u32 __iomem *map;
97 int err;
98
99 err = i915_gem_object_set_to_gtt_domain(obj, true);
100 if (err)
101 return err;
102
103 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
104 if (IS_ERR(vma))
105 return PTR_ERR(vma);
106
107 map = i915_vma_pin_iomap(vma);
108 i915_vma_unpin(vma);
109 if (IS_ERR(map))
110 return PTR_ERR(map);
111
112 iowrite32(v, &map[offset / sizeof(*map)]);
113 i915_vma_unpin_iomap(vma);
114
115 return 0;
116}
117
118static int gtt_get(struct drm_i915_gem_object *obj,
119 unsigned long offset,
120 u32 *v)
121{
122 struct i915_vma *vma;
123 u32 __iomem *map;
124 int err;
125
126 err = i915_gem_object_set_to_gtt_domain(obj, false);
127 if (err)
128 return err;
129
130 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
131 if (IS_ERR(vma))
132 return PTR_ERR(vma);
133
134 map = i915_vma_pin_iomap(vma);
135 i915_vma_unpin(vma);
136 if (IS_ERR(map))
137 return PTR_ERR(map);
138
139 *v = ioread32(&map[offset / sizeof(*map)]);
140 i915_vma_unpin_iomap(vma);
141
142 return 0;
143}
144
145static int wc_set(struct drm_i915_gem_object *obj,
146 unsigned long offset,
147 u32 v)
148{
149 u32 *map;
150 int err;
151
152 err = i915_gem_object_set_to_wc_domain(obj, true);
153 if (err)
154 return err;
155
156 map = i915_gem_object_pin_map(obj, I915_MAP_WC);
157 if (IS_ERR(map))
158 return PTR_ERR(map);
159
160 map[offset / sizeof(*map)] = v;
161 i915_gem_object_unpin_map(obj);
162
163 return 0;
164}
165
166static int wc_get(struct drm_i915_gem_object *obj,
167 unsigned long offset,
168 u32 *v)
169{
170 u32 *map;
171 int err;
172
173 err = i915_gem_object_set_to_wc_domain(obj, false);
174 if (err)
175 return err;
176
177 map = i915_gem_object_pin_map(obj, I915_MAP_WC);
178 if (IS_ERR(map))
179 return PTR_ERR(map);
180
181 *v = map[offset / sizeof(*map)];
182 i915_gem_object_unpin_map(obj);
183
184 return 0;
185}
186
187static int gpu_set(struct drm_i915_gem_object *obj,
188 unsigned long offset,
189 u32 v)
190{
191 struct drm_i915_private *i915 = to_i915(obj->base.dev);
192 struct i915_request *rq;
193 struct i915_vma *vma;
194 u32 *cs;
195 int err;
196
197 err = i915_gem_object_set_to_gtt_domain(obj, true);
198 if (err)
199 return err;
200
201 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
202 if (IS_ERR(vma))
203 return PTR_ERR(vma);
204
205 rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
206 if (IS_ERR(rq)) {
207 i915_vma_unpin(vma);
208 return PTR_ERR(rq);
209 }
210
211 cs = intel_ring_begin(rq, 4);
212 if (IS_ERR(cs)) {
213 i915_request_add(rq);
214 i915_vma_unpin(vma);
215 return PTR_ERR(cs);
216 }
217
218 if (INTEL_GEN(i915) >= 8) {
219 *cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
220 *cs++ = lower_32_bits(i915_ggtt_offset(vma) + offset);
221 *cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset);
222 *cs++ = v;
223 } else if (INTEL_GEN(i915) >= 4) {
224 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
225 *cs++ = 0;
226 *cs++ = i915_ggtt_offset(vma) + offset;
227 *cs++ = v;
228 } else {
229 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
230 *cs++ = i915_ggtt_offset(vma) + offset;
231 *cs++ = v;
232 *cs++ = MI_NOOP;
233 }
234 intel_ring_advance(rq, cs);
235
236 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
237 i915_vma_unpin(vma);
238
239 i915_request_add(rq);
240
241 return err;
242}
243
244static bool always_valid(struct drm_i915_private *i915)
245{
246 return true;
247}
248
249static bool needs_fence_registers(struct drm_i915_private *i915)
250{
251 return !i915_terminally_wedged(&i915->gpu_error);
252}
253
254static bool needs_mi_store_dword(struct drm_i915_private *i915)
255{
256 if (i915_terminally_wedged(&i915->gpu_error))
257 return false;
258
259 return intel_engine_can_store_dword(i915->engine[RCS]);
260}
261
262static const struct igt_coherency_mode {
263 const char *name;
264 int (*set)(struct drm_i915_gem_object *, unsigned long offset, u32 v);
265 int (*get)(struct drm_i915_gem_object *, unsigned long offset, u32 *v);
266 bool (*valid)(struct drm_i915_private *i915);
267} igt_coherency_mode[] = {
268 { "cpu", cpu_set, cpu_get, always_valid },
269 { "gtt", gtt_set, gtt_get, needs_fence_registers },
270 { "wc", wc_set, wc_get, always_valid },
271 { "gpu", gpu_set, NULL, needs_mi_store_dword },
272 { },
273};
274
275static int igt_gem_coherency(void *arg)
276{
277 const unsigned int ncachelines = PAGE_SIZE/64;
278 I915_RND_STATE(prng);
279 struct drm_i915_private *i915 = arg;
280 const struct igt_coherency_mode *read, *write, *over;
281 struct drm_i915_gem_object *obj;
282 unsigned long count, n;
283 u32 *offsets, *values;
284 int err = 0;
285
286
287
288
289
290
291
292 offsets = kmalloc_array(ncachelines, 2*sizeof(u32), GFP_KERNEL);
293 if (!offsets)
294 return -ENOMEM;
295 for (count = 0; count < ncachelines; count++)
296 offsets[count] = count * 64 + 4 * (count % 16);
297
298 values = offsets + ncachelines;
299
300 mutex_lock(&i915->drm.struct_mutex);
301 intel_runtime_pm_get(i915);
302 for (over = igt_coherency_mode; over->name; over++) {
303 if (!over->set)
304 continue;
305
306 if (!over->valid(i915))
307 continue;
308
309 for (write = igt_coherency_mode; write->name; write++) {
310 if (!write->set)
311 continue;
312
313 if (!write->valid(i915))
314 continue;
315
316 for (read = igt_coherency_mode; read->name; read++) {
317 if (!read->get)
318 continue;
319
320 if (!read->valid(i915))
321 continue;
322
323 for_each_prime_number_from(count, 1, ncachelines) {
324 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
325 if (IS_ERR(obj)) {
326 err = PTR_ERR(obj);
327 goto unlock;
328 }
329
330 i915_random_reorder(offsets, ncachelines, &prng);
331 for (n = 0; n < count; n++)
332 values[n] = prandom_u32_state(&prng);
333
334 for (n = 0; n < count; n++) {
335 err = over->set(obj, offsets[n], ~values[n]);
336 if (err) {
337 pr_err("Failed to set stale value[%ld/%ld] in object using %s, err=%d\n",
338 n, count, over->name, err);
339 goto put_object;
340 }
341 }
342
343 for (n = 0; n < count; n++) {
344 err = write->set(obj, offsets[n], values[n]);
345 if (err) {
346 pr_err("Failed to set value[%ld/%ld] in object using %s, err=%d\n",
347 n, count, write->name, err);
348 goto put_object;
349 }
350 }
351
352 for (n = 0; n < count; n++) {
353 u32 found;
354
355 err = read->get(obj, offsets[n], &found);
356 if (err) {
357 pr_err("Failed to get value[%ld/%ld] in object using %s, err=%d\n",
358 n, count, read->name, err);
359 goto put_object;
360 }
361
362 if (found != values[n]) {
363 pr_err("Value[%ld/%ld] mismatch, (overwrite with %s) wrote [%s] %x read [%s] %x (inverse %x), at offset %x\n",
364 n, count, over->name,
365 write->name, values[n],
366 read->name, found,
367 ~values[n], offsets[n]);
368 err = -EINVAL;
369 goto put_object;
370 }
371 }
372
373 __i915_gem_object_release_unless_active(obj);
374 }
375 }
376 }
377 }
378unlock:
379 intel_runtime_pm_put(i915);
380 mutex_unlock(&i915->drm.struct_mutex);
381 kfree(offsets);
382 return err;
383
384put_object:
385 __i915_gem_object_release_unless_active(obj);
386 goto unlock;
387}
388
389int i915_gem_coherency_live_selftests(struct drm_i915_private *i915)
390{
391 static const struct i915_subtest tests[] = {
392 SUBTEST(igt_gem_coherency),
393 };
394
395 return i915_subtests(tests, i915);
396}
397
