1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include "uapi/drm/vc4_drm.h"
35#include "vc4_drv.h"
36#include "vc4_packet.h"
37
38struct vc4_rcl_setup {
39 struct drm_gem_cma_object *color_read;
40 struct drm_gem_cma_object *color_write;
41 struct drm_gem_cma_object *zs_read;
42 struct drm_gem_cma_object *zs_write;
43 struct drm_gem_cma_object *msaa_color_write;
44 struct drm_gem_cma_object *msaa_zs_write;
45
46 struct drm_gem_cma_object *rcl;
47 u32 next_offset;
48};
49
50static inline void rcl_u8(struct vc4_rcl_setup *setup, u8 val)
51{
52 *(u8 *)(setup->rcl->vaddr + setup->next_offset) = val;
53 setup->next_offset += 1;
54}
55
56static inline void rcl_u16(struct vc4_rcl_setup *setup, u16 val)
57{
58 *(u16 *)(setup->rcl->vaddr + setup->next_offset) = val;
59 setup->next_offset += 2;
60}
61
62static inline void rcl_u32(struct vc4_rcl_setup *setup, u32 val)
63{
64 *(u32 *)(setup->rcl->vaddr + setup->next_offset) = val;
65 setup->next_offset += 4;
66}
67
68
69
70
71
72
73
74static void vc4_store_before_load(struct vc4_rcl_setup *setup)
75{
76 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
77 rcl_u16(setup,
78 VC4_SET_FIELD(VC4_LOADSTORE_TILE_BUFFER_NONE,
79 VC4_LOADSTORE_TILE_BUFFER_BUFFER) |
80 VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR |
81 VC4_STORE_TILE_BUFFER_DISABLE_ZS_CLEAR |
82 VC4_STORE_TILE_BUFFER_DISABLE_VG_MASK_CLEAR);
83 rcl_u32(setup, 0);
84}
85
86
87
88
89
90
91
92
93static uint32_t vc4_full_res_offset(struct vc4_exec_info *exec,
94 struct drm_gem_cma_object *bo,
95 struct drm_vc4_submit_rcl_surface *surf,
96 uint8_t x, uint8_t y)
97{
98 return bo->paddr + surf->offset + VC4_TILE_BUFFER_SIZE *
99 (DIV_ROUND_UP(exec->args->width, 32) * y + x);
100}
101
102
103
104
105
106
107
108
109static void vc4_tile_coordinates(struct vc4_rcl_setup *setup,
110 uint32_t x, uint32_t y)
111{
112 rcl_u8(setup, VC4_PACKET_TILE_COORDINATES);
113 rcl_u8(setup, x);
114 rcl_u8(setup, y);
115}
116
117static void emit_tile(struct vc4_exec_info *exec,
118 struct vc4_rcl_setup *setup,
119 uint8_t x, uint8_t y, bool first, bool last)
120{
121 struct drm_vc4_submit_cl *args = exec->args;
122 bool has_bin = args->bin_cl_size != 0;
123
124
125
126
127
128 if (setup->color_read) {
129 if (args->color_read.flags &
130 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
131 rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
132 rcl_u32(setup,
133 vc4_full_res_offset(exec, setup->color_read,
134 &args->color_read, x, y) |
135 VC4_LOADSTORE_FULL_RES_DISABLE_ZS);
136 } else {
137 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
138 rcl_u16(setup, args->color_read.bits);
139 rcl_u32(setup, setup->color_read->paddr +
140 args->color_read.offset);
141 }
142 }
143
144 if (setup->zs_read) {
145 if (args->zs_read.flags &
146 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
147 rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
148 rcl_u32(setup,
149 vc4_full_res_offset(exec, setup->zs_read,
150 &args->zs_read, x, y) |
151 VC4_LOADSTORE_FULL_RES_DISABLE_COLOR);
152 } else {
153 if (setup->color_read) {
154
155 vc4_tile_coordinates(setup, x, y);
156 vc4_store_before_load(setup);
157 }
158
159 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
160 rcl_u16(setup, args->zs_read.bits);
161 rcl_u32(setup, setup->zs_read->paddr +
162 args->zs_read.offset);
163 }
164 }
165
166
167
168
169 vc4_tile_coordinates(setup, x, y);
170
171
172
173
174 if (first && has_bin)
175 rcl_u8(setup, VC4_PACKET_WAIT_ON_SEMAPHORE);
176
177 if (has_bin) {
178 rcl_u8(setup, VC4_PACKET_BRANCH_TO_SUB_LIST);
179 rcl_u32(setup, (exec->tile_bo->paddr +
180 exec->tile_alloc_offset +
181 (y * exec->bin_tiles_x + x) * 32));
182 }
183
184 if (setup->msaa_color_write) {
185 bool last_tile_write = (!setup->msaa_zs_write &&
186 !setup->zs_write &&
187 !setup->color_write);
188 uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_ZS;
189
190 if (!last_tile_write)
191 bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
192 else if (last)
193 bits |= VC4_LOADSTORE_FULL_RES_EOF;
194 rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
195 rcl_u32(setup,
196 vc4_full_res_offset(exec, setup->msaa_color_write,
197 &args->msaa_color_write, x, y) |
198 bits);
199 }
200
201 if (setup->msaa_zs_write) {
202 bool last_tile_write = (!setup->zs_write &&
203 !setup->color_write);
204 uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_COLOR;
205
206 if (setup->msaa_color_write)
207 vc4_tile_coordinates(setup, x, y);
208 if (!last_tile_write)
209 bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
210 else if (last)
211 bits |= VC4_LOADSTORE_FULL_RES_EOF;
212 rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
213 rcl_u32(setup,
214 vc4_full_res_offset(exec, setup->msaa_zs_write,
215 &args->msaa_zs_write, x, y) |
216 bits);
217 }
218
219 if (setup->zs_write) {
220 bool last_tile_write = !setup->color_write;
221
222 if (setup->msaa_color_write || setup->msaa_zs_write)
223 vc4_tile_coordinates(setup, x, y);
224
225 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
226 rcl_u16(setup, args->zs_write.bits |
227 (last_tile_write ?
228 0 : VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR));
229 rcl_u32(setup,
230 (setup->zs_write->paddr + args->zs_write.offset) |
231 ((last && last_tile_write) ?
232 VC4_LOADSTORE_TILE_BUFFER_EOF : 0));
233 }
234
235 if (setup->color_write) {
236 if (setup->msaa_color_write || setup->msaa_zs_write ||
237 setup->zs_write) {
238 vc4_tile_coordinates(setup, x, y);
239 }
240
241 if (last)
242 rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER_AND_EOF);
243 else
244 rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER);
245 }
246}
247
248static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
249 struct vc4_rcl_setup *setup)
250{
251 struct drm_vc4_submit_cl *args = exec->args;
252 bool has_bin = args->bin_cl_size != 0;
253 uint8_t min_x_tile = args->min_x_tile;
254 uint8_t min_y_tile = args->min_y_tile;
255 uint8_t max_x_tile = args->max_x_tile;
256 uint8_t max_y_tile = args->max_y_tile;
257 uint8_t xtiles = max_x_tile - min_x_tile + 1;
258 uint8_t ytiles = max_y_tile - min_y_tile + 1;
259 uint8_t x, y;
260 uint32_t size, loop_body_size;
261
262 size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE;
263 loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE;
264
265 if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
266 size += VC4_PACKET_CLEAR_COLORS_SIZE +
267 VC4_PACKET_TILE_COORDINATES_SIZE +
268 VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
269 }
270
271 if (setup->color_read) {
272 if (args->color_read.flags &
273 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
274 loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
275 } else {
276 loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
277 }
278 }
279 if (setup->zs_read) {
280 if (args->zs_read.flags &
281 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
282 loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
283 } else {
284 if (setup->color_read &&
285 !(args->color_read.flags &
286 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES)) {
287 loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
288 loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
289 }
290 loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
291 }
292 }
293
294 if (has_bin) {
295 size += VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE;
296 loop_body_size += VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE;
297 }
298
299 if (setup->msaa_color_write)
300 loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
301 if (setup->msaa_zs_write)
302 loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
303
304 if (setup->zs_write)
305 loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
306 if (setup->color_write)
307 loop_body_size += VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE;
308
309
310 loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE *
311 ((setup->msaa_color_write != NULL) +
312 (setup->msaa_zs_write != NULL) +
313 (setup->color_write != NULL) +
314 (setup->zs_write != NULL) - 1);
315
316 size += xtiles * ytiles * loop_body_size;
317
318 setup->rcl = &vc4_bo_create(dev, size, true)->base;
319 if (IS_ERR(setup->rcl))
320 return PTR_ERR(setup->rcl);
321 list_add_tail(&to_vc4_bo(&setup->rcl->base)->unref_head,
322 &exec->unref_list);
323
324
325
326
327
328
329 if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
330 rcl_u8(setup, VC4_PACKET_CLEAR_COLORS);
331 rcl_u32(setup, args->clear_color[0]);
332 rcl_u32(setup, args->clear_color[1]);
333 rcl_u32(setup, args->clear_z);
334 rcl_u8(setup, args->clear_s);
335
336 vc4_tile_coordinates(setup, 0, 0);
337
338 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
339 rcl_u16(setup, VC4_LOADSTORE_TILE_BUFFER_NONE);
340 rcl_u32(setup, 0);
341 }
342
343 rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG);
344 rcl_u32(setup,
345 (setup->color_write ? (setup->color_write->paddr +
346 args->color_write.offset) :
347 0));
348 rcl_u16(setup, args->width);
349 rcl_u16(setup, args->height);
350 rcl_u16(setup, args->color_write.bits);
351
352 for (y = min_y_tile; y <= max_y_tile; y++) {
353 for (x = min_x_tile; x <= max_x_tile; x++) {
354 bool first = (x == min_x_tile && y == min_y_tile);
355 bool last = (x == max_x_tile && y == max_y_tile);
356
357 emit_tile(exec, setup, x, y, first, last);
358 }
359 }
360
361 BUG_ON(setup->next_offset != size);
362 exec->ct1ca = setup->rcl->paddr;
363 exec->ct1ea = setup->rcl->paddr + setup->next_offset;
364
365 return 0;
366}
367
368static int vc4_full_res_bounds_check(struct vc4_exec_info *exec,
369 struct drm_gem_cma_object *obj,
370 struct drm_vc4_submit_rcl_surface *surf)
371{
372 struct drm_vc4_submit_cl *args = exec->args;
373 u32 render_tiles_stride = DIV_ROUND_UP(exec->args->width, 32);
374
375 if (surf->offset > obj->base.size) {
376 DRM_ERROR("surface offset %d > BO size %zd\n",
377 surf->offset, obj->base.size);
378 return -EINVAL;
379 }
380
381 if ((obj->base.size - surf->offset) / VC4_TILE_BUFFER_SIZE <
382 render_tiles_stride * args->max_y_tile + args->max_x_tile) {
383 DRM_ERROR("MSAA tile %d, %d out of bounds "
384 "(bo size %zd, offset %d).\n",
385 args->max_x_tile, args->max_y_tile,
386 obj->base.size,
387 surf->offset);
388 return -EINVAL;
389 }
390
391 return 0;
392}
393
394static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
395 struct drm_gem_cma_object **obj,
396 struct drm_vc4_submit_rcl_surface *surf)
397{
398 if (surf->flags != 0 || surf->bits != 0) {
399 DRM_ERROR("MSAA surface had nonzero flags/bits\n");
400 return -EINVAL;
401 }
402
403 if (surf->hindex == ~0)
404 return 0;
405
406 *obj = vc4_use_bo(exec, surf->hindex);
407 if (!*obj)
408 return -EINVAL;
409
410 if (surf->offset & 0xf) {
411 DRM_ERROR("MSAA write must be 16b aligned.\n");
412 return -EINVAL;
413 }
414
415 return vc4_full_res_bounds_check(exec, *obj, surf);
416}
417
418static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
419 struct drm_gem_cma_object **obj,
420 struct drm_vc4_submit_rcl_surface *surf)
421{
422 uint8_t tiling = VC4_GET_FIELD(surf->bits,
423 VC4_LOADSTORE_TILE_BUFFER_TILING);
424 uint8_t buffer = VC4_GET_FIELD(surf->bits,
425 VC4_LOADSTORE_TILE_BUFFER_BUFFER);
426 uint8_t format = VC4_GET_FIELD(surf->bits,
427 VC4_LOADSTORE_TILE_BUFFER_FORMAT);
428 int cpp;
429 int ret;
430
431 if (surf->flags & ~VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
432 DRM_ERROR("Extra flags set\n");
433 return -EINVAL;
434 }
435
436 if (surf->hindex == ~0)
437 return 0;
438
439 *obj = vc4_use_bo(exec, surf->hindex);
440 if (!*obj)
441 return -EINVAL;
442
443 if (surf->flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
444 if (surf == &exec->args->zs_write) {
445 DRM_ERROR("general zs write may not be a full-res.\n");
446 return -EINVAL;
447 }
448
449 if (surf->bits != 0) {
450 DRM_ERROR("load/store general bits set with "
451 "full res load/store.\n");
452 return -EINVAL;
453 }
454
455 ret = vc4_full_res_bounds_check(exec, *obj, surf);
456 if (!ret)
457 return ret;
458
459 return 0;
460 }
461
462 if (surf->bits & ~(VC4_LOADSTORE_TILE_BUFFER_TILING_MASK |
463 VC4_LOADSTORE_TILE_BUFFER_BUFFER_MASK |
464 VC4_LOADSTORE_TILE_BUFFER_FORMAT_MASK)) {
465 DRM_ERROR("Unknown bits in load/store: 0x%04x\n",
466 surf->bits);
467 return -EINVAL;
468 }
469
470 if (tiling > VC4_TILING_FORMAT_LT) {
471 DRM_ERROR("Bad tiling format\n");
472 return -EINVAL;
473 }
474
475 if (buffer == VC4_LOADSTORE_TILE_BUFFER_ZS) {
476 if (format != 0) {
477 DRM_ERROR("No color format should be set for ZS\n");
478 return -EINVAL;
479 }
480 cpp = 4;
481 } else if (buffer == VC4_LOADSTORE_TILE_BUFFER_COLOR) {
482 switch (format) {
483 case VC4_LOADSTORE_TILE_BUFFER_BGR565:
484 case VC4_LOADSTORE_TILE_BUFFER_BGR565_DITHER:
485 cpp = 2;
486 break;
487 case VC4_LOADSTORE_TILE_BUFFER_RGBA8888:
488 cpp = 4;
489 break;
490 default:
491 DRM_ERROR("Bad tile buffer format\n");
492 return -EINVAL;
493 }
494 } else {
495 DRM_ERROR("Bad load/store buffer %d.\n", buffer);
496 return -EINVAL;
497 }
498
499 if (surf->offset & 0xf) {
500 DRM_ERROR("load/store buffer must be 16b aligned.\n");
501 return -EINVAL;
502 }
503
504 if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
505 exec->args->width, exec->args->height, cpp)) {
506 return -EINVAL;
507 }
508
509 return 0;
510}
511
512static int
513vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
514 struct vc4_rcl_setup *setup,
515 struct drm_gem_cma_object **obj,
516 struct drm_vc4_submit_rcl_surface *surf)
517{
518 uint8_t tiling = VC4_GET_FIELD(surf->bits,
519 VC4_RENDER_CONFIG_MEMORY_FORMAT);
520 uint8_t format = VC4_GET_FIELD(surf->bits,
521 VC4_RENDER_CONFIG_FORMAT);
522 int cpp;
523
524 if (surf->flags != 0) {
525 DRM_ERROR("No flags supported on render config.\n");
526 return -EINVAL;
527 }
528
529 if (surf->bits & ~(VC4_RENDER_CONFIG_MEMORY_FORMAT_MASK |
530 VC4_RENDER_CONFIG_FORMAT_MASK |
531 VC4_RENDER_CONFIG_MS_MODE_4X |
532 VC4_RENDER_CONFIG_DECIMATE_MODE_4X)) {
533 DRM_ERROR("Unknown bits in render config: 0x%04x\n",
534 surf->bits);
535 return -EINVAL;
536 }
537
538 if (surf->hindex == ~0)
539 return 0;
540
541 *obj = vc4_use_bo(exec, surf->hindex);
542 if (!*obj)
543 return -EINVAL;
544
545 if (tiling > VC4_TILING_FORMAT_LT) {
546 DRM_ERROR("Bad tiling format\n");
547 return -EINVAL;
548 }
549
550 switch (format) {
551 case VC4_RENDER_CONFIG_FORMAT_BGR565_DITHERED:
552 case VC4_RENDER_CONFIG_FORMAT_BGR565:
553 cpp = 2;
554 break;
555 case VC4_RENDER_CONFIG_FORMAT_RGBA8888:
556 cpp = 4;
557 break;
558 default:
559 DRM_ERROR("Bad tile buffer format\n");
560 return -EINVAL;
561 }
562
563 if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
564 exec->args->width, exec->args->height, cpp)) {
565 return -EINVAL;
566 }
567
568 return 0;
569}
570
571int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
572{
573 struct vc4_rcl_setup setup = {0};
574 struct drm_vc4_submit_cl *args = exec->args;
575 bool has_bin = args->bin_cl_size != 0;
576 int ret;
577
578 if (args->min_x_tile > args->max_x_tile ||
579 args->min_y_tile > args->max_y_tile) {
580 DRM_ERROR("Bad render tile set (%d,%d)-(%d,%d)\n",
581 args->min_x_tile, args->min_y_tile,
582 args->max_x_tile, args->max_y_tile);
583 return -EINVAL;
584 }
585
586 if (has_bin &&
587 (args->max_x_tile > exec->bin_tiles_x ||
588 args->max_y_tile > exec->bin_tiles_y)) {
589 DRM_ERROR("Render tiles (%d,%d) outside of bin config "
590 "(%d,%d)\n",
591 args->max_x_tile, args->max_y_tile,
592 exec->bin_tiles_x, exec->bin_tiles_y);
593 return -EINVAL;
594 }
595
596 ret = vc4_rcl_render_config_surface_setup(exec, &setup,
597 &setup.color_write,
598 &args->color_write);
599 if (ret)
600 return ret;
601
602 ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read);
603 if (ret)
604 return ret;
605
606 ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read);
607 if (ret)
608 return ret;
609
610 ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write);
611 if (ret)
612 return ret;
613
614 ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_color_write,
615 &args->msaa_color_write);
616 if (ret)
617 return ret;
618
619 ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_zs_write,
620 &args->msaa_zs_write);
621 if (ret)
622 return ret;
623
624
625
626
627 if (!setup.color_write && !setup.zs_write &&
628 !setup.msaa_color_write && !setup.msaa_zs_write) {
629 DRM_ERROR("RCL requires color or Z/S write\n");
630 return -EINVAL;
631 }
632
633 return vc4_create_rcl_bo(dev, exec, &setup);
634}
635
