1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38#include "uapi/drm/vc4_drm.h"
39#include "vc4_drv.h"
40#include "vc4_packet.h"
41
42struct vc4_rcl_setup {
43 struct drm_gem_cma_object *color_read;
44 struct drm_gem_cma_object *color_write;
45 struct drm_gem_cma_object *zs_read;
46 struct drm_gem_cma_object *zs_write;
47 struct drm_gem_cma_object *msaa_color_write;
48 struct drm_gem_cma_object *msaa_zs_write;
49
50 struct drm_gem_cma_object *rcl;
51 u32 next_offset;
52
53 u32 next_write_bo_index;
54};
55
56static inline void rcl_u8(struct vc4_rcl_setup *setup, u8 val)
57{
58 *(u8 *)(setup->rcl->vaddr + setup->next_offset) = val;
59 setup->next_offset += 1;
60}
61
62static inline void rcl_u16(struct vc4_rcl_setup *setup, u16 val)
63{
64 *(u16 *)(setup->rcl->vaddr + setup->next_offset) = val;
65 setup->next_offset += 2;
66}
67
68static inline void rcl_u32(struct vc4_rcl_setup *setup, u32 val)
69{
70 *(u32 *)(setup->rcl->vaddr + setup->next_offset) = val;
71 setup->next_offset += 4;
72}
73
74
75
76
77
78
79
80static void vc4_store_before_load(struct vc4_rcl_setup *setup)
81{
82 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
83 rcl_u16(setup,
84 VC4_SET_FIELD(VC4_LOADSTORE_TILE_BUFFER_NONE,
85 VC4_LOADSTORE_TILE_BUFFER_BUFFER) |
86 VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR |
87 VC4_STORE_TILE_BUFFER_DISABLE_ZS_CLEAR |
88 VC4_STORE_TILE_BUFFER_DISABLE_VG_MASK_CLEAR);
89 rcl_u32(setup, 0);
90}
91
92
93
94
95
96
97
98
99static uint32_t vc4_full_res_offset(struct vc4_exec_info *exec,
100 struct drm_gem_cma_object *bo,
101 struct drm_vc4_submit_rcl_surface *surf,
102 uint8_t x, uint8_t y)
103{
104 return bo->paddr + surf->offset + VC4_TILE_BUFFER_SIZE *
105 (DIV_ROUND_UP(exec->args->width, 32) * y + x);
106}
107
108
109
110
111
112
113
114
115static void vc4_tile_coordinates(struct vc4_rcl_setup *setup,
116 uint32_t x, uint32_t y)
117{
118 rcl_u8(setup, VC4_PACKET_TILE_COORDINATES);
119 rcl_u8(setup, x);
120 rcl_u8(setup, y);
121}
122
123static void emit_tile(struct vc4_exec_info *exec,
124 struct vc4_rcl_setup *setup,
125 uint8_t x, uint8_t y, bool first, bool last)
126{
127 struct drm_vc4_submit_cl *args = exec->args;
128 bool has_bin = args->bin_cl_size != 0;
129
130
131
132
133
134 if (setup->color_read) {
135 if (args->color_read.flags &
136 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
137 rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
138 rcl_u32(setup,
139 vc4_full_res_offset(exec, setup->color_read,
140 &args->color_read, x, y) |
141 VC4_LOADSTORE_FULL_RES_DISABLE_ZS);
142 } else {
143 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
144 rcl_u16(setup, args->color_read.bits);
145 rcl_u32(setup, setup->color_read->paddr +
146 args->color_read.offset);
147 }
148 }
149
150 if (setup->zs_read) {
151 if (setup->color_read) {
152
153 vc4_tile_coordinates(setup, x, y);
154 vc4_store_before_load(setup);
155 }
156
157 if (args->zs_read.flags &
158 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
159 rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
160 rcl_u32(setup,
161 vc4_full_res_offset(exec, setup->zs_read,
162 &args->zs_read, x, y) |
163 VC4_LOADSTORE_FULL_RES_DISABLE_COLOR);
164 } else {
165 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
166 rcl_u16(setup, args->zs_read.bits);
167 rcl_u32(setup, setup->zs_read->paddr +
168 args->zs_read.offset);
169 }
170 }
171
172
173
174
175 vc4_tile_coordinates(setup, x, y);
176
177
178
179
180 if (first && has_bin)
181 rcl_u8(setup, VC4_PACKET_WAIT_ON_SEMAPHORE);
182
183 if (has_bin) {
184 rcl_u8(setup, VC4_PACKET_BRANCH_TO_SUB_LIST);
185 rcl_u32(setup, (exec->tile_alloc_offset +
186 (y * exec->bin_tiles_x + x) * 32));
187 }
188
189 if (setup->msaa_color_write) {
190 bool last_tile_write = (!setup->msaa_zs_write &&
191 !setup->zs_write &&
192 !setup->color_write);
193 uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_ZS;
194
195 if (!last_tile_write)
196 bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
197 else if (last)
198 bits |= VC4_LOADSTORE_FULL_RES_EOF;
199 rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
200 rcl_u32(setup,
201 vc4_full_res_offset(exec, setup->msaa_color_write,
202 &args->msaa_color_write, x, y) |
203 bits);
204 }
205
206 if (setup->msaa_zs_write) {
207 bool last_tile_write = (!setup->zs_write &&
208 !setup->color_write);
209 uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_COLOR;
210
211 if (setup->msaa_color_write)
212 vc4_tile_coordinates(setup, x, y);
213 if (!last_tile_write)
214 bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
215 else if (last)
216 bits |= VC4_LOADSTORE_FULL_RES_EOF;
217 rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
218 rcl_u32(setup,
219 vc4_full_res_offset(exec, setup->msaa_zs_write,
220 &args->msaa_zs_write, x, y) |
221 bits);
222 }
223
224 if (setup->zs_write) {
225 bool last_tile_write = !setup->color_write;
226
227 if (setup->msaa_color_write || setup->msaa_zs_write)
228 vc4_tile_coordinates(setup, x, y);
229
230 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
231 rcl_u16(setup, args->zs_write.bits |
232 (last_tile_write ?
233 0 : VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR));
234 rcl_u32(setup,
235 (setup->zs_write->paddr + args->zs_write.offset) |
236 ((last && last_tile_write) ?
237 VC4_LOADSTORE_TILE_BUFFER_EOF : 0));
238 }
239
240 if (setup->color_write) {
241 if (setup->msaa_color_write || setup->msaa_zs_write ||
242 setup->zs_write) {
243 vc4_tile_coordinates(setup, x, y);
244 }
245
246 if (last)
247 rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER_AND_EOF);
248 else
249 rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER);
250 }
251}
252
253static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
254 struct vc4_rcl_setup *setup)
255{
256 struct drm_vc4_submit_cl *args = exec->args;
257 bool has_bin = args->bin_cl_size != 0;
258 uint8_t min_x_tile = args->min_x_tile;
259 uint8_t min_y_tile = args->min_y_tile;
260 uint8_t max_x_tile = args->max_x_tile;
261 uint8_t max_y_tile = args->max_y_tile;
262 uint8_t xtiles = max_x_tile - min_x_tile + 1;
263 uint8_t ytiles = max_y_tile - min_y_tile + 1;
264 uint8_t xi, yi;
265 uint32_t size, loop_body_size;
266 bool positive_x = true;
267 bool positive_y = true;
268
269 if (args->flags & VC4_SUBMIT_CL_FIXED_RCL_ORDER) {
270 if (!(args->flags & VC4_SUBMIT_CL_RCL_ORDER_INCREASING_X))
271 positive_x = false;
272 if (!(args->flags & VC4_SUBMIT_CL_RCL_ORDER_INCREASING_Y))
273 positive_y = false;
274 }
275
276 size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE;
277 loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE;
278
279 if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
280 size += VC4_PACKET_CLEAR_COLORS_SIZE +
281 VC4_PACKET_TILE_COORDINATES_SIZE +
282 VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
283 }
284
285 if (setup->color_read) {
286 if (args->color_read.flags &
287 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
288 loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
289 } else {
290 loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
291 }
292 }
293 if (setup->zs_read) {
294 if (setup->color_read) {
295 loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
296 loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
297 }
298
299 if (args->zs_read.flags &
300 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
301 loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
302 } else {
303 loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
304 }
305 }
306
307 if (has_bin) {
308 size += VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE;
309 loop_body_size += VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE;
310 }
311
312 if (setup->msaa_color_write)
313 loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
314 if (setup->msaa_zs_write)
315 loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
316
317 if (setup->zs_write)
318 loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
319 if (setup->color_write)
320 loop_body_size += VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE;
321
322
323 loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE *
324 ((setup->msaa_color_write != NULL) +
325 (setup->msaa_zs_write != NULL) +
326 (setup->color_write != NULL) +
327 (setup->zs_write != NULL) - 1);
328
329 size += xtiles * ytiles * loop_body_size;
330
331 setup->rcl = &vc4_bo_create(dev, size, true, VC4_BO_TYPE_RCL)->base;
332 if (IS_ERR(setup->rcl))
333 return PTR_ERR(setup->rcl);
334 list_add_tail(&to_vc4_bo(&setup->rcl->base)->unref_head,
335 &exec->unref_list);
336
337
338
339
340
341
342 if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
343 rcl_u8(setup, VC4_PACKET_CLEAR_COLORS);
344 rcl_u32(setup, args->clear_color[0]);
345 rcl_u32(setup, args->clear_color[1]);
346 rcl_u32(setup, args->clear_z);
347 rcl_u8(setup, args->clear_s);
348
349 vc4_tile_coordinates(setup, 0, 0);
350
351 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
352 rcl_u16(setup, VC4_LOADSTORE_TILE_BUFFER_NONE);
353 rcl_u32(setup, 0);
354 }
355
356 rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG);
357 rcl_u32(setup,
358 (setup->color_write ? (setup->color_write->paddr +
359 args->color_write.offset) :
360 0));
361 rcl_u16(setup, args->width);
362 rcl_u16(setup, args->height);
363 rcl_u16(setup, args->color_write.bits);
364
365 for (yi = 0; yi < ytiles; yi++) {
366 int y = positive_y ? min_y_tile + yi : max_y_tile - yi;
367 for (xi = 0; xi < xtiles; xi++) {
368 int x = positive_x ? min_x_tile + xi : max_x_tile - xi;
369 bool first = (xi == 0 && yi == 0);
370 bool last = (xi == xtiles - 1 && yi == ytiles - 1);
371
372 emit_tile(exec, setup, x, y, first, last);
373 }
374 }
375
376 BUG_ON(setup->next_offset != size);
377 exec->ct1ca = setup->rcl->paddr;
378 exec->ct1ea = setup->rcl->paddr + setup->next_offset;
379
380 return 0;
381}
382
383static int vc4_full_res_bounds_check(struct vc4_exec_info *exec,
384 struct drm_gem_cma_object *obj,
385 struct drm_vc4_submit_rcl_surface *surf)
386{
387 struct drm_vc4_submit_cl *args = exec->args;
388 u32 render_tiles_stride = DIV_ROUND_UP(exec->args->width, 32);
389
390 if (surf->offset > obj->base.size) {
391 DRM_DEBUG("surface offset %d > BO size %zd\n",
392 surf->offset, obj->base.size);
393 return -EINVAL;
394 }
395
396 if ((obj->base.size - surf->offset) / VC4_TILE_BUFFER_SIZE <
397 render_tiles_stride * args->max_y_tile + args->max_x_tile) {
398 DRM_DEBUG("MSAA tile %d, %d out of bounds "
399 "(bo size %zd, offset %d).\n",
400 args->max_x_tile, args->max_y_tile,
401 obj->base.size,
402 surf->offset);
403 return -EINVAL;
404 }
405
406 return 0;
407}
408
409static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
410 struct drm_gem_cma_object **obj,
411 struct drm_vc4_submit_rcl_surface *surf)
412{
413 if (surf->flags != 0 || surf->bits != 0) {
414 DRM_DEBUG("MSAA surface had nonzero flags/bits\n");
415 return -EINVAL;
416 }
417
418 if (surf->hindex == ~0)
419 return 0;
420
421 *obj = vc4_use_bo(exec, surf->hindex);
422 if (!*obj)
423 return -EINVAL;
424
425 exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
426
427 if (surf->offset & 0xf) {
428 DRM_DEBUG("MSAA write must be 16b aligned.\n");
429 return -EINVAL;
430 }
431
432 return vc4_full_res_bounds_check(exec, *obj, surf);
433}
434
435static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
436 struct drm_gem_cma_object **obj,
437 struct drm_vc4_submit_rcl_surface *surf,
438 bool is_write)
439{
440 uint8_t tiling = VC4_GET_FIELD(surf->bits,
441 VC4_LOADSTORE_TILE_BUFFER_TILING);
442 uint8_t buffer = VC4_GET_FIELD(surf->bits,
443 VC4_LOADSTORE_TILE_BUFFER_BUFFER);
444 uint8_t format = VC4_GET_FIELD(surf->bits,
445 VC4_LOADSTORE_TILE_BUFFER_FORMAT);
446 int cpp;
447 int ret;
448
449 if (surf->flags & ~VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
450 DRM_DEBUG("Extra flags set\n");
451 return -EINVAL;
452 }
453
454 if (surf->hindex == ~0)
455 return 0;
456
457 *obj = vc4_use_bo(exec, surf->hindex);
458 if (!*obj)
459 return -EINVAL;
460
461 if (is_write)
462 exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
463
464 if (surf->flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
465 if (surf == &exec->args->zs_write) {
466 DRM_DEBUG("general zs write may not be a full-res.\n");
467 return -EINVAL;
468 }
469
470 if (surf->bits != 0) {
471 DRM_DEBUG("load/store general bits set with "
472 "full res load/store.\n");
473 return -EINVAL;
474 }
475
476 ret = vc4_full_res_bounds_check(exec, *obj, surf);
477 if (ret)
478 return ret;
479
480 return 0;
481 }
482
483 if (surf->bits & ~(VC4_LOADSTORE_TILE_BUFFER_TILING_MASK |
484 VC4_LOADSTORE_TILE_BUFFER_BUFFER_MASK |
485 VC4_LOADSTORE_TILE_BUFFER_FORMAT_MASK)) {
486 DRM_DEBUG("Unknown bits in load/store: 0x%04x\n",
487 surf->bits);
488 return -EINVAL;
489 }
490
491 if (tiling > VC4_TILING_FORMAT_LT) {
492 DRM_DEBUG("Bad tiling format\n");
493 return -EINVAL;
494 }
495
496 if (buffer == VC4_LOADSTORE_TILE_BUFFER_ZS) {
497 if (format != 0) {
498 DRM_DEBUG("No color format should be set for ZS\n");
499 return -EINVAL;
500 }
501 cpp = 4;
502 } else if (buffer == VC4_LOADSTORE_TILE_BUFFER_COLOR) {
503 switch (format) {
504 case VC4_LOADSTORE_TILE_BUFFER_BGR565:
505 case VC4_LOADSTORE_TILE_BUFFER_BGR565_DITHER:
506 cpp = 2;
507 break;
508 case VC4_LOADSTORE_TILE_BUFFER_RGBA8888:
509 cpp = 4;
510 break;
511 default:
512 DRM_DEBUG("Bad tile buffer format\n");
513 return -EINVAL;
514 }
515 } else {
516 DRM_DEBUG("Bad load/store buffer %d.\n", buffer);
517 return -EINVAL;
518 }
519
520 if (surf->offset & 0xf) {
521 DRM_DEBUG("load/store buffer must be 16b aligned.\n");
522 return -EINVAL;
523 }
524
525 if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
526 exec->args->width, exec->args->height, cpp)) {
527 return -EINVAL;
528 }
529
530 return 0;
531}
532
533static int
534vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
535 struct vc4_rcl_setup *setup,
536 struct drm_gem_cma_object **obj,
537 struct drm_vc4_submit_rcl_surface *surf)
538{
539 uint8_t tiling = VC4_GET_FIELD(surf->bits,
540 VC4_RENDER_CONFIG_MEMORY_FORMAT);
541 uint8_t format = VC4_GET_FIELD(surf->bits,
542 VC4_RENDER_CONFIG_FORMAT);
543 int cpp;
544
545 if (surf->flags != 0) {
546 DRM_DEBUG("No flags supported on render config.\n");
547 return -EINVAL;
548 }
549
550 if (surf->bits & ~(VC4_RENDER_CONFIG_MEMORY_FORMAT_MASK |
551 VC4_RENDER_CONFIG_FORMAT_MASK |
552 VC4_RENDER_CONFIG_MS_MODE_4X |
553 VC4_RENDER_CONFIG_DECIMATE_MODE_4X)) {
554 DRM_DEBUG("Unknown bits in render config: 0x%04x\n",
555 surf->bits);
556 return -EINVAL;
557 }
558
559 if (surf->hindex == ~0)
560 return 0;
561
562 *obj = vc4_use_bo(exec, surf->hindex);
563 if (!*obj)
564 return -EINVAL;
565
566 exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
567
568 if (tiling > VC4_TILING_FORMAT_LT) {
569 DRM_DEBUG("Bad tiling format\n");
570 return -EINVAL;
571 }
572
573 switch (format) {
574 case VC4_RENDER_CONFIG_FORMAT_BGR565_DITHERED:
575 case VC4_RENDER_CONFIG_FORMAT_BGR565:
576 cpp = 2;
577 break;
578 case VC4_RENDER_CONFIG_FORMAT_RGBA8888:
579 cpp = 4;
580 break;
581 default:
582 DRM_DEBUG("Bad tile buffer format\n");
583 return -EINVAL;
584 }
585
586 if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
587 exec->args->width, exec->args->height, cpp)) {
588 return -EINVAL;
589 }
590
591 return 0;
592}
593
594int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
595{
596 struct vc4_rcl_setup setup = {0};
597 struct drm_vc4_submit_cl *args = exec->args;
598 bool has_bin = args->bin_cl_size != 0;
599 int ret;
600
601 if (args->min_x_tile > args->max_x_tile ||
602 args->min_y_tile > args->max_y_tile) {
603 DRM_DEBUG("Bad render tile set (%d,%d)-(%d,%d)\n",
604 args->min_x_tile, args->min_y_tile,
605 args->max_x_tile, args->max_y_tile);
606 return -EINVAL;
607 }
608
609 if (has_bin &&
610 (args->max_x_tile > exec->bin_tiles_x ||
611 args->max_y_tile > exec->bin_tiles_y)) {
612 DRM_DEBUG("Render tiles (%d,%d) outside of bin config "
613 "(%d,%d)\n",
614 args->max_x_tile, args->max_y_tile,
615 exec->bin_tiles_x, exec->bin_tiles_y);
616 return -EINVAL;
617 }
618
619 ret = vc4_rcl_render_config_surface_setup(exec, &setup,
620 &setup.color_write,
621 &args->color_write);
622 if (ret)
623 return ret;
624
625 ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read,
626 false);
627 if (ret)
628 return ret;
629
630 ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read,
631 false);
632 if (ret)
633 return ret;
634
635 ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write,
636 true);
637 if (ret)
638 return ret;
639
640 ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_color_write,
641 &args->msaa_color_write);
642 if (ret)
643 return ret;
644
645 ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_zs_write,
646 &args->msaa_zs_write);
647 if (ret)
648 return ret;
649
650
651
652
653 if (!setup.color_write && !setup.zs_write &&
654 !setup.msaa_color_write && !setup.msaa_zs_write) {
655 DRM_DEBUG("RCL requires color or Z/S write\n");
656 return -EINVAL;
657 }
658
659 return vc4_create_rcl_bo(dev, exec, &setup);
660}
661