linux/drivers/gpu/drm/radeon/radeon_ring.c
<<
>>
Prefs
   1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 *          Christian König
  28 */
  29#include <drm/drmP.h>
  30#include "radeon.h"
  31
  32/*
  33 * Rings
  34 * Most engines on the GPU are fed via ring buffers.  Ring
  35 * buffers are areas of GPU accessible memory that the host
  36 * writes commands into and the GPU reads commands out of.
  37 * There is a rptr (read pointer) that determines where the
  38 * GPU is currently reading, and a wptr (write pointer)
  39 * which determines where the host has written.  When the
  40 * pointers are equal, the ring is idle.  When the host
  41 * writes commands to the ring buffer, it increments the
  42 * wptr.  The GPU then starts fetching commands and executes
  43 * them until the pointers are equal again.
  44 */
  45static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
  46
  47/**
  48 * radeon_ring_supports_scratch_reg - check if the ring supports
  49 * writing to scratch registers
  50 *
  51 * @rdev: radeon_device pointer
  52 * @ring: radeon_ring structure holding ring information
  53 *
  54 * Check if a specific ring supports writing to scratch registers (all asics).
  55 * Returns true if the ring supports writing to scratch regs, false if not.
  56 */
  57bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
  58                                      struct radeon_ring *ring)
  59{
  60        switch (ring->idx) {
  61        case RADEON_RING_TYPE_GFX_INDEX:
  62        case CAYMAN_RING_TYPE_CP1_INDEX:
  63        case CAYMAN_RING_TYPE_CP2_INDEX:
  64                return true;
  65        default:
  66                return false;
  67        }
  68}
  69
  70/**
  71 * radeon_ring_free_size - update the free size
  72 *
  73 * @rdev: radeon_device pointer
  74 * @ring: radeon_ring structure holding ring information
  75 *
  76 * Update the free dw slots in the ring buffer (all asics).
  77 */
  78void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
  79{
  80        uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
  81
  82        /* This works because ring_size is a power of 2 */
  83        ring->ring_free_dw = rptr + (ring->ring_size / 4);
  84        ring->ring_free_dw -= ring->wptr;
  85        ring->ring_free_dw &= ring->ptr_mask;
  86        if (!ring->ring_free_dw) {
  87                /* this is an empty ring */
  88                ring->ring_free_dw = ring->ring_size / 4;
  89                /*  update lockup info to avoid false positive */
  90                radeon_ring_lockup_update(rdev, ring);
  91        }
  92}
  93
  94/**
  95 * radeon_ring_alloc - allocate space on the ring buffer
  96 *
  97 * @rdev: radeon_device pointer
  98 * @ring: radeon_ring structure holding ring information
  99 * @ndw: number of dwords to allocate in the ring buffer
 100 *
 101 * Allocate @ndw dwords in the ring buffer (all asics).
 102 * Returns 0 on success, error on failure.
 103 */
 104int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
 105{
 106        int r;
 107
 108        /* make sure we aren't trying to allocate more space than there is on the ring */
 109        if (ndw > (ring->ring_size / 4))
 110                return -ENOMEM;
 111        /* Align requested size with padding so unlock_commit can
 112         * pad safely */
 113        radeon_ring_free_size(rdev, ring);
 114        ndw = (ndw + ring->align_mask) & ~ring->align_mask;
 115        while (ndw > (ring->ring_free_dw - 1)) {
 116                radeon_ring_free_size(rdev, ring);
 117                if (ndw < ring->ring_free_dw) {
 118                        break;
 119                }
 120                r = radeon_fence_wait_next(rdev, ring->idx);
 121                if (r)
 122                        return r;
 123        }
 124        ring->count_dw = ndw;
 125        ring->wptr_old = ring->wptr;
 126        return 0;
 127}
 128
 129/**
 130 * radeon_ring_lock - lock the ring and allocate space on it
 131 *
 132 * @rdev: radeon_device pointer
 133 * @ring: radeon_ring structure holding ring information
 134 * @ndw: number of dwords to allocate in the ring buffer
 135 *
 136 * Lock the ring and allocate @ndw dwords in the ring buffer
 137 * (all asics).
 138 * Returns 0 on success, error on failure.
 139 */
 140int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
 141{
 142        int r;
 143
 144        mutex_lock(&rdev->ring_lock);
 145        r = radeon_ring_alloc(rdev, ring, ndw);
 146        if (r) {
 147                mutex_unlock(&rdev->ring_lock);
 148                return r;
 149        }
 150        return 0;
 151}
 152
 153/**
 154 * radeon_ring_commit - tell the GPU to execute the new
 155 * commands on the ring buffer
 156 *
 157 * @rdev: radeon_device pointer
 158 * @ring: radeon_ring structure holding ring information
 159 * @hdp_flush: Whether or not to perform an HDP cache flush
 160 *
 161 * Update the wptr (write pointer) to tell the GPU to
 162 * execute new commands on the ring buffer (all asics).
 163 */
 164void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring,
 165                        bool hdp_flush)
 166{
 167        /* If we are emitting the HDP flush via the ring buffer, we need to
 168         * do it before padding.
 169         */
 170        if (hdp_flush && rdev->asic->ring[ring->idx]->hdp_flush)
 171                rdev->asic->ring[ring->idx]->hdp_flush(rdev, ring);
 172        /* We pad to match fetch size */
 173        while (ring->wptr & ring->align_mask) {
 174                radeon_ring_write(ring, ring->nop);
 175        }
 176        mb();
 177        /* If we are emitting the HDP flush via MMIO, we need to do it after
 178         * all CPU writes to VRAM finished.
 179         */
 180        if (hdp_flush && rdev->asic->mmio_hdp_flush)
 181                rdev->asic->mmio_hdp_flush(rdev);
 182        radeon_ring_set_wptr(rdev, ring);
 183}
 184
 185/**
 186 * radeon_ring_unlock_commit - tell the GPU to execute the new
 187 * commands on the ring buffer and unlock it
 188 *
 189 * @rdev: radeon_device pointer
 190 * @ring: radeon_ring structure holding ring information
 191 * @hdp_flush: Whether or not to perform an HDP cache flush
 192 *
 193 * Call radeon_ring_commit() then unlock the ring (all asics).
 194 */
 195void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring,
 196                               bool hdp_flush)
 197{
 198        radeon_ring_commit(rdev, ring, hdp_flush);
 199        mutex_unlock(&rdev->ring_lock);
 200}
 201
 202/**
 203 * radeon_ring_undo - reset the wptr
 204 *
 205 * @ring: radeon_ring structure holding ring information
 206 *
 207 * Reset the driver's copy of the wptr (all asics).
 208 */
 209void radeon_ring_undo(struct radeon_ring *ring)
 210{
 211        ring->wptr = ring->wptr_old;
 212}
 213
 214/**
 215 * radeon_ring_unlock_undo - reset the wptr and unlock the ring
 216 *
 217 * @ring: radeon_ring structure holding ring information
 218 *
 219 * Call radeon_ring_undo() then unlock the ring (all asics).
 220 */
 221void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
 222{
 223        radeon_ring_undo(ring);
 224        mutex_unlock(&rdev->ring_lock);
 225}
 226
 227/**
 228 * radeon_ring_lockup_update - update lockup variables
 229 *
 230 * @ring: radeon_ring structure holding ring information
 231 *
 232 * Update the last rptr value and timestamp (all asics).
 233 */
 234void radeon_ring_lockup_update(struct radeon_device *rdev,
 235                               struct radeon_ring *ring)
 236{
 237        atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring));
 238        atomic64_set(&ring->last_activity, jiffies_64);
 239}
 240
 241/**
 242 * radeon_ring_test_lockup() - check if ring is lockedup by recording information
 243 * @rdev:       radeon device structure
 244 * @ring:       radeon_ring structure holding ring information
 245 *
 246 */
 247bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
 248{
 249        uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
 250        uint64_t last = atomic64_read(&ring->last_activity);
 251        uint64_t elapsed;
 252
 253        if (rptr != atomic_read(&ring->last_rptr)) {
 254                /* ring is still working, no lockup */
 255                radeon_ring_lockup_update(rdev, ring);
 256                return false;
 257        }
 258
 259        elapsed = jiffies_to_msecs(jiffies_64 - last);
 260        if (radeon_lockup_timeout && elapsed >= radeon_lockup_timeout) {
 261                dev_err(rdev->dev, "ring %d stalled for more than %llumsec\n",
 262                        ring->idx, elapsed);
 263                return true;
 264        }
 265        /* give a chance to the GPU ... */
 266        return false;
 267}
 268
 269/**
 270 * radeon_ring_backup - Back up the content of a ring
 271 *
 272 * @rdev: radeon_device pointer
 273 * @ring: the ring we want to back up
 274 *
 275 * Saves all unprocessed commits from a ring, returns the number of dwords saved.
 276 */
 277unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
 278                            uint32_t **data)
 279{
 280        unsigned size, ptr, i;
 281
 282        /* just in case lock the ring */
 283        mutex_lock(&rdev->ring_lock);
 284        *data = NULL;
 285
 286        if (ring->ring_obj == NULL) {
 287                mutex_unlock(&rdev->ring_lock);
 288                return 0;
 289        }
 290
 291        /* it doesn't make sense to save anything if all fences are signaled */
 292        if (!radeon_fence_count_emitted(rdev, ring->idx)) {
 293                mutex_unlock(&rdev->ring_lock);
 294                return 0;
 295        }
 296
 297        /* calculate the number of dw on the ring */
 298        if (ring->rptr_save_reg)
 299                ptr = RREG32(ring->rptr_save_reg);
 300        else if (rdev->wb.enabled)
 301                ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
 302        else {
 303                /* no way to read back the next rptr */
 304                mutex_unlock(&rdev->ring_lock);
 305                return 0;
 306        }
 307
 308        size = ring->wptr + (ring->ring_size / 4);
 309        size -= ptr;
 310        size &= ring->ptr_mask;
 311        if (size == 0) {
 312                mutex_unlock(&rdev->ring_lock);
 313                return 0;
 314        }
 315
 316        /* and then save the content of the ring */
 317        *data = drm_malloc_ab(size, sizeof(uint32_t));
 318        if (!*data) {
 319                mutex_unlock(&rdev->ring_lock);
 320                return 0;
 321        }
 322        for (i = 0; i < size; ++i) {
 323                (*data)[i] = ring->ring[ptr++];
 324                ptr &= ring->ptr_mask;
 325        }
 326
 327        mutex_unlock(&rdev->ring_lock);
 328        return size;
 329}
 330
 331/**
 332 * radeon_ring_restore - append saved commands to the ring again
 333 *
 334 * @rdev: radeon_device pointer
 335 * @ring: ring to append commands to
 336 * @size: number of dwords we want to write
 337 * @data: saved commands
 338 *
 339 * Allocates space on the ring and restore the previously saved commands.
 340 */
 341int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
 342                        unsigned size, uint32_t *data)
 343{
 344        int i, r;
 345
 346        if (!size || !data)
 347                return 0;
 348
 349        /* restore the saved ring content */
 350        r = radeon_ring_lock(rdev, ring, size);
 351        if (r)
 352                return r;
 353
 354        for (i = 0; i < size; ++i) {
 355                radeon_ring_write(ring, data[i]);
 356        }
 357
 358        radeon_ring_unlock_commit(rdev, ring, false);
 359        drm_free_large(data);
 360        return 0;
 361}
 362
 363/**
 364 * radeon_ring_init - init driver ring struct.
 365 *
 366 * @rdev: radeon_device pointer
 367 * @ring: radeon_ring structure holding ring information
 368 * @ring_size: size of the ring
 369 * @rptr_offs: offset of the rptr writeback location in the WB buffer
 370 * @nop: nop packet for this ring
 371 *
 372 * Initialize the driver information for the selected ring (all asics).
 373 * Returns 0 on success, error on failure.
 374 */
 375int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
 376                     unsigned rptr_offs, u32 nop)
 377{
 378        int r;
 379
 380        ring->ring_size = ring_size;
 381        ring->rptr_offs = rptr_offs;
 382        ring->nop = nop;
 383        /* Allocate ring buffer */
 384        if (ring->ring_obj == NULL) {
 385                r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
 386                                     RADEON_GEM_DOMAIN_GTT, 0, NULL,
 387                                     NULL, &ring->ring_obj);
 388                if (r) {
 389                        dev_err(rdev->dev, "(%d) ring create failed\n", r);
 390                        return r;
 391                }
 392                r = radeon_bo_reserve(ring->ring_obj, false);
 393                if (unlikely(r != 0))
 394                        return r;
 395                r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
 396                                        &ring->gpu_addr);
 397                if (r) {
 398                        radeon_bo_unreserve(ring->ring_obj);
 399                        dev_err(rdev->dev, "(%d) ring pin failed\n", r);
 400                        return r;
 401                }
 402                r = radeon_bo_kmap(ring->ring_obj,
 403                                       (void **)&ring->ring);
 404                radeon_bo_unreserve(ring->ring_obj);
 405                if (r) {
 406                        dev_err(rdev->dev, "(%d) ring map failed\n", r);
 407                        return r;
 408                }
 409        }
 410        ring->ptr_mask = (ring->ring_size / 4) - 1;
 411        ring->ring_free_dw = ring->ring_size / 4;
 412        if (rdev->wb.enabled) {
 413                u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);
 414                ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;
 415                ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];
 416        }
 417        if (radeon_debugfs_ring_init(rdev, ring)) {
 418                DRM_ERROR("Failed to register debugfs file for rings !\n");
 419        }
 420        radeon_ring_lockup_update(rdev, ring);
 421        return 0;
 422}
 423
 424/**
 425 * radeon_ring_fini - tear down the driver ring struct.
 426 *
 427 * @rdev: radeon_device pointer
 428 * @ring: radeon_ring structure holding ring information
 429 *
 430 * Tear down the driver information for the selected ring (all asics).
 431 */
 432void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
 433{
 434        int r;
 435        struct radeon_bo *ring_obj;
 436
 437        mutex_lock(&rdev->ring_lock);
 438        ring_obj = ring->ring_obj;
 439        ring->ready = false;
 440        ring->ring = NULL;
 441        ring->ring_obj = NULL;
 442        mutex_unlock(&rdev->ring_lock);
 443
 444        if (ring_obj) {
 445                r = radeon_bo_reserve(ring_obj, false);
 446                if (likely(r == 0)) {
 447                        radeon_bo_kunmap(ring_obj);
 448                        radeon_bo_unpin(ring_obj);
 449                        radeon_bo_unreserve(ring_obj);
 450                }
 451                radeon_bo_unref(&ring_obj);
 452        }
 453}
 454
 455/*
 456 * Debugfs info
 457 */
 458#if defined(CONFIG_DEBUG_FS)
 459
 460static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
 461{
 462        struct drm_info_node *node = (struct drm_info_node *) m->private;
 463        struct drm_device *dev = node->minor->dev;
 464        struct radeon_device *rdev = dev->dev_private;
 465        int ridx = *(int*)node->info_ent->data;
 466        struct radeon_ring *ring = &rdev->ring[ridx];
 467
 468        uint32_t rptr, wptr, rptr_next;
 469        unsigned count, i, j;
 470
 471        radeon_ring_free_size(rdev, ring);
 472        count = (ring->ring_size / 4) - ring->ring_free_dw;
 473
 474        wptr = radeon_ring_get_wptr(rdev, ring);
 475        seq_printf(m, "wptr: 0x%08x [%5d]\n",
 476                   wptr, wptr);
 477
 478        rptr = radeon_ring_get_rptr(rdev, ring);
 479        seq_printf(m, "rptr: 0x%08x [%5d]\n",
 480                   rptr, rptr);
 481
 482        if (ring->rptr_save_reg) {
 483                rptr_next = RREG32(ring->rptr_save_reg);
 484                seq_printf(m, "rptr next(0x%04x): 0x%08x [%5d]\n",
 485                           ring->rptr_save_reg, rptr_next, rptr_next);
 486        } else
 487                rptr_next = ~0;
 488
 489        seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n",
 490                   ring->wptr, ring->wptr);
 491        seq_printf(m, "last semaphore signal addr : 0x%016llx\n",
 492                   ring->last_semaphore_signal_addr);
 493        seq_printf(m, "last semaphore wait addr   : 0x%016llx\n",
 494                   ring->last_semaphore_wait_addr);
 495        seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
 496        seq_printf(m, "%u dwords in ring\n", count);
 497
 498        if (!ring->ring)
 499                return 0;
 500
 501        /* print 8 dw before current rptr as often it's the last executed
 502         * packet that is the root issue
 503         */
 504        i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;
 505        for (j = 0; j <= (count + 32); j++) {
 506                seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);
 507                if (rptr == i)
 508                        seq_puts(m, " *");
 509                if (rptr_next == i)
 510                        seq_puts(m, " #");
 511                seq_puts(m, "\n");
 512                i = (i + 1) & ring->ptr_mask;
 513        }
 514        return 0;
 515}
 516
 517static int radeon_gfx_index = RADEON_RING_TYPE_GFX_INDEX;
 518static int cayman_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX;
 519static int cayman_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX;
 520static int radeon_dma1_index = R600_RING_TYPE_DMA_INDEX;
 521static int radeon_dma2_index = CAYMAN_RING_TYPE_DMA1_INDEX;
 522static int r600_uvd_index = R600_RING_TYPE_UVD_INDEX;
 523static int si_vce1_index = TN_RING_TYPE_VCE1_INDEX;
 524static int si_vce2_index = TN_RING_TYPE_VCE2_INDEX;
 525
 526static struct drm_info_list radeon_debugfs_ring_info_list[] = {
 527        {"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_gfx_index},
 528        {"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_cp1_index},
 529        {"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_cp2_index},
 530        {"radeon_ring_dma1", radeon_debugfs_ring_info, 0, &radeon_dma1_index},
 531        {"radeon_ring_dma2", radeon_debugfs_ring_info, 0, &radeon_dma2_index},
 532        {"radeon_ring_uvd", radeon_debugfs_ring_info, 0, &r600_uvd_index},
 533        {"radeon_ring_vce1", radeon_debugfs_ring_info, 0, &si_vce1_index},
 534        {"radeon_ring_vce2", radeon_debugfs_ring_info, 0, &si_vce2_index},
 535};
 536
 537#endif
 538
 539static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
 540{
 541#if defined(CONFIG_DEBUG_FS)
 542        unsigned i;
 543        for (i = 0; i < ARRAY_SIZE(radeon_debugfs_ring_info_list); ++i) {
 544                struct drm_info_list *info = &radeon_debugfs_ring_info_list[i];
 545                int ridx = *(int*)radeon_debugfs_ring_info_list[i].data;
 546                unsigned r;
 547
 548                if (&rdev->ring[ridx] != ring)
 549                        continue;
 550
 551                r = radeon_debugfs_add_files(rdev, info, 1);
 552                if (r)
 553                        return r;
 554        }
 555#endif
 556        return 0;
 557}
 558