LXR linux/drivers/gpu/drm/radeon/radeon

   1/*
   2 * Copyright 2009 Jerome Glisse.
   3 * All Rights Reserved.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the
   7 * "Software"), to deal in the Software without restriction, including
   8 * without limitation the rights to use, copy, modify, merge, publish,
   9 * distribute, sub license, and/or sell copies of the Software, and to
  10 * permit persons to whom the Software is furnished to do so, subject to
  11 * the following conditions:
  12 *
  13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20 *
  21 * The above copyright notice and this permission notice (including the
  22 * next paragraph) shall be included in all copies or substantial portions
  23 * of the Software.
  24 *
  25 */
  26/*
  27 * Authors:
  28 *    Jerome Glisse <glisse@freedesktop.org>
  29 *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30 *    Dave Airlie
  31 */
  32#include <linux/list.h>
  33#include <linux/slab.h>
  34#include <drm/drmP.h>
  35#include <drm/radeon_drm.h>
  36#include "radeon.h"
  37#include "radeon_trace.h"
  38
  39
  40int radeon_ttm_init(struct radeon_device *rdev);
  41void radeon_ttm_fini(struct radeon_device *rdev);
  42static void radeon_bo_clear_surface_reg(struct radeon_bo *bo);
  43
  44/*
  45 * To exclude mutual BO access we rely on bo_reserve exclusion, as all
  46 * function are calling it.
  47 */
  48
  49static void radeon_update_memory_usage(struct radeon_bo *bo,
  50                                       unsigned mem_type, int sign)
  51{
  52        struct radeon_device *rdev = bo->rdev;
  53        u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT;
  54
  55        switch (mem_type) {
  56        case TTM_PL_TT:
  57                if (sign > 0)
  58                        atomic64_add(size, &rdev->gtt_usage);
  59                else
  60                        atomic64_sub(size, &rdev->gtt_usage);
  61                break;
  62        case TTM_PL_VRAM:
  63                if (sign > 0)
  64                        atomic64_add(size, &rdev->vram_usage);
  65                else
  66                        atomic64_sub(size, &rdev->vram_usage);
  67                break;
  68        }
  69}
  70
  71static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  72{
  73        struct radeon_bo *bo;
  74
  75        bo = container_of(tbo, struct radeon_bo, tbo);
  76
  77        radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1);
  78        radeon_mn_unregister(bo);
  79
  80        mutex_lock(&bo->rdev->gem.mutex);
  81        list_del_init(&bo->list);
  82        mutex_unlock(&bo->rdev->gem.mutex);
  83        radeon_bo_clear_surface_reg(bo);
  84        WARN_ON(!list_empty(&bo->va));
  85        drm_gem_object_release(&bo->gem_base);
  86        kfree(bo);
  87}
  88
  89bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo)
  90{
  91        if (bo->destroy == &radeon_ttm_bo_destroy)
  92                return true;
  93        return false;
  94}
  95
  96void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
  97{
  98        u32 c = 0, i;
  99
 100        rbo->placement.placement = rbo->placements;
 101        rbo->placement.busy_placement = rbo->placements;
 102        if (domain & RADEON_GEM_DOMAIN_VRAM) {
 103                /* Try placing BOs which don't need CPU access outside of the
 104                 * CPU accessible part of VRAM
 105                 */
 106                if ((rbo->flags & RADEON_GEM_NO_CPU_ACCESS) &&
 107                    rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size) {
 108                        rbo->placements[c].fpfn =
 109                                rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
 110                        rbo->placements[c++].flags = TTM_PL_FLAG_WC |
 111                                                     TTM_PL_FLAG_UNCACHED |
 112                                                     TTM_PL_FLAG_VRAM;
 113                }
 114
 115                rbo->placements[c].fpfn = 0;
 116                rbo->placements[c++].flags = TTM_PL_FLAG_WC |
 117                                             TTM_PL_FLAG_UNCACHED |
 118                                             TTM_PL_FLAG_VRAM;
 119        }
 120
 121        if (domain & RADEON_GEM_DOMAIN_GTT) {
 122                if (rbo->flags & RADEON_GEM_GTT_UC) {
 123                        rbo->placements[c].fpfn = 0;
 124                        rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED |
 125                                TTM_PL_FLAG_TT;
 126
 127                } else if ((rbo->flags & RADEON_GEM_GTT_WC) ||
 128                           (rbo->rdev->flags & RADEON_IS_AGP)) {
 129                        rbo->placements[c].fpfn = 0;
 130                        rbo->placements[c++].flags = TTM_PL_FLAG_WC |
 131                                TTM_PL_FLAG_UNCACHED |
 132                                TTM_PL_FLAG_TT;
 133                } else {
 134                        rbo->placements[c].fpfn = 0;
 135                        rbo->placements[c++].flags = TTM_PL_FLAG_CACHED |
 136                                                     TTM_PL_FLAG_TT;
 137                }
 138        }
 139
 140        if (domain & RADEON_GEM_DOMAIN_CPU) {
 141                if (rbo->flags & RADEON_GEM_GTT_UC) {
 142                        rbo->placements[c].fpfn = 0;
 143                        rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED |
 144                                TTM_PL_FLAG_SYSTEM;
 145
 146                } else if ((rbo->flags & RADEON_GEM_GTT_WC) ||
 147                    rbo->rdev->flags & RADEON_IS_AGP) {
 148                        rbo->placements[c].fpfn = 0;
 149                        rbo->placements[c++].flags = TTM_PL_FLAG_WC |
 150                                TTM_PL_FLAG_UNCACHED |
 151                                TTM_PL_FLAG_SYSTEM;
 152                } else {
 153                        rbo->placements[c].fpfn = 0;
 154                        rbo->placements[c++].flags = TTM_PL_FLAG_CACHED |
 155                                                     TTM_PL_FLAG_SYSTEM;
 156                }
 157        }
 158        if (!c) {
 159                rbo->placements[c].fpfn = 0;
 160                rbo->placements[c++].flags = TTM_PL_MASK_CACHING |
 161                                             TTM_PL_FLAG_SYSTEM;
 162        }
 163
 164        rbo->placement.num_placement = c;
 165        rbo->placement.num_busy_placement = c;
 166
 167        for (i = 0; i < c; ++i) {
 168                if ((rbo->flags & RADEON_GEM_CPU_ACCESS) &&
 169                    (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
 170                    !rbo->placements[i].fpfn)
 171                        rbo->placements[i].lpfn =
 172                                rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
 173                else
 174                        rbo->placements[i].lpfn = 0;
 175        }
 176}
 177
 178int radeon_bo_create(struct radeon_device *rdev,
 179                     unsigned long size, int byte_align, bool kernel,
 180                     u32 domain, u32 flags, struct sg_table *sg,
 181                     struct reservation_object *resv,
 182                     struct radeon_bo **bo_ptr)
 183{
 184        struct radeon_bo *bo;
 185        enum ttm_bo_type type;
 186        unsigned long page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
 187        size_t acc_size;
 188        int r;
 189
 190        size = ALIGN(size, PAGE_SIZE);
 191
 192        if (kernel) {
 193                type = ttm_bo_type_kernel;
 194        } else if (sg) {
 195                type = ttm_bo_type_sg;
 196        } else {
 197                type = ttm_bo_type_device;
 198        }
 199        *bo_ptr = NULL;
 200
 201        acc_size = ttm_bo_dma_acc_size(&rdev->mman.bdev, size,
 202                                       sizeof(struct radeon_bo));
 203
 204        bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);
 205        if (bo == NULL)
 206                return -ENOMEM;
 207        r = drm_gem_object_init(rdev->ddev, &bo->gem_base, size);
 208        if (unlikely(r)) {
 209                kfree(bo);
 210                return r;
 211        }
 212        bo->rdev = rdev;
 213        bo->surface_reg = -1;
 214        INIT_LIST_HEAD(&bo->list);
 215        INIT_LIST_HEAD(&bo->va);
 216        bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM |
 217                                       RADEON_GEM_DOMAIN_GTT |
 218                                       RADEON_GEM_DOMAIN_CPU);
 219
 220        bo->flags = flags;
 221        /* PCI GART is always snooped */
 222        if (!(rdev->flags & RADEON_IS_PCIE))
 223                bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);
 224
 225#ifdef CONFIG_X86_32
 226        /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
 227         * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
 228         */
 229        bo->flags &= ~RADEON_GEM_GTT_WC;
 230#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
 231        /* Don't try to enable write-combining when it can't work, or things
 232         * may be slow
 233         * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
 234         */
 235
 236#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
 237         thanks to write-combining
 238
 239        DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
 240                      "better performance thanks to write-combining\n");
 241        bo->flags &= ~RADEON_GEM_GTT_WC;
 242#endif
 243
 244        radeon_ttm_placement_from_domain(bo, domain);
 245        /* Kernel allocation are uninterruptible */
 246        down_read(&rdev->pm.mclk_lock);
 247        r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,
 248                        &bo->placement, page_align, !kernel, NULL,
 249                        acc_size, sg, resv, &radeon_ttm_bo_destroy);
 250        up_read(&rdev->pm.mclk_lock);
 251        if (unlikely(r != 0)) {
 252                return r;
 253        }
 254        *bo_ptr = bo;
 255
 256        trace_radeon_bo_create(bo);
 257
 258        return 0;
 259}
 260
 261int radeon_bo_kmap(struct radeon_bo *bo, void **ptr)
 262{
 263        bool is_iomem;
 264        int r;
 265
 266        if (bo->kptr) {
 267                if (ptr) {
 268                        *ptr = bo->kptr;
 269                }
 270                return 0;
 271        }
 272        r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
 273        if (r) {
 274                return r;
 275        }
 276        bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
 277        if (ptr) {
 278                *ptr = bo->kptr;
 279        }
 280        radeon_bo_check_tiling(bo, 0, 0);
 281        return 0;
 282}
 283
 284void radeon_bo_kunmap(struct radeon_bo *bo)
 285{
 286        if (bo->kptr == NULL)
 287                return;
 288        bo->kptr = NULL;
 289        radeon_bo_check_tiling(bo, 0, 0);
 290        ttm_bo_kunmap(&bo->kmap);
 291}
 292
 293struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo)
 294{
 295        if (bo == NULL)
 296                return NULL;
 297
 298        ttm_bo_reference(&bo->tbo);
 299        return bo;
 300}
 301
 302void radeon_bo_unref(struct radeon_bo **bo)
 303{
 304        struct ttm_buffer_object *tbo;
 305        struct radeon_device *rdev;
 306
 307        if ((*bo) == NULL)
 308                return;
 309        rdev = (*bo)->rdev;
 310        tbo = &((*bo)->tbo);
 311        ttm_bo_unref(&tbo);
 312        if (tbo == NULL)
 313                *bo = NULL;
 314}
 315
 316int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,
 317                             u64 *gpu_addr)
 318{
 319        int r, i;
 320
 321        if (radeon_ttm_tt_has_userptr(bo->tbo.ttm))
 322                return -EPERM;
 323
 324        if (bo->pin_count) {
 325                bo->pin_count++;
 326                if (gpu_addr)
 327                        *gpu_addr = radeon_bo_gpu_offset(bo);
 328
 329                if (max_offset != 0) {
 330                        u64 domain_start;
 331
 332                        if (domain == RADEON_GEM_DOMAIN_VRAM)
 333                                domain_start = bo->rdev->mc.vram_start;
 334                        else
 335                                domain_start = bo->rdev->mc.gtt_start;
 336                        WARN_ON_ONCE(max_offset <
 337                                     (radeon_bo_gpu_offset(bo) - domain_start));
 338                }
 339
 340                return 0;
 341        }
 342        radeon_ttm_placement_from_domain(bo, domain);
 343        for (i = 0; i < bo->placement.num_placement; i++) {
 344                /* force to pin into visible video ram */
 345                if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
 346                    !(bo->flags & RADEON_GEM_NO_CPU_ACCESS) &&
 347                    (!max_offset || max_offset > bo->rdev->mc.visible_vram_size))
 348                        bo->placements[i].lpfn =
 349                                bo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
 350                else
 351                        bo->placements[i].lpfn = max_offset >> PAGE_SHIFT;
 352
 353                bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 354        }
 355
 356        r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 357        if (likely(r == 0)) {
 358                bo->pin_count = 1;
 359                if (gpu_addr != NULL)
 360                        *gpu_addr = radeon_bo_gpu_offset(bo);
 361                if (domain == RADEON_GEM_DOMAIN_VRAM)
 362                        bo->rdev->vram_pin_size += radeon_bo_size(bo);
 363                else
 364                        bo->rdev->gart_pin_size += radeon_bo_size(bo);
 365        } else {
 366                dev_err(bo->rdev->dev, "%p pin failed\n", bo);
 367        }
 368        return r;
 369}
 370
 371int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr)
 372{
 373        return radeon_bo_pin_restricted(bo, domain, 0, gpu_addr);
 374}
 375
 376int radeon_bo_unpin(struct radeon_bo *bo)
 377{
 378        int r, i;
 379
 380        if (!bo->pin_count) {
 381                dev_warn(bo->rdev->dev, "%p unpin not necessary\n", bo);
 382                return 0;
 383        }
 384        bo->pin_count--;
 385        if (bo->pin_count)
 386                return 0;
 387        for (i = 0; i < bo->placement.num_placement; i++) {
 388                bo->placements[i].lpfn = 0;
 389                bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
 390        }
 391        r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 392        if (likely(r == 0)) {
 393                if (bo->tbo.mem.mem_type == TTM_PL_VRAM)
 394                        bo->rdev->vram_pin_size -= radeon_bo_size(bo);
 395                else
 396                        bo->rdev->gart_pin_size -= radeon_bo_size(bo);
 397        } else {
 398                dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo);
 399        }
 400        return r;
 401}
 402
 403int radeon_bo_evict_vram(struct radeon_device *rdev)
 404{
 405        /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
 406        if (0 && (rdev->flags & RADEON_IS_IGP)) {
 407                if (rdev->mc.igp_sideport_enabled == false)
 408                        /* Useless to evict on IGP chips */
 409                        return 0;
 410        }
 411        return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM);
 412}
 413
 414void radeon_bo_force_delete(struct radeon_device *rdev)
 415{
 416        struct radeon_bo *bo, *n;
 417
 418        if (list_empty(&rdev->gem.objects)) {
 419                return;
 420        }
 421        dev_err(rdev->dev, "Userspace still has active objects !\n");
 422        list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {
 423                mutex_lock(&rdev->ddev->struct_mutex);
 424                dev_err(rdev->dev, "%p %p %lu %lu force free\n",
 425                        &bo->gem_base, bo, (unsigned long)bo->gem_base.size,
 426                        *((unsigned long *)&bo->gem_base.refcount));
 427                mutex_lock(&bo->rdev->gem.mutex);
 428                list_del_init(&bo->list);
 429                mutex_unlock(&bo->rdev->gem.mutex);
 430                /* this should unref the ttm bo */
 431                drm_gem_object_unreference(&bo->gem_base);
 432                mutex_unlock(&rdev->ddev->struct_mutex);
 433        }
 434}
 435
 436int radeon_bo_init(struct radeon_device *rdev)
 437{
 438        /* Add an MTRR for the VRAM */
 439        if (!rdev->fastfb_working) {
 440                rdev->mc.vram_mtrr = arch_phys_wc_add(rdev->mc.aper_base,
 441                                                      rdev->mc.aper_size);
 442        }
 443        DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
 444                rdev->mc.mc_vram_size >> 20,
 445                (unsigned long long)rdev->mc.aper_size >> 20);
 446        DRM_INFO("RAM width %dbits %cDR\n",
 447                        rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S');
 448        return radeon_ttm_init(rdev);
 449}
 450
 451void radeon_bo_fini(struct radeon_device *rdev)
 452{
 453        radeon_ttm_fini(rdev);
 454        arch_phys_wc_del(rdev->mc.vram_mtrr);
 455}
 456
 457/* Returns how many bytes TTM can move per IB.
 458 */
 459static u64 radeon_bo_get_threshold_for_moves(struct radeon_device *rdev)
 460{
 461        u64 real_vram_size = rdev->mc.real_vram_size;
 462        u64 vram_usage = atomic64_read(&rdev->vram_usage);
 463
 464        /* This function is based on the current VRAM usage.
 465         *
 466         * - If all of VRAM is free, allow relocating the number of bytes that
 467         *   is equal to 1/4 of the size of VRAM for this IB.
 468
 469         * - If more than one half of VRAM is occupied, only allow relocating
 470         *   1 MB of data for this IB.
 471         *
 472         * - From 0 to one half of used VRAM, the threshold decreases
 473         *   linearly.
 474         *         __________________
 475         * 1/4 of -|\               |
 476         * VRAM    | \              |
 477         *         |  \             |
 478         *         |   \            |
 479         *         |    \           |
 480         *         |     \          |
 481         *         |      \         |
 482         *         |       \________|1 MB
 483         *         |----------------|
 484         *    VRAM 0 %             100 %
 485         *         used            used
 486         *
 487         * Note: It's a threshold, not a limit. The threshold must be crossed
 488         * for buffer relocations to stop, so any buffer of an arbitrary size
 489         * can be moved as long as the threshold isn't crossed before
 490         * the relocation takes place. We don't want to disable buffer
 491         * relocations completely.
 492         *
 493         * The idea is that buffers should be placed in VRAM at creation time
 494         * and TTM should only do a minimum number of relocations during
 495         * command submission. In practice, you need to submit at least
 496         * a dozen IBs to move all buffers to VRAM if they are in GTT.
 497         *
 498         * Also, things can get pretty crazy under memory pressure and actual
 499         * VRAM usage can change a lot, so playing safe even at 50% does
 500         * consistently increase performance.
 501         */
 502
 503        u64 half_vram = real_vram_size >> 1;
 504        u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
 505        u64 bytes_moved_threshold = half_free_vram >> 1;
 506        return max(bytes_moved_threshold, 1024*1024ull);
 507}
 508
 509int radeon_bo_list_validate(struct radeon_device *rdev,
 510                            struct ww_acquire_ctx *ticket,
 511                            struct list_head *head, int ring)
 512{
 513        struct radeon_bo_list *lobj;
 514        struct list_head duplicates;
 515        int r;
 516        u64 bytes_moved = 0, initial_bytes_moved;
 517        u64 bytes_moved_threshold = radeon_bo_get_threshold_for_moves(rdev);
 518
 519        INIT_LIST_HEAD(&duplicates);
 520        r = ttm_eu_reserve_buffers(ticket, head, true, &duplicates);
 521        if (unlikely(r != 0)) {
 522                return r;
 523        }
 524
 525        list_for_each_entry(lobj, head, tv.head) {
 526                struct radeon_bo *bo = lobj->robj;
 527                if (!bo->pin_count) {
 528                        u32 domain = lobj->prefered_domains;
 529                        u32 allowed = lobj->allowed_domains;
 530                        u32 current_domain =
 531                                radeon_mem_type_to_domain(bo->tbo.mem.mem_type);
 532
 533                        /* Check if this buffer will be moved and don't move it
 534                         * if we have moved too many buffers for this IB already.
 535                         *
 536                         * Note that this allows moving at least one buffer of
 537                         * any size, because it doesn't take the current "bo"
 538                         * into account. We don't want to disallow buffer moves
 539                         * completely.
 540                         */
 541                        if ((allowed & current_domain) != 0 &&
 542                            (domain & current_domain) == 0 && /* will be moved */
 543                            bytes_moved > bytes_moved_threshold) {
 544                                /* don't move it */
 545                                domain = current_domain;
 546                        }
 547
 548                retry:
 549                        radeon_ttm_placement_from_domain(bo, domain);
 550                        if (ring == R600_RING_TYPE_UVD_INDEX)
 551                                radeon_uvd_force_into_uvd_segment(bo, allowed);
 552
 553                        initial_bytes_moved = atomic64_read(&rdev->num_bytes_moved);
 554                        r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
 555                        bytes_moved += atomic64_read(&rdev->num_bytes_moved) -
 556                                       initial_bytes_moved;
 557
 558                        if (unlikely(r)) {
 559                                if (r != -ERESTARTSYS &&
 560                                    domain != lobj->allowed_domains) {
 561                                        domain = lobj->allowed_domains;
 562                                        goto retry;
 563                                }
 564                                ttm_eu_backoff_reservation(ticket, head);
 565                                return r;
 566                        }
 567                }
 568                lobj->gpu_offset = radeon_bo_gpu_offset(bo);
 569                lobj->tiling_flags = bo->tiling_flags;
 570        }
 571
 572        list_for_each_entry(lobj, &duplicates, tv.head) {
 573                lobj->gpu_offset = radeon_bo_gpu_offset(lobj->robj);
 574                lobj->tiling_flags = lobj->robj->tiling_flags;
 575        }
 576
 577        return 0;
 578}
 579
 580int radeon_bo_get_surface_reg(struct radeon_bo *bo)
 581{
 582        struct radeon_device *rdev = bo->rdev;
 583        struct radeon_surface_reg *reg;
 584        struct radeon_bo *old_object;
 585        int steal;
 586        int i;
 587
 588        lockdep_assert_held(&bo->tbo.resv->lock.base);
 589
 590        if (!bo->tiling_flags)
 591                return 0;
 592
 593        if (bo->surface_reg >= 0) {
 594                reg = &rdev->surface_regs[bo->surface_reg];
 595                i = bo->surface_reg;
 596                goto out;
 597        }
 598
 599        steal = -1;
 600        for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {
 601
 602                reg = &rdev->surface_regs[i];
 603                if (!reg->bo)
 604                        break;
 605
 606                old_object = reg->bo;
 607                if (old_object->pin_count == 0)
 608                        steal = i;
 609        }
 610
 611        /* if we are all out */
 612        if (i == RADEON_GEM_MAX_SURFACES) {
 613                if (steal == -1)
 614                        return -ENOMEM;
 615                /* find someone with a surface reg and nuke their BO */
 616                reg = &rdev->surface_regs[steal];
 617                old_object = reg->bo;
 618                /* blow away the mapping */
 619                DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object);
 620                ttm_bo_unmap_virtual(&old_object->tbo);
 621                old_object->surface_reg = -1;
 622                i = steal;
 623        }
 624
 625        bo->surface_reg = i;
 626        reg->bo = bo;
 627
 628out:
 629        radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch,
 630                               bo->tbo.mem.start << PAGE_SHIFT,
 631                               bo->tbo.num_pages << PAGE_SHIFT);
 632        return 0;
 633}
 634
 635static void radeon_bo_clear_surface_reg(struct radeon_bo *bo)
 636{
 637        struct radeon_device *rdev = bo->rdev;
 638        struct radeon_surface_reg *reg;
 639
 640        if (bo->surface_reg == -1)
 641                return;
 642
 643        reg = &rdev->surface_regs[bo->surface_reg];
 644        radeon_clear_surface_reg(rdev, bo->surface_reg);
 645
 646        reg->bo = NULL;
 647        bo->surface_reg = -1;
 648}
 649
 650int radeon_bo_set_tiling_flags(struct radeon_bo *bo,
 651                                uint32_t tiling_flags, uint32_t pitch)
 652{
 653        struct radeon_device *rdev = bo->rdev;
 654        int r;
 655
 656        if (rdev->family >= CHIP_CEDAR) {
 657                unsigned bankw, bankh, mtaspect, tilesplit, stilesplit;
 658
 659                bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
 660                bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
 661                mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
 662                tilesplit = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
 663                stilesplit = (tiling_flags >> RADEON_TILING_EG_STENCIL_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_STENCIL_TILE_SPLIT_MASK;
 664                switch (bankw) {
 665                case 0:
 666                case 1:
 667                case 2:
 668                case 4:
 669                case 8:
 670                        break;
 671                default:
 672                        return -EINVAL;
 673                }
 674                switch (bankh) {
 675                case 0:
 676                case 1:
 677                case 2:
 678                case 4:
 679                case 8:
 680                        break;
 681                default:
 682                        return -EINVAL;
 683                }
 684                switch (mtaspect) {
 685                case 0:
 686                case 1:
 687                case 2:
 688                case 4:
 689                case 8:
 690                        break;
 691                default:
 692                        return -EINVAL;
 693                }
 694                if (tilesplit > 6) {
 695                        return -EINVAL;
 696                }
 697                if (stilesplit > 6) {
 698                        return -EINVAL;
 699                }
 700        }
 701        r = radeon_bo_reserve(bo, false);
 702        if (unlikely(r != 0))
 703                return r;
 704        bo->tiling_flags = tiling_flags;
 705        bo->pitch = pitch;
 706        radeon_bo_unreserve(bo);
 707        return 0;
 708}
 709
 710void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
 711                                uint32_t *tiling_flags,
 712                                uint32_t *pitch)
 713{
 714        lockdep_assert_held(&bo->tbo.resv->lock.base);
 715
 716        if (tiling_flags)
 717                *tiling_flags = bo->tiling_flags;
 718        if (pitch)
 719                *pitch = bo->pitch;
 720}
 721
 722int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
 723                                bool force_drop)
 724{
 725        if (!force_drop)
 726                lockdep_assert_held(&bo->tbo.resv->lock.base);
 727
 728        if (!(bo->tiling_flags & RADEON_TILING_SURFACE))
 729                return 0;
 730
 731        if (force_drop) {
 732                radeon_bo_clear_surface_reg(bo);
 733                return 0;
 734        }
 735
 736        if (bo->tbo.mem.mem_type != TTM_PL_VRAM) {
 737                if (!has_moved)
 738                        return 0;
 739
 740                if (bo->surface_reg >= 0)
 741                        radeon_bo_clear_surface_reg(bo);
 742                return 0;
 743        }
 744
 745        if ((bo->surface_reg >= 0) && !has_moved)
 746                return 0;
 747
 748        return radeon_bo_get_surface_reg(bo);
 749}
 750
 751void radeon_bo_move_notify(struct ttm_buffer_object *bo,
 752                           struct ttm_mem_reg *new_mem)
 753{
 754        struct radeon_bo *rbo;
 755
 756        if (!radeon_ttm_bo_is_radeon_bo(bo))
 757                return;
 758
 759        rbo = container_of(bo, struct radeon_bo, tbo);
 760        radeon_bo_check_tiling(rbo, 0, 1);
 761        radeon_vm_bo_invalidate(rbo->rdev, rbo);
 762
 763        /* update statistics */
 764        if (!new_mem)
 765                return;
 766
 767        radeon_update_memory_usage(rbo, bo->mem.mem_type, -1);
 768        radeon_update_memory_usage(rbo, new_mem->mem_type, 1);
 769}
 770
 771int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 772{
 773        struct radeon_device *rdev;
 774        struct radeon_bo *rbo;
 775        unsigned long offset, size, lpfn;
 776        int i, r;
 777
 778        if (!radeon_ttm_bo_is_radeon_bo(bo))
 779                return 0;
 780        rbo = container_of(bo, struct radeon_bo, tbo);
 781        radeon_bo_check_tiling(rbo, 0, 0);
 782        rdev = rbo->rdev;
 783        if (bo->mem.mem_type != TTM_PL_VRAM)
 784                return 0;
 785
 786        size = bo->mem.num_pages << PAGE_SHIFT;
 787        offset = bo->mem.start << PAGE_SHIFT;
 788        if ((offset + size) <= rdev->mc.visible_vram_size)
 789                return 0;
 790
 791        /* hurrah the memory is not visible ! */
 792        radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM);
 793        lpfn =  rdev->mc.visible_vram_size >> PAGE_SHIFT;
 794        for (i = 0; i < rbo->placement.num_placement; i++) {
 795                /* Force into visible VRAM */
 796                if ((rbo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
 797                    (!rbo->placements[i].lpfn || rbo->placements[i].lpfn > lpfn))
 798                        rbo->placements[i].lpfn = lpfn;
 799        }
 800        r = ttm_bo_validate(bo, &rbo->placement, false, false);
 801        if (unlikely(r == -ENOMEM)) {
 802                radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
 803                return ttm_bo_validate(bo, &rbo->placement, false, false);
 804        } else if (unlikely(r != 0)) {
 805                return r;
 806        }
 807
 808        offset = bo->mem.start << PAGE_SHIFT;
 809        /* this should never happen */
 810        if ((offset + size) > rdev->mc.visible_vram_size)
 811                return -EINVAL;
 812
 813        return 0;
 814}
 815
 816int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait)
 817{
 818        int r;
 819
 820        r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, NULL);
 821        if (unlikely(r != 0))
 822                return r;
 823        if (mem_type)
 824                *mem_type = bo->tbo.mem.mem_type;
 825
 826        r = ttm_bo_wait(&bo->tbo, true, true, no_wait);
 827        ttm_bo_unreserve(&bo->tbo);
 828        return r;
 829}
 830
 831/**
 832 * radeon_bo_fence - add fence to buffer object
 833 *
 834 * @bo: buffer object in question
 835 * @fence: fence to add
 836 * @shared: true if fence should be added shared
 837 *
 838 */
 839void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence,
 840                     bool shared)
 841{
 842        struct reservation_object *resv = bo->tbo.resv;
 843
 844        if (shared)
 845                reservation_object_add_shared_fence(resv, &fence->base);
 846        else
 847                reservation_object_add_excl_fence(resv, &fence->base);
 848}
 849