LXR linux/drivers/gpu/drm/radeon/cik.c

   1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include "drmP.h"
  28#include "radeon.h"
  29#include "radeon_asic.h"
  30#include "radeon_audio.h"
  31#include "cikd.h"
  32#include "atom.h"
  33#include "cik_blit_shaders.h"
  34#include "radeon_ucode.h"
  35#include "clearstate_ci.h"
  36#include "radeon_kfd.h"
  37
  38MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  39MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  40MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  41MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  42MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  43MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  44MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  45MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  46MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  47
  48MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
  49MODULE_FIRMWARE("radeon/bonaire_me.bin");
  50MODULE_FIRMWARE("radeon/bonaire_ce.bin");
  51MODULE_FIRMWARE("radeon/bonaire_mec.bin");
  52MODULE_FIRMWARE("radeon/bonaire_mc.bin");
  53MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
  54MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
  55MODULE_FIRMWARE("radeon/bonaire_smc.bin");
  56
  57MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  58MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  59MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  60MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  61MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  62MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  63MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  64MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  65MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  66
  67MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
  68MODULE_FIRMWARE("radeon/hawaii_me.bin");
  69MODULE_FIRMWARE("radeon/hawaii_ce.bin");
  70MODULE_FIRMWARE("radeon/hawaii_mec.bin");
  71MODULE_FIRMWARE("radeon/hawaii_mc.bin");
  72MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
  73MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
  74MODULE_FIRMWARE("radeon/hawaii_smc.bin");
  75
  76MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  77MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  78MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  79MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  80MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  81MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  82
  83MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
  84MODULE_FIRMWARE("radeon/kaveri_me.bin");
  85MODULE_FIRMWARE("radeon/kaveri_ce.bin");
  86MODULE_FIRMWARE("radeon/kaveri_mec.bin");
  87MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
  88MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
  89MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
  90
  91MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  92MODULE_FIRMWARE("radeon/KABINI_me.bin");
  93MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  94MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  95MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  96MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  97
  98MODULE_FIRMWARE("radeon/kabini_pfp.bin");
  99MODULE_FIRMWARE("radeon/kabini_me.bin");
 100MODULE_FIRMWARE("radeon/kabini_ce.bin");
 101MODULE_FIRMWARE("radeon/kabini_mec.bin");
 102MODULE_FIRMWARE("radeon/kabini_rlc.bin");
 103MODULE_FIRMWARE("radeon/kabini_sdma.bin");
 104
 105MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
 106MODULE_FIRMWARE("radeon/MULLINS_me.bin");
 107MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
 108MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
 109MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
 110MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
 111
 112MODULE_FIRMWARE("radeon/mullins_pfp.bin");
 113MODULE_FIRMWARE("radeon/mullins_me.bin");
 114MODULE_FIRMWARE("radeon/mullins_ce.bin");
 115MODULE_FIRMWARE("radeon/mullins_mec.bin");
 116MODULE_FIRMWARE("radeon/mullins_rlc.bin");
 117MODULE_FIRMWARE("radeon/mullins_sdma.bin");
 118
 119extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 120extern void r600_ih_ring_fini(struct radeon_device *rdev);
 121extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 122extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 123extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 124extern void sumo_rlc_fini(struct radeon_device *rdev);
 125extern int sumo_rlc_init(struct radeon_device *rdev);
 126extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
 127extern void si_rlc_reset(struct radeon_device *rdev);
 128extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
 129static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 130extern int cik_sdma_resume(struct radeon_device *rdev);
 131extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
 132extern void cik_sdma_fini(struct radeon_device *rdev);
 133extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
 134static void cik_rlc_stop(struct radeon_device *rdev);
 135static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 136static void cik_program_aspm(struct radeon_device *rdev);
 137static void cik_init_pg(struct radeon_device *rdev);
 138static void cik_init_cg(struct radeon_device *rdev);
 139static void cik_fini_pg(struct radeon_device *rdev);
 140static void cik_fini_cg(struct radeon_device *rdev);
 141static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
 142                                          bool enable);
 143
 144/**
 145 * cik_get_allowed_info_register - fetch the register for the info ioctl
 146 *
 147 * @rdev: radeon_device pointer
 148 * @reg: register offset in bytes
 149 * @val: register value
 150 *
 151 * Returns 0 for success or -EINVAL for an invalid register
 152 *
 153 */
 154int cik_get_allowed_info_register(struct radeon_device *rdev,
 155                                  u32 reg, u32 *val)
 156{
 157        switch (reg) {
 158        case GRBM_STATUS:
 159        case GRBM_STATUS2:
 160        case GRBM_STATUS_SE0:
 161        case GRBM_STATUS_SE1:
 162        case GRBM_STATUS_SE2:
 163        case GRBM_STATUS_SE3:
 164        case SRBM_STATUS:
 165        case SRBM_STATUS2:
 166        case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
 167        case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
 168        case UVD_STATUS:
 169        /* TODO VCE */
 170                *val = RREG32(reg);
 171                return 0;
 172        default:
 173                return -EINVAL;
 174        }
 175}
 176
 177/*
 178 * Indirect registers accessor
 179 */
 180u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
 181{
 182        unsigned long flags;
 183        u32 r;
 184
 185        spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 186        WREG32(CIK_DIDT_IND_INDEX, (reg));
 187        r = RREG32(CIK_DIDT_IND_DATA);
 188        spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 189        return r;
 190}
 191
 192void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 193{
 194        unsigned long flags;
 195
 196        spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 197        WREG32(CIK_DIDT_IND_INDEX, (reg));
 198        WREG32(CIK_DIDT_IND_DATA, (v));
 199        spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 200}
 201
 202/* get temperature in millidegrees */
 203int ci_get_temp(struct radeon_device *rdev)
 204{
 205        u32 temp;
 206        int actual_temp = 0;
 207
 208        temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 209                CTF_TEMP_SHIFT;
 210
 211        if (temp & 0x200)
 212                actual_temp = 255;
 213        else
 214                actual_temp = temp & 0x1ff;
 215
 216        actual_temp = actual_temp * 1000;
 217
 218        return actual_temp;
 219}
 220
 221/* get temperature in millidegrees */
 222int kv_get_temp(struct radeon_device *rdev)
 223{
 224        u32 temp;
 225        int actual_temp = 0;
 226
 227        temp = RREG32_SMC(0xC0300E0C);
 228
 229        if (temp)
 230                actual_temp = (temp / 8) - 49;
 231        else
 232                actual_temp = 0;
 233
 234        actual_temp = actual_temp * 1000;
 235
 236        return actual_temp;
 237}
 238
 239/*
 240 * Indirect registers accessor
 241 */
 242u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 243{
 244        unsigned long flags;
 245        u32 r;
 246
 247        spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 248        WREG32(PCIE_INDEX, reg);
 249        (void)RREG32(PCIE_INDEX);
 250        r = RREG32(PCIE_DATA);
 251        spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 252        return r;
 253}
 254
 255void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 256{
 257        unsigned long flags;
 258
 259        spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 260        WREG32(PCIE_INDEX, reg);
 261        (void)RREG32(PCIE_INDEX);
 262        WREG32(PCIE_DATA, v);
 263        (void)RREG32(PCIE_DATA);
 264        spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 265}
 266
 267static const u32 spectre_rlc_save_restore_register_list[] =
 268{
 269        (0x0e00 << 16) | (0xc12c >> 2),
 270        0x00000000,
 271        (0x0e00 << 16) | (0xc140 >> 2),
 272        0x00000000,
 273        (0x0e00 << 16) | (0xc150 >> 2),
 274        0x00000000,
 275        (0x0e00 << 16) | (0xc15c >> 2),
 276        0x00000000,
 277        (0x0e00 << 16) | (0xc168 >> 2),
 278        0x00000000,
 279        (0x0e00 << 16) | (0xc170 >> 2),
 280        0x00000000,
 281        (0x0e00 << 16) | (0xc178 >> 2),
 282        0x00000000,
 283        (0x0e00 << 16) | (0xc204 >> 2),
 284        0x00000000,
 285        (0x0e00 << 16) | (0xc2b4 >> 2),
 286        0x00000000,
 287        (0x0e00 << 16) | (0xc2b8 >> 2),
 288        0x00000000,
 289        (0x0e00 << 16) | (0xc2bc >> 2),
 290        0x00000000,
 291        (0x0e00 << 16) | (0xc2c0 >> 2),
 292        0x00000000,
 293        (0x0e00 << 16) | (0x8228 >> 2),
 294        0x00000000,
 295        (0x0e00 << 16) | (0x829c >> 2),
 296        0x00000000,
 297        (0x0e00 << 16) | (0x869c >> 2),
 298        0x00000000,
 299        (0x0600 << 16) | (0x98f4 >> 2),
 300        0x00000000,
 301        (0x0e00 << 16) | (0x98f8 >> 2),
 302        0x00000000,
 303        (0x0e00 << 16) | (0x9900 >> 2),
 304        0x00000000,
 305        (0x0e00 << 16) | (0xc260 >> 2),
 306        0x00000000,
 307        (0x0e00 << 16) | (0x90e8 >> 2),
 308        0x00000000,
 309        (0x0e00 << 16) | (0x3c000 >> 2),
 310        0x00000000,
 311        (0x0e00 << 16) | (0x3c00c >> 2),
 312        0x00000000,
 313        (0x0e00 << 16) | (0x8c1c >> 2),
 314        0x00000000,
 315        (0x0e00 << 16) | (0x9700 >> 2),
 316        0x00000000,
 317        (0x0e00 << 16) | (0xcd20 >> 2),
 318        0x00000000,
 319        (0x4e00 << 16) | (0xcd20 >> 2),
 320        0x00000000,
 321        (0x5e00 << 16) | (0xcd20 >> 2),
 322        0x00000000,
 323        (0x6e00 << 16) | (0xcd20 >> 2),
 324        0x00000000,
 325        (0x7e00 << 16) | (0xcd20 >> 2),
 326        0x00000000,
 327        (0x8e00 << 16) | (0xcd20 >> 2),
 328        0x00000000,
 329        (0x9e00 << 16) | (0xcd20 >> 2),
 330        0x00000000,
 331        (0xae00 << 16) | (0xcd20 >> 2),
 332        0x00000000,
 333        (0xbe00 << 16) | (0xcd20 >> 2),
 334        0x00000000,
 335        (0x0e00 << 16) | (0x89bc >> 2),
 336        0x00000000,
 337        (0x0e00 << 16) | (0x8900 >> 2),
 338        0x00000000,
 339        0x3,
 340        (0x0e00 << 16) | (0xc130 >> 2),
 341        0x00000000,
 342        (0x0e00 << 16) | (0xc134 >> 2),
 343        0x00000000,
 344        (0x0e00 << 16) | (0xc1fc >> 2),
 345        0x00000000,
 346        (0x0e00 << 16) | (0xc208 >> 2),
 347        0x00000000,
 348        (0x0e00 << 16) | (0xc264 >> 2),
 349        0x00000000,
 350        (0x0e00 << 16) | (0xc268 >> 2),
 351        0x00000000,
 352        (0x0e00 << 16) | (0xc26c >> 2),
 353        0x00000000,
 354        (0x0e00 << 16) | (0xc270 >> 2),
 355        0x00000000,
 356        (0x0e00 << 16) | (0xc274 >> 2),
 357        0x00000000,
 358        (0x0e00 << 16) | (0xc278 >> 2),
 359        0x00000000,
 360        (0x0e00 << 16) | (0xc27c >> 2),
 361        0x00000000,
 362        (0x0e00 << 16) | (0xc280 >> 2),
 363        0x00000000,
 364        (0x0e00 << 16) | (0xc284 >> 2),
 365        0x00000000,
 366        (0x0e00 << 16) | (0xc288 >> 2),
 367        0x00000000,
 368        (0x0e00 << 16) | (0xc28c >> 2),
 369        0x00000000,
 370        (0x0e00 << 16) | (0xc290 >> 2),
 371        0x00000000,
 372        (0x0e00 << 16) | (0xc294 >> 2),
 373        0x00000000,
 374        (0x0e00 << 16) | (0xc298 >> 2),
 375        0x00000000,
 376        (0x0e00 << 16) | (0xc29c >> 2),
 377        0x00000000,
 378        (0x0e00 << 16) | (0xc2a0 >> 2),
 379        0x00000000,
 380        (0x0e00 << 16) | (0xc2a4 >> 2),
 381        0x00000000,
 382        (0x0e00 << 16) | (0xc2a8 >> 2),
 383        0x00000000,
 384        (0x0e00 << 16) | (0xc2ac  >> 2),
 385        0x00000000,
 386        (0x0e00 << 16) | (0xc2b0 >> 2),
 387        0x00000000,
 388        (0x0e00 << 16) | (0x301d0 >> 2),
 389        0x00000000,
 390        (0x0e00 << 16) | (0x30238 >> 2),
 391        0x00000000,
 392        (0x0e00 << 16) | (0x30250 >> 2),
 393        0x00000000,
 394        (0x0e00 << 16) | (0x30254 >> 2),
 395        0x00000000,
 396        (0x0e00 << 16) | (0x30258 >> 2),
 397        0x00000000,
 398        (0x0e00 << 16) | (0x3025c >> 2),
 399        0x00000000,
 400        (0x4e00 << 16) | (0xc900 >> 2),
 401        0x00000000,
 402        (0x5e00 << 16) | (0xc900 >> 2),
 403        0x00000000,
 404        (0x6e00 << 16) | (0xc900 >> 2),
 405        0x00000000,
 406        (0x7e00 << 16) | (0xc900 >> 2),
 407        0x00000000,
 408        (0x8e00 << 16) | (0xc900 >> 2),
 409        0x00000000,
 410        (0x9e00 << 16) | (0xc900 >> 2),
 411        0x00000000,
 412        (0xae00 << 16) | (0xc900 >> 2),
 413        0x00000000,
 414        (0xbe00 << 16) | (0xc900 >> 2),
 415        0x00000000,
 416        (0x4e00 << 16) | (0xc904 >> 2),
 417        0x00000000,
 418        (0x5e00 << 16) | (0xc904 >> 2),
 419        0x00000000,
 420        (0x6e00 << 16) | (0xc904 >> 2),
 421        0x00000000,
 422        (0x7e00 << 16) | (0xc904 >> 2),
 423        0x00000000,
 424        (0x8e00 << 16) | (0xc904 >> 2),
 425        0x00000000,
 426        (0x9e00 << 16) | (0xc904 >> 2),
 427        0x00000000,
 428        (0xae00 << 16) | (0xc904 >> 2),
 429        0x00000000,
 430        (0xbe00 << 16) | (0xc904 >> 2),
 431        0x00000000,
 432        (0x4e00 << 16) | (0xc908 >> 2),
 433        0x00000000,
 434        (0x5e00 << 16) | (0xc908 >> 2),
 435        0x00000000,
 436        (0x6e00 << 16) | (0xc908 >> 2),
 437        0x00000000,
 438        (0x7e00 << 16) | (0xc908 >> 2),
 439        0x00000000,
 440        (0x8e00 << 16) | (0xc908 >> 2),
 441        0x00000000,
 442        (0x9e00 << 16) | (0xc908 >> 2),
 443        0x00000000,
 444        (0xae00 << 16) | (0xc908 >> 2),
 445        0x00000000,
 446        (0xbe00 << 16) | (0xc908 >> 2),
 447        0x00000000,
 448        (0x4e00 << 16) | (0xc90c >> 2),
 449        0x00000000,
 450        (0x5e00 << 16) | (0xc90c >> 2),
 451        0x00000000,
 452        (0x6e00 << 16) | (0xc90c >> 2),
 453        0x00000000,
 454        (0x7e00 << 16) | (0xc90c >> 2),
 455        0x00000000,
 456        (0x8e00 << 16) | (0xc90c >> 2),
 457        0x00000000,
 458        (0x9e00 << 16) | (0xc90c >> 2),
 459        0x00000000,
 460        (0xae00 << 16) | (0xc90c >> 2),
 461        0x00000000,
 462        (0xbe00 << 16) | (0xc90c >> 2),
 463        0x00000000,
 464        (0x4e00 << 16) | (0xc910 >> 2),
 465        0x00000000,
 466        (0x5e00 << 16) | (0xc910 >> 2),
 467        0x00000000,
 468        (0x6e00 << 16) | (0xc910 >> 2),
 469        0x00000000,
 470        (0x7e00 << 16) | (0xc910 >> 2),
 471        0x00000000,
 472        (0x8e00 << 16) | (0xc910 >> 2),
 473        0x00000000,
 474        (0x9e00 << 16) | (0xc910 >> 2),
 475        0x00000000,
 476        (0xae00 << 16) | (0xc910 >> 2),
 477        0x00000000,
 478        (0xbe00 << 16) | (0xc910 >> 2),
 479        0x00000000,
 480        (0x0e00 << 16) | (0xc99c >> 2),
 481        0x00000000,
 482        (0x0e00 << 16) | (0x9834 >> 2),
 483        0x00000000,
 484        (0x0000 << 16) | (0x30f00 >> 2),
 485        0x00000000,
 486        (0x0001 << 16) | (0x30f00 >> 2),
 487        0x00000000,
 488        (0x0000 << 16) | (0x30f04 >> 2),
 489        0x00000000,
 490        (0x0001 << 16) | (0x30f04 >> 2),
 491        0x00000000,
 492        (0x0000 << 16) | (0x30f08 >> 2),
 493        0x00000000,
 494        (0x0001 << 16) | (0x30f08 >> 2),
 495        0x00000000,
 496        (0x0000 << 16) | (0x30f0c >> 2),
 497        0x00000000,
 498        (0x0001 << 16) | (0x30f0c >> 2),
 499        0x00000000,
 500        (0x0600 << 16) | (0x9b7c >> 2),
 501        0x00000000,
 502        (0x0e00 << 16) | (0x8a14 >> 2),
 503        0x00000000,
 504        (0x0e00 << 16) | (0x8a18 >> 2),
 505        0x00000000,
 506        (0x0600 << 16) | (0x30a00 >> 2),
 507        0x00000000,
 508        (0x0e00 << 16) | (0x8bf0 >> 2),
 509        0x00000000,
 510        (0x0e00 << 16) | (0x8bcc >> 2),
 511        0x00000000,
 512        (0x0e00 << 16) | (0x8b24 >> 2),
 513        0x00000000,
 514        (0x0e00 << 16) | (0x30a04 >> 2),
 515        0x00000000,
 516        (0x0600 << 16) | (0x30a10 >> 2),
 517        0x00000000,
 518        (0x0600 << 16) | (0x30a14 >> 2),
 519        0x00000000,
 520        (0x0600 << 16) | (0x30a18 >> 2),
 521        0x00000000,
 522        (0x0600 << 16) | (0x30a2c >> 2),
 523        0x00000000,
 524        (0x0e00 << 16) | (0xc700 >> 2),
 525        0x00000000,
 526        (0x0e00 << 16) | (0xc704 >> 2),
 527        0x00000000,
 528        (0x0e00 << 16) | (0xc708 >> 2),
 529        0x00000000,
 530        (0x0e00 << 16) | (0xc768 >> 2),
 531        0x00000000,
 532        (0x0400 << 16) | (0xc770 >> 2),
 533        0x00000000,
 534        (0x0400 << 16) | (0xc774 >> 2),
 535        0x00000000,
 536        (0x0400 << 16) | (0xc778 >> 2),
 537        0x00000000,
 538        (0x0400 << 16) | (0xc77c >> 2),
 539        0x00000000,
 540        (0x0400 << 16) | (0xc780 >> 2),
 541        0x00000000,
 542        (0x0400 << 16) | (0xc784 >> 2),
 543        0x00000000,
 544        (0x0400 << 16) | (0xc788 >> 2),
 545        0x00000000,
 546        (0x0400 << 16) | (0xc78c >> 2),
 547        0x00000000,
 548        (0x0400 << 16) | (0xc798 >> 2),
 549        0x00000000,
 550        (0x0400 << 16) | (0xc79c >> 2),
 551        0x00000000,
 552        (0x0400 << 16) | (0xc7a0 >> 2),
 553        0x00000000,
 554        (0x0400 << 16) | (0xc7a4 >> 2),
 555        0x00000000,
 556        (0x0400 << 16) | (0xc7a8 >> 2),
 557        0x00000000,
 558        (0x0400 << 16) | (0xc7ac >> 2),
 559        0x00000000,
 560        (0x0400 << 16) | (0xc7b0 >> 2),
 561        0x00000000,
 562        (0x0400 << 16) | (0xc7b4 >> 2),
 563        0x00000000,
 564        (0x0e00 << 16) | (0x9100 >> 2),
 565        0x00000000,
 566        (0x0e00 << 16) | (0x3c010 >> 2),
 567        0x00000000,
 568        (0x0e00 << 16) | (0x92a8 >> 2),
 569        0x00000000,
 570        (0x0e00 << 16) | (0x92ac >> 2),
 571        0x00000000,
 572        (0x0e00 << 16) | (0x92b4 >> 2),
 573        0x00000000,
 574        (0x0e00 << 16) | (0x92b8 >> 2),
 575        0x00000000,
 576        (0x0e00 << 16) | (0x92bc >> 2),
 577        0x00000000,
 578        (0x0e00 << 16) | (0x92c0 >> 2),
 579        0x00000000,
 580        (0x0e00 << 16) | (0x92c4 >> 2),
 581        0x00000000,
 582        (0x0e00 << 16) | (0x92c8 >> 2),
 583        0x00000000,
 584        (0x0e00 << 16) | (0x92cc >> 2),
 585        0x00000000,
 586        (0x0e00 << 16) | (0x92d0 >> 2),
 587        0x00000000,
 588        (0x0e00 << 16) | (0x8c00 >> 2),
 589        0x00000000,
 590        (0x0e00 << 16) | (0x8c04 >> 2),
 591        0x00000000,
 592        (0x0e00 << 16) | (0x8c20 >> 2),
 593        0x00000000,
 594        (0x0e00 << 16) | (0x8c38 >> 2),
 595        0x00000000,
 596        (0x0e00 << 16) | (0x8c3c >> 2),
 597        0x00000000,
 598        (0x0e00 << 16) | (0xae00 >> 2),
 599        0x00000000,
 600        (0x0e00 << 16) | (0x9604 >> 2),
 601        0x00000000,
 602        (0x0e00 << 16) | (0xac08 >> 2),
 603        0x00000000,
 604        (0x0e00 << 16) | (0xac0c >> 2),
 605        0x00000000,
 606        (0x0e00 << 16) | (0xac10 >> 2),
 607        0x00000000,
 608        (0x0e00 << 16) | (0xac14 >> 2),
 609        0x00000000,
 610        (0x0e00 << 16) | (0xac58 >> 2),
 611        0x00000000,
 612        (0x0e00 << 16) | (0xac68 >> 2),
 613        0x00000000,
 614        (0x0e00 << 16) | (0xac6c >> 2),
 615        0x00000000,
 616        (0x0e00 << 16) | (0xac70 >> 2),
 617        0x00000000,
 618        (0x0e00 << 16) | (0xac74 >> 2),
 619        0x00000000,
 620        (0x0e00 << 16) | (0xac78 >> 2),
 621        0x00000000,
 622        (0x0e00 << 16) | (0xac7c >> 2),
 623        0x00000000,
 624        (0x0e00 << 16) | (0xac80 >> 2),
 625        0x00000000,
 626        (0x0e00 << 16) | (0xac84 >> 2),
 627        0x00000000,
 628        (0x0e00 << 16) | (0xac88 >> 2),
 629        0x00000000,
 630        (0x0e00 << 16) | (0xac8c >> 2),
 631        0x00000000,
 632        (0x0e00 << 16) | (0x970c >> 2),
 633        0x00000000,
 634        (0x0e00 << 16) | (0x9714 >> 2),
 635        0x00000000,
 636        (0x0e00 << 16) | (0x9718 >> 2),
 637        0x00000000,
 638        (0x0e00 << 16) | (0x971c >> 2),
 639        0x00000000,
 640        (0x0e00 << 16) | (0x31068 >> 2),
 641        0x00000000,
 642        (0x4e00 << 16) | (0x31068 >> 2),
 643        0x00000000,
 644        (0x5e00 << 16) | (0x31068 >> 2),
 645        0x00000000,
 646        (0x6e00 << 16) | (0x31068 >> 2),
 647        0x00000000,
 648        (0x7e00 << 16) | (0x31068 >> 2),
 649        0x00000000,
 650        (0x8e00 << 16) | (0x31068 >> 2),
 651        0x00000000,
 652        (0x9e00 << 16) | (0x31068 >> 2),
 653        0x00000000,
 654        (0xae00 << 16) | (0x31068 >> 2),
 655        0x00000000,
 656        (0xbe00 << 16) | (0x31068 >> 2),
 657        0x00000000,
 658        (0x0e00 << 16) | (0xcd10 >> 2),
 659        0x00000000,
 660        (0x0e00 << 16) | (0xcd14 >> 2),
 661        0x00000000,
 662        (0x0e00 << 16) | (0x88b0 >> 2),
 663        0x00000000,
 664        (0x0e00 << 16) | (0x88b4 >> 2),
 665        0x00000000,
 666        (0x0e00 << 16) | (0x88b8 >> 2),
 667        0x00000000,
 668        (0x0e00 << 16) | (0x88bc >> 2),
 669        0x00000000,
 670        (0x0400 << 16) | (0x89c0 >> 2),
 671        0x00000000,
 672        (0x0e00 << 16) | (0x88c4 >> 2),
 673        0x00000000,
 674        (0x0e00 << 16) | (0x88c8 >> 2),
 675        0x00000000,
 676        (0x0e00 << 16) | (0x88d0 >> 2),
 677        0x00000000,
 678        (0x0e00 << 16) | (0x88d4 >> 2),
 679        0x00000000,
 680        (0x0e00 << 16) | (0x88d8 >> 2),
 681        0x00000000,
 682        (0x0e00 << 16) | (0x8980 >> 2),
 683        0x00000000,
 684        (0x0e00 << 16) | (0x30938 >> 2),
 685        0x00000000,
 686        (0x0e00 << 16) | (0x3093c >> 2),
 687        0x00000000,
 688        (0x0e00 << 16) | (0x30940 >> 2),
 689        0x00000000,
 690        (0x0e00 << 16) | (0x89a0 >> 2),
 691        0x00000000,
 692        (0x0e00 << 16) | (0x30900 >> 2),
 693        0x00000000,
 694        (0x0e00 << 16) | (0x30904 >> 2),
 695        0x00000000,
 696        (0x0e00 << 16) | (0x89b4 >> 2),
 697        0x00000000,
 698        (0x0e00 << 16) | (0x3c210 >> 2),
 699        0x00000000,
 700        (0x0e00 << 16) | (0x3c214 >> 2),
 701        0x00000000,
 702        (0x0e00 << 16) | (0x3c218 >> 2),
 703        0x00000000,
 704        (0x0e00 << 16) | (0x8904 >> 2),
 705        0x00000000,
 706        0x5,
 707        (0x0e00 << 16) | (0x8c28 >> 2),
 708        (0x0e00 << 16) | (0x8c2c >> 2),
 709        (0x0e00 << 16) | (0x8c30 >> 2),
 710        (0x0e00 << 16) | (0x8c34 >> 2),
 711        (0x0e00 << 16) | (0x9600 >> 2),
 712};
 713
 714static const u32 kalindi_rlc_save_restore_register_list[] =
 715{
 716        (0x0e00 << 16) | (0xc12c >> 2),
 717        0x00000000,
 718        (0x0e00 << 16) | (0xc140 >> 2),
 719        0x00000000,
 720        (0x0e00 << 16) | (0xc150 >> 2),
 721        0x00000000,
 722        (0x0e00 << 16) | (0xc15c >> 2),
 723        0x00000000,
 724        (0x0e00 << 16) | (0xc168 >> 2),
 725        0x00000000,
 726        (0x0e00 << 16) | (0xc170 >> 2),
 727        0x00000000,
 728        (0x0e00 << 16) | (0xc204 >> 2),
 729        0x00000000,
 730        (0x0e00 << 16) | (0xc2b4 >> 2),
 731        0x00000000,
 732        (0x0e00 << 16) | (0xc2b8 >> 2),
 733        0x00000000,
 734        (0x0e00 << 16) | (0xc2bc >> 2),
 735        0x00000000,
 736        (0x0e00 << 16) | (0xc2c0 >> 2),
 737        0x00000000,
 738        (0x0e00 << 16) | (0x8228 >> 2),
 739        0x00000000,
 740        (0x0e00 << 16) | (0x829c >> 2),
 741        0x00000000,
 742        (0x0e00 << 16) | (0x869c >> 2),
 743        0x00000000,
 744        (0x0600 << 16) | (0x98f4 >> 2),
 745        0x00000000,
 746        (0x0e00 << 16) | (0x98f8 >> 2),
 747        0x00000000,
 748        (0x0e00 << 16) | (0x9900 >> 2),
 749        0x00000000,
 750        (0x0e00 << 16) | (0xc260 >> 2),
 751        0x00000000,
 752        (0x0e00 << 16) | (0x90e8 >> 2),
 753        0x00000000,
 754        (0x0e00 << 16) | (0x3c000 >> 2),
 755        0x00000000,
 756        (0x0e00 << 16) | (0x3c00c >> 2),
 757        0x00000000,
 758        (0x0e00 << 16) | (0x8c1c >> 2),
 759        0x00000000,
 760        (0x0e00 << 16) | (0x9700 >> 2),
 761        0x00000000,
 762        (0x0e00 << 16) | (0xcd20 >> 2),
 763        0x00000000,
 764        (0x4e00 << 16) | (0xcd20 >> 2),
 765        0x00000000,
 766        (0x5e00 << 16) | (0xcd20 >> 2),
 767        0x00000000,
 768        (0x6e00 << 16) | (0xcd20 >> 2),
 769        0x00000000,
 770        (0x7e00 << 16) | (0xcd20 >> 2),
 771        0x00000000,
 772        (0x0e00 << 16) | (0x89bc >> 2),
 773        0x00000000,
 774        (0x0e00 << 16) | (0x8900 >> 2),
 775        0x00000000,
 776        0x3,
 777        (0x0e00 << 16) | (0xc130 >> 2),
 778        0x00000000,
 779        (0x0e00 << 16) | (0xc134 >> 2),
 780        0x00000000,
 781        (0x0e00 << 16) | (0xc1fc >> 2),
 782        0x00000000,
 783        (0x0e00 << 16) | (0xc208 >> 2),
 784        0x00000000,
 785        (0x0e00 << 16) | (0xc264 >> 2),
 786        0x00000000,
 787        (0x0e00 << 16) | (0xc268 >> 2),
 788        0x00000000,
 789        (0x0e00 << 16) | (0xc26c >> 2),
 790        0x00000000,
 791        (0x0e00 << 16) | (0xc270 >> 2),
 792        0x00000000,
 793        (0x0e00 << 16) | (0xc274 >> 2),
 794        0x00000000,
 795        (0x0e00 << 16) | (0xc28c >> 2),
 796        0x00000000,
 797        (0x0e00 << 16) | (0xc290 >> 2),
 798        0x00000000,
 799        (0x0e00 << 16) | (0xc294 >> 2),
 800        0x00000000,
 801        (0x0e00 << 16) | (0xc298 >> 2),
 802        0x00000000,
 803        (0x0e00 << 16) | (0xc2a0 >> 2),
 804        0x00000000,
 805        (0x0e00 << 16) | (0xc2a4 >> 2),
 806        0x00000000,
 807        (0x0e00 << 16) | (0xc2a8 >> 2),
 808        0x00000000,
 809        (0x0e00 << 16) | (0xc2ac >> 2),
 810        0x00000000,
 811        (0x0e00 << 16) | (0x301d0 >> 2),
 812        0x00000000,
 813        (0x0e00 << 16) | (0x30238 >> 2),
 814        0x00000000,
 815        (0x0e00 << 16) | (0x30250 >> 2),
 816        0x00000000,
 817        (0x0e00 << 16) | (0x30254 >> 2),
 818        0x00000000,
 819        (0x0e00 << 16) | (0x30258 >> 2),
 820        0x00000000,
 821        (0x0e00 << 16) | (0x3025c >> 2),
 822        0x00000000,
 823        (0x4e00 << 16) | (0xc900 >> 2),
 824        0x00000000,
 825        (0x5e00 << 16) | (0xc900 >> 2),
 826        0x00000000,
 827        (0x6e00 << 16) | (0xc900 >> 2),
 828        0x00000000,
 829        (0x7e00 << 16) | (0xc900 >> 2),
 830        0x00000000,
 831        (0x4e00 << 16) | (0xc904 >> 2),
 832        0x00000000,
 833        (0x5e00 << 16) | (0xc904 >> 2),
 834        0x00000000,
 835        (0x6e00 << 16) | (0xc904 >> 2),
 836        0x00000000,
 837        (0x7e00 << 16) | (0xc904 >> 2),
 838        0x00000000,
 839        (0x4e00 << 16) | (0xc908 >> 2),
 840        0x00000000,
 841        (0x5e00 << 16) | (0xc908 >> 2),
 842        0x00000000,
 843        (0x6e00 << 16) | (0xc908 >> 2),
 844        0x00000000,
 845        (0x7e00 << 16) | (0xc908 >> 2),
 846        0x00000000,
 847        (0x4e00 << 16) | (0xc90c >> 2),
 848        0x00000000,
 849        (0x5e00 << 16) | (0xc90c >> 2),
 850        0x00000000,
 851        (0x6e00 << 16) | (0xc90c >> 2),
 852        0x00000000,
 853        (0x7e00 << 16) | (0xc90c >> 2),
 854        0x00000000,
 855        (0x4e00 << 16) | (0xc910 >> 2),
 856        0x00000000,
 857        (0x5e00 << 16) | (0xc910 >> 2),
 858        0x00000000,
 859        (0x6e00 << 16) | (0xc910 >> 2),
 860        0x00000000,
 861        (0x7e00 << 16) | (0xc910 >> 2),
 862        0x00000000,
 863        (0x0e00 << 16) | (0xc99c >> 2),
 864        0x00000000,
 865        (0x0e00 << 16) | (0x9834 >> 2),
 866        0x00000000,
 867        (0x0000 << 16) | (0x30f00 >> 2),
 868        0x00000000,
 869        (0x0000 << 16) | (0x30f04 >> 2),
 870        0x00000000,
 871        (0x0000 << 16) | (0x30f08 >> 2),
 872        0x00000000,
 873        (0x0000 << 16) | (0x30f0c >> 2),
 874        0x00000000,
 875        (0x0600 << 16) | (0x9b7c >> 2),
 876        0x00000000,
 877        (0x0e00 << 16) | (0x8a14 >> 2),
 878        0x00000000,
 879        (0x0e00 << 16) | (0x8a18 >> 2),
 880        0x00000000,
 881        (0x0600 << 16) | (0x30a00 >> 2),
 882        0x00000000,
 883        (0x0e00 << 16) | (0x8bf0 >> 2),
 884        0x00000000,
 885        (0x0e00 << 16) | (0x8bcc >> 2),
 886        0x00000000,
 887        (0x0e00 << 16) | (0x8b24 >> 2),
 888        0x00000000,
 889        (0x0e00 << 16) | (0x30a04 >> 2),
 890        0x00000000,
 891        (0x0600 << 16) | (0x30a10 >> 2),
 892        0x00000000,
 893        (0x0600 << 16) | (0x30a14 >> 2),
 894        0x00000000,
 895        (0x0600 << 16) | (0x30a18 >> 2),
 896        0x00000000,
 897        (0x0600 << 16) | (0x30a2c >> 2),
 898        0x00000000,
 899        (0x0e00 << 16) | (0xc700 >> 2),
 900        0x00000000,
 901        (0x0e00 << 16) | (0xc704 >> 2),
 902        0x00000000,
 903        (0x0e00 << 16) | (0xc708 >> 2),
 904        0x00000000,
 905        (0x0e00 << 16) | (0xc768 >> 2),
 906        0x00000000,
 907        (0x0400 << 16) | (0xc770 >> 2),
 908        0x00000000,
 909        (0x0400 << 16) | (0xc774 >> 2),
 910        0x00000000,
 911        (0x0400 << 16) | (0xc798 >> 2),
 912        0x00000000,
 913        (0x0400 << 16) | (0xc79c >> 2),
 914        0x00000000,
 915        (0x0e00 << 16) | (0x9100 >> 2),
 916        0x00000000,
 917        (0x0e00 << 16) | (0x3c010 >> 2),
 918        0x00000000,
 919        (0x0e00 << 16) | (0x8c00 >> 2),
 920        0x00000000,
 921        (0x0e00 << 16) | (0x8c04 >> 2),
 922        0x00000000,
 923        (0x0e00 << 16) | (0x8c20 >> 2),
 924        0x00000000,
 925        (0x0e00 << 16) | (0x8c38 >> 2),
 926        0x00000000,
 927        (0x0e00 << 16) | (0x8c3c >> 2),
 928        0x00000000,
 929        (0x0e00 << 16) | (0xae00 >> 2),
 930        0x00000000,
 931        (0x0e00 << 16) | (0x9604 >> 2),
 932        0x00000000,
 933        (0x0e00 << 16) | (0xac08 >> 2),
 934        0x00000000,
 935        (0x0e00 << 16) | (0xac0c >> 2),
 936        0x00000000,
 937        (0x0e00 << 16) | (0xac10 >> 2),
 938        0x00000000,
 939        (0x0e00 << 16) | (0xac14 >> 2),
 940        0x00000000,
 941        (0x0e00 << 16) | (0xac58 >> 2),
 942        0x00000000,
 943        (0x0e00 << 16) | (0xac68 >> 2),
 944        0x00000000,
 945        (0x0e00 << 16) | (0xac6c >> 2),
 946        0x00000000,
 947        (0x0e00 << 16) | (0xac70 >> 2),
 948        0x00000000,
 949        (0x0e00 << 16) | (0xac74 >> 2),
 950        0x00000000,
 951        (0x0e00 << 16) | (0xac78 >> 2),
 952        0x00000000,
 953        (0x0e00 << 16) | (0xac7c >> 2),
 954        0x00000000,
 955        (0x0e00 << 16) | (0xac80 >> 2),
 956        0x00000000,
 957        (0x0e00 << 16) | (0xac84 >> 2),
 958        0x00000000,
 959        (0x0e00 << 16) | (0xac88 >> 2),
 960        0x00000000,
 961        (0x0e00 << 16) | (0xac8c >> 2),
 962        0x00000000,
 963        (0x0e00 << 16) | (0x970c >> 2),
 964        0x00000000,
 965        (0x0e00 << 16) | (0x9714 >> 2),
 966        0x00000000,
 967        (0x0e00 << 16) | (0x9718 >> 2),
 968        0x00000000,
 969        (0x0e00 << 16) | (0x971c >> 2),
 970        0x00000000,
 971        (0x0e00 << 16) | (0x31068 >> 2),
 972        0x00000000,
 973        (0x4e00 << 16) | (0x31068 >> 2),
 974        0x00000000,
 975        (0x5e00 << 16) | (0x31068 >> 2),
 976        0x00000000,
 977        (0x6e00 << 16) | (0x31068 >> 2),
 978        0x00000000,
 979        (0x7e00 << 16) | (0x31068 >> 2),
 980        0x00000000,
 981        (0x0e00 << 16) | (0xcd10 >> 2),
 982        0x00000000,
 983        (0x0e00 << 16) | (0xcd14 >> 2),
 984        0x00000000,
 985        (0x0e00 << 16) | (0x88b0 >> 2),
 986        0x00000000,
 987        (0x0e00 << 16) | (0x88b4 >> 2),
 988        0x00000000,
 989        (0x0e00 << 16) | (0x88b8 >> 2),
 990        0x00000000,
 991        (0x0e00 << 16) | (0x88bc >> 2),
 992        0x00000000,
 993        (0x0400 << 16) | (0x89c0 >> 2),
 994        0x00000000,
 995        (0x0e00 << 16) | (0x88c4 >> 2),
 996        0x00000000,
 997        (0x0e00 << 16) | (0x88c8 >> 2),
 998        0x00000000,
 999        (0x0e00 << 16) | (0x88d0 >> 2),
1000        0x00000000,

1001        (0x0e00 << 16) | (0x88d4 >> 2),
1002        0x00000000,
1003        (0x0e00 << 16) | (0x88d8 >> 2),
1004        0x00000000,
1005        (0x0e00 << 16) | (0x8980 >> 2),
1006        0x00000000,
1007        (0x0e00 << 16) | (0x30938 >> 2),
1008        0x00000000,
1009        (0x0e00 << 16) | (0x3093c >> 2),
1010        0x00000000,
1011        (0x0e00 << 16) | (0x30940 >> 2),
1012        0x00000000,
1013        (0x0e00 << 16) | (0x89a0 >> 2),
1014        0x00000000,
1015        (0x0e00 << 16) | (0x30900 >> 2),
1016        0x00000000,
1017        (0x0e00 << 16) | (0x30904 >> 2),
1018        0x00000000,
1019        (0x0e00 << 16) | (0x89b4 >> 2),
1020        0x00000000,
1021        (0x0e00 << 16) | (0x3e1fc >> 2),
1022        0x00000000,
1023        (0x0e00 << 16) | (0x3c210 >> 2),
1024        0x00000000,
1025        (0x0e00 << 16) | (0x3c214 >> 2),
1026        0x00000000,
1027        (0x0e00 << 16) | (0x3c218 >> 2),
1028        0x00000000,
1029        (0x0e00 << 16) | (0x8904 >> 2),
1030        0x00000000,
1031        0x5,
1032        (0x0e00 << 16) | (0x8c28 >> 2),
1033        (0x0e00 << 16) | (0x8c2c >> 2),
1034        (0x0e00 << 16) | (0x8c30 >> 2),
1035        (0x0e00 << 16) | (0x8c34 >> 2),
1036        (0x0e00 << 16) | (0x9600 >> 2),
1037};
1038
1039static const u32 bonaire_golden_spm_registers[] =
1040{
1041        0x30800, 0xe0ffffff, 0xe0000000
1042};
1043
1044static const u32 bonaire_golden_common_registers[] =
1045{
1046        0xc770, 0xffffffff, 0x00000800,
1047        0xc774, 0xffffffff, 0x00000800,
1048        0xc798, 0xffffffff, 0x00007fbf,
1049        0xc79c, 0xffffffff, 0x00007faf
1050};
1051
1052static const u32 bonaire_golden_registers[] =
1053{
1054        0x3354, 0x00000333, 0x00000333,
1055        0x3350, 0x000c0fc0, 0x00040200,
1056        0x9a10, 0x00010000, 0x00058208,
1057        0x3c000, 0xffff1fff, 0x00140000,
1058        0x3c200, 0xfdfc0fff, 0x00000100,
1059        0x3c234, 0x40000000, 0x40000200,
1060        0x9830, 0xffffffff, 0x00000000,
1061        0x9834, 0xf00fffff, 0x00000400,
1062        0x9838, 0x0002021c, 0x00020200,
1063        0xc78, 0x00000080, 0x00000000,
1064        0x5bb0, 0x000000f0, 0x00000070,
1065        0x5bc0, 0xf0311fff, 0x80300000,
1066        0x98f8, 0x73773777, 0x12010001,
1067        0x350c, 0x00810000, 0x408af000,
1068        0x7030, 0x31000111, 0x00000011,
1069        0x2f48, 0x73773777, 0x12010001,
1070        0x220c, 0x00007fb6, 0x0021a1b1,
1071        0x2210, 0x00007fb6, 0x002021b1,
1072        0x2180, 0x00007fb6, 0x00002191,
1073        0x2218, 0x00007fb6, 0x002121b1,
1074        0x221c, 0x00007fb6, 0x002021b1,
1075        0x21dc, 0x00007fb6, 0x00002191,
1076        0x21e0, 0x00007fb6, 0x00002191,
1077        0x3628, 0x0000003f, 0x0000000a,
1078        0x362c, 0x0000003f, 0x0000000a,
1079        0x2ae4, 0x00073ffe, 0x000022a2,
1080        0x240c, 0x000007ff, 0x00000000,
1081        0x8a14, 0xf000003f, 0x00000007,
1082        0x8bf0, 0x00002001, 0x00000001,
1083        0x8b24, 0xffffffff, 0x00ffffff,
1084        0x30a04, 0x0000ff0f, 0x00000000,
1085        0x28a4c, 0x07ffffff, 0x06000000,
1086        0x4d8, 0x00000fff, 0x00000100,
1087        0x3e78, 0x00000001, 0x00000002,
1088        0x9100, 0x03000000, 0x0362c688,
1089        0x8c00, 0x000000ff, 0x00000001,
1090        0xe40, 0x00001fff, 0x00001fff,
1091        0x9060, 0x0000007f, 0x00000020,
1092        0x9508, 0x00010000, 0x00010000,
1093        0xac14, 0x000003ff, 0x000000f3,
1094        0xac0c, 0xffffffff, 0x00001032
1095};
1096
1097static const u32 bonaire_mgcg_cgcg_init[] =
1098{
1099        0xc420, 0xffffffff, 0xfffffffc,
1100        0x30800, 0xffffffff, 0xe0000000,
1101        0x3c2a0, 0xffffffff, 0x00000100,
1102        0x3c208, 0xffffffff, 0x00000100,
1103        0x3c2c0, 0xffffffff, 0xc0000100,
1104        0x3c2c8, 0xffffffff, 0xc0000100,
1105        0x3c2c4, 0xffffffff, 0xc0000100,
1106        0x55e4, 0xffffffff, 0x00600100,
1107        0x3c280, 0xffffffff, 0x00000100,
1108        0x3c214, 0xffffffff, 0x06000100,
1109        0x3c220, 0xffffffff, 0x00000100,
1110        0x3c218, 0xffffffff, 0x06000100,
1111        0x3c204, 0xffffffff, 0x00000100,
1112        0x3c2e0, 0xffffffff, 0x00000100,
1113        0x3c224, 0xffffffff, 0x00000100,
1114        0x3c200, 0xffffffff, 0x00000100,
1115        0x3c230, 0xffffffff, 0x00000100,
1116        0x3c234, 0xffffffff, 0x00000100,
1117        0x3c250, 0xffffffff, 0x00000100,
1118        0x3c254, 0xffffffff, 0x00000100,
1119        0x3c258, 0xffffffff, 0x00000100,
1120        0x3c25c, 0xffffffff, 0x00000100,
1121        0x3c260, 0xffffffff, 0x00000100,
1122        0x3c27c, 0xffffffff, 0x00000100,
1123        0x3c278, 0xffffffff, 0x00000100,
1124        0x3c210, 0xffffffff, 0x06000100,
1125        0x3c290, 0xffffffff, 0x00000100,
1126        0x3c274, 0xffffffff, 0x00000100,
1127        0x3c2b4, 0xffffffff, 0x00000100,
1128        0x3c2b0, 0xffffffff, 0x00000100,
1129        0x3c270, 0xffffffff, 0x00000100,
1130        0x30800, 0xffffffff, 0xe0000000,
1131        0x3c020, 0xffffffff, 0x00010000,
1132        0x3c024, 0xffffffff, 0x00030002,
1133        0x3c028, 0xffffffff, 0x00040007,
1134        0x3c02c, 0xffffffff, 0x00060005,
1135        0x3c030, 0xffffffff, 0x00090008,
1136        0x3c034, 0xffffffff, 0x00010000,
1137        0x3c038, 0xffffffff, 0x00030002,
1138        0x3c03c, 0xffffffff, 0x00040007,
1139        0x3c040, 0xffffffff, 0x00060005,
1140        0x3c044, 0xffffffff, 0x00090008,
1141        0x3c048, 0xffffffff, 0x00010000,
1142        0x3c04c, 0xffffffff, 0x00030002,
1143        0x3c050, 0xffffffff, 0x00040007,
1144        0x3c054, 0xffffffff, 0x00060005,
1145        0x3c058, 0xffffffff, 0x00090008,
1146        0x3c05c, 0xffffffff, 0x00010000,
1147        0x3c060, 0xffffffff, 0x00030002,
1148        0x3c064, 0xffffffff, 0x00040007,
1149        0x3c068, 0xffffffff, 0x00060005,
1150        0x3c06c, 0xffffffff, 0x00090008,
1151        0x3c070, 0xffffffff, 0x00010000,
1152        0x3c074, 0xffffffff, 0x00030002,
1153        0x3c078, 0xffffffff, 0x00040007,
1154        0x3c07c, 0xffffffff, 0x00060005,
1155        0x3c080, 0xffffffff, 0x00090008,
1156        0x3c084, 0xffffffff, 0x00010000,
1157        0x3c088, 0xffffffff, 0x00030002,
1158        0x3c08c, 0xffffffff, 0x00040007,
1159        0x3c090, 0xffffffff, 0x00060005,
1160        0x3c094, 0xffffffff, 0x00090008,
1161        0x3c098, 0xffffffff, 0x00010000,
1162        0x3c09c, 0xffffffff, 0x00030002,
1163        0x3c0a0, 0xffffffff, 0x00040007,
1164        0x3c0a4, 0xffffffff, 0x00060005,
1165        0x3c0a8, 0xffffffff, 0x00090008,
1166        0x3c000, 0xffffffff, 0x96e00200,
1167        0x8708, 0xffffffff, 0x00900100,
1168        0xc424, 0xffffffff, 0x0020003f,
1169        0x38, 0xffffffff, 0x0140001c,
1170        0x3c, 0x000f0000, 0x000f0000,
1171        0x220, 0xffffffff, 0xC060000C,
1172        0x224, 0xc0000fff, 0x00000100,
1173        0xf90, 0xffffffff, 0x00000100,
1174        0xf98, 0x00000101, 0x00000000,
1175        0x20a8, 0xffffffff, 0x00000104,
1176        0x55e4, 0xff000fff, 0x00000100,
1177        0x30cc, 0xc0000fff, 0x00000104,
1178        0xc1e4, 0x00000001, 0x00000001,
1179        0xd00c, 0xff000ff0, 0x00000100,
1180        0xd80c, 0xff000ff0, 0x00000100
1181};
1182
1183static const u32 spectre_golden_spm_registers[] =
1184{
1185        0x30800, 0xe0ffffff, 0xe0000000
1186};
1187
1188static const u32 spectre_golden_common_registers[] =
1189{
1190        0xc770, 0xffffffff, 0x00000800,
1191        0xc774, 0xffffffff, 0x00000800,
1192        0xc798, 0xffffffff, 0x00007fbf,
1193        0xc79c, 0xffffffff, 0x00007faf
1194};
1195
1196static const u32 spectre_golden_registers[] =
1197{
1198        0x3c000, 0xffff1fff, 0x96940200,
1199        0x3c00c, 0xffff0001, 0xff000000,
1200        0x3c200, 0xfffc0fff, 0x00000100,
1201        0x6ed8, 0x00010101, 0x00010000,
1202        0x9834, 0xf00fffff, 0x00000400,
1203        0x9838, 0xfffffffc, 0x00020200,
1204        0x5bb0, 0x000000f0, 0x00000070,
1205        0x5bc0, 0xf0311fff, 0x80300000,
1206        0x98f8, 0x73773777, 0x12010001,
1207        0x9b7c, 0x00ff0000, 0x00fc0000,
1208        0x2f48, 0x73773777, 0x12010001,
1209        0x8a14, 0xf000003f, 0x00000007,
1210        0x8b24, 0xffffffff, 0x00ffffff,
1211        0x28350, 0x3f3f3fff, 0x00000082,
1212        0x28354, 0x0000003f, 0x00000000,
1213        0x3e78, 0x00000001, 0x00000002,
1214        0x913c, 0xffff03df, 0x00000004,
1215        0xc768, 0x00000008, 0x00000008,
1216        0x8c00, 0x000008ff, 0x00000800,
1217        0x9508, 0x00010000, 0x00010000,
1218        0xac0c, 0xffffffff, 0x54763210,
1219        0x214f8, 0x01ff01ff, 0x00000002,
1220        0x21498, 0x007ff800, 0x00200000,
1221        0x2015c, 0xffffffff, 0x00000f40,
1222        0x30934, 0xffffffff, 0x00000001
1223};
1224
1225static const u32 spectre_mgcg_cgcg_init[] =
1226{
1227        0xc420, 0xffffffff, 0xfffffffc,
1228        0x30800, 0xffffffff, 0xe0000000,
1229        0x3c2a0, 0xffffffff, 0x00000100,
1230        0x3c208, 0xffffffff, 0x00000100,
1231        0x3c2c0, 0xffffffff, 0x00000100,
1232        0x3c2c8, 0xffffffff, 0x00000100,
1233        0x3c2c4, 0xffffffff, 0x00000100,
1234        0x55e4, 0xffffffff, 0x00600100,
1235        0x3c280, 0xffffffff, 0x00000100,
1236        0x3c214, 0xffffffff, 0x06000100,
1237        0x3c220, 0xffffffff, 0x00000100,
1238        0x3c218, 0xffffffff, 0x06000100,
1239        0x3c204, 0xffffffff, 0x00000100,
1240        0x3c2e0, 0xffffffff, 0x00000100,
1241        0x3c224, 0xffffffff, 0x00000100,
1242        0x3c200, 0xffffffff, 0x00000100,
1243        0x3c230, 0xffffffff, 0x00000100,
1244        0x3c234, 0xffffffff, 0x00000100,
1245        0x3c250, 0xffffffff, 0x00000100,
1246        0x3c254, 0xffffffff, 0x00000100,
1247        0x3c258, 0xffffffff, 0x00000100,
1248        0x3c25c, 0xffffffff, 0x00000100,
1249        0x3c260, 0xffffffff, 0x00000100,
1250        0x3c27c, 0xffffffff, 0x00000100,
1251        0x3c278, 0xffffffff, 0x00000100,
1252        0x3c210, 0xffffffff, 0x06000100,
1253        0x3c290, 0xffffffff, 0x00000100,
1254        0x3c274, 0xffffffff, 0x00000100,
1255        0x3c2b4, 0xffffffff, 0x00000100,
1256        0x3c2b0, 0xffffffff, 0x00000100,
1257        0x3c270, 0xffffffff, 0x00000100,
1258        0x30800, 0xffffffff, 0xe0000000,
1259        0x3c020, 0xffffffff, 0x00010000,
1260        0x3c024, 0xffffffff, 0x00030002,
1261        0x3c028, 0xffffffff, 0x00040007,
1262        0x3c02c, 0xffffffff, 0x00060005,
1263        0x3c030, 0xffffffff, 0x00090008,
1264        0x3c034, 0xffffffff, 0x00010000,
1265        0x3c038, 0xffffffff, 0x00030002,
1266        0x3c03c, 0xffffffff, 0x00040007,
1267        0x3c040, 0xffffffff, 0x00060005,
1268        0x3c044, 0xffffffff, 0x00090008,
1269        0x3c048, 0xffffffff, 0x00010000,
1270        0x3c04c, 0xffffffff, 0x00030002,
1271        0x3c050, 0xffffffff, 0x00040007,
1272        0x3c054, 0xffffffff, 0x00060005,
1273        0x3c058, 0xffffffff, 0x00090008,
1274        0x3c05c, 0xffffffff, 0x00010000,
1275        0x3c060, 0xffffffff, 0x00030002,
1276        0x3c064, 0xffffffff, 0x00040007,
1277        0x3c068, 0xffffffff, 0x00060005,
1278        0x3c06c, 0xffffffff, 0x00090008,
1279        0x3c070, 0xffffffff, 0x00010000,
1280        0x3c074, 0xffffffff, 0x00030002,
1281        0x3c078, 0xffffffff, 0x00040007,
1282        0x3c07c, 0xffffffff, 0x00060005,
1283        0x3c080, 0xffffffff, 0x00090008,
1284        0x3c084, 0xffffffff, 0x00010000,
1285        0x3c088, 0xffffffff, 0x00030002,
1286        0x3c08c, 0xffffffff, 0x00040007,
1287        0x3c090, 0xffffffff, 0x00060005,
1288        0x3c094, 0xffffffff, 0x00090008,
1289        0x3c098, 0xffffffff, 0x00010000,
1290        0x3c09c, 0xffffffff, 0x00030002,
1291        0x3c0a0, 0xffffffff, 0x00040007,
1292        0x3c0a4, 0xffffffff, 0x00060005,
1293        0x3c0a8, 0xffffffff, 0x00090008,
1294        0x3c0ac, 0xffffffff, 0x00010000,
1295        0x3c0b0, 0xffffffff, 0x00030002,
1296        0x3c0b4, 0xffffffff, 0x00040007,
1297        0x3c0b8, 0xffffffff, 0x00060005,
1298        0x3c0bc, 0xffffffff, 0x00090008,
1299        0x3c000, 0xffffffff, 0x96e00200,
1300        0x8708, 0xffffffff, 0x00900100,
1301        0xc424, 0xffffffff, 0x0020003f,
1302        0x38, 0xffffffff, 0x0140001c,
1303        0x3c, 0x000f0000, 0x000f0000,
1304        0x220, 0xffffffff, 0xC060000C,
1305        0x224, 0xc0000fff, 0x00000100,
1306        0xf90, 0xffffffff, 0x00000100,
1307        0xf98, 0x00000101, 0x00000000,
1308        0x20a8, 0xffffffff, 0x00000104,
1309        0x55e4, 0xff000fff, 0x00000100,
1310        0x30cc, 0xc0000fff, 0x00000104,
1311        0xc1e4, 0x00000001, 0x00000001,
1312        0xd00c, 0xff000ff0, 0x00000100,
1313        0xd80c, 0xff000ff0, 0x00000100
1314};
1315
1316static const u32 kalindi_golden_spm_registers[] =
1317{
1318        0x30800, 0xe0ffffff, 0xe0000000
1319};
1320
1321static const u32 kalindi_golden_common_registers[] =
1322{
1323        0xc770, 0xffffffff, 0x00000800,
1324        0xc774, 0xffffffff, 0x00000800,
1325        0xc798, 0xffffffff, 0x00007fbf,
1326        0xc79c, 0xffffffff, 0x00007faf
1327};
1328
1329static const u32 kalindi_golden_registers[] =
1330{
1331        0x3c000, 0xffffdfff, 0x6e944040,
1332        0x55e4, 0xff607fff, 0xfc000100,
1333        0x3c220, 0xff000fff, 0x00000100,
1334        0x3c224, 0xff000fff, 0x00000100,
1335        0x3c200, 0xfffc0fff, 0x00000100,
1336        0x6ed8, 0x00010101, 0x00010000,
1337        0x9830, 0xffffffff, 0x00000000,
1338        0x9834, 0xf00fffff, 0x00000400,
1339        0x5bb0, 0x000000f0, 0x00000070,
1340        0x5bc0, 0xf0311fff, 0x80300000,
1341        0x98f8, 0x73773777, 0x12010001,
1342        0x98fc, 0xffffffff, 0x00000010,
1343        0x9b7c, 0x00ff0000, 0x00fc0000,
1344        0x8030, 0x00001f0f, 0x0000100a,
1345        0x2f48, 0x73773777, 0x12010001,
1346        0x2408, 0x000fffff, 0x000c007f,
1347        0x8a14, 0xf000003f, 0x00000007,
1348        0x8b24, 0x3fff3fff, 0x00ffcfff,
1349        0x30a04, 0x0000ff0f, 0x00000000,
1350        0x28a4c, 0x07ffffff, 0x06000000,
1351        0x4d8, 0x00000fff, 0x00000100,
1352        0x3e78, 0x00000001, 0x00000002,
1353        0xc768, 0x00000008, 0x00000008,
1354        0x8c00, 0x000000ff, 0x00000003,
1355        0x214f8, 0x01ff01ff, 0x00000002,
1356        0x21498, 0x007ff800, 0x00200000,
1357        0x2015c, 0xffffffff, 0x00000f40,
1358        0x88c4, 0x001f3ae3, 0x00000082,
1359        0x88d4, 0x0000001f, 0x00000010,
1360        0x30934, 0xffffffff, 0x00000000
1361};
1362
1363static const u32 kalindi_mgcg_cgcg_init[] =
1364{
1365        0xc420, 0xffffffff, 0xfffffffc,
1366        0x30800, 0xffffffff, 0xe0000000,
1367        0x3c2a0, 0xffffffff, 0x00000100,
1368        0x3c208, 0xffffffff, 0x00000100,
1369        0x3c2c0, 0xffffffff, 0x00000100,
1370        0x3c2c8, 0xffffffff, 0x00000100,
1371        0x3c2c4, 0xffffffff, 0x00000100,
1372        0x55e4, 0xffffffff, 0x00600100,
1373        0x3c280, 0xffffffff, 0x00000100,
1374        0x3c214, 0xffffffff, 0x06000100,
1375        0x3c220, 0xffffffff, 0x00000100,
1376        0x3c218, 0xffffffff, 0x06000100,
1377        0x3c204, 0xffffffff, 0x00000100,
1378        0x3c2e0, 0xffffffff, 0x00000100,
1379        0x3c224, 0xffffffff, 0x00000100,
1380        0x3c200, 0xffffffff, 0x00000100,
1381        0x3c230, 0xffffffff, 0x00000100,
1382        0x3c234, 0xffffffff, 0x00000100,
1383        0x3c250, 0xffffffff, 0x00000100,
1384        0x3c254, 0xffffffff, 0x00000100,
1385        0x3c258, 0xffffffff, 0x00000100,
1386        0x3c25c, 0xffffffff, 0x00000100,
1387        0x3c260, 0xffffffff, 0x00000100,
1388        0x3c27c, 0xffffffff, 0x00000100,
1389        0x3c278, 0xffffffff, 0x00000100,
1390        0x3c210, 0xffffffff, 0x06000100,
1391        0x3c290, 0xffffffff, 0x00000100,
1392        0x3c274, 0xffffffff, 0x00000100,
1393        0x3c2b4, 0xffffffff, 0x00000100,
1394        0x3c2b0, 0xffffffff, 0x00000100,
1395        0x3c270, 0xffffffff, 0x00000100,
1396        0x30800, 0xffffffff, 0xe0000000,
1397        0x3c020, 0xffffffff, 0x00010000,
1398        0x3c024, 0xffffffff, 0x00030002,
1399        0x3c028, 0xffffffff, 0x00040007,
1400        0x3c02c, 0xffffffff, 0x00060005,
1401        0x3c030, 0xffffffff, 0x00090008,
1402        0x3c034, 0xffffffff, 0x00010000,
1403        0x3c038, 0xffffffff, 0x00030002,
1404        0x3c03c, 0xffffffff, 0x00040007,
1405        0x3c040, 0xffffffff, 0x00060005,
1406        0x3c044, 0xffffffff, 0x00090008,
1407        0x3c000, 0xffffffff, 0x96e00200,
1408        0x8708, 0xffffffff, 0x00900100,
1409        0xc424, 0xffffffff, 0x0020003f,
1410        0x38, 0xffffffff, 0x0140001c,
1411        0x3c, 0x000f0000, 0x000f0000,
1412        0x220, 0xffffffff, 0xC060000C,
1413        0x224, 0xc0000fff, 0x00000100,
1414        0x20a8, 0xffffffff, 0x00000104,
1415        0x55e4, 0xff000fff, 0x00000100,
1416        0x30cc, 0xc0000fff, 0x00000104,
1417        0xc1e4, 0x00000001, 0x00000001,
1418        0xd00c, 0xff000ff0, 0x00000100,
1419        0xd80c, 0xff000ff0, 0x00000100
1420};
1421
1422static const u32 hawaii_golden_spm_registers[] =
1423{
1424        0x30800, 0xe0ffffff, 0xe0000000
1425};
1426
1427static const u32 hawaii_golden_common_registers[] =
1428{
1429        0x30800, 0xffffffff, 0xe0000000,
1430        0x28350, 0xffffffff, 0x3a00161a,
1431        0x28354, 0xffffffff, 0x0000002e,
1432        0x9a10, 0xffffffff, 0x00018208,
1433        0x98f8, 0xffffffff, 0x12011003
1434};
1435
1436static const u32 hawaii_golden_registers[] =
1437{
1438        0x3354, 0x00000333, 0x00000333,
1439        0x9a10, 0x00010000, 0x00058208,
1440        0x9830, 0xffffffff, 0x00000000,
1441        0x9834, 0xf00fffff, 0x00000400,
1442        0x9838, 0x0002021c, 0x00020200,
1443        0xc78, 0x00000080, 0x00000000,
1444        0x5bb0, 0x000000f0, 0x00000070,
1445        0x5bc0, 0xf0311fff, 0x80300000,
1446        0x350c, 0x00810000, 0x408af000,
1447        0x7030, 0x31000111, 0x00000011,
1448        0x2f48, 0x73773777, 0x12010001,
1449        0x2120, 0x0000007f, 0x0000001b,
1450        0x21dc, 0x00007fb6, 0x00002191,
1451        0x3628, 0x0000003f, 0x0000000a,
1452        0x362c, 0x0000003f, 0x0000000a,
1453        0x2ae4, 0x00073ffe, 0x000022a2,
1454        0x240c, 0x000007ff, 0x00000000,
1455        0x8bf0, 0x00002001, 0x00000001,
1456        0x8b24, 0xffffffff, 0x00ffffff,
1457        0x30a04, 0x0000ff0f, 0x00000000,
1458        0x28a4c, 0x07ffffff, 0x06000000,
1459        0x3e78, 0x00000001, 0x00000002,
1460        0xc768, 0x00000008, 0x00000008,
1461        0xc770, 0x00000f00, 0x00000800,
1462        0xc774, 0x00000f00, 0x00000800,
1463        0xc798, 0x00ffffff, 0x00ff7fbf,
1464        0xc79c, 0x00ffffff, 0x00ff7faf,
1465        0x8c00, 0x000000ff, 0x00000800,
1466        0xe40, 0x00001fff, 0x00001fff,
1467        0x9060, 0x0000007f, 0x00000020,
1468        0x9508, 0x00010000, 0x00010000,
1469        0xae00, 0x00100000, 0x000ff07c,
1470        0xac14, 0x000003ff, 0x0000000f,
1471        0xac10, 0xffffffff, 0x7564fdec,
1472        0xac0c, 0xffffffff, 0x3120b9a8,
1473        0xac08, 0x20000000, 0x0f9c0000
1474};
1475
1476static const u32 hawaii_mgcg_cgcg_init[] =
1477{
1478        0xc420, 0xffffffff, 0xfffffffd,
1479        0x30800, 0xffffffff, 0xe0000000,
1480        0x3c2a0, 0xffffffff, 0x00000100,
1481        0x3c208, 0xffffffff, 0x00000100,
1482        0x3c2c0, 0xffffffff, 0x00000100,
1483        0x3c2c8, 0xffffffff, 0x00000100,
1484        0x3c2c4, 0xffffffff, 0x00000100,
1485        0x55e4, 0xffffffff, 0x00200100,
1486        0x3c280, 0xffffffff, 0x00000100,
1487        0x3c214, 0xffffffff, 0x06000100,
1488        0x3c220, 0xffffffff, 0x00000100,
1489        0x3c218, 0xffffffff, 0x06000100,
1490        0x3c204, 0xffffffff, 0x00000100,
1491        0x3c2e0, 0xffffffff, 0x00000100,
1492        0x3c224, 0xffffffff, 0x00000100,
1493        0x3c200, 0xffffffff, 0x00000100,
1494        0x3c230, 0xffffffff, 0x00000100,
1495        0x3c234, 0xffffffff, 0x00000100,
1496        0x3c250, 0xffffffff, 0x00000100,
1497        0x3c254, 0xffffffff, 0x00000100,
1498        0x3c258, 0xffffffff, 0x00000100,
1499        0x3c25c, 0xffffffff, 0x00000100,
1500        0x3c260, 0xffffffff, 0x00000100,
1501        0x3c27c, 0xffffffff, 0x00000100,
1502        0x3c278, 0xffffffff, 0x00000100,
1503        0x3c210, 0xffffffff, 0x06000100,
1504        0x3c290, 0xffffffff, 0x00000100,
1505        0x3c274, 0xffffffff, 0x00000100,
1506        0x3c2b4, 0xffffffff, 0x00000100,
1507        0x3c2b0, 0xffffffff, 0x00000100,
1508        0x3c270, 0xffffffff, 0x00000100,
1509        0x30800, 0xffffffff, 0xe0000000,
1510        0x3c020, 0xffffffff, 0x00010000,
1511        0x3c024, 0xffffffff, 0x00030002,
1512        0x3c028, 0xffffffff, 0x00040007,
1513        0x3c02c, 0xffffffff, 0x00060005,
1514        0x3c030, 0xffffffff, 0x00090008,
1515        0x3c034, 0xffffffff, 0x00010000,
1516        0x3c038, 0xffffffff, 0x00030002,
1517        0x3c03c, 0xffffffff, 0x00040007,
1518        0x3c040, 0xffffffff, 0x00060005,
1519        0x3c044, 0xffffffff, 0x00090008,
1520        0x3c048, 0xffffffff, 0x00010000,
1521        0x3c04c, 0xffffffff, 0x00030002,
1522        0x3c050, 0xffffffff, 0x00040007,
1523        0x3c054, 0xffffffff, 0x00060005,
1524        0x3c058, 0xffffffff, 0x00090008,
1525        0x3c05c, 0xffffffff, 0x00010000,
1526        0x3c060, 0xffffffff, 0x00030002,
1527        0x3c064, 0xffffffff, 0x00040007,
1528        0x3c068, 0xffffffff, 0x00060005,
1529        0x3c06c, 0xffffffff, 0x00090008,
1530        0x3c070, 0xffffffff, 0x00010000,
1531        0x3c074, 0xffffffff, 0x00030002,
1532        0x3c078, 0xffffffff, 0x00040007,
1533        0x3c07c, 0xffffffff, 0x00060005,
1534        0x3c080, 0xffffffff, 0x00090008,
1535        0x3c084, 0xffffffff, 0x00010000,
1536        0x3c088, 0xffffffff, 0x00030002,
1537        0x3c08c, 0xffffffff, 0x00040007,
1538        0x3c090, 0xffffffff, 0x00060005,
1539        0x3c094, 0xffffffff, 0x00090008,
1540        0x3c098, 0xffffffff, 0x00010000,
1541        0x3c09c, 0xffffffff, 0x00030002,
1542        0x3c0a0, 0xffffffff, 0x00040007,
1543        0x3c0a4, 0xffffffff, 0x00060005,
1544        0x3c0a8, 0xffffffff, 0x00090008,
1545        0x3c0ac, 0xffffffff, 0x00010000,
1546        0x3c0b0, 0xffffffff, 0x00030002,
1547        0x3c0b4, 0xffffffff, 0x00040007,
1548        0x3c0b8, 0xffffffff, 0x00060005,
1549        0x3c0bc, 0xffffffff, 0x00090008,
1550        0x3c0c0, 0xffffffff, 0x00010000,
1551        0x3c0c4, 0xffffffff, 0x00030002,
1552        0x3c0c8, 0xffffffff, 0x00040007,
1553        0x3c0cc, 0xffffffff, 0x00060005,
1554        0x3c0d0, 0xffffffff, 0x00090008,
1555        0x3c0d4, 0xffffffff, 0x00010000,
1556        0x3c0d8, 0xffffffff, 0x00030002,
1557        0x3c0dc, 0xffffffff, 0x00040007,
1558        0x3c0e0, 0xffffffff, 0x00060005,
1559        0x3c0e4, 0xffffffff, 0x00090008,
1560        0x3c0e8, 0xffffffff, 0x00010000,
1561        0x3c0ec, 0xffffffff, 0x00030002,
1562        0x3c0f0, 0xffffffff, 0x00040007,
1563        0x3c0f4, 0xffffffff, 0x00060005,
1564        0x3c0f8, 0xffffffff, 0x00090008,
1565        0xc318, 0xffffffff, 0x00020200,
1566        0x3350, 0xffffffff, 0x00000200,
1567        0x15c0, 0xffffffff, 0x00000400,
1568        0x55e8, 0xffffffff, 0x00000000,
1569        0x2f50, 0xffffffff, 0x00000902,
1570        0x3c000, 0xffffffff, 0x96940200,
1571        0x8708, 0xffffffff, 0x00900100,
1572        0xc424, 0xffffffff, 0x0020003f,
1573        0x38, 0xffffffff, 0x0140001c,
1574        0x3c, 0x000f0000, 0x000f0000,
1575        0x220, 0xffffffff, 0xc060000c,
1576        0x224, 0xc0000fff, 0x00000100,
1577        0xf90, 0xffffffff, 0x00000100,
1578        0xf98, 0x00000101, 0x00000000,
1579        0x20a8, 0xffffffff, 0x00000104,
1580        0x55e4, 0xff000fff, 0x00000100,
1581        0x30cc, 0xc0000fff, 0x00000104,
1582        0xc1e4, 0x00000001, 0x00000001,
1583        0xd00c, 0xff000ff0, 0x00000100,
1584        0xd80c, 0xff000ff0, 0x00000100
1585};
1586
1587static const u32 godavari_golden_registers[] =
1588{
1589        0x55e4, 0xff607fff, 0xfc000100,
1590        0x6ed8, 0x00010101, 0x00010000,
1591        0x9830, 0xffffffff, 0x00000000,
1592        0x98302, 0xf00fffff, 0x00000400,
1593        0x6130, 0xffffffff, 0x00010000,
1594        0x5bb0, 0x000000f0, 0x00000070,
1595        0x5bc0, 0xf0311fff, 0x80300000,
1596        0x98f8, 0x73773777, 0x12010001,
1597        0x98fc, 0xffffffff, 0x00000010,
1598        0x8030, 0x00001f0f, 0x0000100a,
1599        0x2f48, 0x73773777, 0x12010001,
1600        0x2408, 0x000fffff, 0x000c007f,
1601        0x8a14, 0xf000003f, 0x00000007,
1602        0x8b24, 0xffffffff, 0x00ff0fff,
1603        0x30a04, 0x0000ff0f, 0x00000000,
1604        0x28a4c, 0x07ffffff, 0x06000000,
1605        0x4d8, 0x00000fff, 0x00000100,
1606        0xd014, 0x00010000, 0x00810001,
1607        0xd814, 0x00010000, 0x00810001,
1608        0x3e78, 0x00000001, 0x00000002,
1609        0xc768, 0x00000008, 0x00000008,
1610        0xc770, 0x00000f00, 0x00000800,
1611        0xc774, 0x00000f00, 0x00000800,
1612        0xc798, 0x00ffffff, 0x00ff7fbf,
1613        0xc79c, 0x00ffffff, 0x00ff7faf,
1614        0x8c00, 0x000000ff, 0x00000001,
1615        0x214f8, 0x01ff01ff, 0x00000002,
1616        0x21498, 0x007ff800, 0x00200000,
1617        0x2015c, 0xffffffff, 0x00000f40,
1618        0x88c4, 0x001f3ae3, 0x00000082,
1619        0x88d4, 0x0000001f, 0x00000010,
1620        0x30934, 0xffffffff, 0x00000000
1621};
1622
1623
1624static void cik_init_golden_registers(struct radeon_device *rdev)
1625{
1626        /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1627        mutex_lock(&rdev->grbm_idx_mutex);
1628        switch (rdev->family) {
1629        case CHIP_BONAIRE:
1630                radeon_program_register_sequence(rdev,
1631                                                 bonaire_mgcg_cgcg_init,
1632                                                 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1633                radeon_program_register_sequence(rdev,
1634                                                 bonaire_golden_registers,
1635                                                 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1636                radeon_program_register_sequence(rdev,
1637                                                 bonaire_golden_common_registers,
1638                                                 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1639                radeon_program_register_sequence(rdev,
1640                                                 bonaire_golden_spm_registers,
1641                                                 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1642                break;
1643        case CHIP_KABINI:
1644                radeon_program_register_sequence(rdev,
1645                                                 kalindi_mgcg_cgcg_init,
1646                                                 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1647                radeon_program_register_sequence(rdev,
1648                                                 kalindi_golden_registers,
1649                                                 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1650                radeon_program_register_sequence(rdev,
1651                                                 kalindi_golden_common_registers,
1652                                                 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1653                radeon_program_register_sequence(rdev,
1654                                                 kalindi_golden_spm_registers,
1655                                                 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1656                break;
1657        case CHIP_MULLINS:
1658                radeon_program_register_sequence(rdev,
1659                                                 kalindi_mgcg_cgcg_init,
1660                                                 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1661                radeon_program_register_sequence(rdev,
1662                                                 godavari_golden_registers,
1663                                                 (const u32)ARRAY_SIZE(godavari_golden_registers));
1664                radeon_program_register_sequence(rdev,
1665                                                 kalindi_golden_common_registers,
1666                                                 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1667                radeon_program_register_sequence(rdev,
1668                                                 kalindi_golden_spm_registers,
1669                                                 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1670                break;
1671        case CHIP_KAVERI:
1672                radeon_program_register_sequence(rdev,
1673                                                 spectre_mgcg_cgcg_init,
1674                                                 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1675                radeon_program_register_sequence(rdev,
1676                                                 spectre_golden_registers,
1677                                                 (const u32)ARRAY_SIZE(spectre_golden_registers));
1678                radeon_program_register_sequence(rdev,
1679                                                 spectre_golden_common_registers,
1680                                                 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1681                radeon_program_register_sequence(rdev,
1682                                                 spectre_golden_spm_registers,
1683                                                 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1684                break;
1685        case CHIP_HAWAII:
1686                radeon_program_register_sequence(rdev,
1687                                                 hawaii_mgcg_cgcg_init,
1688                                                 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1689                radeon_program_register_sequence(rdev,
1690                                                 hawaii_golden_registers,
1691                                                 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1692                radeon_program_register_sequence(rdev,
1693                                                 hawaii_golden_common_registers,
1694                                                 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1695                radeon_program_register_sequence(rdev,
1696                                                 hawaii_golden_spm_registers,
1697                                                 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1698                break;
1699        default:
1700                break;
1701        }
1702        mutex_unlock(&rdev->grbm_idx_mutex);
1703}
1704
1705/**
1706 * cik_get_xclk - get the xclk
1707 *
1708 * @rdev: radeon_device pointer
1709 *
1710 * Returns the reference clock used by the gfx engine
1711 * (CIK).
1712 */
1713u32 cik_get_xclk(struct radeon_device *rdev)
1714{
1715        u32 reference_clock = rdev->clock.spll.reference_freq;
1716
1717        if (rdev->flags & RADEON_IS_IGP) {
1718                if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1719                        return reference_clock / 2;
1720        } else {
1721                if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1722                        return reference_clock / 4;
1723        }
1724        return reference_clock;
1725}
1726
1727/**
1728 * cik_mm_rdoorbell - read a doorbell dword
1729 *
1730 * @rdev: radeon_device pointer
1731 * @index: doorbell index
1732 *
1733 * Returns the value in the doorbell aperture at the
1734 * requested doorbell index (CIK).
1735 */
1736u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1737{
1738        if (index < rdev->doorbell.num_doorbells) {
1739                return readl(rdev->doorbell.ptr + index);
1740        } else {
1741                DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1742                return 0;
1743        }
1744}
1745
1746/**
1747 * cik_mm_wdoorbell - write a doorbell dword
1748 *
1749 * @rdev: radeon_device pointer
1750 * @index: doorbell index
1751 * @v: value to write
1752 *
1753 * Writes @v to the doorbell aperture at the
1754 * requested doorbell index (CIK).
1755 */
1756void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1757{
1758        if (index < rdev->doorbell.num_doorbells) {
1759                writel(v, rdev->doorbell.ptr + index);
1760        } else {
1761                DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1762        }
1763}
1764
1765#define BONAIRE_IO_MC_REGS_SIZE 36
1766
1767static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1768{
1769        {0x00000070, 0x04400000},
1770        {0x00000071, 0x80c01803},
1771        {0x00000072, 0x00004004},
1772        {0x00000073, 0x00000100},
1773        {0x00000074, 0x00ff0000},
1774        {0x00000075, 0x34000000},
1775        {0x00000076, 0x08000014},
1776        {0x00000077, 0x00cc08ec},
1777        {0x00000078, 0x00000400},
1778        {0x00000079, 0x00000000},
1779        {0x0000007a, 0x04090000},
1780        {0x0000007c, 0x00000000},
1781        {0x0000007e, 0x4408a8e8},
1782        {0x0000007f, 0x00000304},
1783        {0x00000080, 0x00000000},
1784        {0x00000082, 0x00000001},
1785        {0x00000083, 0x00000002},
1786        {0x00000084, 0xf3e4f400},
1787        {0x00000085, 0x052024e3},
1788        {0x00000087, 0x00000000},
1789        {0x00000088, 0x01000000},
1790        {0x0000008a, 0x1c0a0000},
1791        {0x0000008b, 0xff010000},
1792        {0x0000008d, 0xffffefff},
1793        {0x0000008e, 0xfff3efff},
1794        {0x0000008f, 0xfff3efbf},
1795        {0x00000092, 0xf7ffffff},
1796        {0x00000093, 0xffffff7f},
1797        {0x00000095, 0x00101101},
1798        {0x00000096, 0x00000fff},
1799        {0x00000097, 0x00116fff},
1800        {0x00000098, 0x60010000},
1801        {0x00000099, 0x10010000},
1802        {0x0000009a, 0x00006000},
1803        {0x0000009b, 0x00001000},
1804        {0x0000009f, 0x00b48000}
1805};
1806
1807#define HAWAII_IO_MC_REGS_SIZE 22
1808
1809static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1810{
1811        {0x0000007d, 0x40000000},
1812        {0x0000007e, 0x40180304},
1813        {0x0000007f, 0x0000ff00},
1814        {0x00000081, 0x00000000},
1815        {0x00000083, 0x00000800},
1816        {0x00000086, 0x00000000},
1817        {0x00000087, 0x00000100},
1818        {0x00000088, 0x00020100},
1819        {0x00000089, 0x00000000},
1820        {0x0000008b, 0x00040000},
1821        {0x0000008c, 0x00000100},
1822        {0x0000008e, 0xff010000},
1823        {0x00000090, 0xffffefff},
1824        {0x00000091, 0xfff3efff},
1825        {0x00000092, 0xfff3efbf},
1826        {0x00000093, 0xf7ffffff},
1827        {0x00000094, 0xffffff7f},
1828        {0x00000095, 0x00000fff},
1829        {0x00000096, 0x00116fff},
1830        {0x00000097, 0x60010000},
1831        {0x00000098, 0x10010000},
1832        {0x0000009f, 0x00c79000}
1833};
1834
1835
1836/**
1837 * cik_srbm_select - select specific register instances
1838 *
1839 * @rdev: radeon_device pointer
1840 * @me: selected ME (micro engine)
1841 * @pipe: pipe
1842 * @queue: queue
1843 * @vmid: VMID
1844 *
1845 * Switches the currently active registers instances.  Some
1846 * registers are instanced per VMID, others are instanced per
1847 * me/pipe/queue combination.
1848 */
1849static void cik_srbm_select(struct radeon_device *rdev,
1850                            u32 me, u32 pipe, u32 queue, u32 vmid)
1851{
1852        u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1853                             MEID(me & 0x3) |
1854                             VMID(vmid & 0xf) |
1855                             QUEUEID(queue & 0x7));
1856        WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1857}
1858
1859/* ucode loading */
1860/**
1861 * ci_mc_load_microcode - load MC ucode into the hw
1862 *
1863 * @rdev: radeon_device pointer
1864 *
1865 * Load the GDDR MC ucode into the hw (CIK).
1866 * Returns 0 on success, error on failure.
1867 */
1868int ci_mc_load_microcode(struct radeon_device *rdev)
1869{
1870        const __be32 *fw_data = NULL;
1871        const __le32 *new_fw_data = NULL;
1872        u32 running, blackout = 0, tmp;
1873        u32 *io_mc_regs = NULL;
1874        const __le32 *new_io_mc_regs = NULL;
1875        int i, regs_size, ucode_size;
1876
1877        if (!rdev->mc_fw)
1878                return -EINVAL;
1879
1880        if (rdev->new_fw) {
1881                const struct mc_firmware_header_v1_0 *hdr =
1882                        (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1883
1884                radeon_ucode_print_mc_hdr(&hdr->header);
1885
1886                regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1887                new_io_mc_regs = (const __le32 *)
1888                        (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1889                ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1890                new_fw_data = (const __le32 *)
1891                        (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1892        } else {
1893                ucode_size = rdev->mc_fw->size / 4;
1894
1895                switch (rdev->family) {
1896                case CHIP_BONAIRE:
1897                        io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1898                        regs_size = BONAIRE_IO_MC_REGS_SIZE;
1899                        break;
1900                case CHIP_HAWAII:
1901                        io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1902                        regs_size = HAWAII_IO_MC_REGS_SIZE;
1903                        break;
1904                default:
1905                        return -EINVAL;
1906                }
1907                fw_data = (const __be32 *)rdev->mc_fw->data;
1908        }
1909
1910        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1911
1912        if (running == 0) {
1913                if (running) {
1914                        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1915                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1916                }
1917
1918                /* reset the engine and set to writable */
1919                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1920                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1921
1922                /* load mc io regs */
1923                for (i = 0; i < regs_size; i++) {
1924                        if (rdev->new_fw) {
1925                                WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1926                                WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1927                        } else {
1928                                WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1929                                WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1930                        }
1931                }
1932
1933                tmp = RREG32(MC_SEQ_MISC0);
1934                if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1935                        WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1936                        WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1937                        WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1938                        WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1939                }
1940
1941                /* load the MC ucode */
1942                for (i = 0; i < ucode_size; i++) {
1943                        if (rdev->new_fw)
1944                                WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1945                        else
1946                                WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1947                }
1948
1949                /* put the engine back into the active state */
1950                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1951                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1952                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1953
1954                /* wait for training to complete */
1955                for (i = 0; i < rdev->usec_timeout; i++) {
1956                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1957                                break;
1958                        udelay(1);
1959                }
1960                for (i = 0; i < rdev->usec_timeout; i++) {
1961                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1962                                break;
1963                        udelay(1);
1964                }
1965
1966                if (running)
1967                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1968        }
1969
1970        return 0;
1971}
1972
1973/**
1974 * cik_init_microcode - load ucode images from disk
1975 *
1976 * @rdev: radeon_device pointer
1977 *
1978 * Use the firmware interface to load the ucode images into
1979 * the driver (not loaded into hw).
1980 * Returns 0 on success, error on failure.
1981 */
1982static int cik_init_microcode(struct radeon_device *rdev)
1983{
1984        const char *chip_name;
1985        const char *new_chip_name;
1986        size_t pfp_req_size, me_req_size, ce_req_size,
1987                mec_req_size, rlc_req_size, mc_req_size = 0,
1988                sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1989        char fw_name[30];
1990        int new_fw = 0;
1991        int err;
1992        int num_fw;
1993
1994        DRM_DEBUG("\n");
1995
1996        switch (rdev->family) {
1997        case CHIP_BONAIRE:
1998                chip_name = "BONAIRE";
1999                new_chip_name = "bonaire";
2000                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;

2001                me_req_size = CIK_ME_UCODE_SIZE * 4;
2002                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2003                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2004                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2005                mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2006                mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
2007                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2008                smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
2009                num_fw = 8;
2010                break;
2011        case CHIP_HAWAII:
2012                chip_name = "HAWAII";
2013                new_chip_name = "hawaii";
2014                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2015                me_req_size = CIK_ME_UCODE_SIZE * 4;
2016                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2017                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2018                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2019                mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2020                mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2021                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2022                smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2023                num_fw = 8;
2024                break;
2025        case CHIP_KAVERI:
2026                chip_name = "KAVERI";
2027                new_chip_name = "kaveri";
2028                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2029                me_req_size = CIK_ME_UCODE_SIZE * 4;
2030                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2031                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2032                rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2033                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2034                num_fw = 7;
2035                break;
2036        case CHIP_KABINI:
2037                chip_name = "KABINI";
2038                new_chip_name = "kabini";
2039                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2040                me_req_size = CIK_ME_UCODE_SIZE * 4;
2041                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2042                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2043                rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2044                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2045                num_fw = 6;
2046                break;
2047        case CHIP_MULLINS:
2048                chip_name = "MULLINS";
2049                new_chip_name = "mullins";
2050                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2051                me_req_size = CIK_ME_UCODE_SIZE * 4;
2052                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2053                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2054                rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2055                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2056                num_fw = 6;
2057                break;
2058        default: BUG();
2059        }
2060
2061        DRM_INFO("Loading %s Microcode\n", new_chip_name);
2062
2063        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2064        err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2065        if (err) {
2066                snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2067                err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2068                if (err)
2069                        goto out;
2070                if (rdev->pfp_fw->size != pfp_req_size) {
2071                        printk(KERN_ERR
2072                               "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2073                               rdev->pfp_fw->size, fw_name);
2074                        err = -EINVAL;
2075                        goto out;
2076                }
2077        } else {
2078                err = radeon_ucode_validate(rdev->pfp_fw);
2079                if (err) {
2080                        printk(KERN_ERR
2081                               "cik_fw: validation failed for firmware \"%s\"\n",
2082                               fw_name);
2083                        goto out;
2084                } else {
2085                        new_fw++;
2086                }
2087        }
2088
2089        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2090        err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2091        if (err) {
2092                snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2093                err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2094                if (err)
2095                        goto out;
2096                if (rdev->me_fw->size != me_req_size) {
2097                        printk(KERN_ERR
2098                               "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2099                               rdev->me_fw->size, fw_name);
2100                        err = -EINVAL;
2101                }
2102        } else {
2103                err = radeon_ucode_validate(rdev->me_fw);
2104                if (err) {
2105                        printk(KERN_ERR
2106                               "cik_fw: validation failed for firmware \"%s\"\n",
2107                               fw_name);
2108                        goto out;
2109                } else {
2110                        new_fw++;
2111                }
2112        }
2113
2114        snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2115        err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2116        if (err) {
2117                snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2118                err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2119                if (err)
2120                        goto out;
2121                if (rdev->ce_fw->size != ce_req_size) {
2122                        printk(KERN_ERR
2123                               "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2124                               rdev->ce_fw->size, fw_name);
2125                        err = -EINVAL;
2126                }
2127        } else {
2128                err = radeon_ucode_validate(rdev->ce_fw);
2129                if (err) {
2130                        printk(KERN_ERR
2131                               "cik_fw: validation failed for firmware \"%s\"\n",
2132                               fw_name);
2133                        goto out;
2134                } else {
2135                        new_fw++;
2136                }
2137        }
2138
2139        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2140        err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2141        if (err) {
2142                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2143                err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2144                if (err)
2145                        goto out;
2146                if (rdev->mec_fw->size != mec_req_size) {
2147                        printk(KERN_ERR
2148                               "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2149                               rdev->mec_fw->size, fw_name);
2150                        err = -EINVAL;
2151                }
2152        } else {
2153                err = radeon_ucode_validate(rdev->mec_fw);
2154                if (err) {
2155                        printk(KERN_ERR
2156                               "cik_fw: validation failed for firmware \"%s\"\n",
2157                               fw_name);
2158                        goto out;
2159                } else {
2160                        new_fw++;
2161                }
2162        }
2163
2164        if (rdev->family == CHIP_KAVERI) {
2165                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2166                err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2167                if (err) {
2168                        goto out;
2169                } else {
2170                        err = radeon_ucode_validate(rdev->mec2_fw);
2171                        if (err) {
2172                                goto out;
2173                        } else {
2174                                new_fw++;
2175                        }
2176                }
2177        }
2178
2179        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2180        err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2181        if (err) {
2182                snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2183                err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2184                if (err)
2185                        goto out;
2186                if (rdev->rlc_fw->size != rlc_req_size) {
2187                        printk(KERN_ERR
2188                               "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2189                               rdev->rlc_fw->size, fw_name);
2190                        err = -EINVAL;
2191                }
2192        } else {
2193                err = radeon_ucode_validate(rdev->rlc_fw);
2194                if (err) {
2195                        printk(KERN_ERR
2196                               "cik_fw: validation failed for firmware \"%s\"\n",
2197                               fw_name);
2198                        goto out;
2199                } else {
2200                        new_fw++;
2201                }
2202        }
2203
2204        snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2205        err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2206        if (err) {
2207                snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2208                err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2209                if (err)
2210                        goto out;
2211                if (rdev->sdma_fw->size != sdma_req_size) {
2212                        printk(KERN_ERR
2213                               "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2214                               rdev->sdma_fw->size, fw_name);
2215                        err = -EINVAL;
2216                }
2217        } else {
2218                err = radeon_ucode_validate(rdev->sdma_fw);
2219                if (err) {
2220                        printk(KERN_ERR
2221                               "cik_fw: validation failed for firmware \"%s\"\n",
2222                               fw_name);
2223                        goto out;
2224                } else {
2225                        new_fw++;
2226                }
2227        }
2228
2229        /* No SMC, MC ucode on APUs */
2230        if (!(rdev->flags & RADEON_IS_IGP)) {
2231                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2232                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2233                if (err) {
2234                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2235                        err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2236                        if (err) {
2237                                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2238                                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2239                                if (err)
2240                                        goto out;
2241                        }
2242                        if ((rdev->mc_fw->size != mc_req_size) &&
2243                            (rdev->mc_fw->size != mc2_req_size)){
2244                                printk(KERN_ERR
2245                                       "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2246                                       rdev->mc_fw->size, fw_name);
2247                                err = -EINVAL;
2248                        }
2249                        DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2250                } else {
2251                        err = radeon_ucode_validate(rdev->mc_fw);
2252                        if (err) {
2253                                printk(KERN_ERR
2254                                       "cik_fw: validation failed for firmware \"%s\"\n",
2255                                       fw_name);
2256                                goto out;
2257                        } else {
2258                                new_fw++;
2259                        }
2260                }
2261
2262                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2263                err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2264                if (err) {
2265                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2266                        err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2267                        if (err) {
2268                                printk(KERN_ERR
2269                                       "smc: error loading firmware \"%s\"\n",
2270                                       fw_name);
2271                                release_firmware(rdev->smc_fw);
2272                                rdev->smc_fw = NULL;
2273                                err = 0;
2274                        } else if (rdev->smc_fw->size != smc_req_size) {
2275                                printk(KERN_ERR
2276                                       "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2277                                       rdev->smc_fw->size, fw_name);
2278                                err = -EINVAL;
2279                        }
2280                } else {
2281                        err = radeon_ucode_validate(rdev->smc_fw);
2282                        if (err) {
2283                                printk(KERN_ERR
2284                                       "cik_fw: validation failed for firmware \"%s\"\n",
2285                                       fw_name);
2286                                goto out;
2287                        } else {
2288                                new_fw++;
2289                        }
2290                }
2291        }
2292
2293        if (new_fw == 0) {
2294                rdev->new_fw = false;
2295        } else if (new_fw < num_fw) {
2296                printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2297                err = -EINVAL;
2298        } else {
2299                rdev->new_fw = true;
2300        }
2301
2302out:
2303        if (err) {
2304                if (err != -EINVAL)
2305                        printk(KERN_ERR
2306                               "cik_cp: Failed to load firmware \"%s\"\n",
2307                               fw_name);
2308                release_firmware(rdev->pfp_fw);
2309                rdev->pfp_fw = NULL;
2310                release_firmware(rdev->me_fw);
2311                rdev->me_fw = NULL;
2312                release_firmware(rdev->ce_fw);
2313                rdev->ce_fw = NULL;
2314                release_firmware(rdev->mec_fw);
2315                rdev->mec_fw = NULL;
2316                release_firmware(rdev->mec2_fw);
2317                rdev->mec2_fw = NULL;
2318                release_firmware(rdev->rlc_fw);
2319                rdev->rlc_fw = NULL;
2320                release_firmware(rdev->sdma_fw);
2321                rdev->sdma_fw = NULL;
2322                release_firmware(rdev->mc_fw);
2323                rdev->mc_fw = NULL;
2324                release_firmware(rdev->smc_fw);
2325                rdev->smc_fw = NULL;
2326        }
2327        return err;
2328}
2329
2330/*
2331 * Core functions
2332 */
2333/**
2334 * cik_tiling_mode_table_init - init the hw tiling table
2335 *
2336 * @rdev: radeon_device pointer
2337 *
2338 * Starting with SI, the tiling setup is done globally in a
2339 * set of 32 tiling modes.  Rather than selecting each set of
2340 * parameters per surface as on older asics, we just select
2341 * which index in the tiling table we want to use, and the
2342 * surface uses those parameters (CIK).
2343 */
2344static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2345{
2346        const u32 num_tile_mode_states = 32;
2347        const u32 num_secondary_tile_mode_states = 16;
2348        u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2349        u32 num_pipe_configs;
2350        u32 num_rbs = rdev->config.cik.max_backends_per_se *
2351                rdev->config.cik.max_shader_engines;
2352
2353        switch (rdev->config.cik.mem_row_size_in_kb) {
2354        case 1:
2355                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2356                break;
2357        case 2:
2358        default:
2359                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2360                break;
2361        case 4:
2362                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2363                break;
2364        }
2365
2366        num_pipe_configs = rdev->config.cik.max_tile_pipes;
2367        if (num_pipe_configs > 8)
2368                num_pipe_configs = 16;
2369
2370        if (num_pipe_configs == 16) {
2371                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2372                        switch (reg_offset) {
2373                        case 0:
2374                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2376                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2377                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2378                                break;
2379                        case 1:
2380                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2381                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2382                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2383                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2384                                break;
2385                        case 2:
2386                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2387                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2388                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2389                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2390                                break;
2391                        case 3:
2392                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2393                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2394                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2395                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2396                                break;
2397                        case 4:
2398                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2399                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2400                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401                                                 TILE_SPLIT(split_equal_to_row_size));
2402                                break;
2403                        case 5:
2404                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2405                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2406                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2407                                break;
2408                        case 6:
2409                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2410                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2411                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2412                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2413                                break;
2414                        case 7:
2415                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2416                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2417                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2418                                                 TILE_SPLIT(split_equal_to_row_size));
2419                                break;
2420                        case 8:
2421                                gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2422                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2423                                break;
2424                        case 9:
2425                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2426                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2427                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2428                                break;
2429                        case 10:
2430                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2431                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2432                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2433                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2434                                break;
2435                        case 11:
2436                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2437                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2438                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2439                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440                                break;
2441                        case 12:
2442                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2443                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2444                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2445                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2446                                break;
2447                        case 13:
2448                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2449                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2450                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2451                                break;
2452                        case 14:
2453                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2454                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2455                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2456                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2457                                break;
2458                        case 16:
2459                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2460                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2461                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2462                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2463                                break;
2464                        case 17:
2465                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2466                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2467                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2468                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2469                                break;
2470                        case 27:
2471                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2472                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2473                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2474                                break;
2475                        case 28:
2476                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2477                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2478                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2479                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2480                                break;
2481                        case 29:
2482                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2483                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2484                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2485                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2486                                break;
2487                        case 30:
2488                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2489                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2490                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2491                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2492                                break;
2493                        default:
2494                                gb_tile_moden = 0;
2495                                break;
2496                        }
2497                        rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2498                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2499                }
2500                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2501                        switch (reg_offset) {
2502                        case 0:
2503                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2505                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2506                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2507                                break;
2508                        case 1:
2509                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2511                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2512                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2513                                break;
2514                        case 2:
2515                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2516                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2517                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2518                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2519                                break;
2520                        case 3:
2521                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2522                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2523                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2524                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2525                                break;
2526                        case 4:
2527                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2528                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2529                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2530                                                 NUM_BANKS(ADDR_SURF_8_BANK));
2531                                break;
2532                        case 5:
2533                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2534                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2535                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2536                                                 NUM_BANKS(ADDR_SURF_4_BANK));
2537                                break;
2538                        case 6:
2539                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2540                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2541                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2542                                                 NUM_BANKS(ADDR_SURF_2_BANK));
2543                                break;
2544                        case 8:
2545                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2546                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2547                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2548                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2549                                break;
2550                        case 9:
2551                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2552                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2553                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2554                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2555                                break;
2556                        case 10:
2557                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2558                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2559                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2560                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2561                                break;
2562                        case 11:
2563                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2564                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2565                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2566                                                 NUM_BANKS(ADDR_SURF_8_BANK));
2567                                break;
2568                        case 12:
2569                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2570                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2571                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2572                                                 NUM_BANKS(ADDR_SURF_4_BANK));
2573                                break;
2574                        case 13:
2575                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2576                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2577                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2578                                                 NUM_BANKS(ADDR_SURF_2_BANK));
2579                                break;
2580                        case 14:
2581                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2582                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2583                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2584                                                 NUM_BANKS(ADDR_SURF_2_BANK));
2585                                break;
2586                        default:
2587                                gb_tile_moden = 0;
2588                                break;
2589                        }
2590                        rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2591                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2592                }
2593        } else if (num_pipe_configs == 8) {
2594                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2595                        switch (reg_offset) {
2596                        case 0:
2597                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2598                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2599                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2600                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2601                                break;
2602                        case 1:
2603                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2605                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2606                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2607                                break;
2608                        case 2:
2609                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2610                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2611                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2612                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2613                                break;
2614                        case 3:
2615                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2616                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2617                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2618                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2619                                break;
2620                        case 4:
2621                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2622                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2623                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624                                                 TILE_SPLIT(split_equal_to_row_size));
2625                                break;
2626                        case 5:
2627                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2628                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2629                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2630                                break;
2631                        case 6:
2632                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2633                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2634                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2635                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2636                                break;
2637                        case 7:
2638                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2639                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2640                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2641                                                 TILE_SPLIT(split_equal_to_row_size));
2642                                break;
2643                        case 8:
2644                                gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2645                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2646                                break;
2647                        case 9:
2648                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2649                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2650                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2651                                break;
2652                        case 10:
2653                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2655                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2656                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2657                                break;
2658                        case 11:
2659                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2660                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2661                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2662                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2663                                break;
2664                        case 12:
2665                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2666                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2667                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2668                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2669                                break;
2670                        case 13:
2671                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2672                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2673                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2674                                break;
2675                        case 14:
2676                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2678                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2679                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2680                                break;
2681                        case 16:
2682                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2683                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2684                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2685                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2686                                break;
2687                        case 17:
2688                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2689                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2690                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2691                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2692                                break;
2693                        case 27:
2694                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2695                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2696                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2697                                break;
2698                        case 28:
2699                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2700                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2701                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2702                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2703                                break;
2704                        case 29:
2705                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2706                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2707                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2708                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2709                                break;
2710                        case 30:
2711                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2712                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2713                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2714                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715                                break;
2716                        default:
2717                                gb_tile_moden = 0;
2718                                break;
2719                        }
2720                        rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2721                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2722                }
2723                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2724                        switch (reg_offset) {
2725                        case 0:
2726                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2728                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2729                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2730                                break;
2731                        case 1:
2732                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2734                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2735                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2736                                break;
2737                        case 2:
2738                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2739                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2740                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2741                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2742                                break;
2743                        case 3:
2744                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2746                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2747                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2748                                break;
2749                        case 4:
2750                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2752                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2753                                                 NUM_BANKS(ADDR_SURF_8_BANK));
2754                                break;
2755                        case 5:
2756                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2759                                                 NUM_BANKS(ADDR_SURF_4_BANK));
2760                                break;
2761                        case 6:
2762                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2765                                                 NUM_BANKS(ADDR_SURF_2_BANK));
2766                                break;
2767                        case 8:
2768                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2770                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2771                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2772                                break;
2773                        case 9:
2774                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2775                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2776                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2777                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2778                                break;
2779                        case 10:
2780                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2782                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2783                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2784                                break;
2785                        case 11:
2786                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2787                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2788                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2789                                                 NUM_BANKS(ADDR_SURF_16_BANK));
2790                                break;
2791                        case 12:
2792                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2793                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2794                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2795                                                 NUM_BANKS(ADDR_SURF_8_BANK));
2796                                break;
2797                        case 13:
2798                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2799                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2800                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2801                                                 NUM_BANKS(ADDR_SURF_4_BANK));
2802                                break;
2803                        case 14:
2804                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2805                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2806                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2807                                                 NUM_BANKS(ADDR_SURF_2_BANK));
2808                                break;
2809                        default:
2810                                gb_tile_moden = 0;
2811                                break;
2812                        }
2813                        rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2814                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2815                }
2816        } else if (num_pipe_configs == 4) {
2817                if (num_rbs == 4) {
2818                        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2819                                switch (reg_offset) {
2820                                case 0:
2821                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2822                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2823                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2824                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2825                                        break;
2826                                case 1:
2827                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2828                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2829                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2830                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2831                                        break;
2832                                case 2:
2833                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2834                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2835                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2836                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2837                                        break;
2838                                case 3:
2839                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2840                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2841                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2842                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2843                                        break;
2844                                case 4:
2845                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2846                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2847                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848                                                         TILE_SPLIT(split_equal_to_row_size));
2849                                        break;
2850                                case 5:
2851                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2852                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2853                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2854                                        break;
2855                                case 6:
2856                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2857                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2858                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2859                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2860                                        break;
2861                                case 7:
2862                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2863                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2864                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2865                                                         TILE_SPLIT(split_equal_to_row_size));
2866                                        break;
2867                                case 8:
2868                                        gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2869                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16));
2870                                        break;
2871                                case 9:
2872                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2873                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2874                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2875                                        break;
2876                                case 10:
2877                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2878                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2879                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2880                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2881                                        break;
2882                                case 11:
2883                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2884                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2885                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2886                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2887                                        break;
2888                                case 12:
2889                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2890                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2891                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2892                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2893                                        break;
2894                                case 13:
2895                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2896                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2897                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2898                                        break;
2899                                case 14:
2900                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2902                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2903                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2904                                        break;
2905                                case 16:
2906                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2907                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2908                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2909                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2910                                        break;
2911                                case 17:
2912                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2913                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2914                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2915                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2916                                        break;
2917                                case 27:
2918                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2919                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2920                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2921                                        break;
2922                                case 28:
2923                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2924                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2925                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2926                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2927                                        break;
2928                                case 29:
2929                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2930                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2931                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2933                                        break;
2934                                case 30:
2935                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2936                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2937                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2938                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939                                        break;
2940                                default:
2941                                        gb_tile_moden = 0;
2942                                        break;
2943                                }
2944                                rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2945                                WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2946                        }
2947                } else if (num_rbs < 4) {
2948                        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2949                                switch (reg_offset) {
2950                                case 0:
2951                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2952                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2953                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2954                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2955                                        break;
2956                                case 1:
2957                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2958                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2959                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2960                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2961                                        break;
2962                                case 2:
2963                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2964                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2965                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2966                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2967                                        break;
2968                                case 3:
2969                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2970                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2971                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2972                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2973                                        break;
2974                                case 4:
2975                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2976                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2977                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978                                                         TILE_SPLIT(split_equal_to_row_size));
2979                                        break;
2980                                case 5:
2981                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2982                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2983                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2984                                        break;
2985                                case 6:
2986                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2987                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2988                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2989                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2990                                        break;
2991                                case 7:
2992                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2993                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2994                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2995                                                         TILE_SPLIT(split_equal_to_row_size));
2996                                        break;
2997                                case 8:
2998                                        gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2999                                                 PIPE_CONFIG(ADDR_SURF_P4_8x16));
3000                                        break;

3001                                case 9:
3002                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3003                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3004                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
3005                                        break;
3006                                case 10:
3007                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3008                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3009                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3010                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3011                                        break;
3012                                case 11:
3013                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3014                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3015                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3016                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3017                                        break;
3018                                case 12:
3019                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3020                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3021                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3022                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3023                                        break;
3024                                case 13:
3025                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3026                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3027                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3028                                        break;
3029                                case 14:
3030                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3031                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3032                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3033                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3034                                        break;
3035                                case 16:
3036                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3037                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3038                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3039                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3040                                        break;
3041                                case 17:
3042                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3043                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3044                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3045                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3046                                        break;
3047                                case 27:
3048                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3049                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3050                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
3051                                        break;
3052                                case 28:
3053                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3054                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3055                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3056                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3057                                        break;
3058                                case 29:
3059                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3060                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3061                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3062                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3063                                        break;
3064                                case 30:
3065                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3066                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3067                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3068                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3069                                        break;
3070                                default:
3071                                        gb_tile_moden = 0;
3072                                        break;
3073                                }
3074                                rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3075                                WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3076                        }
3077                }
3078                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3079                        switch (reg_offset) {
3080                        case 0:
3081                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3083                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3084                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3085                                break;
3086                        case 1:
3087                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3089                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3090                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3091                                break;
3092                        case 2:
3093                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3094                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3095                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3096                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3097                                break;
3098                        case 3:
3099                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3100                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3101                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3102                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3103                                break;
3104                        case 4:
3105                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3106                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3107                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3108                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3109                                break;
3110                        case 5:
3111                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3112                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3113                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3114                                                 NUM_BANKS(ADDR_SURF_8_BANK));
3115                                break;
3116                        case 6:
3117                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3118                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3119                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3120                                                 NUM_BANKS(ADDR_SURF_4_BANK));
3121                                break;
3122                        case 8:
3123                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3124                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3125                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3126                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3127                                break;
3128                        case 9:
3129                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3130                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3131                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3132                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3133                                break;
3134                        case 10:
3135                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3136                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3137                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3138                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3139                                break;
3140                        case 11:
3141                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3142                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3143                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3144                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3145                                break;
3146                        case 12:
3147                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3148                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3149                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3150                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3151                                break;
3152                        case 13:
3153                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3154                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3155                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3156                                                 NUM_BANKS(ADDR_SURF_8_BANK));
3157                                break;
3158                        case 14:
3159                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3160                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3161                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3162                                                 NUM_BANKS(ADDR_SURF_4_BANK));
3163                                break;
3164                        default:
3165                                gb_tile_moden = 0;
3166                                break;
3167                        }
3168                        rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3169                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3170                }
3171        } else if (num_pipe_configs == 2) {
3172                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3173                        switch (reg_offset) {
3174                        case 0:
3175                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3176                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3177                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3178                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3179                                break;
3180                        case 1:
3181                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3182                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3183                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3184                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3185                                break;
3186                        case 2:
3187                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3188                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3189                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3190                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3191                                break;
3192                        case 3:
3193                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3194                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3195                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3196                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3197                                break;
3198                        case 4:
3199                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3200                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3201                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3202                                                 TILE_SPLIT(split_equal_to_row_size));
3203                                break;
3204                        case 5:
3205                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3206                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3207                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3208                                break;
3209                        case 6:
3210                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3211                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3212                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3213                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3214                                break;
3215                        case 7:
3216                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3217                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3218                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3219                                                 TILE_SPLIT(split_equal_to_row_size));
3220                                break;
3221                        case 8:
3222                                gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3223                                                PIPE_CONFIG(ADDR_SURF_P2);
3224                                break;
3225                        case 9:
3226                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3227                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3228                                                 PIPE_CONFIG(ADDR_SURF_P2));
3229                                break;
3230                        case 10:
3231                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3232                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3233                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3234                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3235                                break;
3236                        case 11:
3237                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3238                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3239                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3240                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3241                                break;
3242                        case 12:
3243                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3244                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3245                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3246                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3247                                break;
3248                        case 13:
3249                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3250                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3251                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3252                                break;
3253                        case 14:
3254                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3255                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3256                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3257                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3258                                break;
3259                        case 16:
3260                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3261                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3262                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3263                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3264                                break;
3265                        case 17:
3266                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3267                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3268                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3269                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3270                                break;
3271                        case 27:
3272                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3273                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3274                                                 PIPE_CONFIG(ADDR_SURF_P2));
3275                                break;
3276                        case 28:
3277                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3278                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3279                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3280                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3281                                break;
3282                        case 29:
3283                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3284                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3285                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3286                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3287                                break;
3288                        case 30:
3289                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3290                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3291                                                 PIPE_CONFIG(ADDR_SURF_P2) |
3292                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3293                                break;
3294                        default:
3295                                gb_tile_moden = 0;
3296                                break;
3297                        }
3298                        rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3299                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3300                }
3301                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3302                        switch (reg_offset) {
3303                        case 0:
3304                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3305                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3306                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3308                                break;
3309                        case 1:
3310                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3311                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3312                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3313                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3314                                break;
3315                        case 2:
3316                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3317                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3318                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3319                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3320                                break;
3321                        case 3:
3322                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3323                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3324                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3325                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3326                                break;
3327                        case 4:
3328                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3329                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3330                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3331                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3332                                break;
3333                        case 5:
3334                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3335                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3336                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3337                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3338                                break;
3339                        case 6:
3340                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3341                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3342                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3343                                                 NUM_BANKS(ADDR_SURF_8_BANK));
3344                                break;
3345                        case 8:
3346                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3347                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3348                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3349                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3350                                break;
3351                        case 9:
3352                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3353                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3354                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3355                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3356                                break;
3357                        case 10:
3358                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3359                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3360                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3361                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3362                                break;
3363                        case 11:
3364                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3365                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3366                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3367                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3368                                break;
3369                        case 12:
3370                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3371                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3372                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3373                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3374                                break;
3375                        case 13:
3376                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3377                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3378                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3379                                                 NUM_BANKS(ADDR_SURF_16_BANK));
3380                                break;
3381                        case 14:
3382                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3383                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3384                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3385                                                 NUM_BANKS(ADDR_SURF_8_BANK));
3386                                break;
3387                        default:
3388                                gb_tile_moden = 0;
3389                                break;
3390                        }
3391                        rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3392                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3393                }
3394        } else
3395                DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3396}
3397
3398/**
3399 * cik_select_se_sh - select which SE, SH to address
3400 *
3401 * @rdev: radeon_device pointer
3402 * @se_num: shader engine to address
3403 * @sh_num: sh block to address
3404 *
3405 * Select which SE, SH combinations to address. Certain
3406 * registers are instanced per SE or SH.  0xffffffff means
3407 * broadcast to all SEs or SHs (CIK).
3408 */
3409static void cik_select_se_sh(struct radeon_device *rdev,
3410                             u32 se_num, u32 sh_num)
3411{
3412        u32 data = INSTANCE_BROADCAST_WRITES;
3413
3414        if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3415                data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3416        else if (se_num == 0xffffffff)
3417                data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3418        else if (sh_num == 0xffffffff)
3419                data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3420        else
3421                data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3422        WREG32(GRBM_GFX_INDEX, data);
3423}
3424
3425/**
3426 * cik_create_bitmask - create a bitmask
3427 *
3428 * @bit_width: length of the mask
3429 *
3430 * create a variable length bit mask (CIK).
3431 * Returns the bitmask.
3432 */
3433static u32 cik_create_bitmask(u32 bit_width)
3434{
3435        u32 i, mask = 0;
3436
3437        for (i = 0; i < bit_width; i++) {
3438                mask <<= 1;
3439                mask |= 1;
3440        }
3441        return mask;
3442}
3443
3444/**
3445 * cik_get_rb_disabled - computes the mask of disabled RBs
3446 *
3447 * @rdev: radeon_device pointer
3448 * @max_rb_num: max RBs (render backends) for the asic
3449 * @se_num: number of SEs (shader engines) for the asic
3450 * @sh_per_se: number of SH blocks per SE for the asic
3451 *
3452 * Calculates the bitmask of disabled RBs (CIK).
3453 * Returns the disabled RB bitmask.
3454 */
3455static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3456                              u32 max_rb_num_per_se,
3457                              u32 sh_per_se)
3458{
3459        u32 data, mask;
3460
3461        data = RREG32(CC_RB_BACKEND_DISABLE);
3462        if (data & 1)
3463                data &= BACKEND_DISABLE_MASK;
3464        else
3465                data = 0;
3466        data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3467
3468        data >>= BACKEND_DISABLE_SHIFT;
3469
3470        mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3471
3472        return data & mask;
3473}
3474
3475/**
3476 * cik_setup_rb - setup the RBs on the asic
3477 *
3478 * @rdev: radeon_device pointer
3479 * @se_num: number of SEs (shader engines) for the asic
3480 * @sh_per_se: number of SH blocks per SE for the asic
3481 * @max_rb_num: max RBs (render backends) for the asic
3482 *
3483 * Configures per-SE/SH RB registers (CIK).
3484 */
3485static void cik_setup_rb(struct radeon_device *rdev,
3486                         u32 se_num, u32 sh_per_se,
3487                         u32 max_rb_num_per_se)
3488{
3489        int i, j;
3490        u32 data, mask;
3491        u32 disabled_rbs = 0;
3492        u32 enabled_rbs = 0;
3493
3494        mutex_lock(&rdev->grbm_idx_mutex);
3495        for (i = 0; i < se_num; i++) {
3496                for (j = 0; j < sh_per_se; j++) {
3497                        cik_select_se_sh(rdev, i, j);
3498                        data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3499                        if (rdev->family == CHIP_HAWAII)
3500                                disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3501                        else
3502                                disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3503                }
3504        }
3505        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3506        mutex_unlock(&rdev->grbm_idx_mutex);
3507
3508        mask = 1;
3509        for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3510                if (!(disabled_rbs & mask))
3511                        enabled_rbs |= mask;
3512                mask <<= 1;
3513        }
3514
3515        rdev->config.cik.backend_enable_mask = enabled_rbs;
3516
3517        mutex_lock(&rdev->grbm_idx_mutex);
3518        for (i = 0; i < se_num; i++) {
3519                cik_select_se_sh(rdev, i, 0xffffffff);
3520                data = 0;
3521                for (j = 0; j < sh_per_se; j++) {
3522                        switch (enabled_rbs & 3) {
3523                        case 0:
3524                                if (j == 0)
3525                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3526                                else
3527                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3528                                break;
3529                        case 1:
3530                                data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3531                                break;
3532                        case 2:
3533                                data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3534                                break;
3535                        case 3:
3536                        default:
3537                                data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3538                                break;
3539                        }
3540                        enabled_rbs >>= 2;
3541                }
3542                WREG32(PA_SC_RASTER_CONFIG, data);
3543        }
3544        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3545        mutex_unlock(&rdev->grbm_idx_mutex);
3546}
3547
3548/**
3549 * cik_gpu_init - setup the 3D engine
3550 *
3551 * @rdev: radeon_device pointer
3552 *
3553 * Configures the 3D engine and tiling configuration
3554 * registers so that the 3D engine is usable.
3555 */
3556static void cik_gpu_init(struct radeon_device *rdev)
3557{
3558        u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3559        u32 mc_shared_chmap, mc_arb_ramcfg;
3560        u32 hdp_host_path_cntl;
3561        u32 tmp;
3562        int i, j;
3563
3564        switch (rdev->family) {
3565        case CHIP_BONAIRE:
3566                rdev->config.cik.max_shader_engines = 2;
3567                rdev->config.cik.max_tile_pipes = 4;
3568                rdev->config.cik.max_cu_per_sh = 7;
3569                rdev->config.cik.max_sh_per_se = 1;
3570                rdev->config.cik.max_backends_per_se = 2;
3571                rdev->config.cik.max_texture_channel_caches = 4;
3572                rdev->config.cik.max_gprs = 256;
3573                rdev->config.cik.max_gs_threads = 32;
3574                rdev->config.cik.max_hw_contexts = 8;
3575
3576                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3577                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3578                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3579                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3580                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3581                break;
3582        case CHIP_HAWAII:
3583                rdev->config.cik.max_shader_engines = 4;
3584                rdev->config.cik.max_tile_pipes = 16;
3585                rdev->config.cik.max_cu_per_sh = 11;
3586                rdev->config.cik.max_sh_per_se = 1;
3587                rdev->config.cik.max_backends_per_se = 4;
3588                rdev->config.cik.max_texture_channel_caches = 16;
3589                rdev->config.cik.max_gprs = 256;
3590                rdev->config.cik.max_gs_threads = 32;
3591                rdev->config.cik.max_hw_contexts = 8;
3592
3593                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3594                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3595                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3596                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3597                gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3598                break;
3599        case CHIP_KAVERI:
3600                rdev->config.cik.max_shader_engines = 1;
3601                rdev->config.cik.max_tile_pipes = 4;
3602                if ((rdev->pdev->device == 0x1304) ||
3603                    (rdev->pdev->device == 0x1305) ||
3604                    (rdev->pdev->device == 0x130C) ||
3605                    (rdev->pdev->device == 0x130F) ||
3606                    (rdev->pdev->device == 0x1310) ||
3607                    (rdev->pdev->device == 0x1311) ||
3608                    (rdev->pdev->device == 0x131C)) {
3609                        rdev->config.cik.max_cu_per_sh = 8;
3610                        rdev->config.cik.max_backends_per_se = 2;
3611                } else if ((rdev->pdev->device == 0x1309) ||
3612                           (rdev->pdev->device == 0x130A) ||
3613                           (rdev->pdev->device == 0x130D) ||
3614                           (rdev->pdev->device == 0x1313) ||
3615                           (rdev->pdev->device == 0x131D)) {
3616                        rdev->config.cik.max_cu_per_sh = 6;
3617                        rdev->config.cik.max_backends_per_se = 2;
3618                } else if ((rdev->pdev->device == 0x1306) ||
3619                           (rdev->pdev->device == 0x1307) ||
3620                           (rdev->pdev->device == 0x130B) ||
3621                           (rdev->pdev->device == 0x130E) ||
3622                           (rdev->pdev->device == 0x1315) ||
3623                           (rdev->pdev->device == 0x1318) ||
3624                           (rdev->pdev->device == 0x131B)) {
3625                        rdev->config.cik.max_cu_per_sh = 4;
3626                        rdev->config.cik.max_backends_per_se = 1;
3627                } else {
3628                        rdev->config.cik.max_cu_per_sh = 3;
3629                        rdev->config.cik.max_backends_per_se = 1;
3630                }
3631                rdev->config.cik.max_sh_per_se = 1;
3632                rdev->config.cik.max_texture_channel_caches = 4;
3633                rdev->config.cik.max_gprs = 256;
3634                rdev->config.cik.max_gs_threads = 16;
3635                rdev->config.cik.max_hw_contexts = 8;
3636
3637                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3638                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3639                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3640                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3641                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3642                break;
3643        case CHIP_KABINI:
3644        case CHIP_MULLINS:
3645        default:
3646                rdev->config.cik.max_shader_engines = 1;
3647                rdev->config.cik.max_tile_pipes = 2;
3648                rdev->config.cik.max_cu_per_sh = 2;
3649                rdev->config.cik.max_sh_per_se = 1;
3650                rdev->config.cik.max_backends_per_se = 1;
3651                rdev->config.cik.max_texture_channel_caches = 2;
3652                rdev->config.cik.max_gprs = 256;
3653                rdev->config.cik.max_gs_threads = 16;
3654                rdev->config.cik.max_hw_contexts = 8;
3655
3656                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3657                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3658                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3659                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3660                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3661                break;
3662        }
3663
3664        /* Initialize HDP */
3665        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3666                WREG32((0x2c14 + j), 0x00000000);
3667                WREG32((0x2c18 + j), 0x00000000);
3668                WREG32((0x2c1c + j), 0x00000000);
3669                WREG32((0x2c20 + j), 0x00000000);
3670                WREG32((0x2c24 + j), 0x00000000);
3671        }
3672
3673        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3674        WREG32(SRBM_INT_CNTL, 0x1);
3675        WREG32(SRBM_INT_ACK, 0x1);
3676
3677        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3678
3679        mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3680        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3681
3682        rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3683        rdev->config.cik.mem_max_burst_length_bytes = 256;
3684        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3685        rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3686        if (rdev->config.cik.mem_row_size_in_kb > 4)
3687                rdev->config.cik.mem_row_size_in_kb = 4;
3688        /* XXX use MC settings? */
3689        rdev->config.cik.shader_engine_tile_size = 32;
3690        rdev->config.cik.num_gpus = 1;
3691        rdev->config.cik.multi_gpu_tile_size = 64;
3692
3693        /* fix up row size */
3694        gb_addr_config &= ~ROW_SIZE_MASK;
3695        switch (rdev->config.cik.mem_row_size_in_kb) {
3696        case 1:
3697        default:
3698                gb_addr_config |= ROW_SIZE(0);
3699                break;
3700        case 2:
3701                gb_addr_config |= ROW_SIZE(1);
3702                break;
3703        case 4:
3704                gb_addr_config |= ROW_SIZE(2);
3705                break;
3706        }
3707
3708        /* setup tiling info dword.  gb_addr_config is not adequate since it does
3709         * not have bank info, so create a custom tiling dword.
3710         * bits 3:0   num_pipes
3711         * bits 7:4   num_banks
3712         * bits 11:8  group_size
3713         * bits 15:12 row_size
3714         */
3715        rdev->config.cik.tile_config = 0;
3716        switch (rdev->config.cik.num_tile_pipes) {
3717        case 1:
3718                rdev->config.cik.tile_config |= (0 << 0);
3719                break;
3720        case 2:
3721                rdev->config.cik.tile_config |= (1 << 0);
3722                break;
3723        case 4:
3724                rdev->config.cik.tile_config |= (2 << 0);
3725                break;
3726        case 8:
3727        default:
3728                /* XXX what about 12? */
3729                rdev->config.cik.tile_config |= (3 << 0);
3730                break;
3731        }
3732        rdev->config.cik.tile_config |=
3733                ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3734        rdev->config.cik.tile_config |=
3735                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3736        rdev->config.cik.tile_config |=
3737                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3738
3739        WREG32(GB_ADDR_CONFIG, gb_addr_config);
3740        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3741        WREG32(DMIF_ADDR_CALC, gb_addr_config);
3742        WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3743        WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3744        WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3745        WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3746        WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3747
3748        cik_tiling_mode_table_init(rdev);
3749
3750        cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3751                     rdev->config.cik.max_sh_per_se,
3752                     rdev->config.cik.max_backends_per_se);
3753
3754        rdev->config.cik.active_cus = 0;
3755        for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3756                for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3757                        rdev->config.cik.active_cus +=
3758                                hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3759                }
3760        }
3761
3762        /* set HW defaults for 3D engine */
3763        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3764
3765        mutex_lock(&rdev->grbm_idx_mutex);
3766        /*
3767         * making sure that the following register writes will be broadcasted
3768         * to all the shaders
3769         */
3770        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3771        WREG32(SX_DEBUG_1, 0x20);
3772
3773        WREG32(TA_CNTL_AUX, 0x00010000);
3774
3775        tmp = RREG32(SPI_CONFIG_CNTL);
3776        tmp |= 0x03000000;
3777        WREG32(SPI_CONFIG_CNTL, tmp);
3778
3779        WREG32(SQ_CONFIG, 1);
3780
3781        WREG32(DB_DEBUG, 0);
3782
3783        tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3784        tmp |= 0x00000400;
3785        WREG32(DB_DEBUG2, tmp);
3786
3787        tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3788        tmp |= 0x00020200;
3789        WREG32(DB_DEBUG3, tmp);
3790
3791        tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3792        tmp |= 0x00018208;
3793        WREG32(CB_HW_CONTROL, tmp);
3794
3795        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3796
3797        WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3798                                 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3799                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3800                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3801
3802        WREG32(VGT_NUM_INSTANCES, 1);
3803
3804        WREG32(CP_PERFMON_CNTL, 0);
3805
3806        WREG32(SQ_CONFIG, 0);
3807
3808        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3809                                          FORCE_EOV_MAX_REZ_CNT(255)));
3810
3811        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3812               AUTO_INVLD_EN(ES_AND_GS_AUTO));
3813
3814        WREG32(VGT_GS_VERTEX_REUSE, 16);
3815        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3816
3817        tmp = RREG32(HDP_MISC_CNTL);
3818        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3819        WREG32(HDP_MISC_CNTL, tmp);
3820
3821        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3822        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3823
3824        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3825        WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3826        mutex_unlock(&rdev->grbm_idx_mutex);
3827
3828        udelay(50);
3829}
3830
3831/*
3832 * GPU scratch registers helpers function.
3833 */
3834/**
3835 * cik_scratch_init - setup driver info for CP scratch regs
3836 *
3837 * @rdev: radeon_device pointer
3838 *
3839 * Set up the number and offset of the CP scratch registers.
3840 * NOTE: use of CP scratch registers is a legacy inferface and
3841 * is not used by default on newer asics (r6xx+).  On newer asics,
3842 * memory buffers are used for fences rather than scratch regs.
3843 */
3844static void cik_scratch_init(struct radeon_device *rdev)
3845{
3846        int i;
3847
3848        rdev->scratch.num_reg = 7;
3849        rdev->scratch.reg_base = SCRATCH_REG0;
3850        for (i = 0; i < rdev->scratch.num_reg; i++) {
3851                rdev->scratch.free[i] = true;
3852                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3853        }
3854}
3855
3856/**
3857 * cik_ring_test - basic gfx ring test
3858 *
3859 * @rdev: radeon_device pointer
3860 * @ring: radeon_ring structure holding ring information
3861 *
3862 * Allocate a scratch register and write to it using the gfx ring (CIK).
3863 * Provides a basic gfx ring test to verify that the ring is working.
3864 * Used by cik_cp_gfx_resume();
3865 * Returns 0 on success, error on failure.
3866 */
3867int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3868{
3869        uint32_t scratch;
3870        uint32_t tmp = 0;
3871        unsigned i;
3872        int r;
3873
3874        r = radeon_scratch_get(rdev, &scratch);
3875        if (r) {
3876                DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3877                return r;
3878        }
3879        WREG32(scratch, 0xCAFEDEAD);
3880        r = radeon_ring_lock(rdev, ring, 3);
3881        if (r) {
3882                DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3883                radeon_scratch_free(rdev, scratch);
3884                return r;
3885        }
3886        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3887        radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3888        radeon_ring_write(ring, 0xDEADBEEF);
3889        radeon_ring_unlock_commit(rdev, ring, false);
3890
3891        for (i = 0; i < rdev->usec_timeout; i++) {
3892                tmp = RREG32(scratch);
3893                if (tmp == 0xDEADBEEF)
3894                        break;
3895                DRM_UDELAY(1);
3896        }
3897        if (i < rdev->usec_timeout) {
3898                DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3899        } else {
3900                DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3901                          ring->idx, scratch, tmp);
3902                r = -EINVAL;
3903        }
3904        radeon_scratch_free(rdev, scratch);
3905        return r;
3906}
3907
3908/**
3909 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3910 *
3911 * @rdev: radeon_device pointer
3912 * @ridx: radeon ring index
3913 *
3914 * Emits an hdp flush on the cp.
3915 */
3916static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3917                                       int ridx)
3918{
3919        struct radeon_ring *ring = &rdev->ring[ridx];
3920        u32 ref_and_mask;
3921
3922        switch (ring->idx) {
3923        case CAYMAN_RING_TYPE_CP1_INDEX:
3924        case CAYMAN_RING_TYPE_CP2_INDEX:
3925        default:
3926                switch (ring->me) {
3927                case 0:
3928                        ref_and_mask = CP2 << ring->pipe;
3929                        break;
3930                case 1:
3931                        ref_and_mask = CP6 << ring->pipe;
3932                        break;
3933                default:
3934                        return;
3935                }
3936                break;
3937        case RADEON_RING_TYPE_GFX_INDEX:
3938                ref_and_mask = CP0;
3939                break;
3940        }
3941
3942        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3943        radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3944                                 WAIT_REG_MEM_FUNCTION(3) |  /* == */
3945                                 WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3946        radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3947        radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3948        radeon_ring_write(ring, ref_and_mask);
3949        radeon_ring_write(ring, ref_and_mask);
3950        radeon_ring_write(ring, 0x20); /* poll interval */
3951}
3952
3953/**
3954 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3955 *
3956 * @rdev: radeon_device pointer
3957 * @fence: radeon fence object
3958 *
3959 * Emits a fence sequnce number on the gfx ring and flushes
3960 * GPU caches.
3961 */
3962void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3963                             struct radeon_fence *fence)
3964{
3965        struct radeon_ring *ring = &rdev->ring[fence->ring];
3966        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3967
3968        /* Workaround for cache flush problems. First send a dummy EOP
3969         * event down the pipe with seq one below.
3970         */
3971        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3972        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3973                                 EOP_TC_ACTION_EN |
3974                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3975                                 EVENT_INDEX(5)));
3976        radeon_ring_write(ring, addr & 0xfffffffc);
3977        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3978                                DATA_SEL(1) | INT_SEL(0));
3979        radeon_ring_write(ring, fence->seq - 1);
3980        radeon_ring_write(ring, 0);
3981
3982        /* Then send the real EOP event down the pipe. */
3983        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3984        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3985                                 EOP_TC_ACTION_EN |
3986                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3987                                 EVENT_INDEX(5)));
3988        radeon_ring_write(ring, addr & 0xfffffffc);
3989        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3990        radeon_ring_write(ring, fence->seq);
3991        radeon_ring_write(ring, 0);
3992}
3993
3994/**
3995 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3996 *
3997 * @rdev: radeon_device pointer
3998 * @fence: radeon fence object
3999 *
4000 * Emits a fence sequnce number on the compute ring and flushes

4001 * GPU caches.
4002 */
4003void cik_fence_compute_ring_emit(struct radeon_device *rdev,
4004                                 struct radeon_fence *fence)
4005{
4006        struct radeon_ring *ring = &rdev->ring[fence->ring];
4007        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4008
4009        /* RELEASE_MEM - flush caches, send int */
4010        radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
4011        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
4012                                 EOP_TC_ACTION_EN |
4013                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
4014                                 EVENT_INDEX(5)));
4015        radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
4016        radeon_ring_write(ring, addr & 0xfffffffc);
4017        radeon_ring_write(ring, upper_32_bits(addr));
4018        radeon_ring_write(ring, fence->seq);
4019        radeon_ring_write(ring, 0);
4020}
4021
4022/**
4023 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
4024 *
4025 * @rdev: radeon_device pointer
4026 * @ring: radeon ring buffer object
4027 * @semaphore: radeon semaphore object
4028 * @emit_wait: Is this a sempahore wait?
4029 *
4030 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
4031 * from running ahead of semaphore waits.
4032 */
4033bool cik_semaphore_ring_emit(struct radeon_device *rdev,
4034                             struct radeon_ring *ring,
4035                             struct radeon_semaphore *semaphore,
4036                             bool emit_wait)
4037{
4038        uint64_t addr = semaphore->gpu_addr;
4039        unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
4040
4041        radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
4042        radeon_ring_write(ring, lower_32_bits(addr));
4043        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
4044
4045        if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
4046                /* Prevent the PFP from running ahead of the semaphore wait */
4047                radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4048                radeon_ring_write(ring, 0x0);
4049        }
4050
4051        return true;
4052}
4053
4054/**
4055 * cik_copy_cpdma - copy pages using the CP DMA engine
4056 *
4057 * @rdev: radeon_device pointer
4058 * @src_offset: src GPU address
4059 * @dst_offset: dst GPU address
4060 * @num_gpu_pages: number of GPU pages to xfer
4061 * @resv: reservation object to sync to
4062 *
4063 * Copy GPU paging using the CP DMA engine (CIK+).
4064 * Used by the radeon ttm implementation to move pages if
4065 * registered as the asic copy callback.
4066 */
4067struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
4068                                    uint64_t src_offset, uint64_t dst_offset,
4069                                    unsigned num_gpu_pages,
4070                                    struct reservation_object *resv)
4071{
4072        struct radeon_fence *fence;
4073        struct radeon_sync sync;
4074        int ring_index = rdev->asic->copy.blit_ring_index;
4075        struct radeon_ring *ring = &rdev->ring[ring_index];
4076        u32 size_in_bytes, cur_size_in_bytes, control;
4077        int i, num_loops;
4078        int r = 0;
4079
4080        radeon_sync_create(&sync);
4081
4082        size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
4083        num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4084        r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4085        if (r) {
4086                DRM_ERROR("radeon: moving bo (%d).\n", r);
4087                radeon_sync_free(rdev, &sync, NULL);
4088                return ERR_PTR(r);
4089        }
4090
4091        radeon_sync_resv(rdev, &sync, resv, false);
4092        radeon_sync_rings(rdev, &sync, ring->idx);
4093
4094        for (i = 0; i < num_loops; i++) {
4095                cur_size_in_bytes = size_in_bytes;
4096                if (cur_size_in_bytes > 0x1fffff)
4097                        cur_size_in_bytes = 0x1fffff;
4098                size_in_bytes -= cur_size_in_bytes;
4099                control = 0;
4100                if (size_in_bytes == 0)
4101                        control |= PACKET3_DMA_DATA_CP_SYNC;
4102                radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4103                radeon_ring_write(ring, control);
4104                radeon_ring_write(ring, lower_32_bits(src_offset));
4105                radeon_ring_write(ring, upper_32_bits(src_offset));
4106                radeon_ring_write(ring, lower_32_bits(dst_offset));
4107                radeon_ring_write(ring, upper_32_bits(dst_offset));
4108                radeon_ring_write(ring, cur_size_in_bytes);
4109                src_offset += cur_size_in_bytes;
4110                dst_offset += cur_size_in_bytes;
4111        }
4112
4113        r = radeon_fence_emit(rdev, &fence, ring->idx);
4114        if (r) {
4115                radeon_ring_unlock_undo(rdev, ring);
4116                radeon_sync_free(rdev, &sync, NULL);
4117                return ERR_PTR(r);
4118        }
4119
4120        radeon_ring_unlock_commit(rdev, ring, false);
4121        radeon_sync_free(rdev, &sync, fence);
4122
4123        return fence;
4124}
4125
4126/*
4127 * IB stuff
4128 */
4129/**
4130 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4131 *
4132 * @rdev: radeon_device pointer
4133 * @ib: radeon indirect buffer object
4134 *
4135 * Emits an DE (drawing engine) or CE (constant engine) IB
4136 * on the gfx ring.  IBs are usually generated by userspace
4137 * acceleration drivers and submitted to the kernel for
4138 * sheduling on the ring.  This function schedules the IB
4139 * on the gfx ring for execution by the GPU.
4140 */
4141void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4142{
4143        struct radeon_ring *ring = &rdev->ring[ib->ring];
4144        unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
4145        u32 header, control = INDIRECT_BUFFER_VALID;
4146
4147        if (ib->is_const_ib) {
4148                /* set switch buffer packet before const IB */
4149                radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4150                radeon_ring_write(ring, 0);
4151
4152                header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4153        } else {
4154                u32 next_rptr;
4155                if (ring->rptr_save_reg) {
4156                        next_rptr = ring->wptr + 3 + 4;
4157                        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4158                        radeon_ring_write(ring, ((ring->rptr_save_reg -
4159                                                  PACKET3_SET_UCONFIG_REG_START) >> 2));
4160                        radeon_ring_write(ring, next_rptr);
4161                } else if (rdev->wb.enabled) {
4162                        next_rptr = ring->wptr + 5 + 4;
4163                        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4164                        radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4165                        radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4166                        radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4167                        radeon_ring_write(ring, next_rptr);
4168                }
4169
4170                header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4171        }
4172
4173        control |= ib->length_dw | (vm_id << 24);
4174
4175        radeon_ring_write(ring, header);
4176        radeon_ring_write(ring,
4177#ifdef __BIG_ENDIAN
4178                          (2 << 0) |
4179#endif
4180                          (ib->gpu_addr & 0xFFFFFFFC));
4181        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4182        radeon_ring_write(ring, control);
4183}
4184
4185/**
4186 * cik_ib_test - basic gfx ring IB test
4187 *
4188 * @rdev: radeon_device pointer
4189 * @ring: radeon_ring structure holding ring information
4190 *
4191 * Allocate an IB and execute it on the gfx ring (CIK).
4192 * Provides a basic gfx ring test to verify that IBs are working.
4193 * Returns 0 on success, error on failure.
4194 */
4195int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4196{
4197        struct radeon_ib ib;
4198        uint32_t scratch;
4199        uint32_t tmp = 0;
4200        unsigned i;
4201        int r;
4202
4203        r = radeon_scratch_get(rdev, &scratch);
4204        if (r) {
4205                DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4206                return r;
4207        }
4208        WREG32(scratch, 0xCAFEDEAD);
4209        r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4210        if (r) {
4211                DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4212                radeon_scratch_free(rdev, scratch);
4213                return r;
4214        }
4215        ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4216        ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4217        ib.ptr[2] = 0xDEADBEEF;
4218        ib.length_dw = 3;
4219        r = radeon_ib_schedule(rdev, &ib, NULL, false);
4220        if (r) {
4221                radeon_scratch_free(rdev, scratch);
4222                radeon_ib_free(rdev, &ib);
4223                DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4224                return r;
4225        }
4226        r = radeon_fence_wait(ib.fence, false);
4227        if (r) {
4228                DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4229                radeon_scratch_free(rdev, scratch);
4230                radeon_ib_free(rdev, &ib);
4231                return r;
4232        }
4233        for (i = 0; i < rdev->usec_timeout; i++) {
4234                tmp = RREG32(scratch);
4235                if (tmp == 0xDEADBEEF)
4236                        break;
4237                DRM_UDELAY(1);
4238        }
4239        if (i < rdev->usec_timeout) {
4240                DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4241        } else {
4242                DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4243                          scratch, tmp);
4244                r = -EINVAL;
4245        }
4246        radeon_scratch_free(rdev, scratch);
4247        radeon_ib_free(rdev, &ib);
4248        return r;
4249}
4250
4251/*
4252 * CP.
4253 * On CIK, gfx and compute now have independant command processors.
4254 *
4255 * GFX
4256 * Gfx consists of a single ring and can process both gfx jobs and
4257 * compute jobs.  The gfx CP consists of three microengines (ME):
4258 * PFP - Pre-Fetch Parser
4259 * ME - Micro Engine
4260 * CE - Constant Engine
4261 * The PFP and ME make up what is considered the Drawing Engine (DE).
4262 * The CE is an asynchronous engine used for updating buffer desciptors
4263 * used by the DE so that they can be loaded into cache in parallel
4264 * while the DE is processing state update packets.
4265 *
4266 * Compute
4267 * The compute CP consists of two microengines (ME):
4268 * MEC1 - Compute MicroEngine 1
4269 * MEC2 - Compute MicroEngine 2
4270 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4271 * The queues are exposed to userspace and are programmed directly
4272 * by the compute runtime.
4273 */
4274/**
4275 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4276 *
4277 * @rdev: radeon_device pointer
4278 * @enable: enable or disable the MEs
4279 *
4280 * Halts or unhalts the gfx MEs.
4281 */
4282static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4283{
4284        if (enable)
4285                WREG32(CP_ME_CNTL, 0);
4286        else {
4287                if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4288                        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4289                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4290                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4291        }
4292        udelay(50);
4293}
4294
4295/**
4296 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4297 *
4298 * @rdev: radeon_device pointer
4299 *
4300 * Loads the gfx PFP, ME, and CE ucode.
4301 * Returns 0 for success, -EINVAL if the ucode is not available.
4302 */
4303static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4304{
4305        int i;
4306
4307        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4308                return -EINVAL;
4309
4310        cik_cp_gfx_enable(rdev, false);
4311
4312        if (rdev->new_fw) {
4313                const struct gfx_firmware_header_v1_0 *pfp_hdr =
4314                        (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4315                const struct gfx_firmware_header_v1_0 *ce_hdr =
4316                        (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4317                const struct gfx_firmware_header_v1_0 *me_hdr =
4318                        (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4319                const __le32 *fw_data;
4320                u32 fw_size;
4321
4322                radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4323                radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4324                radeon_ucode_print_gfx_hdr(&me_hdr->header);
4325
4326                /* PFP */
4327                fw_data = (const __le32 *)
4328                        (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4329                fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4330                WREG32(CP_PFP_UCODE_ADDR, 0);
4331                for (i = 0; i < fw_size; i++)
4332                        WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4333                WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
4334
4335                /* CE */
4336                fw_data = (const __le32 *)
4337                        (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4338                fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4339                WREG32(CP_CE_UCODE_ADDR, 0);
4340                for (i = 0; i < fw_size; i++)
4341                        WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4342                WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
4343
4344                /* ME */
4345                fw_data = (const __be32 *)
4346                        (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4347                fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4348                WREG32(CP_ME_RAM_WADDR, 0);
4349                for (i = 0; i < fw_size; i++)
4350                        WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4351                WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4352                WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
4353        } else {
4354                const __be32 *fw_data;
4355
4356                /* PFP */
4357                fw_data = (const __be32 *)rdev->pfp_fw->data;
4358                WREG32(CP_PFP_UCODE_ADDR, 0);
4359                for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4360                        WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4361                WREG32(CP_PFP_UCODE_ADDR, 0);
4362
4363                /* CE */
4364                fw_data = (const __be32 *)rdev->ce_fw->data;
4365                WREG32(CP_CE_UCODE_ADDR, 0);
4366                for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4367                        WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4368                WREG32(CP_CE_UCODE_ADDR, 0);
4369
4370                /* ME */
4371                fw_data = (const __be32 *)rdev->me_fw->data;
4372                WREG32(CP_ME_RAM_WADDR, 0);
4373                for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4374                        WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4375                WREG32(CP_ME_RAM_WADDR, 0);
4376        }
4377
4378        return 0;
4379}
4380
4381/**
4382 * cik_cp_gfx_start - start the gfx ring
4383 *
4384 * @rdev: radeon_device pointer
4385 *
4386 * Enables the ring and loads the clear state context and other
4387 * packets required to init the ring.
4388 * Returns 0 for success, error for failure.
4389 */
4390static int cik_cp_gfx_start(struct radeon_device *rdev)
4391{
4392        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4393        int r, i;
4394
4395        /* init the CP */
4396        WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4397        WREG32(CP_ENDIAN_SWAP, 0);
4398        WREG32(CP_DEVICE_ID, 1);
4399
4400        cik_cp_gfx_enable(rdev, true);
4401
4402        r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4403        if (r) {
4404                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4405                return r;
4406        }
4407
4408        /* init the CE partitions.  CE only used for gfx on CIK */
4409        radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4410        radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4411        radeon_ring_write(ring, 0x8000);
4412        radeon_ring_write(ring, 0x8000);
4413
4414        /* setup clear context state */
4415        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4416        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4417
4418        radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4419        radeon_ring_write(ring, 0x80000000);
4420        radeon_ring_write(ring, 0x80000000);
4421
4422        for (i = 0; i < cik_default_size; i++)
4423                radeon_ring_write(ring, cik_default_state[i]);
4424
4425        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4426        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4427
4428        /* set clear context state */
4429        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4430        radeon_ring_write(ring, 0);
4431
4432        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4433        radeon_ring_write(ring, 0x00000316);
4434        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4435        radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4436
4437        radeon_ring_unlock_commit(rdev, ring, false);
4438
4439        return 0;
4440}
4441
4442/**
4443 * cik_cp_gfx_fini - stop the gfx ring
4444 *
4445 * @rdev: radeon_device pointer
4446 *
4447 * Stop the gfx ring and tear down the driver ring
4448 * info.
4449 */
4450static void cik_cp_gfx_fini(struct radeon_device *rdev)
4451{
4452        cik_cp_gfx_enable(rdev, false);
4453        radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4454}
4455
4456/**
4457 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4458 *
4459 * @rdev: radeon_device pointer
4460 *
4461 * Program the location and size of the gfx ring buffer
4462 * and test it to make sure it's working.
4463 * Returns 0 for success, error for failure.
4464 */
4465static int cik_cp_gfx_resume(struct radeon_device *rdev)
4466{
4467        struct radeon_ring *ring;
4468        u32 tmp;
4469        u32 rb_bufsz;
4470        u64 rb_addr;
4471        int r;
4472
4473        WREG32(CP_SEM_WAIT_TIMER, 0x0);
4474        if (rdev->family != CHIP_HAWAII)
4475                WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4476
4477        /* Set the write pointer delay */
4478        WREG32(CP_RB_WPTR_DELAY, 0);
4479
4480        /* set the RB to use vmid 0 */
4481        WREG32(CP_RB_VMID, 0);
4482
4483        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4484
4485        /* ring 0 - compute and gfx */
4486        /* Set ring buffer size */
4487        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4488        rb_bufsz = order_base_2(ring->ring_size / 8);
4489        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4490#ifdef __BIG_ENDIAN
4491        tmp |= BUF_SWAP_32BIT;
4492#endif
4493        WREG32(CP_RB0_CNTL, tmp);
4494
4495        /* Initialize the ring buffer's read and write pointers */
4496        WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4497        ring->wptr = 0;
4498        WREG32(CP_RB0_WPTR, ring->wptr);
4499
4500        /* set the wb address wether it's enabled or not */
4501        WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4502        WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4503
4504        /* scratch register shadowing is no longer supported */
4505        WREG32(SCRATCH_UMSK, 0);
4506
4507        if (!rdev->wb.enabled)
4508                tmp |= RB_NO_UPDATE;
4509
4510        mdelay(1);
4511        WREG32(CP_RB0_CNTL, tmp);
4512
4513        rb_addr = ring->gpu_addr >> 8;
4514        WREG32(CP_RB0_BASE, rb_addr);
4515        WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4516
4517        /* start the ring */
4518        cik_cp_gfx_start(rdev);
4519        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4520        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4521        if (r) {
4522                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4523                return r;
4524        }
4525
4526        if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4527                radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4528
4529        return 0;
4530}
4531
4532u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4533                     struct radeon_ring *ring)
4534{
4535        u32 rptr;
4536
4537        if (rdev->wb.enabled)
4538                rptr = rdev->wb.wb[ring->rptr_offs/4];
4539        else
4540                rptr = RREG32(CP_RB0_RPTR);
4541
4542        return rptr;
4543}
4544
4545u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4546                     struct radeon_ring *ring)
4547{
4548        u32 wptr;
4549
4550        wptr = RREG32(CP_RB0_WPTR);
4551
4552        return wptr;
4553}
4554
4555void cik_gfx_set_wptr(struct radeon_device *rdev,
4556                      struct radeon_ring *ring)
4557{
4558        WREG32(CP_RB0_WPTR, ring->wptr);
4559        (void)RREG32(CP_RB0_WPTR);
4560}
4561
4562u32 cik_compute_get_rptr(struct radeon_device *rdev,
4563                         struct radeon_ring *ring)
4564{
4565        u32 rptr;
4566
4567        if (rdev->wb.enabled) {
4568                rptr = rdev->wb.wb[ring->rptr_offs/4];
4569        } else {
4570                mutex_lock(&rdev->srbm_mutex);
4571                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4572                rptr = RREG32(CP_HQD_PQ_RPTR);
4573                cik_srbm_select(rdev, 0, 0, 0, 0);
4574                mutex_unlock(&rdev->srbm_mutex);
4575        }
4576
4577        return rptr;
4578}
4579
4580u32 cik_compute_get_wptr(struct radeon_device *rdev,
4581                         struct radeon_ring *ring)
4582{
4583        u32 wptr;
4584
4585        if (rdev->wb.enabled) {
4586                /* XXX check if swapping is necessary on BE */
4587                wptr = rdev->wb.wb[ring->wptr_offs/4];
4588        } else {
4589                mutex_lock(&rdev->srbm_mutex);
4590                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4591                wptr = RREG32(CP_HQD_PQ_WPTR);
4592                cik_srbm_select(rdev, 0, 0, 0, 0);
4593                mutex_unlock(&rdev->srbm_mutex);
4594        }
4595
4596        return wptr;
4597}
4598
4599void cik_compute_set_wptr(struct radeon_device *rdev,
4600                          struct radeon_ring *ring)
4601{
4602        /* XXX check if swapping is necessary on BE */
4603        rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4604        WDOORBELL32(ring->doorbell_index, ring->wptr);
4605}
4606
4607static void cik_compute_stop(struct radeon_device *rdev,
4608                             struct radeon_ring *ring)
4609{
4610        u32 j, tmp;
4611
4612        cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4613        /* Disable wptr polling. */
4614        tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4615        tmp &= ~WPTR_POLL_EN;
4616        WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4617        /* Disable HQD. */
4618        if (RREG32(CP_HQD_ACTIVE) & 1) {
4619                WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4620                for (j = 0; j < rdev->usec_timeout; j++) {
4621                        if (!(RREG32(CP_HQD_ACTIVE) & 1))
4622                                break;
4623                        udelay(1);
4624                }
4625                WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4626                WREG32(CP_HQD_PQ_RPTR, 0);
4627                WREG32(CP_HQD_PQ_WPTR, 0);
4628        }
4629        cik_srbm_select(rdev, 0, 0, 0, 0);
4630}
4631
4632/**
4633 * cik_cp_compute_enable - enable/disable the compute CP MEs
4634 *
4635 * @rdev: radeon_device pointer
4636 * @enable: enable or disable the MEs
4637 *
4638 * Halts or unhalts the compute MEs.
4639 */
4640static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4641{
4642        if (enable)
4643                WREG32(CP_MEC_CNTL, 0);
4644        else {
4645                /*
4646                 * To make hibernation reliable we need to clear compute ring
4647                 * configuration before halting the compute ring.
4648                 */
4649                mutex_lock(&rdev->srbm_mutex);
4650                cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4651                cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4652                mutex_unlock(&rdev->srbm_mutex);
4653
4654                WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4655                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4656                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4657        }
4658        udelay(50);
4659}
4660
4661/**
4662 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4663 *
4664 * @rdev: radeon_device pointer
4665 *
4666 * Loads the compute MEC1&2 ucode.
4667 * Returns 0 for success, -EINVAL if the ucode is not available.
4668 */
4669static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4670{
4671        int i;
4672
4673        if (!rdev->mec_fw)
4674                return -EINVAL;
4675
4676        cik_cp_compute_enable(rdev, false);
4677
4678        if (rdev->new_fw) {
4679                const struct gfx_firmware_header_v1_0 *mec_hdr =
4680                        (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4681                const __le32 *fw_data;
4682                u32 fw_size;
4683
4684                radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4685
4686                /* MEC1 */
4687                fw_data = (const __le32 *)
4688                        (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4689                fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4690                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4691                for (i = 0; i < fw_size; i++)
4692                        WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4693                WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4694
4695                /* MEC2 */
4696                if (rdev->family == CHIP_KAVERI) {
4697                        const struct gfx_firmware_header_v1_0 *mec2_hdr =
4698                                (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4699
4700                        fw_data = (const __le32 *)
4701                                (rdev->mec2_fw->data +
4702                                 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4703                        fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4704                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4705                        for (i = 0; i < fw_size; i++)
4706                                WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4707                        WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4708                }
4709        } else {
4710                const __be32 *fw_data;
4711
4712                /* MEC1 */
4713                fw_data = (const __be32 *)rdev->mec_fw->data;
4714                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4715                for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4716                        WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4717                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4718
4719                if (rdev->family == CHIP_KAVERI) {
4720                        /* MEC2 */
4721                        fw_data = (const __be32 *)rdev->mec_fw->data;
4722                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4723                        for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4724                                WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4725                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4726                }
4727        }
4728
4729        return 0;
4730}
4731
4732/**
4733 * cik_cp_compute_start - start the compute queues
4734 *
4735 * @rdev: radeon_device pointer
4736 *
4737 * Enable the compute queues.
4738 * Returns 0 for success, error for failure.
4739 */
4740static int cik_cp_compute_start(struct radeon_device *rdev)
4741{
4742        cik_cp_compute_enable(rdev, true);
4743
4744        return 0;
4745}
4746
4747/**
4748 * cik_cp_compute_fini - stop the compute queues
4749 *
4750 * @rdev: radeon_device pointer
4751 *
4752 * Stop the compute queues and tear down the driver queue
4753 * info.
4754 */
4755static void cik_cp_compute_fini(struct radeon_device *rdev)
4756{
4757        int i, idx, r;
4758
4759        cik_cp_compute_enable(rdev, false);
4760
4761        for (i = 0; i < 2; i++) {
4762                if (i == 0)
4763                        idx = CAYMAN_RING_TYPE_CP1_INDEX;
4764                else
4765                        idx = CAYMAN_RING_TYPE_CP2_INDEX;
4766
4767                if (rdev->ring[idx].mqd_obj) {
4768                        r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4769                        if (unlikely(r != 0))
4770                                dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4771
4772                        radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4773                        radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4774
4775                        radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4776                        rdev->ring[idx].mqd_obj = NULL;
4777                }
4778        }
4779}
4780
4781static void cik_mec_fini(struct radeon_device *rdev)
4782{
4783        int r;
4784
4785        if (rdev->mec.hpd_eop_obj) {
4786                r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4787                if (unlikely(r != 0))
4788                        dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4789                radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4790                radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4791
4792                radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4793                rdev->mec.hpd_eop_obj = NULL;
4794        }
4795}
4796
4797#define MEC_HPD_SIZE 2048
4798
4799static int cik_mec_init(struct radeon_device *rdev)
4800{
4801        int r;
4802        u32 *hpd;
4803
4804        /*
4805         * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4806         * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4807         * Nonetheless, we assign only 1 pipe because all other pipes will
4808         * be handled by KFD
4809         */
4810        rdev->mec.num_mec = 1;
4811        rdev->mec.num_pipe = 1;
4812        rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4813
4814        if (rdev->mec.hpd_eop_obj == NULL) {
4815                r = radeon_bo_create(rdev,
4816                                     rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4817                                     PAGE_SIZE, true,
4818                                     RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4819                                     &rdev->mec.hpd_eop_obj);
4820                if (r) {
4821                        dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4822                        return r;
4823                }
4824        }
4825
4826        r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4827        if (unlikely(r != 0)) {
4828                cik_mec_fini(rdev);
4829                return r;
4830        }
4831        r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4832                          &rdev->mec.hpd_eop_gpu_addr);
4833        if (r) {
4834                dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4835                cik_mec_fini(rdev);
4836                return r;
4837        }
4838        r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4839        if (r) {
4840                dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4841                cik_mec_fini(rdev);
4842                return r;
4843        }
4844
4845        /* clear memory.  Not sure if this is required or not */
4846        memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4847
4848        radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4849        radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4850
4851        return 0;
4852}
4853
4854struct hqd_registers
4855{
4856        u32 cp_mqd_base_addr;
4857        u32 cp_mqd_base_addr_hi;
4858        u32 cp_hqd_active;
4859        u32 cp_hqd_vmid;
4860        u32 cp_hqd_persistent_state;
4861        u32 cp_hqd_pipe_priority;
4862        u32 cp_hqd_queue_priority;
4863        u32 cp_hqd_quantum;
4864        u32 cp_hqd_pq_base;
4865        u32 cp_hqd_pq_base_hi;
4866        u32 cp_hqd_pq_rptr;
4867        u32 cp_hqd_pq_rptr_report_addr;
4868        u32 cp_hqd_pq_rptr_report_addr_hi;
4869        u32 cp_hqd_pq_wptr_poll_addr;
4870        u32 cp_hqd_pq_wptr_poll_addr_hi;
4871        u32 cp_hqd_pq_doorbell_control;
4872        u32 cp_hqd_pq_wptr;
4873        u32 cp_hqd_pq_control;
4874        u32 cp_hqd_ib_base_addr;
4875        u32 cp_hqd_ib_base_addr_hi;
4876        u32 cp_hqd_ib_rptr;
4877        u32 cp_hqd_ib_control;
4878        u32 cp_hqd_iq_timer;
4879        u32 cp_hqd_iq_rptr;
4880        u32 cp_hqd_dequeue_request;
4881        u32 cp_hqd_dma_offload;
4882        u32 cp_hqd_sema_cmd;
4883        u32 cp_hqd_msg_type;
4884        u32 cp_hqd_atomic0_preop_lo;
4885        u32 cp_hqd_atomic0_preop_hi;
4886        u32 cp_hqd_atomic1_preop_lo;
4887        u32 cp_hqd_atomic1_preop_hi;
4888        u32 cp_hqd_hq_scheduler0;
4889        u32 cp_hqd_hq_scheduler1;
4890        u32 cp_mqd_control;
4891};
4892
4893struct bonaire_mqd
4894{
4895        u32 header;
4896        u32 dispatch_initiator;
4897        u32 dimensions[3];
4898        u32 start_idx[3];
4899        u32 num_threads[3];
4900        u32 pipeline_stat_enable;
4901        u32 perf_counter_enable;
4902        u32 pgm[2];
4903        u32 tba[2];
4904        u32 tma[2];
4905        u32 pgm_rsrc[2];
4906        u32 vmid;
4907        u32 resource_limits;
4908        u32 static_thread_mgmt01[2];
4909        u32 tmp_ring_size;
4910        u32 static_thread_mgmt23[2];
4911        u32 restart[3];
4912        u32 thread_trace_enable;
4913        u32 reserved1;
4914        u32 user_data[16];
4915        u32 vgtcs_invoke_count[2];
4916        struct hqd_registers queue_state;
4917        u32 dequeue_cntr;
4918        u32 interrupt_queue[64];
4919};
4920
4921/**
4922 * cik_cp_compute_resume - setup the compute queue registers
4923 *
4924 * @rdev: radeon_device pointer
4925 *
4926 * Program the compute queues and test them to make sure they
4927 * are working.
4928 * Returns 0 for success, error for failure.
4929 */
4930static int cik_cp_compute_resume(struct radeon_device *rdev)
4931{
4932        int r, i, j, idx;
4933        u32 tmp;
4934        bool use_doorbell = true;
4935        u64 hqd_gpu_addr;
4936        u64 mqd_gpu_addr;
4937        u64 eop_gpu_addr;
4938        u64 wb_gpu_addr;
4939        u32 *buf;
4940        struct bonaire_mqd *mqd;
4941
4942        r = cik_cp_compute_start(rdev);
4943        if (r)
4944                return r;
4945
4946        /* fix up chicken bits */
4947        tmp = RREG32(CP_CPF_DEBUG);
4948        tmp |= (1 << 23);
4949        WREG32(CP_CPF_DEBUG, tmp);
4950
4951        /* init the pipes */
4952        mutex_lock(&rdev->srbm_mutex);
4953
4954        eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
4955
4956        cik_srbm_select(rdev, 0, 0, 0, 0);
4957
4958        /* write the EOP addr */
4959        WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4960        WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4961
4962        /* set the VMID assigned */
4963        WREG32(CP_HPD_EOP_VMID, 0);
4964
4965        /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4966        tmp = RREG32(CP_HPD_EOP_CONTROL);
4967        tmp &= ~EOP_SIZE_MASK;
4968        tmp |= order_base_2(MEC_HPD_SIZE / 8);
4969        WREG32(CP_HPD_EOP_CONTROL, tmp);
4970
4971        mutex_unlock(&rdev->srbm_mutex);
4972
4973        /* init the queues.  Just two for now. */
4974        for (i = 0; i < 2; i++) {
4975                if (i == 0)
4976                        idx = CAYMAN_RING_TYPE_CP1_INDEX;
4977                else
4978                        idx = CAYMAN_RING_TYPE_CP2_INDEX;
4979
4980                if (rdev->ring[idx].mqd_obj == NULL) {
4981                        r = radeon_bo_create(rdev,
4982                                             sizeof(struct bonaire_mqd),
4983                                             PAGE_SIZE, true,
4984                                             RADEON_GEM_DOMAIN_GTT, 0, NULL,
4985                                             NULL, &rdev->ring[idx].mqd_obj);
4986                        if (r) {
4987                                dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4988                                return r;
4989                        }
4990                }
4991
4992                r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4993                if (unlikely(r != 0)) {
4994                        cik_cp_compute_fini(rdev);
4995                        return r;
4996                }
4997                r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4998                                  &mqd_gpu_addr);
4999                if (r) {
5000                        dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);

5001                        cik_cp_compute_fini(rdev);
5002                        return r;
5003                }
5004                r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
5005                if (r) {
5006                        dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
5007                        cik_cp_compute_fini(rdev);
5008                        return r;
5009                }
5010
5011                /* init the mqd struct */
5012                memset(buf, 0, sizeof(struct bonaire_mqd));
5013
5014                mqd = (struct bonaire_mqd *)buf;
5015                mqd->header = 0xC0310800;
5016                mqd->static_thread_mgmt01[0] = 0xffffffff;
5017                mqd->static_thread_mgmt01[1] = 0xffffffff;
5018                mqd->static_thread_mgmt23[0] = 0xffffffff;
5019                mqd->static_thread_mgmt23[1] = 0xffffffff;
5020
5021                mutex_lock(&rdev->srbm_mutex);
5022                cik_srbm_select(rdev, rdev->ring[idx].me,
5023                                rdev->ring[idx].pipe,
5024                                rdev->ring[idx].queue, 0);
5025
5026                /* disable wptr polling */
5027                tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
5028                tmp &= ~WPTR_POLL_EN;
5029                WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
5030
5031                /* enable doorbell? */
5032                mqd->queue_state.cp_hqd_pq_doorbell_control =
5033                        RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5034                if (use_doorbell)
5035                        mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5036                else
5037                        mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
5038                WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5039                       mqd->queue_state.cp_hqd_pq_doorbell_control);
5040
5041                /* disable the queue if it's active */
5042                mqd->queue_state.cp_hqd_dequeue_request = 0;
5043                mqd->queue_state.cp_hqd_pq_rptr = 0;
5044                mqd->queue_state.cp_hqd_pq_wptr= 0;
5045                if (RREG32(CP_HQD_ACTIVE) & 1) {
5046                        WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
5047                        for (j = 0; j < rdev->usec_timeout; j++) {
5048                                if (!(RREG32(CP_HQD_ACTIVE) & 1))
5049                                        break;
5050                                udelay(1);
5051                        }
5052                        WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
5053                        WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
5054                        WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5055                }
5056
5057                /* set the pointer to the MQD */
5058                mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
5059                mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
5060                WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
5061                WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
5062                /* set MQD vmid to 0 */
5063                mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
5064                mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
5065                WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
5066
5067                /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
5068                hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
5069                mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
5070                mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
5071                WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
5072                WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
5073
5074                /* set up the HQD, this is similar to CP_RB0_CNTL */
5075                mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
5076                mqd->queue_state.cp_hqd_pq_control &=
5077                        ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
5078
5079                mqd->queue_state.cp_hqd_pq_control |=
5080                        order_base_2(rdev->ring[idx].ring_size / 8);
5081                mqd->queue_state.cp_hqd_pq_control |=
5082                        (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
5083#ifdef __BIG_ENDIAN
5084                mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
5085#endif
5086                mqd->queue_state.cp_hqd_pq_control &=
5087                        ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
5088                mqd->queue_state.cp_hqd_pq_control |=
5089                        PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
5090                WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
5091
5092                /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
5093                if (i == 0)
5094                        wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
5095                else
5096                        wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
5097                mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
5098                mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
5099                WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
5100                WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
5101                       mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
5102
5103                /* set the wb address wether it's enabled or not */
5104                if (i == 0)
5105                        wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
5106                else
5107                        wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
5108                mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
5109                mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
5110                        upper_32_bits(wb_gpu_addr) & 0xffff;
5111                WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
5112                       mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5113                WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5114                       mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5115
5116                /* enable the doorbell if requested */
5117                if (use_doorbell) {
5118                        mqd->queue_state.cp_hqd_pq_doorbell_control =
5119                                RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5120                        mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5121                        mqd->queue_state.cp_hqd_pq_doorbell_control |=
5122                                DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5123                        mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5124                        mqd->queue_state.cp_hqd_pq_doorbell_control &=
5125                                ~(DOORBELL_SOURCE | DOORBELL_HIT);
5126
5127                } else {
5128                        mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5129                }
5130                WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5131                       mqd->queue_state.cp_hqd_pq_doorbell_control);
5132
5133                /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5134                rdev->ring[idx].wptr = 0;
5135                mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5136                WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5137                mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5138
5139                /* set the vmid for the queue */
5140                mqd->queue_state.cp_hqd_vmid = 0;
5141                WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5142
5143                /* activate the queue */
5144                mqd->queue_state.cp_hqd_active = 1;
5145                WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5146
5147                cik_srbm_select(rdev, 0, 0, 0, 0);
5148                mutex_unlock(&rdev->srbm_mutex);
5149
5150                radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5151                radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5152
5153                rdev->ring[idx].ready = true;
5154                r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5155                if (r)
5156                        rdev->ring[idx].ready = false;
5157        }
5158
5159        return 0;
5160}
5161
5162static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5163{
5164        cik_cp_gfx_enable(rdev, enable);
5165        cik_cp_compute_enable(rdev, enable);
5166}
5167
5168static int cik_cp_load_microcode(struct radeon_device *rdev)
5169{
5170        int r;
5171
5172        r = cik_cp_gfx_load_microcode(rdev);
5173        if (r)
5174                return r;
5175        r = cik_cp_compute_load_microcode(rdev);
5176        if (r)
5177                return r;
5178
5179        return 0;
5180}
5181
5182static void cik_cp_fini(struct radeon_device *rdev)
5183{
5184        cik_cp_gfx_fini(rdev);
5185        cik_cp_compute_fini(rdev);
5186}
5187
5188static int cik_cp_resume(struct radeon_device *rdev)
5189{
5190        int r;
5191
5192        cik_enable_gui_idle_interrupt(rdev, false);
5193
5194        r = cik_cp_load_microcode(rdev);
5195        if (r)
5196                return r;
5197
5198        r = cik_cp_gfx_resume(rdev);
5199        if (r)
5200                return r;
5201        r = cik_cp_compute_resume(rdev);
5202        if (r)
5203                return r;
5204
5205        cik_enable_gui_idle_interrupt(rdev, true);
5206
5207        return 0;
5208}
5209
5210static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5211{
5212        dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
5213                RREG32(GRBM_STATUS));
5214        dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
5215                RREG32(GRBM_STATUS2));
5216        dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
5217                RREG32(GRBM_STATUS_SE0));
5218        dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
5219                RREG32(GRBM_STATUS_SE1));
5220        dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
5221                RREG32(GRBM_STATUS_SE2));
5222        dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
5223                RREG32(GRBM_STATUS_SE3));
5224        dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
5225                RREG32(SRBM_STATUS));
5226        dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
5227                RREG32(SRBM_STATUS2));
5228        dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
5229                RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5230        dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
5231                 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5232        dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5233        dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
5234                 RREG32(CP_STALLED_STAT1));
5235        dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
5236                 RREG32(CP_STALLED_STAT2));
5237        dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
5238                 RREG32(CP_STALLED_STAT3));
5239        dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
5240                 RREG32(CP_CPF_BUSY_STAT));
5241        dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
5242                 RREG32(CP_CPF_STALLED_STAT1));
5243        dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5244        dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5245        dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
5246                 RREG32(CP_CPC_STALLED_STAT1));
5247        dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5248}
5249
5250/**
5251 * cik_gpu_check_soft_reset - check which blocks are busy
5252 *
5253 * @rdev: radeon_device pointer
5254 *
5255 * Check which blocks are busy and return the relevant reset
5256 * mask to be used by cik_gpu_soft_reset().
5257 * Returns a mask of the blocks to be reset.
5258 */
5259u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5260{
5261        u32 reset_mask = 0;
5262        u32 tmp;
5263
5264        /* GRBM_STATUS */
5265        tmp = RREG32(GRBM_STATUS);
5266        if (tmp & (PA_BUSY | SC_BUSY |
5267                   BCI_BUSY | SX_BUSY |
5268                   TA_BUSY | VGT_BUSY |
5269                   DB_BUSY | CB_BUSY |
5270                   GDS_BUSY | SPI_BUSY |
5271                   IA_BUSY | IA_BUSY_NO_DMA))
5272                reset_mask |= RADEON_RESET_GFX;
5273
5274        if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5275                reset_mask |= RADEON_RESET_CP;
5276
5277        /* GRBM_STATUS2 */
5278        tmp = RREG32(GRBM_STATUS2);
5279        if (tmp & RLC_BUSY)
5280                reset_mask |= RADEON_RESET_RLC;
5281
5282        /* SDMA0_STATUS_REG */
5283        tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5284        if (!(tmp & SDMA_IDLE))
5285                reset_mask |= RADEON_RESET_DMA;
5286
5287        /* SDMA1_STATUS_REG */
5288        tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5289        if (!(tmp & SDMA_IDLE))
5290                reset_mask |= RADEON_RESET_DMA1;
5291
5292        /* SRBM_STATUS2 */
5293        tmp = RREG32(SRBM_STATUS2);
5294        if (tmp & SDMA_BUSY)
5295                reset_mask |= RADEON_RESET_DMA;
5296
5297        if (tmp & SDMA1_BUSY)
5298                reset_mask |= RADEON_RESET_DMA1;
5299
5300        /* SRBM_STATUS */
5301        tmp = RREG32(SRBM_STATUS);
5302
5303        if (tmp & IH_BUSY)
5304                reset_mask |= RADEON_RESET_IH;
5305
5306        if (tmp & SEM_BUSY)
5307                reset_mask |= RADEON_RESET_SEM;
5308
5309        if (tmp & GRBM_RQ_PENDING)
5310                reset_mask |= RADEON_RESET_GRBM;
5311
5312        if (tmp & VMC_BUSY)
5313                reset_mask |= RADEON_RESET_VMC;
5314
5315        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5316                   MCC_BUSY | MCD_BUSY))
5317                reset_mask |= RADEON_RESET_MC;
5318
5319        if (evergreen_is_display_hung(rdev))
5320                reset_mask |= RADEON_RESET_DISPLAY;
5321
5322        /* Skip MC reset as it's mostly likely not hung, just busy */
5323        if (reset_mask & RADEON_RESET_MC) {
5324                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5325                reset_mask &= ~RADEON_RESET_MC;
5326        }
5327
5328        return reset_mask;
5329}
5330
5331/**
5332 * cik_gpu_soft_reset - soft reset GPU
5333 *
5334 * @rdev: radeon_device pointer
5335 * @reset_mask: mask of which blocks to reset
5336 *
5337 * Soft reset the blocks specified in @reset_mask.
5338 */
5339static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5340{
5341        struct evergreen_mc_save save;
5342        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5343        u32 tmp;
5344
5345        if (reset_mask == 0)
5346                return;
5347
5348        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5349
5350        cik_print_gpu_status_regs(rdev);
5351        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5352                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5353        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5354                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5355
5356        /* disable CG/PG */
5357        cik_fini_pg(rdev);
5358        cik_fini_cg(rdev);
5359
5360        /* stop the rlc */
5361        cik_rlc_stop(rdev);
5362
5363        /* Disable GFX parsing/prefetching */
5364        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5365
5366        /* Disable MEC parsing/prefetching */
5367        WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5368
5369        if (reset_mask & RADEON_RESET_DMA) {
5370                /* sdma0 */
5371                tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5372                tmp |= SDMA_HALT;
5373                WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5374        }
5375        if (reset_mask & RADEON_RESET_DMA1) {
5376                /* sdma1 */
5377                tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5378                tmp |= SDMA_HALT;
5379                WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5380        }
5381
5382        evergreen_mc_stop(rdev, &save);
5383        if (evergreen_mc_wait_for_idle(rdev)) {
5384                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5385        }
5386
5387        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5388                grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5389
5390        if (reset_mask & RADEON_RESET_CP) {
5391                grbm_soft_reset |= SOFT_RESET_CP;
5392
5393                srbm_soft_reset |= SOFT_RESET_GRBM;
5394        }
5395
5396        if (reset_mask & RADEON_RESET_DMA)
5397                srbm_soft_reset |= SOFT_RESET_SDMA;
5398
5399        if (reset_mask & RADEON_RESET_DMA1)
5400                srbm_soft_reset |= SOFT_RESET_SDMA1;
5401
5402        if (reset_mask & RADEON_RESET_DISPLAY)
5403                srbm_soft_reset |= SOFT_RESET_DC;
5404
5405        if (reset_mask & RADEON_RESET_RLC)
5406                grbm_soft_reset |= SOFT_RESET_RLC;
5407
5408        if (reset_mask & RADEON_RESET_SEM)
5409                srbm_soft_reset |= SOFT_RESET_SEM;
5410
5411        if (reset_mask & RADEON_RESET_IH)
5412                srbm_soft_reset |= SOFT_RESET_IH;
5413
5414        if (reset_mask & RADEON_RESET_GRBM)
5415                srbm_soft_reset |= SOFT_RESET_GRBM;
5416
5417        if (reset_mask & RADEON_RESET_VMC)
5418                srbm_soft_reset |= SOFT_RESET_VMC;
5419
5420        if (!(rdev->flags & RADEON_IS_IGP)) {
5421                if (reset_mask & RADEON_RESET_MC)
5422                        srbm_soft_reset |= SOFT_RESET_MC;
5423        }
5424
5425        if (grbm_soft_reset) {
5426                tmp = RREG32(GRBM_SOFT_RESET);
5427                tmp |= grbm_soft_reset;
5428                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5429                WREG32(GRBM_SOFT_RESET, tmp);
5430                tmp = RREG32(GRBM_SOFT_RESET);
5431
5432                udelay(50);
5433
5434                tmp &= ~grbm_soft_reset;
5435                WREG32(GRBM_SOFT_RESET, tmp);
5436                tmp = RREG32(GRBM_SOFT_RESET);
5437        }
5438
5439        if (srbm_soft_reset) {
5440                tmp = RREG32(SRBM_SOFT_RESET);
5441                tmp |= srbm_soft_reset;
5442                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5443                WREG32(SRBM_SOFT_RESET, tmp);
5444                tmp = RREG32(SRBM_SOFT_RESET);
5445
5446                udelay(50);
5447
5448                tmp &= ~srbm_soft_reset;
5449                WREG32(SRBM_SOFT_RESET, tmp);
5450                tmp = RREG32(SRBM_SOFT_RESET);
5451        }
5452
5453        /* Wait a little for things to settle down */
5454        udelay(50);
5455
5456        evergreen_mc_resume(rdev, &save);
5457        udelay(50);
5458
5459        cik_print_gpu_status_regs(rdev);
5460}
5461
5462struct kv_reset_save_regs {
5463        u32 gmcon_reng_execute;
5464        u32 gmcon_misc;
5465        u32 gmcon_misc3;
5466};
5467
5468static void kv_save_regs_for_reset(struct radeon_device *rdev,
5469                                   struct kv_reset_save_regs *save)
5470{
5471        save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5472        save->gmcon_misc = RREG32(GMCON_MISC);
5473        save->gmcon_misc3 = RREG32(GMCON_MISC3);
5474
5475        WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5476        WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5477                                                STCTRL_STUTTER_EN));
5478}
5479
5480static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5481                                      struct kv_reset_save_regs *save)
5482{
5483        int i;
5484
5485        WREG32(GMCON_PGFSM_WRITE, 0);
5486        WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5487
5488        for (i = 0; i < 5; i++)
5489                WREG32(GMCON_PGFSM_WRITE, 0);
5490
5491        WREG32(GMCON_PGFSM_WRITE, 0);
5492        WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5493
5494        for (i = 0; i < 5; i++)
5495                WREG32(GMCON_PGFSM_WRITE, 0);
5496
5497        WREG32(GMCON_PGFSM_WRITE, 0x210000);
5498        WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5499
5500        for (i = 0; i < 5; i++)
5501                WREG32(GMCON_PGFSM_WRITE, 0);
5502
5503        WREG32(GMCON_PGFSM_WRITE, 0x21003);
5504        WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5505
5506        for (i = 0; i < 5; i++)
5507                WREG32(GMCON_PGFSM_WRITE, 0);
5508
5509        WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5510        WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5511
5512        for (i = 0; i < 5; i++)
5513                WREG32(GMCON_PGFSM_WRITE, 0);
5514
5515        WREG32(GMCON_PGFSM_WRITE, 0);
5516        WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5517
5518        for (i = 0; i < 5; i++)
5519                WREG32(GMCON_PGFSM_WRITE, 0);
5520
5521        WREG32(GMCON_PGFSM_WRITE, 0x420000);
5522        WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5523
5524        for (i = 0; i < 5; i++)
5525                WREG32(GMCON_PGFSM_WRITE, 0);
5526
5527        WREG32(GMCON_PGFSM_WRITE, 0x120202);
5528        WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5529
5530        for (i = 0; i < 5; i++)
5531                WREG32(GMCON_PGFSM_WRITE, 0);
5532
5533        WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5534        WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5535
5536        for (i = 0; i < 5; i++)
5537                WREG32(GMCON_PGFSM_WRITE, 0);
5538
5539        WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5540        WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5541
5542        for (i = 0; i < 5; i++)
5543                WREG32(GMCON_PGFSM_WRITE, 0);
5544
5545        WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5546        WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5547
5548        WREG32(GMCON_MISC3, save->gmcon_misc3);
5549        WREG32(GMCON_MISC, save->gmcon_misc);
5550        WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5551}
5552
5553static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5554{
5555        struct evergreen_mc_save save;
5556        struct kv_reset_save_regs kv_save = { 0 };
5557        u32 tmp, i;
5558
5559        dev_info(rdev->dev, "GPU pci config reset\n");
5560
5561        /* disable dpm? */
5562
5563        /* disable cg/pg */
5564        cik_fini_pg(rdev);
5565        cik_fini_cg(rdev);
5566
5567        /* Disable GFX parsing/prefetching */
5568        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5569
5570        /* Disable MEC parsing/prefetching */
5571        WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5572
5573        /* sdma0 */
5574        tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5575        tmp |= SDMA_HALT;
5576        WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5577        /* sdma1 */
5578        tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5579        tmp |= SDMA_HALT;
5580        WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5581        /* XXX other engines? */
5582
5583        /* halt the rlc, disable cp internal ints */
5584        cik_rlc_stop(rdev);
5585
5586        udelay(50);
5587
5588        /* disable mem access */
5589        evergreen_mc_stop(rdev, &save);
5590        if (evergreen_mc_wait_for_idle(rdev)) {
5591                dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5592        }
5593
5594        if (rdev->flags & RADEON_IS_IGP)
5595                kv_save_regs_for_reset(rdev, &kv_save);
5596
5597        /* disable BM */
5598        pci_clear_master(rdev->pdev);
5599        /* reset */
5600        radeon_pci_config_reset(rdev);
5601
5602        udelay(100);
5603
5604        /* wait for asic to come out of reset */
5605        for (i = 0; i < rdev->usec_timeout; i++) {
5606                if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5607                        break;
5608                udelay(1);
5609        }
5610
5611        /* does asic init need to be run first??? */
5612        if (rdev->flags & RADEON_IS_IGP)
5613                kv_restore_regs_for_reset(rdev, &kv_save);
5614}
5615
5616/**
5617 * cik_asic_reset - soft reset GPU
5618 *
5619 * @rdev: radeon_device pointer
5620 *
5621 * Look up which blocks are hung and attempt
5622 * to reset them.
5623 * Returns 0 for success.
5624 */
5625int cik_asic_reset(struct radeon_device *rdev)
5626{
5627        u32 reset_mask;
5628
5629        reset_mask = cik_gpu_check_soft_reset(rdev);
5630
5631        if (reset_mask)
5632                r600_set_bios_scratch_engine_hung(rdev, true);
5633
5634        /* try soft reset */
5635        cik_gpu_soft_reset(rdev, reset_mask);
5636
5637        reset_mask = cik_gpu_check_soft_reset(rdev);
5638
5639        /* try pci config reset */
5640        if (reset_mask && radeon_hard_reset)
5641                cik_gpu_pci_config_reset(rdev);
5642
5643        reset_mask = cik_gpu_check_soft_reset(rdev);
5644
5645        if (!reset_mask)
5646                r600_set_bios_scratch_engine_hung(rdev, false);
5647
5648        return 0;
5649}
5650
5651/**
5652 * cik_gfx_is_lockup - check if the 3D engine is locked up
5653 *
5654 * @rdev: radeon_device pointer
5655 * @ring: radeon_ring structure holding ring information
5656 *
5657 * Check if the 3D engine is locked up (CIK).
5658 * Returns true if the engine is locked, false if not.
5659 */
5660bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5661{
5662        u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5663
5664        if (!(reset_mask & (RADEON_RESET_GFX |
5665                            RADEON_RESET_COMPUTE |
5666                            RADEON_RESET_CP))) {
5667                radeon_ring_lockup_update(rdev, ring);
5668                return false;
5669        }
5670        return radeon_ring_test_lockup(rdev, ring);
5671}
5672
5673/* MC */
5674/**
5675 * cik_mc_program - program the GPU memory controller
5676 *
5677 * @rdev: radeon_device pointer
5678 *
5679 * Set the location of vram, gart, and AGP in the GPU's
5680 * physical address space (CIK).
5681 */
5682static void cik_mc_program(struct radeon_device *rdev)
5683{
5684        struct evergreen_mc_save save;
5685        u32 tmp;
5686        int i, j;
5687
5688        /* Initialize HDP */
5689        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5690                WREG32((0x2c14 + j), 0x00000000);
5691                WREG32((0x2c18 + j), 0x00000000);
5692                WREG32((0x2c1c + j), 0x00000000);
5693                WREG32((0x2c20 + j), 0x00000000);
5694                WREG32((0x2c24 + j), 0x00000000);
5695        }
5696        WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5697
5698        evergreen_mc_stop(rdev, &save);
5699        if (radeon_mc_wait_for_idle(rdev)) {
5700                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5701        }
5702        /* Lockout access through VGA aperture*/
5703        WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5704        /* Update configuration */
5705        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5706               rdev->mc.vram_start >> 12);
5707        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5708               rdev->mc.vram_end >> 12);
5709        WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5710               rdev->vram_scratch.gpu_addr >> 12);
5711        tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5712        tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5713        WREG32(MC_VM_FB_LOCATION, tmp);
5714        /* XXX double check these! */
5715        WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5716        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5717        WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5718        WREG32(MC_VM_AGP_BASE, 0);
5719        WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5720        WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5721        if (radeon_mc_wait_for_idle(rdev)) {
5722                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5723        }
5724        evergreen_mc_resume(rdev, &save);
5725        /* we need to own VRAM, so turn off the VGA renderer here
5726         * to stop it overwriting our objects */
5727        rv515_vga_render_disable(rdev);
5728}
5729
5730/**
5731 * cik_mc_init - initialize the memory controller driver params
5732 *
5733 * @rdev: radeon_device pointer
5734 *
5735 * Look up the amount of vram, vram width, and decide how to place
5736 * vram and gart within the GPU's physical address space (CIK).
5737 * Returns 0 for success.
5738 */
5739static int cik_mc_init(struct radeon_device *rdev)
5740{
5741        u32 tmp;
5742        int chansize, numchan;
5743
5744        /* Get VRAM informations */
5745        rdev->mc.vram_is_ddr = true;
5746        tmp = RREG32(MC_ARB_RAMCFG);
5747        if (tmp & CHANSIZE_MASK) {
5748                chansize = 64;
5749        } else {
5750                chansize = 32;
5751        }
5752        tmp = RREG32(MC_SHARED_CHMAP);
5753        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5754        case 0:
5755        default:
5756                numchan = 1;
5757                break;
5758        case 1:
5759                numchan = 2;
5760                break;
5761        case 2:
5762                numchan = 4;
5763                break;
5764        case 3:
5765                numchan = 8;
5766                break;
5767        case 4:
5768                numchan = 3;
5769                break;
5770        case 5:
5771                numchan = 6;
5772                break;
5773        case 6:
5774                numchan = 10;
5775                break;
5776        case 7:
5777                numchan = 12;
5778                break;
5779        case 8:
5780                numchan = 16;
5781                break;
5782        }
5783        rdev->mc.vram_width = numchan * chansize;
5784        /* Could aper size report 0 ? */
5785        rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5786        rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5787        /* size in MB on si */
5788        rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5789        rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5790        rdev->mc.visible_vram_size = rdev->mc.aper_size;
5791        si_vram_gtt_location(rdev, &rdev->mc);
5792        radeon_update_bandwidth_info(rdev);
5793
5794        return 0;
5795}
5796
5797/*
5798 * GART
5799 * VMID 0 is the physical GPU addresses as used by the kernel.
5800 * VMIDs 1-15 are used for userspace clients and are handled
5801 * by the radeon vm/hsa code.
5802 */
5803/**
5804 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5805 *
5806 * @rdev: radeon_device pointer
5807 *
5808 * Flush the TLB for the VMID 0 page table (CIK).
5809 */
5810void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5811{
5812        /* flush hdp cache */
5813        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5814
5815        /* bits 0-15 are the VM contexts0-15 */
5816        WREG32(VM_INVALIDATE_REQUEST, 0x1);
5817}
5818
5819static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5820{
5821        int i;
5822        uint32_t sh_mem_bases, sh_mem_config;
5823
5824        sh_mem_bases = 0x6000 | 0x6000 << 16;
5825        sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5826        sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5827
5828        mutex_lock(&rdev->srbm_mutex);
5829        for (i = 8; i < 16; i++) {
5830                cik_srbm_select(rdev, 0, 0, 0, i);
5831                /* CP and shaders */
5832                WREG32(SH_MEM_CONFIG, sh_mem_config);
5833                WREG32(SH_MEM_APE1_BASE, 1);
5834                WREG32(SH_MEM_APE1_LIMIT, 0);
5835                WREG32(SH_MEM_BASES, sh_mem_bases);
5836        }
5837        cik_srbm_select(rdev, 0, 0, 0, 0);
5838        mutex_unlock(&rdev->srbm_mutex);
5839}
5840
5841/**
5842 * cik_pcie_gart_enable - gart enable
5843 *
5844 * @rdev: radeon_device pointer
5845 *
5846 * This sets up the TLBs, programs the page tables for VMID0,
5847 * sets up the hw for VMIDs 1-15 which are allocated on
5848 * demand, and sets up the global locations for the LDS, GDS,
5849 * and GPUVM for FSA64 clients (CIK).
5850 * Returns 0 for success, errors for failure.
5851 */
5852static int cik_pcie_gart_enable(struct radeon_device *rdev)
5853{
5854        int r, i;
5855
5856        if (rdev->gart.robj == NULL) {
5857                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5858                return -EINVAL;
5859        }
5860        r = radeon_gart_table_vram_pin(rdev);
5861        if (r)
5862                return r;
5863        /* Setup TLB control */
5864        WREG32(MC_VM_MX_L1_TLB_CNTL,
5865               (0xA << 7) |
5866               ENABLE_L1_TLB |
5867               ENABLE_L1_FRAGMENT_PROCESSING |
5868               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5869               ENABLE_ADVANCED_DRIVER_MODEL |
5870               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5871        /* Setup L2 cache */
5872        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5873               ENABLE_L2_FRAGMENT_PROCESSING |
5874               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5875               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5876               EFFECTIVE_L2_QUEUE_SIZE(7) |
5877               CONTEXT1_IDENTITY_ACCESS_MODE(1));
5878        WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5879        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5880               BANK_SELECT(4) |
5881               L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5882        /* setup context0 */
5883        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5884        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5885        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5886        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5887                        (u32)(rdev->dummy_page.addr >> 12));
5888        WREG32(VM_CONTEXT0_CNTL2, 0);
5889        WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5890                                  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5891
5892        WREG32(0x15D4, 0);
5893        WREG32(0x15D8, 0);
5894        WREG32(0x15DC, 0);
5895
5896        /* restore context1-15 */
5897        /* set vm size, must be a multiple of 4 */
5898        WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5899        WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
5900        for (i = 1; i < 16; i++) {
5901                if (i < 8)
5902                        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5903                               rdev->vm_manager.saved_table_addr[i]);
5904                else
5905                        WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5906                               rdev->vm_manager.saved_table_addr[i]);
5907        }
5908
5909        /* enable context1-15 */
5910        WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5911               (u32)(rdev->dummy_page.addr >> 12));
5912        WREG32(VM_CONTEXT1_CNTL2, 4);
5913        WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5914                                PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5915                                RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5916                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5917                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5918                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5919                                PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5920                                PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5921                                VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5922                                VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5923                                READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5924                                READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5925                                WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5926                                WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5927
5928        if (rdev->family == CHIP_KAVERI) {
5929                u32 tmp = RREG32(CHUB_CONTROL);
5930                tmp &= ~BYPASS_VM;
5931                WREG32(CHUB_CONTROL, tmp);
5932        }
5933
5934        /* XXX SH_MEM regs */
5935        /* where to put LDS, scratch, GPUVM in FSA64 space */
5936        mutex_lock(&rdev->srbm_mutex);
5937        for (i = 0; i < 16; i++) {
5938                cik_srbm_select(rdev, 0, 0, 0, i);
5939                /* CP and shaders */
5940                WREG32(SH_MEM_CONFIG, 0);
5941                WREG32(SH_MEM_APE1_BASE, 1);
5942                WREG32(SH_MEM_APE1_LIMIT, 0);
5943                WREG32(SH_MEM_BASES, 0);
5944                /* SDMA GFX */
5945                WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5946                WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5947                WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5948                WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5949                /* XXX SDMA RLC - todo */
5950        }
5951        cik_srbm_select(rdev, 0, 0, 0, 0);
5952        mutex_unlock(&rdev->srbm_mutex);
5953
5954        cik_pcie_init_compute_vmid(rdev);
5955
5956        cik_pcie_gart_tlb_flush(rdev);
5957        DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5958                 (unsigned)(rdev->mc.gtt_size >> 20),
5959                 (unsigned long long)rdev->gart.table_addr);
5960        rdev->gart.ready = true;
5961        return 0;
5962}
5963
5964/**
5965 * cik_pcie_gart_disable - gart disable
5966 *
5967 * @rdev: radeon_device pointer
5968 *
5969 * This disables all VM page table (CIK).
5970 */
5971static void cik_pcie_gart_disable(struct radeon_device *rdev)
5972{
5973        unsigned i;
5974
5975        for (i = 1; i < 16; ++i) {
5976                uint32_t reg;
5977                if (i < 8)
5978                        reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5979                else
5980                        reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5981                rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5982        }
5983
5984        /* Disable all tables */
5985        WREG32(VM_CONTEXT0_CNTL, 0);
5986        WREG32(VM_CONTEXT1_CNTL, 0);
5987        /* Setup TLB control */
5988        WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5989               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5990        /* Setup L2 cache */
5991        WREG32(VM_L2_CNTL,
5992               ENABLE_L2_FRAGMENT_PROCESSING |
5993               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5994               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5995               EFFECTIVE_L2_QUEUE_SIZE(7) |
5996               CONTEXT1_IDENTITY_ACCESS_MODE(1));
5997        WREG32(VM_L2_CNTL2, 0);
5998        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5999               L2_CACHE_BIGK_FRAGMENT_SIZE(6));
6000        radeon_gart_table_vram_unpin(rdev);

6001}
6002
6003/**
6004 * cik_pcie_gart_fini - vm fini callback
6005 *
6006 * @rdev: radeon_device pointer
6007 *
6008 * Tears down the driver GART/VM setup (CIK).
6009 */
6010static void cik_pcie_gart_fini(struct radeon_device *rdev)
6011{
6012        cik_pcie_gart_disable(rdev);
6013        radeon_gart_table_vram_free(rdev);
6014        radeon_gart_fini(rdev);
6015}
6016
6017/* vm parser */
6018/**
6019 * cik_ib_parse - vm ib_parse callback
6020 *
6021 * @rdev: radeon_device pointer
6022 * @ib: indirect buffer pointer
6023 *
6024 * CIK uses hw IB checking so this is a nop (CIK).
6025 */
6026int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
6027{
6028        return 0;
6029}
6030
6031/*
6032 * vm
6033 * VMID 0 is the physical GPU addresses as used by the kernel.
6034 * VMIDs 1-15 are used for userspace clients and are handled
6035 * by the radeon vm/hsa code.
6036 */
6037/**
6038 * cik_vm_init - cik vm init callback
6039 *
6040 * @rdev: radeon_device pointer
6041 *
6042 * Inits cik specific vm parameters (number of VMs, base of vram for
6043 * VMIDs 1-15) (CIK).
6044 * Returns 0 for success.
6045 */
6046int cik_vm_init(struct radeon_device *rdev)
6047{
6048        /*
6049         * number of VMs
6050         * VMID 0 is reserved for System
6051         * radeon graphics/compute will use VMIDs 1-7
6052         * amdkfd will use VMIDs 8-15
6053         */
6054        rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
6055        /* base offset of vram pages */
6056        if (rdev->flags & RADEON_IS_IGP) {
6057                u64 tmp = RREG32(MC_VM_FB_OFFSET);
6058                tmp <<= 22;
6059                rdev->vm_manager.vram_base_offset = tmp;
6060        } else
6061                rdev->vm_manager.vram_base_offset = 0;
6062
6063        return 0;
6064}
6065
6066/**
6067 * cik_vm_fini - cik vm fini callback
6068 *
6069 * @rdev: radeon_device pointer
6070 *
6071 * Tear down any asic specific VM setup (CIK).
6072 */
6073void cik_vm_fini(struct radeon_device *rdev)
6074{
6075}
6076
6077/**
6078 * cik_vm_decode_fault - print human readable fault info
6079 *
6080 * @rdev: radeon_device pointer
6081 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
6082 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
6083 *
6084 * Print human readable fault information (CIK).
6085 */
6086static void cik_vm_decode_fault(struct radeon_device *rdev,
6087                                u32 status, u32 addr, u32 mc_client)
6088{
6089        u32 mc_id;
6090        u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
6091        u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
6092        char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
6093                (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
6094
6095        if (rdev->family == CHIP_HAWAII)
6096                mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6097        else
6098                mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6099
6100        printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
6101               protections, vmid, addr,
6102               (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
6103               block, mc_client, mc_id);
6104}
6105
6106/**
6107 * cik_vm_flush - cik vm flush using the CP
6108 *
6109 * @rdev: radeon_device pointer
6110 *
6111 * Update the page table base and flush the VM TLB
6112 * using the CP (CIK).
6113 */
6114void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
6115                  unsigned vm_id, uint64_t pd_addr)
6116{
6117        int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
6118
6119        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6120        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6121                                 WRITE_DATA_DST_SEL(0)));
6122        if (vm_id < 8) {
6123                radeon_ring_write(ring,
6124                                  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
6125        } else {
6126                radeon_ring_write(ring,
6127                                  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
6128        }
6129        radeon_ring_write(ring, 0);
6130        radeon_ring_write(ring, pd_addr >> 12);
6131
6132        /* update SH_MEM_* regs */
6133        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6134        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6135                                 WRITE_DATA_DST_SEL(0)));
6136        radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6137        radeon_ring_write(ring, 0);
6138        radeon_ring_write(ring, VMID(vm_id));
6139
6140        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
6141        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6142                                 WRITE_DATA_DST_SEL(0)));
6143        radeon_ring_write(ring, SH_MEM_BASES >> 2);
6144        radeon_ring_write(ring, 0);
6145
6146        radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6147        radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6148        radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6149        radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
6150
6151        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6152        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6153                                 WRITE_DATA_DST_SEL(0)));
6154        radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6155        radeon_ring_write(ring, 0);
6156        radeon_ring_write(ring, VMID(0));
6157
6158        /* HDP flush */
6159        cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
6160
6161        /* bits 0-15 are the VM contexts0-15 */
6162        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6163        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6164                                 WRITE_DATA_DST_SEL(0)));
6165        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6166        radeon_ring_write(ring, 0);
6167        radeon_ring_write(ring, 1 << vm_id);
6168
6169        /* wait for the invalidate to complete */
6170        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6171        radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6172                                 WAIT_REG_MEM_FUNCTION(0) |  /* always */
6173                                 WAIT_REG_MEM_ENGINE(0))); /* me */
6174        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6175        radeon_ring_write(ring, 0);
6176        radeon_ring_write(ring, 0); /* ref */
6177        radeon_ring_write(ring, 0); /* mask */
6178        radeon_ring_write(ring, 0x20); /* poll interval */
6179
6180        /* compute doesn't have PFP */
6181        if (usepfp) {
6182                /* sync PFP to ME, otherwise we might get invalid PFP reads */
6183                radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6184                radeon_ring_write(ring, 0x0);
6185        }
6186}
6187
6188/*
6189 * RLC
6190 * The RLC is a multi-purpose microengine that handles a
6191 * variety of functions, the most important of which is
6192 * the interrupt controller.
6193 */
6194static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6195                                          bool enable)
6196{
6197        u32 tmp = RREG32(CP_INT_CNTL_RING0);
6198
6199        if (enable)
6200                tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6201        else
6202                tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6203        WREG32(CP_INT_CNTL_RING0, tmp);
6204}
6205
6206static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6207{
6208        u32 tmp;
6209
6210        tmp = RREG32(RLC_LB_CNTL);
6211        if (enable)
6212                tmp |= LOAD_BALANCE_ENABLE;
6213        else
6214                tmp &= ~LOAD_BALANCE_ENABLE;
6215        WREG32(RLC_LB_CNTL, tmp);
6216}
6217
6218static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6219{
6220        u32 i, j, k;
6221        u32 mask;
6222
6223        mutex_lock(&rdev->grbm_idx_mutex);
6224        for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6225                for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6226                        cik_select_se_sh(rdev, i, j);
6227                        for (k = 0; k < rdev->usec_timeout; k++) {
6228                                if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6229                                        break;
6230                                udelay(1);
6231                        }
6232                }
6233        }
6234        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6235        mutex_unlock(&rdev->grbm_idx_mutex);
6236
6237        mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6238        for (k = 0; k < rdev->usec_timeout; k++) {
6239                if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6240                        break;
6241                udelay(1);
6242        }
6243}
6244
6245static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6246{
6247        u32 tmp;
6248
6249        tmp = RREG32(RLC_CNTL);
6250        if (tmp != rlc)
6251                WREG32(RLC_CNTL, rlc);
6252}
6253
6254static u32 cik_halt_rlc(struct radeon_device *rdev)
6255{
6256        u32 data, orig;
6257
6258        orig = data = RREG32(RLC_CNTL);
6259
6260        if (data & RLC_ENABLE) {
6261                u32 i;
6262
6263                data &= ~RLC_ENABLE;
6264                WREG32(RLC_CNTL, data);
6265
6266                for (i = 0; i < rdev->usec_timeout; i++) {
6267                        if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6268                                break;
6269                        udelay(1);
6270                }
6271
6272                cik_wait_for_rlc_serdes(rdev);
6273        }
6274
6275        return orig;
6276}
6277
6278void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6279{
6280        u32 tmp, i, mask;
6281
6282        tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6283        WREG32(RLC_GPR_REG2, tmp);
6284
6285        mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6286        for (i = 0; i < rdev->usec_timeout; i++) {
6287                if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6288                        break;
6289                udelay(1);
6290        }
6291
6292        for (i = 0; i < rdev->usec_timeout; i++) {
6293                if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6294                        break;
6295                udelay(1);
6296        }
6297}
6298
6299void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6300{
6301        u32 tmp;
6302
6303        tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6304        WREG32(RLC_GPR_REG2, tmp);
6305}
6306
6307/**
6308 * cik_rlc_stop - stop the RLC ME
6309 *
6310 * @rdev: radeon_device pointer
6311 *
6312 * Halt the RLC ME (MicroEngine) (CIK).
6313 */
6314static void cik_rlc_stop(struct radeon_device *rdev)
6315{
6316        WREG32(RLC_CNTL, 0);
6317
6318        cik_enable_gui_idle_interrupt(rdev, false);
6319
6320        cik_wait_for_rlc_serdes(rdev);
6321}
6322
6323/**
6324 * cik_rlc_start - start the RLC ME
6325 *
6326 * @rdev: radeon_device pointer
6327 *
6328 * Unhalt the RLC ME (MicroEngine) (CIK).
6329 */
6330static void cik_rlc_start(struct radeon_device *rdev)
6331{
6332        WREG32(RLC_CNTL, RLC_ENABLE);
6333
6334        cik_enable_gui_idle_interrupt(rdev, true);
6335
6336        udelay(50);
6337}
6338
6339/**
6340 * cik_rlc_resume - setup the RLC hw
6341 *
6342 * @rdev: radeon_device pointer
6343 *
6344 * Initialize the RLC registers, load the ucode,
6345 * and start the RLC (CIK).
6346 * Returns 0 for success, -EINVAL if the ucode is not available.
6347 */
6348static int cik_rlc_resume(struct radeon_device *rdev)
6349{
6350        u32 i, size, tmp;
6351
6352        if (!rdev->rlc_fw)
6353                return -EINVAL;
6354
6355        cik_rlc_stop(rdev);
6356
6357        /* disable CG */
6358        tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6359        WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6360
6361        si_rlc_reset(rdev);
6362
6363        cik_init_pg(rdev);
6364
6365        cik_init_cg(rdev);
6366
6367        WREG32(RLC_LB_CNTR_INIT, 0);
6368        WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6369
6370        mutex_lock(&rdev->grbm_idx_mutex);
6371        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6372        WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6373        WREG32(RLC_LB_PARAMS, 0x00600408);
6374        WREG32(RLC_LB_CNTL, 0x80000004);
6375        mutex_unlock(&rdev->grbm_idx_mutex);
6376
6377        WREG32(RLC_MC_CNTL, 0);
6378        WREG32(RLC_UCODE_CNTL, 0);
6379
6380        if (rdev->new_fw) {
6381                const struct rlc_firmware_header_v1_0 *hdr =
6382                        (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6383                const __le32 *fw_data = (const __le32 *)
6384                        (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6385
6386                radeon_ucode_print_rlc_hdr(&hdr->header);
6387
6388                size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6389                WREG32(RLC_GPM_UCODE_ADDR, 0);
6390                for (i = 0; i < size; i++)
6391                        WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6392                WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
6393        } else {
6394                const __be32 *fw_data;
6395
6396                switch (rdev->family) {
6397                case CHIP_BONAIRE:
6398                case CHIP_HAWAII:
6399                default:
6400                        size = BONAIRE_RLC_UCODE_SIZE;
6401                        break;
6402                case CHIP_KAVERI:
6403                        size = KV_RLC_UCODE_SIZE;
6404                        break;
6405                case CHIP_KABINI:
6406                        size = KB_RLC_UCODE_SIZE;
6407                        break;
6408                case CHIP_MULLINS:
6409                        size = ML_RLC_UCODE_SIZE;
6410                        break;
6411                }
6412
6413                fw_data = (const __be32 *)rdev->rlc_fw->data;
6414                WREG32(RLC_GPM_UCODE_ADDR, 0);
6415                for (i = 0; i < size; i++)
6416                        WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6417                WREG32(RLC_GPM_UCODE_ADDR, 0);
6418        }
6419
6420        /* XXX - find out what chips support lbpw */
6421        cik_enable_lbpw(rdev, false);
6422
6423        if (rdev->family == CHIP_BONAIRE)
6424                WREG32(RLC_DRIVER_DMA_STATUS, 0);
6425
6426        cik_rlc_start(rdev);
6427
6428        return 0;
6429}
6430
6431static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6432{
6433        u32 data, orig, tmp, tmp2;
6434
6435        orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6436
6437        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6438                cik_enable_gui_idle_interrupt(rdev, true);
6439
6440                tmp = cik_halt_rlc(rdev);
6441
6442                mutex_lock(&rdev->grbm_idx_mutex);
6443                cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6444                WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6445                WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6446                tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6447                WREG32(RLC_SERDES_WR_CTRL, tmp2);
6448                mutex_unlock(&rdev->grbm_idx_mutex);
6449
6450                cik_update_rlc(rdev, tmp);
6451
6452                data |= CGCG_EN | CGLS_EN;
6453        } else {
6454                cik_enable_gui_idle_interrupt(rdev, false);
6455
6456                RREG32(CB_CGTT_SCLK_CTRL);
6457                RREG32(CB_CGTT_SCLK_CTRL);
6458                RREG32(CB_CGTT_SCLK_CTRL);
6459                RREG32(CB_CGTT_SCLK_CTRL);
6460
6461                data &= ~(CGCG_EN | CGLS_EN);
6462        }
6463
6464        if (orig != data)
6465                WREG32(RLC_CGCG_CGLS_CTRL, data);
6466
6467}
6468
6469static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6470{
6471        u32 data, orig, tmp = 0;
6472
6473        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6474                if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6475                        if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6476                                orig = data = RREG32(CP_MEM_SLP_CNTL);
6477                                data |= CP_MEM_LS_EN;
6478                                if (orig != data)
6479                                        WREG32(CP_MEM_SLP_CNTL, data);
6480                        }
6481                }
6482
6483                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6484                data |= 0x00000001;
6485                data &= 0xfffffffd;
6486                if (orig != data)
6487                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6488
6489                tmp = cik_halt_rlc(rdev);
6490
6491                mutex_lock(&rdev->grbm_idx_mutex);
6492                cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6493                WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6494                WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6495                data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6496                WREG32(RLC_SERDES_WR_CTRL, data);
6497                mutex_unlock(&rdev->grbm_idx_mutex);
6498
6499                cik_update_rlc(rdev, tmp);
6500
6501                if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6502                        orig = data = RREG32(CGTS_SM_CTRL_REG);
6503                        data &= ~SM_MODE_MASK;
6504                        data |= SM_MODE(0x2);
6505                        data |= SM_MODE_ENABLE;
6506                        data &= ~CGTS_OVERRIDE;
6507                        if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6508                            (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6509                                data &= ~CGTS_LS_OVERRIDE;
6510                        data &= ~ON_MONITOR_ADD_MASK;
6511                        data |= ON_MONITOR_ADD_EN;
6512                        data |= ON_MONITOR_ADD(0x96);
6513                        if (orig != data)
6514                                WREG32(CGTS_SM_CTRL_REG, data);
6515                }
6516        } else {
6517                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6518                data |= 0x00000003;
6519                if (orig != data)
6520                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6521
6522                data = RREG32(RLC_MEM_SLP_CNTL);
6523                if (data & RLC_MEM_LS_EN) {
6524                        data &= ~RLC_MEM_LS_EN;
6525                        WREG32(RLC_MEM_SLP_CNTL, data);
6526                }
6527
6528                data = RREG32(CP_MEM_SLP_CNTL);
6529                if (data & CP_MEM_LS_EN) {
6530                        data &= ~CP_MEM_LS_EN;
6531                        WREG32(CP_MEM_SLP_CNTL, data);
6532                }
6533
6534                orig = data = RREG32(CGTS_SM_CTRL_REG);
6535                data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6536                if (orig != data)
6537                        WREG32(CGTS_SM_CTRL_REG, data);
6538
6539                tmp = cik_halt_rlc(rdev);
6540
6541                mutex_lock(&rdev->grbm_idx_mutex);
6542                cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6543                WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6544                WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6545                data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6546                WREG32(RLC_SERDES_WR_CTRL, data);
6547                mutex_unlock(&rdev->grbm_idx_mutex);
6548
6549                cik_update_rlc(rdev, tmp);
6550        }
6551}
6552
6553static const u32 mc_cg_registers[] =
6554{
6555        MC_HUB_MISC_HUB_CG,
6556        MC_HUB_MISC_SIP_CG,
6557        MC_HUB_MISC_VM_CG,
6558        MC_XPB_CLK_GAT,
6559        ATC_MISC_CG,
6560        MC_CITF_MISC_WR_CG,
6561        MC_CITF_MISC_RD_CG,
6562        MC_CITF_MISC_VM_CG,
6563        VM_L2_CG,
6564};
6565
6566static void cik_enable_mc_ls(struct radeon_device *rdev,
6567                             bool enable)
6568{
6569        int i;
6570        u32 orig, data;
6571
6572        for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6573                orig = data = RREG32(mc_cg_registers[i]);
6574                if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6575                        data |= MC_LS_ENABLE;
6576                else
6577                        data &= ~MC_LS_ENABLE;
6578                if (data != orig)
6579                        WREG32(mc_cg_registers[i], data);
6580        }
6581}
6582
6583static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6584                               bool enable)
6585{
6586        int i;
6587        u32 orig, data;
6588
6589        for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6590                orig = data = RREG32(mc_cg_registers[i]);
6591                if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6592                        data |= MC_CG_ENABLE;
6593                else
6594                        data &= ~MC_CG_ENABLE;
6595                if (data != orig)
6596                        WREG32(mc_cg_registers[i], data);
6597        }
6598}
6599
6600static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6601                                 bool enable)
6602{
6603        u32 orig, data;
6604
6605        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6606                WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6607                WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6608        } else {
6609                orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6610                data |= 0xff000000;
6611                if (data != orig)
6612                        WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6613
6614                orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6615                data |= 0xff000000;
6616                if (data != orig)
6617                        WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6618        }
6619}
6620
6621static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6622                                 bool enable)
6623{
6624        u32 orig, data;
6625
6626        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6627                orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6628                data |= 0x100;
6629                if (orig != data)
6630                        WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6631
6632                orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6633                data |= 0x100;
6634                if (orig != data)
6635                        WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6636        } else {
6637                orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6638                data &= ~0x100;
6639                if (orig != data)
6640                        WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6641
6642                orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6643                data &= ~0x100;
6644                if (orig != data)
6645                        WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6646        }
6647}
6648
6649static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6650                                bool enable)
6651{
6652        u32 orig, data;
6653
6654        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6655                data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6656                data = 0xfff;
6657                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6658
6659                orig = data = RREG32(UVD_CGC_CTRL);
6660                data |= DCM;
6661                if (orig != data)
6662                        WREG32(UVD_CGC_CTRL, data);
6663        } else {
6664                data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6665                data &= ~0xfff;
6666                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6667
6668                orig = data = RREG32(UVD_CGC_CTRL);
6669                data &= ~DCM;
6670                if (orig != data)
6671                        WREG32(UVD_CGC_CTRL, data);
6672        }
6673}
6674
6675static void cik_enable_bif_mgls(struct radeon_device *rdev,
6676                               bool enable)
6677{
6678        u32 orig, data;
6679
6680        orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6681
6682        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6683                data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6684                        REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6685        else
6686                data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6687                          REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6688
6689        if (orig != data)
6690                WREG32_PCIE_PORT(PCIE_CNTL2, data);
6691}
6692
6693static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6694                                bool enable)
6695{
6696        u32 orig, data;
6697
6698        orig = data = RREG32(HDP_HOST_PATH_CNTL);
6699
6700        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6701                data &= ~CLOCK_GATING_DIS;
6702        else
6703                data |= CLOCK_GATING_DIS;
6704
6705        if (orig != data)
6706                WREG32(HDP_HOST_PATH_CNTL, data);
6707}
6708
6709static void cik_enable_hdp_ls(struct radeon_device *rdev,
6710                              bool enable)
6711{
6712        u32 orig, data;
6713
6714        orig = data = RREG32(HDP_MEM_POWER_LS);
6715
6716        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6717                data |= HDP_LS_ENABLE;
6718        else
6719                data &= ~HDP_LS_ENABLE;
6720
6721        if (orig != data)
6722                WREG32(HDP_MEM_POWER_LS, data);
6723}
6724
6725void cik_update_cg(struct radeon_device *rdev,
6726                   u32 block, bool enable)
6727{
6728
6729        if (block & RADEON_CG_BLOCK_GFX) {
6730                cik_enable_gui_idle_interrupt(rdev, false);
6731                /* order matters! */
6732                if (enable) {
6733                        cik_enable_mgcg(rdev, true);
6734                        cik_enable_cgcg(rdev, true);
6735                } else {
6736                        cik_enable_cgcg(rdev, false);
6737                        cik_enable_mgcg(rdev, false);
6738                }
6739                cik_enable_gui_idle_interrupt(rdev, true);
6740        }
6741
6742        if (block & RADEON_CG_BLOCK_MC) {
6743                if (!(rdev->flags & RADEON_IS_IGP)) {
6744                        cik_enable_mc_mgcg(rdev, enable);
6745                        cik_enable_mc_ls(rdev, enable);
6746                }
6747        }
6748
6749        if (block & RADEON_CG_BLOCK_SDMA) {
6750                cik_enable_sdma_mgcg(rdev, enable);
6751                cik_enable_sdma_mgls(rdev, enable);
6752        }
6753
6754        if (block & RADEON_CG_BLOCK_BIF) {
6755                cik_enable_bif_mgls(rdev, enable);
6756        }
6757
6758        if (block & RADEON_CG_BLOCK_UVD) {
6759                if (rdev->has_uvd)
6760                        cik_enable_uvd_mgcg(rdev, enable);
6761        }
6762
6763        if (block & RADEON_CG_BLOCK_HDP) {
6764                cik_enable_hdp_mgcg(rdev, enable);
6765                cik_enable_hdp_ls(rdev, enable);
6766        }
6767
6768        if (block & RADEON_CG_BLOCK_VCE) {
6769                vce_v2_0_enable_mgcg(rdev, enable);
6770        }
6771}
6772
6773static void cik_init_cg(struct radeon_device *rdev)
6774{
6775
6776        cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6777
6778        if (rdev->has_uvd)
6779                si_init_uvd_internal_cg(rdev);
6780
6781        cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6782                             RADEON_CG_BLOCK_SDMA |
6783                             RADEON_CG_BLOCK_BIF |
6784                             RADEON_CG_BLOCK_UVD |
6785                             RADEON_CG_BLOCK_HDP), true);
6786}
6787
6788static void cik_fini_cg(struct radeon_device *rdev)
6789{
6790        cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6791                             RADEON_CG_BLOCK_SDMA |
6792                             RADEON_CG_BLOCK_BIF |
6793                             RADEON_CG_BLOCK_UVD |
6794                             RADEON_CG_BLOCK_HDP), false);
6795
6796        cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6797}
6798
6799static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6800                                          bool enable)
6801{
6802        u32 data, orig;
6803
6804        orig = data = RREG32(RLC_PG_CNTL);
6805        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6806                data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6807        else
6808                data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6809        if (orig != data)
6810                WREG32(RLC_PG_CNTL, data);
6811}
6812
6813static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6814                                          bool enable)
6815{
6816        u32 data, orig;
6817
6818        orig = data = RREG32(RLC_PG_CNTL);
6819        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6820                data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6821        else
6822                data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6823        if (orig != data)
6824                WREG32(RLC_PG_CNTL, data);
6825}
6826
6827static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6828{
6829        u32 data, orig;
6830
6831        orig = data = RREG32(RLC_PG_CNTL);
6832        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6833                data &= ~DISABLE_CP_PG;
6834        else
6835                data |= DISABLE_CP_PG;
6836        if (orig != data)
6837                WREG32(RLC_PG_CNTL, data);
6838}
6839
6840static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6841{
6842        u32 data, orig;
6843
6844        orig = data = RREG32(RLC_PG_CNTL);
6845        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6846                data &= ~DISABLE_GDS_PG;
6847        else
6848                data |= DISABLE_GDS_PG;
6849        if (orig != data)
6850                WREG32(RLC_PG_CNTL, data);
6851}
6852
6853#define CP_ME_TABLE_SIZE    96
6854#define CP_ME_TABLE_OFFSET  2048
6855#define CP_MEC_TABLE_OFFSET 4096
6856
6857void cik_init_cp_pg_table(struct radeon_device *rdev)
6858{
6859        volatile u32 *dst_ptr;
6860        int me, i, max_me = 4;
6861        u32 bo_offset = 0;
6862        u32 table_offset, table_size;
6863
6864        if (rdev->family == CHIP_KAVERI)
6865                max_me = 5;
6866
6867        if (rdev->rlc.cp_table_ptr == NULL)
6868                return;
6869
6870        /* write the cp table buffer */
6871        dst_ptr = rdev->rlc.cp_table_ptr;
6872        for (me = 0; me < max_me; me++) {
6873                if (rdev->new_fw) {
6874                        const __le32 *fw_data;
6875                        const struct gfx_firmware_header_v1_0 *hdr;
6876
6877                        if (me == 0) {
6878                                hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6879                                fw_data = (const __le32 *)
6880                                        (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6881                                table_offset = le32_to_cpu(hdr->jt_offset);
6882                                table_size = le32_to_cpu(hdr->jt_size);
6883                        } else if (me == 1) {
6884                                hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6885                                fw_data = (const __le32 *)
6886                                        (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6887                                table_offset = le32_to_cpu(hdr->jt_offset);
6888                                table_size = le32_to_cpu(hdr->jt_size);
6889                        } else if (me == 2) {
6890                                hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6891                                fw_data = (const __le32 *)
6892                                        (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6893                                table_offset = le32_to_cpu(hdr->jt_offset);
6894                                table_size = le32_to_cpu(hdr->jt_size);
6895                        } else if (me == 3) {
6896                                hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6897                                fw_data = (const __le32 *)
6898                                        (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6899                                table_offset = le32_to_cpu(hdr->jt_offset);
6900                                table_size = le32_to_cpu(hdr->jt_size);
6901                        } else {
6902                                hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6903                                fw_data = (const __le32 *)
6904                                        (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6905                                table_offset = le32_to_cpu(hdr->jt_offset);
6906                                table_size = le32_to_cpu(hdr->jt_size);
6907                        }
6908
6909                        for (i = 0; i < table_size; i ++) {
6910                                dst_ptr[bo_offset + i] =
6911                                        cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6912                        }
6913                        bo_offset += table_size;
6914                } else {
6915                        const __be32 *fw_data;
6916                        table_size = CP_ME_TABLE_SIZE;
6917
6918                        if (me == 0) {
6919                                fw_data = (const __be32 *)rdev->ce_fw->data;
6920                                table_offset = CP_ME_TABLE_OFFSET;
6921                        } else if (me == 1) {
6922                                fw_data = (const __be32 *)rdev->pfp_fw->data;
6923                                table_offset = CP_ME_TABLE_OFFSET;
6924                        } else if (me == 2) {
6925                                fw_data = (const __be32 *)rdev->me_fw->data;
6926                                table_offset = CP_ME_TABLE_OFFSET;
6927                        } else {
6928                                fw_data = (const __be32 *)rdev->mec_fw->data;
6929                                table_offset = CP_MEC_TABLE_OFFSET;
6930                        }
6931
6932                        for (i = 0; i < table_size; i ++) {
6933                                dst_ptr[bo_offset + i] =
6934                                        cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6935                        }
6936                        bo_offset += table_size;
6937                }
6938        }
6939}
6940
6941static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6942                                bool enable)
6943{
6944        u32 data, orig;
6945
6946        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6947                orig = data = RREG32(RLC_PG_CNTL);
6948                data |= GFX_PG_ENABLE;
6949                if (orig != data)
6950                        WREG32(RLC_PG_CNTL, data);
6951
6952                orig = data = RREG32(RLC_AUTO_PG_CTRL);
6953                data |= AUTO_PG_EN;
6954                if (orig != data)
6955                        WREG32(RLC_AUTO_PG_CTRL, data);
6956        } else {
6957                orig = data = RREG32(RLC_PG_CNTL);
6958                data &= ~GFX_PG_ENABLE;
6959                if (orig != data)
6960                        WREG32(RLC_PG_CNTL, data);
6961
6962                orig = data = RREG32(RLC_AUTO_PG_CTRL);
6963                data &= ~AUTO_PG_EN;
6964                if (orig != data)
6965                        WREG32(RLC_AUTO_PG_CTRL, data);
6966
6967                data = RREG32(DB_RENDER_CONTROL);
6968        }
6969}
6970
6971static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6972{
6973        u32 mask = 0, tmp, tmp1;
6974        int i;
6975
6976        mutex_lock(&rdev->grbm_idx_mutex);
6977        cik_select_se_sh(rdev, se, sh);
6978        tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6979        tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6980        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6981        mutex_unlock(&rdev->grbm_idx_mutex);
6982
6983        tmp &= 0xffff0000;
6984
6985        tmp |= tmp1;
6986        tmp >>= 16;
6987
6988        for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6989                mask <<= 1;
6990                mask |= 1;
6991        }
6992
6993        return (~tmp) & mask;
6994}
6995
6996static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6997{
6998        u32 i, j, k, active_cu_number = 0;
6999        u32 mask, counter, cu_bitmap;
7000        u32 tmp = 0;

7001
7002        for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
7003                for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
7004                        mask = 1;
7005                        cu_bitmap = 0;
7006                        counter = 0;
7007                        for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
7008                                if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
7009                                        if (counter < 2)
7010                                                cu_bitmap |= mask;
7011                                        counter ++;
7012                                }
7013                                mask <<= 1;
7014                        }
7015
7016                        active_cu_number += counter;
7017                        tmp |= (cu_bitmap << (i * 16 + j * 8));
7018                }
7019        }
7020
7021        WREG32(RLC_PG_AO_CU_MASK, tmp);
7022
7023        tmp = RREG32(RLC_MAX_PG_CU);
7024        tmp &= ~MAX_PU_CU_MASK;
7025        tmp |= MAX_PU_CU(active_cu_number);
7026        WREG32(RLC_MAX_PG_CU, tmp);
7027}
7028
7029static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
7030                                       bool enable)
7031{
7032        u32 data, orig;
7033
7034        orig = data = RREG32(RLC_PG_CNTL);
7035        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
7036                data |= STATIC_PER_CU_PG_ENABLE;
7037        else
7038                data &= ~STATIC_PER_CU_PG_ENABLE;
7039        if (orig != data)
7040                WREG32(RLC_PG_CNTL, data);
7041}
7042
7043static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
7044                                        bool enable)
7045{
7046        u32 data, orig;
7047
7048        orig = data = RREG32(RLC_PG_CNTL);
7049        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
7050                data |= DYN_PER_CU_PG_ENABLE;
7051        else
7052                data &= ~DYN_PER_CU_PG_ENABLE;
7053        if (orig != data)
7054                WREG32(RLC_PG_CNTL, data);
7055}
7056
7057#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
7058#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
7059
7060static void cik_init_gfx_cgpg(struct radeon_device *rdev)
7061{
7062        u32 data, orig;
7063        u32 i;
7064
7065        if (rdev->rlc.cs_data) {
7066                WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
7067                WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
7068                WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
7069                WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
7070        } else {
7071                WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
7072                for (i = 0; i < 3; i++)
7073                        WREG32(RLC_GPM_SCRATCH_DATA, 0);
7074        }
7075        if (rdev->rlc.reg_list) {
7076                WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
7077                for (i = 0; i < rdev->rlc.reg_list_size; i++)
7078                        WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
7079        }
7080
7081        orig = data = RREG32(RLC_PG_CNTL);
7082        data |= GFX_PG_SRC;
7083        if (orig != data)
7084                WREG32(RLC_PG_CNTL, data);
7085
7086        WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
7087        WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
7088
7089        data = RREG32(CP_RB_WPTR_POLL_CNTL);
7090        data &= ~IDLE_POLL_COUNT_MASK;
7091        data |= IDLE_POLL_COUNT(0x60);
7092        WREG32(CP_RB_WPTR_POLL_CNTL, data);
7093
7094        data = 0x10101010;
7095        WREG32(RLC_PG_DELAY, data);
7096
7097        data = RREG32(RLC_PG_DELAY_2);
7098        data &= ~0xff;
7099        data |= 0x3;
7100        WREG32(RLC_PG_DELAY_2, data);
7101
7102        data = RREG32(RLC_AUTO_PG_CTRL);
7103        data &= ~GRBM_REG_SGIT_MASK;
7104        data |= GRBM_REG_SGIT(0x700);
7105        WREG32(RLC_AUTO_PG_CTRL, data);
7106
7107}
7108
7109static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
7110{
7111        cik_enable_gfx_cgpg(rdev, enable);
7112        cik_enable_gfx_static_mgpg(rdev, enable);
7113        cik_enable_gfx_dynamic_mgpg(rdev, enable);
7114}
7115
7116u32 cik_get_csb_size(struct radeon_device *rdev)
7117{
7118        u32 count = 0;
7119        const struct cs_section_def *sect = NULL;
7120        const struct cs_extent_def *ext = NULL;
7121
7122        if (rdev->rlc.cs_data == NULL)
7123                return 0;
7124
7125        /* begin clear state */
7126        count += 2;
7127        /* context control state */
7128        count += 3;
7129
7130        for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7131                for (ext = sect->section; ext->extent != NULL; ++ext) {
7132                        if (sect->id == SECT_CONTEXT)
7133                                count += 2 + ext->reg_count;
7134                        else
7135                                return 0;
7136                }
7137        }
7138        /* pa_sc_raster_config/pa_sc_raster_config1 */
7139        count += 4;
7140        /* end clear state */
7141        count += 2;
7142        /* clear state */
7143        count += 2;
7144
7145        return count;
7146}
7147
7148void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
7149{
7150        u32 count = 0, i;
7151        const struct cs_section_def *sect = NULL;
7152        const struct cs_extent_def *ext = NULL;
7153
7154        if (rdev->rlc.cs_data == NULL)
7155                return;
7156        if (buffer == NULL)
7157                return;
7158
7159        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7160        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
7161
7162        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7163        buffer[count++] = cpu_to_le32(0x80000000);
7164        buffer[count++] = cpu_to_le32(0x80000000);
7165
7166        for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7167                for (ext = sect->section; ext->extent != NULL; ++ext) {
7168                        if (sect->id == SECT_CONTEXT) {
7169                                buffer[count++] =
7170                                        cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7171                                buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
7172                                for (i = 0; i < ext->reg_count; i++)
7173                                        buffer[count++] = cpu_to_le32(ext->extent[i]);
7174                        } else {
7175                                return;
7176                        }
7177                }
7178        }
7179
7180        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7181        buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7182        switch (rdev->family) {
7183        case CHIP_BONAIRE:
7184                buffer[count++] = cpu_to_le32(0x16000012);
7185                buffer[count++] = cpu_to_le32(0x00000000);
7186                break;
7187        case CHIP_KAVERI:
7188                buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7189                buffer[count++] = cpu_to_le32(0x00000000);
7190                break;
7191        case CHIP_KABINI:
7192        case CHIP_MULLINS:
7193                buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7194                buffer[count++] = cpu_to_le32(0x00000000);
7195                break;
7196        case CHIP_HAWAII:
7197                buffer[count++] = cpu_to_le32(0x3a00161a);
7198                buffer[count++] = cpu_to_le32(0x0000002e);
7199                break;
7200        default:
7201                buffer[count++] = cpu_to_le32(0x00000000);
7202                buffer[count++] = cpu_to_le32(0x00000000);
7203                break;
7204        }
7205
7206        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7207        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7208
7209        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7210        buffer[count++] = cpu_to_le32(0);
7211}
7212
7213static void cik_init_pg(struct radeon_device *rdev)
7214{
7215        if (rdev->pg_flags) {
7216                cik_enable_sck_slowdown_on_pu(rdev, true);
7217                cik_enable_sck_slowdown_on_pd(rdev, true);
7218                if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7219                        cik_init_gfx_cgpg(rdev);
7220                        cik_enable_cp_pg(rdev, true);
7221                        cik_enable_gds_pg(rdev, true);
7222                }
7223                cik_init_ao_cu_mask(rdev);
7224                cik_update_gfx_pg(rdev, true);
7225        }
7226}
7227
7228static void cik_fini_pg(struct radeon_device *rdev)
7229{
7230        if (rdev->pg_flags) {
7231                cik_update_gfx_pg(rdev, false);
7232                if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7233                        cik_enable_cp_pg(rdev, false);
7234                        cik_enable_gds_pg(rdev, false);
7235                }
7236        }
7237}
7238
7239/*
7240 * Interrupts
7241 * Starting with r6xx, interrupts are handled via a ring buffer.
7242 * Ring buffers are areas of GPU accessible memory that the GPU
7243 * writes interrupt vectors into and the host reads vectors out of.
7244 * There is a rptr (read pointer) that determines where the
7245 * host is currently reading, and a wptr (write pointer)
7246 * which determines where the GPU has written.  When the
7247 * pointers are equal, the ring is idle.  When the GPU
7248 * writes vectors to the ring buffer, it increments the
7249 * wptr.  When there is an interrupt, the host then starts
7250 * fetching commands and processing them until the pointers are
7251 * equal again at which point it updates the rptr.
7252 */
7253
7254/**
7255 * cik_enable_interrupts - Enable the interrupt ring buffer
7256 *
7257 * @rdev: radeon_device pointer
7258 *
7259 * Enable the interrupt ring buffer (CIK).
7260 */
7261static void cik_enable_interrupts(struct radeon_device *rdev)
7262{
7263        u32 ih_cntl = RREG32(IH_CNTL);
7264        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7265
7266        ih_cntl |= ENABLE_INTR;
7267        ih_rb_cntl |= IH_RB_ENABLE;
7268        WREG32(IH_CNTL, ih_cntl);
7269        WREG32(IH_RB_CNTL, ih_rb_cntl);
7270        rdev->ih.enabled = true;
7271}
7272
7273/**
7274 * cik_disable_interrupts - Disable the interrupt ring buffer
7275 *
7276 * @rdev: radeon_device pointer
7277 *
7278 * Disable the interrupt ring buffer (CIK).
7279 */
7280static void cik_disable_interrupts(struct radeon_device *rdev)
7281{
7282        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7283        u32 ih_cntl = RREG32(IH_CNTL);
7284
7285        ih_rb_cntl &= ~IH_RB_ENABLE;
7286        ih_cntl &= ~ENABLE_INTR;
7287        WREG32(IH_RB_CNTL, ih_rb_cntl);
7288        WREG32(IH_CNTL, ih_cntl);
7289        /* set rptr, wptr to 0 */
7290        WREG32(IH_RB_RPTR, 0);
7291        WREG32(IH_RB_WPTR, 0);
7292        rdev->ih.enabled = false;
7293        rdev->ih.rptr = 0;
7294}
7295
7296/**
7297 * cik_disable_interrupt_state - Disable all interrupt sources
7298 *
7299 * @rdev: radeon_device pointer
7300 *
7301 * Clear all interrupt enable bits used by the driver (CIK).
7302 */
7303static void cik_disable_interrupt_state(struct radeon_device *rdev)
7304{
7305        u32 tmp;
7306
7307        /* gfx ring */
7308        tmp = RREG32(CP_INT_CNTL_RING0) &
7309                (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7310        WREG32(CP_INT_CNTL_RING0, tmp);
7311        /* sdma */
7312        tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7313        WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7314        tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7315        WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7316        /* compute queues */
7317        WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7318        WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7319        WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7320        WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7321        WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7322        WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7323        WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7324        WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7325        /* grbm */
7326        WREG32(GRBM_INT_CNTL, 0);
7327        /* SRBM */
7328        WREG32(SRBM_INT_CNTL, 0);
7329        /* vline/vblank, etc. */
7330        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7331        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7332        if (rdev->num_crtc >= 4) {
7333                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7334                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7335        }
7336        if (rdev->num_crtc >= 6) {
7337                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7338                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7339        }
7340        /* pflip */
7341        if (rdev->num_crtc >= 2) {
7342                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7343                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7344        }
7345        if (rdev->num_crtc >= 4) {
7346                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7347                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7348        }
7349        if (rdev->num_crtc >= 6) {
7350                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7351                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7352        }
7353
7354        /* dac hotplug */
7355        WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7356
7357        /* digital hotplug */
7358        tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7359        WREG32(DC_HPD1_INT_CONTROL, tmp);
7360        tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7361        WREG32(DC_HPD2_INT_CONTROL, tmp);
7362        tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7363        WREG32(DC_HPD3_INT_CONTROL, tmp);
7364        tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7365        WREG32(DC_HPD4_INT_CONTROL, tmp);
7366        tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7367        WREG32(DC_HPD5_INT_CONTROL, tmp);
7368        tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7369        WREG32(DC_HPD6_INT_CONTROL, tmp);
7370
7371}
7372
7373/**
7374 * cik_irq_init - init and enable the interrupt ring
7375 *
7376 * @rdev: radeon_device pointer
7377 *
7378 * Allocate a ring buffer for the interrupt controller,
7379 * enable the RLC, disable interrupts, enable the IH
7380 * ring buffer and enable it (CIK).
7381 * Called at device load and reume.
7382 * Returns 0 for success, errors for failure.
7383 */
7384static int cik_irq_init(struct radeon_device *rdev)
7385{
7386        int ret = 0;
7387        int rb_bufsz;
7388        u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7389
7390        /* allocate ring */
7391        ret = r600_ih_ring_alloc(rdev);
7392        if (ret)
7393                return ret;
7394
7395        /* disable irqs */
7396        cik_disable_interrupts(rdev);
7397
7398        /* init rlc */
7399        ret = cik_rlc_resume(rdev);
7400        if (ret) {
7401                r600_ih_ring_fini(rdev);
7402                return ret;
7403        }
7404
7405        /* setup interrupt control */
7406        /* XXX this should actually be a bus address, not an MC address. same on older asics */
7407        WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7408        interrupt_cntl = RREG32(INTERRUPT_CNTL);
7409        /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7410         * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7411         */
7412        interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7413        /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7414        interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7415        WREG32(INTERRUPT_CNTL, interrupt_cntl);
7416
7417        WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7418        rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7419
7420        ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7421                      IH_WPTR_OVERFLOW_CLEAR |
7422                      (rb_bufsz << 1));
7423
7424        if (rdev->wb.enabled)
7425                ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7426
7427        /* set the writeback address whether it's enabled or not */
7428        WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7429        WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7430
7431        WREG32(IH_RB_CNTL, ih_rb_cntl);
7432
7433        /* set rptr, wptr to 0 */
7434        WREG32(IH_RB_RPTR, 0);
7435        WREG32(IH_RB_WPTR, 0);
7436
7437        /* Default settings for IH_CNTL (disabled at first) */
7438        ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7439        /* RPTR_REARM only works if msi's are enabled */
7440        if (rdev->msi_enabled)
7441                ih_cntl |= RPTR_REARM;
7442        WREG32(IH_CNTL, ih_cntl);
7443
7444        /* force the active interrupt state to all disabled */
7445        cik_disable_interrupt_state(rdev);
7446
7447        pci_set_master(rdev->pdev);
7448
7449        /* enable irqs */
7450        cik_enable_interrupts(rdev);
7451
7452        return ret;
7453}
7454
7455/**
7456 * cik_irq_set - enable/disable interrupt sources
7457 *
7458 * @rdev: radeon_device pointer
7459 *
7460 * Enable interrupt sources on the GPU (vblanks, hpd,
7461 * etc.) (CIK).
7462 * Returns 0 for success, errors for failure.
7463 */
7464int cik_irq_set(struct radeon_device *rdev)
7465{
7466        u32 cp_int_cntl;
7467        u32 cp_m1p0;
7468        u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7469        u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7470        u32 grbm_int_cntl = 0;
7471        u32 dma_cntl, dma_cntl1;
7472
7473        if (!rdev->irq.installed) {
7474                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7475                return -EINVAL;
7476        }
7477        /* don't enable anything if the ih is disabled */
7478        if (!rdev->ih.enabled) {
7479                cik_disable_interrupts(rdev);
7480                /* force the active interrupt state to all disabled */
7481                cik_disable_interrupt_state(rdev);
7482                return 0;
7483        }
7484
7485        cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7486                (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7487        cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7488
7489        hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7490        hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7491        hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7492        hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7493        hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7494        hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7495
7496        dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7497        dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7498
7499        cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7500
7501        /* enable CP interrupts on all rings */
7502        if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7503                DRM_DEBUG("cik_irq_set: sw int gfx\n");
7504                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7505        }
7506        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7507                struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7508                DRM_DEBUG("si_irq_set: sw int cp1\n");
7509                if (ring->me == 1) {
7510                        switch (ring->pipe) {
7511                        case 0:
7512                                cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7513                                break;
7514                        default:
7515                                DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7516                                break;
7517                        }
7518                } else {
7519                        DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7520                }
7521        }
7522        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7523                struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7524                DRM_DEBUG("si_irq_set: sw int cp2\n");
7525                if (ring->me == 1) {
7526                        switch (ring->pipe) {
7527                        case 0:
7528                                cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7529                                break;
7530                        default:
7531                                DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7532                                break;
7533                        }
7534                } else {
7535                        DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7536                }
7537        }
7538
7539        if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7540                DRM_DEBUG("cik_irq_set: sw int dma\n");
7541                dma_cntl |= TRAP_ENABLE;
7542        }
7543
7544        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7545                DRM_DEBUG("cik_irq_set: sw int dma1\n");
7546                dma_cntl1 |= TRAP_ENABLE;
7547        }
7548
7549        if (rdev->irq.crtc_vblank_int[0] ||
7550            atomic_read(&rdev->irq.pflip[0])) {
7551                DRM_DEBUG("cik_irq_set: vblank 0\n");
7552                crtc1 |= VBLANK_INTERRUPT_MASK;
7553        }
7554        if (rdev->irq.crtc_vblank_int[1] ||
7555            atomic_read(&rdev->irq.pflip[1])) {
7556                DRM_DEBUG("cik_irq_set: vblank 1\n");
7557                crtc2 |= VBLANK_INTERRUPT_MASK;
7558        }
7559        if (rdev->irq.crtc_vblank_int[2] ||
7560            atomic_read(&rdev->irq.pflip[2])) {
7561                DRM_DEBUG("cik_irq_set: vblank 2\n");
7562                crtc3 |= VBLANK_INTERRUPT_MASK;
7563        }
7564        if (rdev->irq.crtc_vblank_int[3] ||
7565            atomic_read(&rdev->irq.pflip[3])) {
7566                DRM_DEBUG("cik_irq_set: vblank 3\n");
7567                crtc4 |= VBLANK_INTERRUPT_MASK;
7568        }
7569        if (rdev->irq.crtc_vblank_int[4] ||
7570            atomic_read(&rdev->irq.pflip[4])) {
7571                DRM_DEBUG("cik_irq_set: vblank 4\n");
7572                crtc5 |= VBLANK_INTERRUPT_MASK;
7573        }
7574        if (rdev->irq.crtc_vblank_int[5] ||
7575            atomic_read(&rdev->irq.pflip[5])) {
7576                DRM_DEBUG("cik_irq_set: vblank 5\n");
7577                crtc6 |= VBLANK_INTERRUPT_MASK;
7578        }
7579        if (rdev->irq.hpd[0]) {
7580                DRM_DEBUG("cik_irq_set: hpd 1\n");
7581                hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7582        }
7583        if (rdev->irq.hpd[1]) {
7584                DRM_DEBUG("cik_irq_set: hpd 2\n");
7585                hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7586        }
7587        if (rdev->irq.hpd[2]) {
7588                DRM_DEBUG("cik_irq_set: hpd 3\n");
7589                hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7590        }
7591        if (rdev->irq.hpd[3]) {
7592                DRM_DEBUG("cik_irq_set: hpd 4\n");
7593                hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7594        }
7595        if (rdev->irq.hpd[4]) {
7596                DRM_DEBUG("cik_irq_set: hpd 5\n");
7597                hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7598        }
7599        if (rdev->irq.hpd[5]) {
7600                DRM_DEBUG("cik_irq_set: hpd 6\n");
7601                hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7602        }
7603
7604        WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7605
7606        WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7607        WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7608
7609        WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7610
7611        WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7612
7613        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7614        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7615        if (rdev->num_crtc >= 4) {
7616                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7617                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7618        }
7619        if (rdev->num_crtc >= 6) {
7620                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7621                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7622        }
7623
7624        if (rdev->num_crtc >= 2) {
7625                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7626                       GRPH_PFLIP_INT_MASK);
7627                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7628                       GRPH_PFLIP_INT_MASK);
7629        }
7630        if (rdev->num_crtc >= 4) {
7631                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7632                       GRPH_PFLIP_INT_MASK);
7633                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7634                       GRPH_PFLIP_INT_MASK);
7635        }
7636        if (rdev->num_crtc >= 6) {
7637                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7638                       GRPH_PFLIP_INT_MASK);
7639                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7640                       GRPH_PFLIP_INT_MASK);
7641        }
7642
7643        WREG32(DC_HPD1_INT_CONTROL, hpd1);
7644        WREG32(DC_HPD2_INT_CONTROL, hpd2);
7645        WREG32(DC_HPD3_INT_CONTROL, hpd3);
7646        WREG32(DC_HPD4_INT_CONTROL, hpd4);
7647        WREG32(DC_HPD5_INT_CONTROL, hpd5);
7648        WREG32(DC_HPD6_INT_CONTROL, hpd6);
7649
7650        /* posting read */
7651        RREG32(SRBM_STATUS);
7652
7653        return 0;
7654}
7655
7656/**
7657 * cik_irq_ack - ack interrupt sources
7658 *
7659 * @rdev: radeon_device pointer
7660 *
7661 * Ack interrupt sources on the GPU (vblanks, hpd,
7662 * etc.) (CIK).  Certain interrupts sources are sw
7663 * generated and do not require an explicit ack.
7664 */
7665static inline void cik_irq_ack(struct radeon_device *rdev)
7666{
7667        u32 tmp;
7668
7669        rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7670        rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7671        rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7672        rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7673        rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7674        rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7675        rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7676
7677        rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7678                EVERGREEN_CRTC0_REGISTER_OFFSET);
7679        rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7680                EVERGREEN_CRTC1_REGISTER_OFFSET);
7681        if (rdev->num_crtc >= 4) {
7682                rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7683                        EVERGREEN_CRTC2_REGISTER_OFFSET);
7684                rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7685                        EVERGREEN_CRTC3_REGISTER_OFFSET);
7686        }
7687        if (rdev->num_crtc >= 6) {
7688                rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7689                        EVERGREEN_CRTC4_REGISTER_OFFSET);
7690                rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7691                        EVERGREEN_CRTC5_REGISTER_OFFSET);
7692        }
7693
7694        if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7695                WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7696                       GRPH_PFLIP_INT_CLEAR);
7697        if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7698                WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7699                       GRPH_PFLIP_INT_CLEAR);
7700        if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7701                WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7702        if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7703                WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7704        if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7705                WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7706        if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7707                WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7708
7709        if (rdev->num_crtc >= 4) {
7710                if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7711                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7712                               GRPH_PFLIP_INT_CLEAR);
7713                if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7714                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7715                               GRPH_PFLIP_INT_CLEAR);
7716                if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7717                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7718                if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7719                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7720                if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7721                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7722                if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7723                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7724        }
7725
7726        if (rdev->num_crtc >= 6) {
7727                if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7728                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7729                               GRPH_PFLIP_INT_CLEAR);
7730                if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7731                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7732                               GRPH_PFLIP_INT_CLEAR);
7733                if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7734                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7735                if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7736                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7737                if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7738                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7739                if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7740                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7741        }
7742
7743        if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7744                tmp = RREG32(DC_HPD1_INT_CONTROL);
7745                tmp |= DC_HPDx_INT_ACK;
7746                WREG32(DC_HPD1_INT_CONTROL, tmp);
7747        }
7748        if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7749                tmp = RREG32(DC_HPD2_INT_CONTROL);
7750                tmp |= DC_HPDx_INT_ACK;
7751                WREG32(DC_HPD2_INT_CONTROL, tmp);
7752        }
7753        if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7754                tmp = RREG32(DC_HPD3_INT_CONTROL);
7755                tmp |= DC_HPDx_INT_ACK;
7756                WREG32(DC_HPD3_INT_CONTROL, tmp);
7757        }
7758        if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7759                tmp = RREG32(DC_HPD4_INT_CONTROL);
7760                tmp |= DC_HPDx_INT_ACK;
7761                WREG32(DC_HPD4_INT_CONTROL, tmp);
7762        }
7763        if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7764                tmp = RREG32(DC_HPD5_INT_CONTROL);
7765                tmp |= DC_HPDx_INT_ACK;
7766                WREG32(DC_HPD5_INT_CONTROL, tmp);
7767        }
7768        if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7769                tmp = RREG32(DC_HPD5_INT_CONTROL);
7770                tmp |= DC_HPDx_INT_ACK;
7771                WREG32(DC_HPD6_INT_CONTROL, tmp);
7772        }
7773        if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7774                tmp = RREG32(DC_HPD1_INT_CONTROL);
7775                tmp |= DC_HPDx_RX_INT_ACK;
7776                WREG32(DC_HPD1_INT_CONTROL, tmp);
7777        }
7778        if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7779                tmp = RREG32(DC_HPD2_INT_CONTROL);
7780                tmp |= DC_HPDx_RX_INT_ACK;
7781                WREG32(DC_HPD2_INT_CONTROL, tmp);
7782        }
7783        if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7784                tmp = RREG32(DC_HPD3_INT_CONTROL);
7785                tmp |= DC_HPDx_RX_INT_ACK;
7786                WREG32(DC_HPD3_INT_CONTROL, tmp);
7787        }
7788        if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7789                tmp = RREG32(DC_HPD4_INT_CONTROL);
7790                tmp |= DC_HPDx_RX_INT_ACK;
7791                WREG32(DC_HPD4_INT_CONTROL, tmp);
7792        }
7793        if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7794                tmp = RREG32(DC_HPD5_INT_CONTROL);
7795                tmp |= DC_HPDx_RX_INT_ACK;
7796                WREG32(DC_HPD5_INT_CONTROL, tmp);
7797        }
7798        if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7799                tmp = RREG32(DC_HPD5_INT_CONTROL);
7800                tmp |= DC_HPDx_RX_INT_ACK;
7801                WREG32(DC_HPD6_INT_CONTROL, tmp);
7802        }
7803}
7804
7805/**
7806 * cik_irq_disable - disable interrupts
7807 *
7808 * @rdev: radeon_device pointer
7809 *
7810 * Disable interrupts on the hw (CIK).
7811 */
7812static void cik_irq_disable(struct radeon_device *rdev)
7813{
7814        cik_disable_interrupts(rdev);
7815        /* Wait and acknowledge irq */
7816        mdelay(1);
7817        cik_irq_ack(rdev);
7818        cik_disable_interrupt_state(rdev);
7819}
7820
7821/**
7822 * cik_irq_disable - disable interrupts for suspend
7823 *
7824 * @rdev: radeon_device pointer
7825 *
7826 * Disable interrupts and stop the RLC (CIK).
7827 * Used for suspend.
7828 */
7829static void cik_irq_suspend(struct radeon_device *rdev)
7830{
7831        cik_irq_disable(rdev);
7832        cik_rlc_stop(rdev);
7833}
7834
7835/**
7836 * cik_irq_fini - tear down interrupt support
7837 *
7838 * @rdev: radeon_device pointer
7839 *
7840 * Disable interrupts on the hw and free the IH ring
7841 * buffer (CIK).
7842 * Used for driver unload.
7843 */
7844static void cik_irq_fini(struct radeon_device *rdev)
7845{
7846        cik_irq_suspend(rdev);
7847        r600_ih_ring_fini(rdev);
7848}
7849
7850/**
7851 * cik_get_ih_wptr - get the IH ring buffer wptr
7852 *
7853 * @rdev: radeon_device pointer
7854 *
7855 * Get the IH ring buffer wptr from either the register
7856 * or the writeback memory buffer (CIK).  Also check for
7857 * ring buffer overflow and deal with it.
7858 * Used by cik_irq_process().
7859 * Returns the value of the wptr.
7860 */
7861static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7862{
7863        u32 wptr, tmp;
7864
7865        if (rdev->wb.enabled)
7866                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7867        else
7868                wptr = RREG32(IH_RB_WPTR);
7869
7870        if (wptr & RB_OVERFLOW) {
7871                wptr &= ~RB_OVERFLOW;
7872                /* When a ring buffer overflow happen start parsing interrupt
7873                 * from the last not overwritten vector (wptr + 16). Hopefully
7874                 * this should allow us to catchup.
7875                 */
7876                dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7877                         wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7878                rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7879                tmp = RREG32(IH_RB_CNTL);
7880                tmp |= IH_WPTR_OVERFLOW_CLEAR;
7881                WREG32(IH_RB_CNTL, tmp);
7882        }
7883        return (wptr & rdev->ih.ptr_mask);
7884}
7885
7886/*        CIK IV Ring
7887 * Each IV ring entry is 128 bits:
7888 * [7:0]    - interrupt source id
7889 * [31:8]   - reserved
7890 * [59:32]  - interrupt source data
7891 * [63:60]  - reserved
7892 * [71:64]  - RINGID
7893 *            CP:
7894 *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7895 *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7896 *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7897 *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7898 *            PIPE_ID - ME0 0=3D
7899 *                    - ME1&2 compute dispatcher (4 pipes each)
7900 *            SDMA:
7901 *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7902 *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7903 *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7904 * [79:72]  - VMID
7905 * [95:80]  - PASID
7906 * [127:96] - reserved
7907 */
7908/**
7909 * cik_irq_process - interrupt handler
7910 *
7911 * @rdev: radeon_device pointer
7912 *
7913 * Interrupt hander (CIK).  Walk the IH ring,
7914 * ack interrupts and schedule work to handle
7915 * interrupt events.
7916 * Returns irq process return code.
7917 */
7918int cik_irq_process(struct radeon_device *rdev)
7919{
7920        struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7921        struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7922        u32 wptr;
7923        u32 rptr;
7924        u32 src_id, src_data, ring_id;
7925        u8 me_id, pipe_id, queue_id;
7926        u32 ring_index;
7927        bool queue_hotplug = false;
7928        bool queue_dp = false;
7929        bool queue_reset = false;
7930        u32 addr, status, mc_client;
7931        bool queue_thermal = false;
7932
7933        if (!rdev->ih.enabled || rdev->shutdown)
7934                return IRQ_NONE;
7935
7936        wptr = cik_get_ih_wptr(rdev);
7937
7938restart_ih:
7939        /* is somebody else already processing irqs? */
7940        if (atomic_xchg(&rdev->ih.lock, 1))
7941                return IRQ_NONE;
7942
7943        rptr = rdev->ih.rptr;
7944        DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7945
7946        /* Order reading of wptr vs. reading of IH ring data */
7947        rmb();
7948
7949        /* display interrupts */
7950        cik_irq_ack(rdev);
7951
7952        while (rptr != wptr) {
7953                /* wptr/rptr are in bytes! */
7954                ring_index = rptr / 4;
7955
7956                radeon_kfd_interrupt(rdev,
7957                                (const void *) &rdev->ih.ring[ring_index]);
7958
7959                src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7960                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7961                ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7962
7963                switch (src_id) {
7964                case 1: /* D1 vblank/vline */
7965                        switch (src_data) {
7966                        case 0: /* D1 vblank */
7967                                if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7968                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7969
7970                                if (rdev->irq.crtc_vblank_int[0]) {
7971                                        drm_handle_vblank(rdev->ddev, 0);
7972                                        rdev->pm.vblank_sync = true;
7973                                        wake_up(&rdev->irq.vblank_queue);
7974                                }
7975                                if (atomic_read(&rdev->irq.pflip[0]))
7976                                        radeon_crtc_handle_vblank(rdev, 0);
7977                                rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7978                                DRM_DEBUG("IH: D1 vblank\n");
7979
7980                                break;
7981                        case 1: /* D1 vline */
7982                                if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7983                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7984
7985                                rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7986                                DRM_DEBUG("IH: D1 vline\n");
7987
7988                                break;
7989                        default:
7990                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7991                                break;
7992                        }
7993                        break;
7994                case 2: /* D2 vblank/vline */
7995                        switch (src_data) {
7996                        case 0: /* D2 vblank */
7997                                if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7998                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7999
8000                                if (rdev->irq.crtc_vblank_int[1]) {

8001                                        drm_handle_vblank(rdev->ddev, 1);
8002                                        rdev->pm.vblank_sync = true;
8003                                        wake_up(&rdev->irq.vblank_queue);
8004                                }
8005                                if (atomic_read(&rdev->irq.pflip[1]))
8006                                        radeon_crtc_handle_vblank(rdev, 1);
8007                                rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
8008                                DRM_DEBUG("IH: D2 vblank\n");
8009
8010                                break;
8011                        case 1: /* D2 vline */
8012                                if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
8013                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8014
8015                                rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
8016                                DRM_DEBUG("IH: D2 vline\n");
8017
8018                                break;
8019                        default:
8020                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8021                                break;
8022                        }
8023                        break;
8024                case 3: /* D3 vblank/vline */
8025                        switch (src_data) {
8026                        case 0: /* D3 vblank */
8027                                if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
8028                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8029
8030                                if (rdev->irq.crtc_vblank_int[2]) {
8031                                        drm_handle_vblank(rdev->ddev, 2);
8032                                        rdev->pm.vblank_sync = true;
8033                                        wake_up(&rdev->irq.vblank_queue);
8034                                }
8035                                if (atomic_read(&rdev->irq.pflip[2]))
8036                                        radeon_crtc_handle_vblank(rdev, 2);
8037                                rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
8038                                DRM_DEBUG("IH: D3 vblank\n");
8039
8040                                break;
8041                        case 1: /* D3 vline */
8042                                if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
8043                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8044
8045                                rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
8046                                DRM_DEBUG("IH: D3 vline\n");
8047
8048                                break;
8049                        default:
8050                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8051                                break;
8052                        }
8053                        break;
8054                case 4: /* D4 vblank/vline */
8055                        switch (src_data) {
8056                        case 0: /* D4 vblank */
8057                                if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
8058                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8059
8060                                if (rdev->irq.crtc_vblank_int[3]) {
8061                                        drm_handle_vblank(rdev->ddev, 3);
8062                                        rdev->pm.vblank_sync = true;
8063                                        wake_up(&rdev->irq.vblank_queue);
8064                                }
8065                                if (atomic_read(&rdev->irq.pflip[3]))
8066                                        radeon_crtc_handle_vblank(rdev, 3);
8067                                rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
8068                                DRM_DEBUG("IH: D4 vblank\n");
8069
8070                                break;
8071                        case 1: /* D4 vline */
8072                                if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
8073                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8074
8075                                rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
8076                                DRM_DEBUG("IH: D4 vline\n");
8077
8078                                break;
8079                        default:
8080                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8081                                break;
8082                        }
8083                        break;
8084                case 5: /* D5 vblank/vline */
8085                        switch (src_data) {
8086                        case 0: /* D5 vblank */
8087                                if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
8088                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8089
8090                                if (rdev->irq.crtc_vblank_int[4]) {
8091                                        drm_handle_vblank(rdev->ddev, 4);
8092                                        rdev->pm.vblank_sync = true;
8093                                        wake_up(&rdev->irq.vblank_queue);
8094                                }
8095                                if (atomic_read(&rdev->irq.pflip[4]))
8096                                        radeon_crtc_handle_vblank(rdev, 4);
8097                                rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
8098                                DRM_DEBUG("IH: D5 vblank\n");
8099
8100                                break;
8101                        case 1: /* D5 vline */
8102                                if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
8103                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8104
8105                                rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
8106                                DRM_DEBUG("IH: D5 vline\n");
8107
8108                                break;
8109                        default:
8110                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8111                                break;
8112                        }
8113                        break;
8114                case 6: /* D6 vblank/vline */
8115                        switch (src_data) {
8116                        case 0: /* D6 vblank */
8117                                if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
8118                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8119
8120                                if (rdev->irq.crtc_vblank_int[5]) {
8121                                        drm_handle_vblank(rdev->ddev, 5);
8122                                        rdev->pm.vblank_sync = true;
8123                                        wake_up(&rdev->irq.vblank_queue);
8124                                }
8125                                if (atomic_read(&rdev->irq.pflip[5]))
8126                                        radeon_crtc_handle_vblank(rdev, 5);
8127                                rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
8128                                DRM_DEBUG("IH: D6 vblank\n");
8129
8130                                break;
8131                        case 1: /* D6 vline */
8132                                if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
8133                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8134
8135                                rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
8136                                DRM_DEBUG("IH: D6 vline\n");
8137
8138                                break;
8139                        default:
8140                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8141                                break;
8142                        }
8143                        break;
8144                case 8: /* D1 page flip */
8145                case 10: /* D2 page flip */
8146                case 12: /* D3 page flip */
8147                case 14: /* D4 page flip */
8148                case 16: /* D5 page flip */
8149                case 18: /* D6 page flip */
8150                        DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
8151                        if (radeon_use_pflipirq > 0)
8152                                radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
8153                        break;
8154                case 42: /* HPD hotplug */
8155                        switch (src_data) {
8156                        case 0:
8157                                if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
8158                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8159
8160                                rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8161                                queue_hotplug = true;
8162                                DRM_DEBUG("IH: HPD1\n");
8163
8164                                break;
8165                        case 1:
8166                                if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
8167                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8168
8169                                rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8170                                queue_hotplug = true;
8171                                DRM_DEBUG("IH: HPD2\n");
8172
8173                                break;
8174                        case 2:
8175                                if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
8176                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8177
8178                                rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8179                                queue_hotplug = true;
8180                                DRM_DEBUG("IH: HPD3\n");
8181
8182                                break;
8183                        case 3:
8184                                if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
8185                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8186
8187                                rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8188                                queue_hotplug = true;
8189                                DRM_DEBUG("IH: HPD4\n");
8190
8191                                break;
8192                        case 4:
8193                                if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
8194                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8195
8196                                rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8197                                queue_hotplug = true;
8198                                DRM_DEBUG("IH: HPD5\n");
8199
8200                                break;
8201                        case 5:
8202                                if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
8203                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8204
8205                                rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8206                                queue_hotplug = true;
8207                                DRM_DEBUG("IH: HPD6\n");
8208
8209                                break;
8210                        case 6:
8211                                if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
8212                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8213
8214                                rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
8215                                queue_dp = true;
8216                                DRM_DEBUG("IH: HPD_RX 1\n");
8217
8218                                break;
8219                        case 7:
8220                                if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
8221                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8222
8223                                rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
8224                                queue_dp = true;
8225                                DRM_DEBUG("IH: HPD_RX 2\n");
8226
8227                                break;
8228                        case 8:
8229                                if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
8230                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8231
8232                                rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
8233                                queue_dp = true;
8234                                DRM_DEBUG("IH: HPD_RX 3\n");
8235
8236                                break;
8237                        case 9:
8238                                if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
8239                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8240
8241                                rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
8242                                queue_dp = true;
8243                                DRM_DEBUG("IH: HPD_RX 4\n");
8244
8245                                break;
8246                        case 10:
8247                                if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
8248                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8249
8250                                rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
8251                                queue_dp = true;
8252                                DRM_DEBUG("IH: HPD_RX 5\n");
8253
8254                                break;
8255                        case 11:
8256                                if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
8257                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
8258
8259                                rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
8260                                queue_dp = true;
8261                                DRM_DEBUG("IH: HPD_RX 6\n");
8262
8263                                break;
8264                        default:
8265                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8266                                break;
8267                        }
8268                        break;
8269                case 96:
8270                        DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
8271                        WREG32(SRBM_INT_ACK, 0x1);
8272                        break;
8273                case 124: /* UVD */
8274                        DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8275                        radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8276                        break;
8277                case 146:
8278                case 147:
8279                        addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8280                        status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8281                        mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8282                        /* reset addr and status */
8283                        WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8284                        if (addr == 0x0 && status == 0x0)
8285                                break;
8286                        dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8287                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
8288                                addr);
8289                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8290                                status);
8291                        cik_vm_decode_fault(rdev, status, addr, mc_client);
8292                        break;
8293                case 167: /* VCE */
8294                        DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8295                        switch (src_data) {
8296                        case 0:
8297                                radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8298                                break;
8299                        case 1:
8300                                radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8301                                break;
8302                        default:
8303                                DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8304                                break;
8305                        }
8306                        break;
8307                case 176: /* GFX RB CP_INT */
8308                case 177: /* GFX IB CP_INT */
8309                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8310                        break;
8311                case 181: /* CP EOP event */
8312                        DRM_DEBUG("IH: CP EOP\n");
8313                        /* XXX check the bitfield order! */
8314                        me_id = (ring_id & 0x60) >> 5;
8315                        pipe_id = (ring_id & 0x18) >> 3;
8316                        queue_id = (ring_id & 0x7) >> 0;
8317                        switch (me_id) {
8318                        case 0:
8319                                radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8320                                break;
8321                        case 1:
8322                        case 2:
8323                                if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8324                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8325                                if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8326                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8327                                break;
8328                        }
8329                        break;
8330                case 184: /* CP Privileged reg access */
8331                        DRM_ERROR("Illegal register access in command stream\n");
8332                        /* XXX check the bitfield order! */
8333                        me_id = (ring_id & 0x60) >> 5;
8334                        pipe_id = (ring_id & 0x18) >> 3;
8335                        queue_id = (ring_id & 0x7) >> 0;
8336                        switch (me_id) {
8337                        case 0:
8338                                /* This results in a full GPU reset, but all we need to do is soft
8339                                 * reset the CP for gfx
8340                                 */
8341                                queue_reset = true;
8342                                break;
8343                        case 1:
8344                                /* XXX compute */
8345                                queue_reset = true;
8346                                break;
8347                        case 2:
8348                                /* XXX compute */
8349                                queue_reset = true;
8350                                break;
8351                        }
8352                        break;
8353                case 185: /* CP Privileged inst */
8354                        DRM_ERROR("Illegal instruction in command stream\n");
8355                        /* XXX check the bitfield order! */
8356                        me_id = (ring_id & 0x60) >> 5;
8357                        pipe_id = (ring_id & 0x18) >> 3;
8358                        queue_id = (ring_id & 0x7) >> 0;
8359                        switch (me_id) {
8360                        case 0:
8361                                /* This results in a full GPU reset, but all we need to do is soft
8362                                 * reset the CP for gfx
8363                                 */
8364                                queue_reset = true;
8365                                break;
8366                        case 1:
8367                                /* XXX compute */
8368                                queue_reset = true;
8369                                break;
8370                        case 2:
8371                                /* XXX compute */
8372                                queue_reset = true;
8373                                break;
8374                        }
8375                        break;
8376                case 224: /* SDMA trap event */
8377                        /* XXX check the bitfield order! */
8378                        me_id = (ring_id & 0x3) >> 0;
8379                        queue_id = (ring_id & 0xc) >> 2;
8380                        DRM_DEBUG("IH: SDMA trap\n");
8381                        switch (me_id) {
8382                        case 0:
8383                                switch (queue_id) {
8384                                case 0:
8385                                        radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8386                                        break;
8387                                case 1:
8388                                        /* XXX compute */
8389                                        break;
8390                                case 2:
8391                                        /* XXX compute */
8392                                        break;
8393                                }
8394                                break;
8395                        case 1:
8396                                switch (queue_id) {
8397                                case 0:
8398                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8399                                        break;
8400                                case 1:
8401                                        /* XXX compute */
8402                                        break;
8403                                case 2:
8404                                        /* XXX compute */
8405                                        break;
8406                                }
8407                                break;
8408                        }
8409                        break;
8410                case 230: /* thermal low to high */
8411                        DRM_DEBUG("IH: thermal low to high\n");
8412                        rdev->pm.dpm.thermal.high_to_low = false;
8413                        queue_thermal = true;
8414                        break;
8415                case 231: /* thermal high to low */
8416                        DRM_DEBUG("IH: thermal high to low\n");
8417                        rdev->pm.dpm.thermal.high_to_low = true;
8418                        queue_thermal = true;
8419                        break;
8420                case 233: /* GUI IDLE */
8421                        DRM_DEBUG("IH: GUI idle\n");
8422                        break;
8423                case 241: /* SDMA Privileged inst */
8424                case 247: /* SDMA Privileged inst */
8425                        DRM_ERROR("Illegal instruction in SDMA command stream\n");
8426                        /* XXX check the bitfield order! */
8427                        me_id = (ring_id & 0x3) >> 0;
8428                        queue_id = (ring_id & 0xc) >> 2;
8429                        switch (me_id) {
8430                        case 0:
8431                                switch (queue_id) {
8432                                case 0:
8433                                        queue_reset = true;
8434                                        break;
8435                                case 1:
8436                                        /* XXX compute */
8437                                        queue_reset = true;
8438                                        break;
8439                                case 2:
8440                                        /* XXX compute */
8441                                        queue_reset = true;
8442                                        break;
8443                                }
8444                                break;
8445                        case 1:
8446                                switch (queue_id) {
8447                                case 0:
8448                                        queue_reset = true;
8449                                        break;
8450                                case 1:
8451                                        /* XXX compute */
8452                                        queue_reset = true;
8453                                        break;
8454                                case 2:
8455                                        /* XXX compute */
8456                                        queue_reset = true;
8457                                        break;
8458                                }
8459                                break;
8460                        }
8461                        break;
8462                default:
8463                        DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8464                        break;
8465                }
8466
8467                /* wptr/rptr are in bytes! */
8468                rptr += 16;
8469                rptr &= rdev->ih.ptr_mask;
8470                WREG32(IH_RB_RPTR, rptr);
8471        }
8472        if (queue_dp)
8473                schedule_work(&rdev->dp_work);
8474        if (queue_hotplug)
8475                schedule_work(&rdev->hotplug_work);
8476        if (queue_reset) {
8477                rdev->needs_reset = true;
8478                wake_up_all(&rdev->fence_queue);
8479        }
8480        if (queue_thermal)
8481                schedule_work(&rdev->pm.dpm.thermal.work);
8482        rdev->ih.rptr = rptr;
8483        atomic_set(&rdev->ih.lock, 0);
8484
8485        /* make sure wptr hasn't changed while processing */
8486        wptr = cik_get_ih_wptr(rdev);
8487        if (wptr != rptr)
8488                goto restart_ih;
8489
8490        return IRQ_HANDLED;
8491}
8492
8493/*
8494 * startup/shutdown callbacks
8495 */
8496/**
8497 * cik_startup - program the asic to a functional state
8498 *
8499 * @rdev: radeon_device pointer
8500 *
8501 * Programs the asic to a functional state (CIK).
8502 * Called by cik_init() and cik_resume().
8503 * Returns 0 for success, error for failure.
8504 */
8505static int cik_startup(struct radeon_device *rdev)
8506{
8507        struct radeon_ring *ring;
8508        u32 nop;
8509        int r;
8510
8511        /* enable pcie gen2/3 link */
8512        cik_pcie_gen3_enable(rdev);
8513        /* enable aspm */
8514        cik_program_aspm(rdev);
8515
8516        /* scratch needs to be initialized before MC */
8517        r = r600_vram_scratch_init(rdev);
8518        if (r)
8519                return r;
8520
8521        cik_mc_program(rdev);
8522
8523        if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8524                r = ci_mc_load_microcode(rdev);
8525                if (r) {
8526                        DRM_ERROR("Failed to load MC firmware!\n");
8527                        return r;
8528                }
8529        }
8530
8531        r = cik_pcie_gart_enable(rdev);
8532        if (r)
8533                return r;
8534        cik_gpu_init(rdev);
8535
8536        /* allocate rlc buffers */
8537        if (rdev->flags & RADEON_IS_IGP) {
8538                if (rdev->family == CHIP_KAVERI) {
8539                        rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8540                        rdev->rlc.reg_list_size =
8541                                (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8542                } else {
8543                        rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8544                        rdev->rlc.reg_list_size =
8545                                (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8546                }
8547        }
8548        rdev->rlc.cs_data = ci_cs_data;
8549        rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8550        r = sumo_rlc_init(rdev);
8551        if (r) {
8552                DRM_ERROR("Failed to init rlc BOs!\n");
8553                return r;
8554        }
8555
8556        /* allocate wb buffer */
8557        r = radeon_wb_init(rdev);
8558        if (r)
8559                return r;
8560
8561        /* allocate mec buffers */
8562        r = cik_mec_init(rdev);
8563        if (r) {
8564                DRM_ERROR("Failed to init MEC BOs!\n");
8565                return r;
8566        }
8567
8568        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8569        if (r) {
8570                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8571                return r;
8572        }
8573
8574        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8575        if (r) {
8576                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8577                return r;
8578        }
8579
8580        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8581        if (r) {
8582                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8583                return r;
8584        }
8585
8586        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8587        if (r) {
8588                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8589                return r;
8590        }
8591
8592        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8593        if (r) {
8594                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8595                return r;
8596        }
8597
8598        r = radeon_uvd_resume(rdev);
8599        if (!r) {
8600                r = uvd_v4_2_resume(rdev);
8601                if (!r) {
8602                        r = radeon_fence_driver_start_ring(rdev,
8603                                                           R600_RING_TYPE_UVD_INDEX);
8604                        if (r)
8605                                dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8606                }
8607        }
8608        if (r)
8609                rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8610
8611        r = radeon_vce_resume(rdev);
8612        if (!r) {
8613                r = vce_v2_0_resume(rdev);
8614                if (!r)
8615                        r = radeon_fence_driver_start_ring(rdev,
8616                                                           TN_RING_TYPE_VCE1_INDEX);
8617                if (!r)
8618                        r = radeon_fence_driver_start_ring(rdev,
8619                                                           TN_RING_TYPE_VCE2_INDEX);
8620        }
8621        if (r) {
8622                dev_err(rdev->dev, "VCE init error (%d).\n", r);
8623                rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8624                rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8625        }
8626
8627        /* Enable IRQ */
8628        if (!rdev->irq.installed) {
8629                r = radeon_irq_kms_init(rdev);
8630                if (r)
8631                        return r;
8632        }
8633
8634        r = cik_irq_init(rdev);
8635        if (r) {
8636                DRM_ERROR("radeon: IH init failed (%d).\n", r);
8637                radeon_irq_kms_fini(rdev);
8638                return r;
8639        }
8640        cik_irq_set(rdev);
8641
8642        if (rdev->family == CHIP_HAWAII) {
8643                if (rdev->new_fw)
8644                        nop = PACKET3(PACKET3_NOP, 0x3FFF);
8645                else
8646                        nop = RADEON_CP_PACKET2;
8647        } else {
8648                nop = PACKET3(PACKET3_NOP, 0x3FFF);
8649        }
8650
8651        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8652        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8653                             nop);
8654        if (r)
8655                return r;
8656
8657        /* set up the compute queues */
8658        /* type-2 packets are deprecated on MEC, use type-3 instead */
8659        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8660        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8661                             nop);
8662        if (r)
8663                return r;
8664        ring->me = 1; /* first MEC */
8665        ring->pipe = 0; /* first pipe */
8666        ring->queue = 0; /* first queue */
8667        ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8668
8669        /* type-2 packets are deprecated on MEC, use type-3 instead */
8670        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8671        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8672                             nop);
8673        if (r)
8674                return r;
8675        /* dGPU only have 1 MEC */
8676        ring->me = 1; /* first MEC */
8677        ring->pipe = 0; /* first pipe */
8678        ring->queue = 1; /* second queue */
8679        ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8680
8681        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8682        r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8683                             SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8684        if (r)
8685                return r;
8686
8687        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8688        r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8689                             SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8690        if (r)
8691                return r;
8692
8693        r = cik_cp_resume(rdev);
8694        if (r)
8695                return r;
8696
8697        r = cik_sdma_resume(rdev);
8698        if (r)
8699                return r;
8700
8701        ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8702        if (ring->ring_size) {
8703                r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8704                                     RADEON_CP_PACKET2);
8705                if (!r)
8706                        r = uvd_v1_0_init(rdev);
8707                if (r)
8708                        DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8709        }
8710
8711        r = -ENOENT;
8712
8713        ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8714        if (ring->ring_size)
8715                r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8716                                     VCE_CMD_NO_OP);
8717
8718        ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8719        if (ring->ring_size)
8720                r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8721                                     VCE_CMD_NO_OP);
8722
8723        if (!r)
8724                r = vce_v1_0_init(rdev);
8725        else if (r != -ENOENT)
8726                DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8727
8728        r = radeon_ib_pool_init(rdev);
8729        if (r) {
8730                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8731                return r;
8732        }
8733
8734        r = radeon_vm_manager_init(rdev);
8735        if (r) {
8736                dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8737                return r;
8738        }
8739
8740        r = radeon_audio_init(rdev);
8741        if (r)
8742                return r;
8743
8744        r = radeon_kfd_resume(rdev);
8745        if (r)
8746                return r;
8747
8748        return 0;
8749}
8750
8751/**
8752 * cik_resume - resume the asic to a functional state
8753 *
8754 * @rdev: radeon_device pointer
8755 *
8756 * Programs the asic to a functional state (CIK).
8757 * Called at resume.
8758 * Returns 0 for success, error for failure.
8759 */
8760int cik_resume(struct radeon_device *rdev)
8761{
8762        int r;
8763
8764        /* post card */
8765        atom_asic_init(rdev->mode_info.atom_context);
8766
8767        /* init golden registers */
8768        cik_init_golden_registers(rdev);
8769
8770        if (rdev->pm.pm_method == PM_METHOD_DPM)
8771                radeon_pm_resume(rdev);
8772
8773        rdev->accel_working = true;
8774        r = cik_startup(rdev);
8775        if (r) {
8776                DRM_ERROR("cik startup failed on resume\n");
8777                rdev->accel_working = false;
8778                return r;
8779        }
8780
8781        return r;
8782
8783}
8784
8785/**
8786 * cik_suspend - suspend the asic
8787 *
8788 * @rdev: radeon_device pointer
8789 *
8790 * Bring the chip into a state suitable for suspend (CIK).
8791 * Called at suspend.
8792 * Returns 0 for success.
8793 */
8794int cik_suspend(struct radeon_device *rdev)
8795{
8796        radeon_kfd_suspend(rdev);
8797        radeon_pm_suspend(rdev);
8798        radeon_audio_fini(rdev);
8799        radeon_vm_manager_fini(rdev);
8800        cik_cp_enable(rdev, false);
8801        cik_sdma_enable(rdev, false);
8802        uvd_v1_0_fini(rdev);
8803        radeon_uvd_suspend(rdev);
8804        radeon_vce_suspend(rdev);
8805        cik_fini_pg(rdev);
8806        cik_fini_cg(rdev);
8807        cik_irq_suspend(rdev);
8808        radeon_wb_disable(rdev);
8809        cik_pcie_gart_disable(rdev);
8810        return 0;
8811}
8812
8813/* Plan is to move initialization in that function and use
8814 * helper function so that radeon_device_init pretty much
8815 * do nothing more than calling asic specific function. This
8816 * should also allow to remove a bunch of callback function
8817 * like vram_info.
8818 */
8819/**
8820 * cik_init - asic specific driver and hw init
8821 *
8822 * @rdev: radeon_device pointer
8823 *
8824 * Setup asic specific driver variables and program the hw
8825 * to a functional state (CIK).
8826 * Called at driver startup.
8827 * Returns 0 for success, errors for failure.
8828 */
8829int cik_init(struct radeon_device *rdev)
8830{
8831        struct radeon_ring *ring;
8832        int r;
8833
8834        /* Read BIOS */
8835        if (!radeon_get_bios(rdev)) {
8836                if (ASIC_IS_AVIVO(rdev))
8837                        return -EINVAL;
8838        }
8839        /* Must be an ATOMBIOS */
8840        if (!rdev->is_atom_bios) {
8841                dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8842                return -EINVAL;
8843        }
8844        r = radeon_atombios_init(rdev);
8845        if (r)
8846                return r;
8847
8848        /* Post card if necessary */
8849        if (!radeon_card_posted(rdev)) {
8850                if (!rdev->bios) {
8851                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8852                        return -EINVAL;
8853                }
8854                DRM_INFO("GPU not posted. posting now...\n");
8855                atom_asic_init(rdev->mode_info.atom_context);
8856        }
8857        /* init golden registers */
8858        cik_init_golden_registers(rdev);
8859        /* Initialize scratch registers */
8860        cik_scratch_init(rdev);
8861        /* Initialize surface registers */
8862        radeon_surface_init(rdev);
8863        /* Initialize clocks */
8864        radeon_get_clock_info(rdev->ddev);
8865
8866        /* Fence driver */
8867        r = radeon_fence_driver_init(rdev);
8868        if (r)
8869                return r;
8870
8871        /* initialize memory controller */
8872        r = cik_mc_init(rdev);
8873        if (r)
8874                return r;
8875        /* Memory manager */
8876        r = radeon_bo_init(rdev);
8877        if (r)
8878                return r;
8879
8880        if (rdev->flags & RADEON_IS_IGP) {
8881                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8882                    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8883                        r = cik_init_microcode(rdev);
8884                        if (r) {
8885                                DRM_ERROR("Failed to load firmware!\n");
8886                                return r;
8887                        }
8888                }
8889        } else {
8890                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8891                    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8892                    !rdev->mc_fw) {
8893                        r = cik_init_microcode(rdev);
8894                        if (r) {
8895                                DRM_ERROR("Failed to load firmware!\n");
8896                                return r;
8897                        }
8898                }
8899        }
8900
8901        /* Initialize power management */
8902        radeon_pm_init(rdev);
8903
8904        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8905        ring->ring_obj = NULL;
8906        r600_ring_init(rdev, ring, 1024 * 1024);
8907
8908        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8909        ring->ring_obj = NULL;
8910        r600_ring_init(rdev, ring, 1024 * 1024);
8911        r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8912        if (r)
8913                return r;
8914
8915        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8916        ring->ring_obj = NULL;
8917        r600_ring_init(rdev, ring, 1024 * 1024);
8918        r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8919        if (r)
8920                return r;
8921
8922        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8923        ring->ring_obj = NULL;
8924        r600_ring_init(rdev, ring, 256 * 1024);
8925
8926        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8927        ring->ring_obj = NULL;
8928        r600_ring_init(rdev, ring, 256 * 1024);
8929
8930        r = radeon_uvd_init(rdev);
8931        if (!r) {
8932                ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8933                ring->ring_obj = NULL;
8934                r600_ring_init(rdev, ring, 4096);
8935        }
8936
8937        r = radeon_vce_init(rdev);
8938        if (!r) {
8939                ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8940                ring->ring_obj = NULL;
8941                r600_ring_init(rdev, ring, 4096);
8942
8943                ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8944                ring->ring_obj = NULL;
8945                r600_ring_init(rdev, ring, 4096);
8946        }
8947
8948        rdev->ih.ring_obj = NULL;
8949        r600_ih_ring_init(rdev, 64 * 1024);
8950
8951        r = r600_pcie_gart_init(rdev);
8952        if (r)
8953                return r;
8954
8955        rdev->accel_working = true;
8956        r = cik_startup(rdev);
8957        if (r) {
8958                dev_err(rdev->dev, "disabling GPU acceleration\n");
8959                cik_cp_fini(rdev);
8960                cik_sdma_fini(rdev);
8961                cik_irq_fini(rdev);
8962                sumo_rlc_fini(rdev);
8963                cik_mec_fini(rdev);
8964                radeon_wb_fini(rdev);
8965                radeon_ib_pool_fini(rdev);
8966                radeon_vm_manager_fini(rdev);
8967                radeon_irq_kms_fini(rdev);
8968                cik_pcie_gart_fini(rdev);
8969                rdev->accel_working = false;
8970        }
8971
8972        /* Don't start up if the MC ucode is missing.
8973         * The default clocks and voltages before the MC ucode
8974         * is loaded are not suffient for advanced operations.
8975         */
8976        if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8977                DRM_ERROR("radeon: MC ucode required for NI+.\n");
8978                return -EINVAL;
8979        }
8980
8981        return 0;
8982}
8983
8984/**
8985 * cik_fini - asic specific driver and hw fini
8986 *
8987 * @rdev: radeon_device pointer
8988 *
8989 * Tear down the asic specific driver variables and program the hw
8990 * to an idle state (CIK).
8991 * Called at driver unload.
8992 */
8993void cik_fini(struct radeon_device *rdev)
8994{
8995        radeon_pm_fini(rdev);
8996        cik_cp_fini(rdev);
8997        cik_sdma_fini(rdev);
8998        cik_fini_pg(rdev);
8999        cik_fini_cg(rdev);
9000        cik_irq_fini(rdev);

9001        sumo_rlc_fini(rdev);
9002        cik_mec_fini(rdev);
9003        radeon_wb_fini(rdev);
9004        radeon_vm_manager_fini(rdev);
9005        radeon_ib_pool_fini(rdev);
9006        radeon_irq_kms_fini(rdev);
9007        uvd_v1_0_fini(rdev);
9008        radeon_uvd_fini(rdev);
9009        radeon_vce_fini(rdev);
9010        cik_pcie_gart_fini(rdev);
9011        r600_vram_scratch_fini(rdev);
9012        radeon_gem_fini(rdev);
9013        radeon_fence_driver_fini(rdev);
9014        radeon_bo_fini(rdev);
9015        radeon_atombios_fini(rdev);
9016        kfree(rdev->bios);
9017        rdev->bios = NULL;
9018}
9019
9020void dce8_program_fmt(struct drm_encoder *encoder)
9021{
9022        struct drm_device *dev = encoder->dev;
9023        struct radeon_device *rdev = dev->dev_private;
9024        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
9025        struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
9026        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
9027        int bpc = 0;
9028        u32 tmp = 0;
9029        enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
9030
9031        if (connector) {
9032                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
9033                bpc = radeon_get_monitor_bpc(connector);
9034                dither = radeon_connector->dither;
9035        }
9036
9037        /* LVDS/eDP FMT is set up by atom */
9038        if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
9039                return;
9040
9041        /* not needed for analog */
9042        if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
9043            (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
9044                return;
9045
9046        if (bpc == 0)
9047                return;
9048
9049        switch (bpc) {
9050        case 6:
9051                if (dither == RADEON_FMT_DITHER_ENABLE)
9052                        /* XXX sort out optimal dither settings */
9053                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
9054                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
9055                else
9056                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
9057                break;
9058        case 8:
9059                if (dither == RADEON_FMT_DITHER_ENABLE)
9060                        /* XXX sort out optimal dither settings */
9061                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
9062                                FMT_RGB_RANDOM_ENABLE |
9063                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
9064                else
9065                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
9066                break;
9067        case 10:
9068                if (dither == RADEON_FMT_DITHER_ENABLE)
9069                        /* XXX sort out optimal dither settings */
9070                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
9071                                FMT_RGB_RANDOM_ENABLE |
9072                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
9073                else
9074                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
9075                break;
9076        default:
9077                /* not needed */
9078                break;
9079        }
9080
9081        WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
9082}
9083
9084/* display watermark setup */
9085/**
9086 * dce8_line_buffer_adjust - Set up the line buffer
9087 *
9088 * @rdev: radeon_device pointer
9089 * @radeon_crtc: the selected display controller
9090 * @mode: the current display mode on the selected display
9091 * controller
9092 *
9093 * Setup up the line buffer allocation for
9094 * the selected display controller (CIK).
9095 * Returns the line buffer size in pixels.
9096 */
9097static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
9098                                   struct radeon_crtc *radeon_crtc,
9099                                   struct drm_display_mode *mode)
9100{
9101        u32 tmp, buffer_alloc, i;
9102        u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
9103        /*
9104         * Line Buffer Setup
9105         * There are 6 line buffers, one for each display controllers.
9106         * There are 3 partitions per LB. Select the number of partitions
9107         * to enable based on the display width.  For display widths larger
9108         * than 4096, you need use to use 2 display controllers and combine
9109         * them using the stereo blender.
9110         */
9111        if (radeon_crtc->base.enabled && mode) {
9112                if (mode->crtc_hdisplay < 1920) {
9113                        tmp = 1;
9114                        buffer_alloc = 2;
9115                } else if (mode->crtc_hdisplay < 2560) {
9116                        tmp = 2;
9117                        buffer_alloc = 2;
9118                } else if (mode->crtc_hdisplay < 4096) {
9119                        tmp = 0;
9120                        buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
9121                } else {
9122                        DRM_DEBUG_KMS("Mode too big for LB!\n");
9123                        tmp = 0;
9124                        buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
9125                }
9126        } else {
9127                tmp = 1;
9128                buffer_alloc = 0;
9129        }
9130
9131        WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
9132               LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
9133
9134        WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
9135               DMIF_BUFFERS_ALLOCATED(buffer_alloc));
9136        for (i = 0; i < rdev->usec_timeout; i++) {
9137                if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
9138                    DMIF_BUFFERS_ALLOCATED_COMPLETED)
9139                        break;
9140                udelay(1);
9141        }
9142
9143        if (radeon_crtc->base.enabled && mode) {
9144                switch (tmp) {
9145                case 0:
9146                default:
9147                        return 4096 * 2;
9148                case 1:
9149                        return 1920 * 2;
9150                case 2:
9151                        return 2560 * 2;
9152                }
9153        }
9154
9155        /* controller not enabled, so no lb used */
9156        return 0;
9157}
9158
9159/**
9160 * cik_get_number_of_dram_channels - get the number of dram channels
9161 *
9162 * @rdev: radeon_device pointer
9163 *
9164 * Look up the number of video ram channels (CIK).
9165 * Used for display watermark bandwidth calculations
9166 * Returns the number of dram channels
9167 */
9168static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
9169{
9170        u32 tmp = RREG32(MC_SHARED_CHMAP);
9171
9172        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
9173        case 0:
9174        default:
9175                return 1;
9176        case 1:
9177                return 2;
9178        case 2:
9179                return 4;
9180        case 3:
9181                return 8;
9182        case 4:
9183                return 3;
9184        case 5:
9185                return 6;
9186        case 6:
9187                return 10;
9188        case 7:
9189                return 12;
9190        case 8:
9191                return 16;
9192        }
9193}
9194
9195struct dce8_wm_params {
9196        u32 dram_channels; /* number of dram channels */
9197        u32 yclk;          /* bandwidth per dram data pin in kHz */
9198        u32 sclk;          /* engine clock in kHz */
9199        u32 disp_clk;      /* display clock in kHz */
9200        u32 src_width;     /* viewport width */
9201        u32 active_time;   /* active display time in ns */
9202        u32 blank_time;    /* blank time in ns */
9203        bool interlaced;    /* mode is interlaced */
9204        fixed20_12 vsc;    /* vertical scale ratio */
9205        u32 num_heads;     /* number of active crtcs */
9206        u32 bytes_per_pixel; /* bytes per pixel display + overlay */
9207        u32 lb_size;       /* line buffer allocated to pipe */
9208        u32 vtaps;         /* vertical scaler taps */
9209};
9210
9211/**
9212 * dce8_dram_bandwidth - get the dram bandwidth
9213 *
9214 * @wm: watermark calculation data
9215 *
9216 * Calculate the raw dram bandwidth (CIK).
9217 * Used for display watermark bandwidth calculations
9218 * Returns the dram bandwidth in MBytes/s
9219 */
9220static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
9221{
9222        /* Calculate raw DRAM Bandwidth */
9223        fixed20_12 dram_efficiency; /* 0.7 */
9224        fixed20_12 yclk, dram_channels, bandwidth;
9225        fixed20_12 a;
9226
9227        a.full = dfixed_const(1000);
9228        yclk.full = dfixed_const(wm->yclk);
9229        yclk.full = dfixed_div(yclk, a);
9230        dram_channels.full = dfixed_const(wm->dram_channels * 4);
9231        a.full = dfixed_const(10);
9232        dram_efficiency.full = dfixed_const(7);
9233        dram_efficiency.full = dfixed_div(dram_efficiency, a);
9234        bandwidth.full = dfixed_mul(dram_channels, yclk);
9235        bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9236
9237        return dfixed_trunc(bandwidth);
9238}
9239
9240/**
9241 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9242 *
9243 * @wm: watermark calculation data
9244 *
9245 * Calculate the dram bandwidth used for display (CIK).
9246 * Used for display watermark bandwidth calculations
9247 * Returns the dram bandwidth for display in MBytes/s
9248 */
9249static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9250{
9251        /* Calculate DRAM Bandwidth and the part allocated to display. */
9252        fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9253        fixed20_12 yclk, dram_channels, bandwidth;
9254        fixed20_12 a;
9255
9256        a.full = dfixed_const(1000);
9257        yclk.full = dfixed_const(wm->yclk);
9258        yclk.full = dfixed_div(yclk, a);
9259        dram_channels.full = dfixed_const(wm->dram_channels * 4);
9260        a.full = dfixed_const(10);
9261        disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9262        disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9263        bandwidth.full = dfixed_mul(dram_channels, yclk);
9264        bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9265
9266        return dfixed_trunc(bandwidth);
9267}
9268
9269/**
9270 * dce8_data_return_bandwidth - get the data return bandwidth
9271 *
9272 * @wm: watermark calculation data
9273 *
9274 * Calculate the data return bandwidth used for display (CIK).
9275 * Used for display watermark bandwidth calculations
9276 * Returns the data return bandwidth in MBytes/s
9277 */
9278static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9279{
9280        /* Calculate the display Data return Bandwidth */
9281        fixed20_12 return_efficiency; /* 0.8 */
9282        fixed20_12 sclk, bandwidth;
9283        fixed20_12 a;
9284
9285        a.full = dfixed_const(1000);
9286        sclk.full = dfixed_const(wm->sclk);
9287        sclk.full = dfixed_div(sclk, a);
9288        a.full = dfixed_const(10);
9289        return_efficiency.full = dfixed_const(8);
9290        return_efficiency.full = dfixed_div(return_efficiency, a);
9291        a.full = dfixed_const(32);
9292        bandwidth.full = dfixed_mul(a, sclk);
9293        bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9294
9295        return dfixed_trunc(bandwidth);
9296}
9297
9298/**
9299 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9300 *
9301 * @wm: watermark calculation data
9302 *
9303 * Calculate the dmif bandwidth used for display (CIK).
9304 * Used for display watermark bandwidth calculations
9305 * Returns the dmif bandwidth in MBytes/s
9306 */
9307static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9308{
9309        /* Calculate the DMIF Request Bandwidth */
9310        fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9311        fixed20_12 disp_clk, bandwidth;
9312        fixed20_12 a, b;
9313
9314        a.full = dfixed_const(1000);
9315        disp_clk.full = dfixed_const(wm->disp_clk);
9316        disp_clk.full = dfixed_div(disp_clk, a);
9317        a.full = dfixed_const(32);
9318        b.full = dfixed_mul(a, disp_clk);
9319
9320        a.full = dfixed_const(10);
9321        disp_clk_request_efficiency.full = dfixed_const(8);
9322        disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9323
9324        bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9325
9326        return dfixed_trunc(bandwidth);
9327}
9328
9329/**
9330 * dce8_available_bandwidth - get the min available bandwidth
9331 *
9332 * @wm: watermark calculation data
9333 *
9334 * Calculate the min available bandwidth used for display (CIK).
9335 * Used for display watermark bandwidth calculations
9336 * Returns the min available bandwidth in MBytes/s
9337 */
9338static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9339{
9340        /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9341        u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9342        u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9343        u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9344
9345        return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9346}
9347
9348/**
9349 * dce8_average_bandwidth - get the average available bandwidth
9350 *
9351 * @wm: watermark calculation data
9352 *
9353 * Calculate the average available bandwidth used for display (CIK).
9354 * Used for display watermark bandwidth calculations
9355 * Returns the average available bandwidth in MBytes/s
9356 */
9357static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9358{
9359        /* Calculate the display mode Average Bandwidth
9360         * DisplayMode should contain the source and destination dimensions,
9361         * timing, etc.
9362         */
9363        fixed20_12 bpp;
9364        fixed20_12 line_time;
9365        fixed20_12 src_width;
9366        fixed20_12 bandwidth;
9367        fixed20_12 a;
9368
9369        a.full = dfixed_const(1000);
9370        line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9371        line_time.full = dfixed_div(line_time, a);
9372        bpp.full = dfixed_const(wm->bytes_per_pixel);
9373        src_width.full = dfixed_const(wm->src_width);
9374        bandwidth.full = dfixed_mul(src_width, bpp);
9375        bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9376        bandwidth.full = dfixed_div(bandwidth, line_time);
9377
9378        return dfixed_trunc(bandwidth);
9379}
9380
9381/**
9382 * dce8_latency_watermark - get the latency watermark
9383 *
9384 * @wm: watermark calculation data
9385 *
9386 * Calculate the latency watermark (CIK).
9387 * Used for display watermark bandwidth calculations
9388 * Returns the latency watermark in ns
9389 */
9390static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9391{
9392        /* First calculate the latency in ns */
9393        u32 mc_latency = 2000; /* 2000 ns. */
9394        u32 available_bandwidth = dce8_available_bandwidth(wm);
9395        u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9396        u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9397        u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9398        u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9399                (wm->num_heads * cursor_line_pair_return_time);
9400        u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9401        u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9402        u32 tmp, dmif_size = 12288;
9403        fixed20_12 a, b, c;
9404
9405        if (wm->num_heads == 0)
9406                return 0;
9407
9408        a.full = dfixed_const(2);
9409        b.full = dfixed_const(1);
9410        if ((wm->vsc.full > a.full) ||
9411            ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9412            (wm->vtaps >= 5) ||
9413            ((wm->vsc.full >= a.full) && wm->interlaced))
9414                max_src_lines_per_dst_line = 4;
9415        else
9416                max_src_lines_per_dst_line = 2;
9417
9418        a.full = dfixed_const(available_bandwidth);
9419        b.full = dfixed_const(wm->num_heads);
9420        a.full = dfixed_div(a, b);
9421
9422        b.full = dfixed_const(mc_latency + 512);
9423        c.full = dfixed_const(wm->disp_clk);
9424        b.full = dfixed_div(b, c);
9425
9426        c.full = dfixed_const(dmif_size);
9427        b.full = dfixed_div(c, b);
9428
9429        tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9430
9431        b.full = dfixed_const(1000);
9432        c.full = dfixed_const(wm->disp_clk);
9433        b.full = dfixed_div(c, b);
9434        c.full = dfixed_const(wm->bytes_per_pixel);
9435        b.full = dfixed_mul(b, c);
9436
9437        lb_fill_bw = min(tmp, dfixed_trunc(b));
9438
9439        a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9440        b.full = dfixed_const(1000);
9441        c.full = dfixed_const(lb_fill_bw);
9442        b.full = dfixed_div(c, b);
9443        a.full = dfixed_div(a, b);
9444        line_fill_time = dfixed_trunc(a);
9445
9446        if (line_fill_time < wm->active_time)
9447                return latency;
9448        else
9449                return latency + (line_fill_time - wm->active_time);
9450
9451}
9452
9453/**
9454 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9455 * average and available dram bandwidth
9456 *
9457 * @wm: watermark calculation data
9458 *
9459 * Check if the display average bandwidth fits in the display
9460 * dram bandwidth (CIK).
9461 * Used for display watermark bandwidth calculations
9462 * Returns true if the display fits, false if not.
9463 */
9464static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9465{
9466        if (dce8_average_bandwidth(wm) <=
9467            (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9468                return true;
9469        else
9470                return false;
9471}
9472
9473/**
9474 * dce8_average_bandwidth_vs_available_bandwidth - check
9475 * average and available bandwidth
9476 *
9477 * @wm: watermark calculation data
9478 *
9479 * Check if the display average bandwidth fits in the display
9480 * available bandwidth (CIK).
9481 * Used for display watermark bandwidth calculations
9482 * Returns true if the display fits, false if not.
9483 */
9484static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9485{
9486        if (dce8_average_bandwidth(wm) <=
9487            (dce8_available_bandwidth(wm) / wm->num_heads))
9488                return true;
9489        else
9490                return false;
9491}
9492
9493/**
9494 * dce8_check_latency_hiding - check latency hiding
9495 *
9496 * @wm: watermark calculation data
9497 *
9498 * Check latency hiding (CIK).
9499 * Used for display watermark bandwidth calculations
9500 * Returns true if the display fits, false if not.
9501 */
9502static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9503{
9504        u32 lb_partitions = wm->lb_size / wm->src_width;
9505        u32 line_time = wm->active_time + wm->blank_time;
9506        u32 latency_tolerant_lines;
9507        u32 latency_hiding;
9508        fixed20_12 a;
9509
9510        a.full = dfixed_const(1);
9511        if (wm->vsc.full > a.full)
9512                latency_tolerant_lines = 1;
9513        else {
9514                if (lb_partitions <= (wm->vtaps + 1))
9515                        latency_tolerant_lines = 1;
9516                else
9517                        latency_tolerant_lines = 2;
9518        }
9519
9520        latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9521
9522        if (dce8_latency_watermark(wm) <= latency_hiding)
9523                return true;
9524        else
9525                return false;
9526}
9527
9528/**
9529 * dce8_program_watermarks - program display watermarks
9530 *
9531 * @rdev: radeon_device pointer
9532 * @radeon_crtc: the selected display controller
9533 * @lb_size: line buffer size
9534 * @num_heads: number of display controllers in use
9535 *
9536 * Calculate and program the display watermarks for the
9537 * selected display controller (CIK).
9538 */
9539static void dce8_program_watermarks(struct radeon_device *rdev,
9540                                    struct radeon_crtc *radeon_crtc,
9541                                    u32 lb_size, u32 num_heads)
9542{
9543        struct drm_display_mode *mode = &radeon_crtc->base.mode;
9544        struct dce8_wm_params wm_low, wm_high;
9545        u32 pixel_period;
9546        u32 line_time = 0;
9547        u32 latency_watermark_a = 0, latency_watermark_b = 0;
9548        u32 tmp, wm_mask;
9549
9550        if (radeon_crtc->base.enabled && num_heads && mode) {
9551                pixel_period = 1000000 / (u32)mode->clock;
9552                line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9553
9554                /* watermark for high clocks */
9555                if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9556                    rdev->pm.dpm_enabled) {
9557                        wm_high.yclk =
9558                                radeon_dpm_get_mclk(rdev, false) * 10;
9559                        wm_high.sclk =
9560                                radeon_dpm_get_sclk(rdev, false) * 10;
9561                } else {
9562                        wm_high.yclk = rdev->pm.current_mclk * 10;
9563                        wm_high.sclk = rdev->pm.current_sclk * 10;
9564                }
9565
9566                wm_high.disp_clk = mode->clock;
9567                wm_high.src_width = mode->crtc_hdisplay;
9568                wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9569                wm_high.blank_time = line_time - wm_high.active_time;
9570                wm_high.interlaced = false;
9571                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9572                        wm_high.interlaced = true;
9573                wm_high.vsc = radeon_crtc->vsc;
9574                wm_high.vtaps = 1;
9575                if (radeon_crtc->rmx_type != RMX_OFF)
9576                        wm_high.vtaps = 2;
9577                wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9578                wm_high.lb_size = lb_size;
9579                wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9580                wm_high.num_heads = num_heads;
9581
9582                /* set for high clocks */
9583                latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9584
9585                /* possibly force display priority to high */
9586                /* should really do this at mode validation time... */
9587                if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9588                    !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9589                    !dce8_check_latency_hiding(&wm_high) ||
9590                    (rdev->disp_priority == 2)) {
9591                        DRM_DEBUG_KMS("force priority to high\n");
9592                }
9593
9594                /* watermark for low clocks */
9595                if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9596                    rdev->pm.dpm_enabled) {
9597                        wm_low.yclk =
9598                                radeon_dpm_get_mclk(rdev, true) * 10;
9599                        wm_low.sclk =
9600                                radeon_dpm_get_sclk(rdev, true) * 10;
9601                } else {
9602                        wm_low.yclk = rdev->pm.current_mclk * 10;
9603                        wm_low.sclk = rdev->pm.current_sclk * 10;
9604                }
9605
9606                wm_low.disp_clk = mode->clock;
9607                wm_low.src_width = mode->crtc_hdisplay;
9608                wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9609                wm_low.blank_time = line_time - wm_low.active_time;
9610                wm_low.interlaced = false;
9611                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9612                        wm_low.interlaced = true;
9613                wm_low.vsc = radeon_crtc->vsc;
9614                wm_low.vtaps = 1;
9615                if (radeon_crtc->rmx_type != RMX_OFF)
9616                        wm_low.vtaps = 2;
9617                wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9618                wm_low.lb_size = lb_size;
9619                wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9620                wm_low.num_heads = num_heads;
9621
9622                /* set for low clocks */
9623                latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9624
9625                /* possibly force display priority to high */
9626                /* should really do this at mode validation time... */
9627                if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9628                    !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9629                    !dce8_check_latency_hiding(&wm_low) ||
9630                    (rdev->disp_priority == 2)) {
9631                        DRM_DEBUG_KMS("force priority to high\n");
9632                }
9633        }
9634
9635        /* select wm A */
9636        wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9637        tmp = wm_mask;
9638        tmp &= ~LATENCY_WATERMARK_MASK(3);
9639        tmp |= LATENCY_WATERMARK_MASK(1);
9640        WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9641        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9642               (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9643                LATENCY_HIGH_WATERMARK(line_time)));
9644        /* select wm B */
9645        tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9646        tmp &= ~LATENCY_WATERMARK_MASK(3);
9647        tmp |= LATENCY_WATERMARK_MASK(2);
9648        WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9649        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9650               (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9651                LATENCY_HIGH_WATERMARK(line_time)));
9652        /* restore original selection */
9653        WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9654
9655        /* save values for DPM */
9656        radeon_crtc->line_time = line_time;
9657        radeon_crtc->wm_high = latency_watermark_a;
9658        radeon_crtc->wm_low = latency_watermark_b;
9659}
9660
9661/**
9662 * dce8_bandwidth_update - program display watermarks
9663 *
9664 * @rdev: radeon_device pointer
9665 *
9666 * Calculate and program the display watermarks and line
9667 * buffer allocation (CIK).
9668 */
9669void dce8_bandwidth_update(struct radeon_device *rdev)
9670{
9671        struct drm_display_mode *mode = NULL;
9672        u32 num_heads = 0, lb_size;
9673        int i;
9674
9675        if (!rdev->mode_info.mode_config_initialized)
9676                return;
9677
9678        radeon_update_display_priority(rdev);
9679
9680        for (i = 0; i < rdev->num_crtc; i++) {
9681                if (rdev->mode_info.crtcs[i]->base.enabled)
9682                        num_heads++;
9683        }
9684        for (i = 0; i < rdev->num_crtc; i++) {
9685                mode = &rdev->mode_info.crtcs[i]->base.mode;
9686                lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9687                dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9688        }
9689}
9690
9691/**
9692 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9693 *
9694 * @rdev: radeon_device pointer
9695 *
9696 * Fetches a GPU clock counter snapshot (SI).
9697 * Returns the 64 bit clock counter snapshot.
9698 */
9699uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9700{
9701        uint64_t clock;
9702
9703        mutex_lock(&rdev->gpu_clock_mutex);
9704        WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9705        clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9706                ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9707        mutex_unlock(&rdev->gpu_clock_mutex);
9708        return clock;
9709}
9710
9711static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9712                              u32 cntl_reg, u32 status_reg)
9713{
9714        int r, i;
9715        struct atom_clock_dividers dividers;
9716        uint32_t tmp;
9717
9718        r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9719                                           clock, false, &dividers);
9720        if (r)
9721                return r;
9722
9723        tmp = RREG32_SMC(cntl_reg);
9724        tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9725        tmp |= dividers.post_divider;
9726        WREG32_SMC(cntl_reg, tmp);
9727
9728        for (i = 0; i < 100; i++) {
9729                if (RREG32_SMC(status_reg) & DCLK_STATUS)
9730                        break;
9731                mdelay(10);
9732        }
9733        if (i == 100)
9734                return -ETIMEDOUT;
9735
9736        return 0;
9737}
9738
9739int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9740{
9741        int r = 0;
9742
9743        r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9744        if (r)
9745                return r;
9746
9747        r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9748        return r;
9749}
9750
9751int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9752{
9753        int r, i;
9754        struct atom_clock_dividers dividers;
9755        u32 tmp;
9756
9757        r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9758                                           ecclk, false, &dividers);
9759        if (r)
9760                return r;
9761
9762        for (i = 0; i < 100; i++) {
9763                if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9764                        break;
9765                mdelay(10);
9766        }
9767        if (i == 100)
9768                return -ETIMEDOUT;
9769
9770        tmp = RREG32_SMC(CG_ECLK_CNTL);
9771        tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9772        tmp |= dividers.post_divider;
9773        WREG32_SMC(CG_ECLK_CNTL, tmp);
9774
9775        for (i = 0; i < 100; i++) {
9776                if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9777                        break;
9778                mdelay(10);
9779        }
9780        if (i == 100)
9781                return -ETIMEDOUT;
9782
9783        return 0;
9784}
9785
9786static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9787{
9788        struct pci_dev *root = rdev->pdev->bus->self;
9789        int bridge_pos, gpu_pos;
9790        u32 speed_cntl, mask, current_data_rate;
9791        int ret, i;
9792        u16 tmp16;
9793
9794        if (pci_is_root_bus(rdev->pdev->bus))
9795                return;
9796
9797        if (radeon_pcie_gen2 == 0)
9798                return;
9799
9800        if (rdev->flags & RADEON_IS_IGP)
9801                return;
9802
9803        if (!(rdev->flags & RADEON_IS_PCIE))
9804                return;
9805
9806        ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9807        if (ret != 0)
9808                return;
9809
9810        if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9811                return;
9812
9813        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9814        current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9815                LC_CURRENT_DATA_RATE_SHIFT;
9816        if (mask & DRM_PCIE_SPEED_80) {
9817                if (current_data_rate == 2) {
9818                        DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9819                        return;
9820                }
9821                DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9822        } else if (mask & DRM_PCIE_SPEED_50) {
9823                if (current_data_rate == 1) {
9824                        DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9825                        return;
9826                }
9827                DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9828        }
9829
9830        bridge_pos = pci_pcie_cap(root);
9831        if (!bridge_pos)
9832                return;
9833
9834        gpu_pos = pci_pcie_cap(rdev->pdev);
9835        if (!gpu_pos)
9836                return;
9837
9838        if (mask & DRM_PCIE_SPEED_80) {
9839                /* re-try equalization if gen3 is not already enabled */
9840                if (current_data_rate != 2) {
9841                        u16 bridge_cfg, gpu_cfg;
9842                        u16 bridge_cfg2, gpu_cfg2;
9843                        u32 max_lw, current_lw, tmp;
9844
9845                        pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9846                        pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9847
9848                        tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9849                        pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9850
9851                        tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9852                        pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9853
9854                        tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9855                        max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9856                        current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9857
9858                        if (current_lw < max_lw) {
9859                                tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9860                                if (tmp & LC_RENEGOTIATION_SUPPORT) {
9861                                        tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9862                                        tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9863                                        tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9864                                        WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9865                                }
9866                        }
9867
9868                        for (i = 0; i < 10; i++) {
9869                                /* check status */
9870                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9871                                if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9872                                        break;
9873
9874                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9875                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9876
9877                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9878                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9879
9880                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9881                                tmp |= LC_SET_QUIESCE;
9882                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9883
9884                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9885                                tmp |= LC_REDO_EQ;
9886                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9887
9888                                mdelay(100);
9889
9890                                /* linkctl */
9891                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9892                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9893                                tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9894                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9895
9896                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9897                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9898                                tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9899                                pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9900
9901                                /* linkctl2 */
9902                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9903                                tmp16 &= ~((1 << 4) | (7 << 9));
9904                                tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9905                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9906
9907                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9908                                tmp16 &= ~((1 << 4) | (7 << 9));
9909                                tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9910                                pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9911
9912                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9913                                tmp &= ~LC_SET_QUIESCE;
9914                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9915                        }
9916                }
9917        }
9918
9919        /* set the link speed */
9920        speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9921        speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9922        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9923
9924        pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9925        tmp16 &= ~0xf;
9926        if (mask & DRM_PCIE_SPEED_80)
9927                tmp16 |= 3; /* gen3 */
9928        else if (mask & DRM_PCIE_SPEED_50)
9929                tmp16 |= 2; /* gen2 */
9930        else
9931                tmp16 |= 1; /* gen1 */
9932        pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9933
9934        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9935        speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9936        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9937
9938        for (i = 0; i < rdev->usec_timeout; i++) {
9939                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9940                if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9941                        break;
9942                udelay(1);
9943        }
9944}
9945
9946static void cik_program_aspm(struct radeon_device *rdev)
9947{
9948        u32 data, orig;
9949        bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9950        bool disable_clkreq = false;
9951
9952        if (radeon_aspm == 0)
9953                return;
9954
9955        /* XXX double check IGPs */
9956        if (rdev->flags & RADEON_IS_IGP)
9957                return;
9958
9959        if (!(rdev->flags & RADEON_IS_PCIE))
9960                return;
9961
9962        orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9963        data &= ~LC_XMIT_N_FTS_MASK;
9964        data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9965        if (orig != data)
9966                WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9967
9968        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9969        data |= LC_GO_TO_RECOVERY;
9970        if (orig != data)
9971                WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9972
9973        orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9974        data |= P_IGNORE_EDB_ERR;
9975        if (orig != data)
9976                WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9977
9978        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9979        data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9980        data |= LC_PMI_TO_L1_DIS;
9981        if (!disable_l0s)
9982                data |= LC_L0S_INACTIVITY(7);
9983
9984        if (!disable_l1) {
9985                data |= LC_L1_INACTIVITY(7);
9986                data &= ~LC_PMI_TO_L1_DIS;
9987                if (orig != data)
9988                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9989
9990                if (!disable_plloff_in_l1) {
9991                        bool clk_req_support;
9992
9993                        orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9994                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9995                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9996                        if (orig != data)
9997                                WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9998
9999                        orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
10000                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);

10001                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
10002                        if (orig != data)
10003                                WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
10004
10005                        orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
10006                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
10007                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
10008                        if (orig != data)
10009                                WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
10010
10011                        orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
10012                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
10013                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
10014                        if (orig != data)
10015                                WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
10016
10017                        orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
10018                        data &= ~LC_DYN_LANES_PWR_STATE_MASK;
10019                        data |= LC_DYN_LANES_PWR_STATE(3);
10020                        if (orig != data)
10021                                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
10022
10023                        if (!disable_clkreq &&
10024                            !pci_is_root_bus(rdev->pdev->bus)) {
10025                                struct pci_dev *root = rdev->pdev->bus->self;
10026                                u32 lnkcap;
10027
10028                                clk_req_support = false;
10029                                pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
10030                                if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
10031                                        clk_req_support = true;
10032                        } else {
10033                                clk_req_support = false;
10034                        }
10035
10036                        if (clk_req_support) {
10037                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
10038                                data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
10039                                if (orig != data)
10040                                        WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
10041
10042                                orig = data = RREG32_SMC(THM_CLK_CNTL);
10043                                data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
10044                                data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
10045                                if (orig != data)
10046                                        WREG32_SMC(THM_CLK_CNTL, data);
10047
10048                                orig = data = RREG32_SMC(MISC_CLK_CTRL);
10049                                data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
10050                                data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
10051                                if (orig != data)
10052                                        WREG32_SMC(MISC_CLK_CTRL, data);
10053
10054                                orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
10055                                data &= ~BCLK_AS_XCLK;
10056                                if (orig != data)
10057                                        WREG32_SMC(CG_CLKPIN_CNTL, data);
10058
10059                                orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
10060                                data &= ~FORCE_BIF_REFCLK_EN;
10061                                if (orig != data)
10062                                        WREG32_SMC(CG_CLKPIN_CNTL_2, data);
10063
10064                                orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
10065                                data &= ~MPLL_CLKOUT_SEL_MASK;
10066                                data |= MPLL_CLKOUT_SEL(4);
10067                                if (orig != data)
10068                                        WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
10069                        }
10070                }
10071        } else {
10072                if (orig != data)
10073                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
10074        }
10075
10076        orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
10077        data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
10078        if (orig != data)
10079                WREG32_PCIE_PORT(PCIE_CNTL2, data);
10080
10081        if (!disable_l0s) {
10082                data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
10083                if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
10084                        data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
10085                        if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
10086                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
10087                                data &= ~LC_L0S_INACTIVITY_MASK;
10088                                if (orig != data)
10089                                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
10090                        }
10091                }
10092        }
10093}
10094