linux/drivers/gpu/drm/radeon/cik.c
<<
>>
Prefs
   1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/slab.h>
  27#include <linux/module.h>
  28
  29#include <drm/drm_pci.h>
  30#include <drm/drm_vblank.h>
  31
  32#include "atom.h"
  33#include "cik_blit_shaders.h"
  34#include "cikd.h"
  35#include "clearstate_ci.h"
  36#include "radeon.h"
  37#include "radeon_asic.h"
  38#include "radeon_audio.h"
  39#include "radeon_ucode.h"
  40
  41#define SH_MEM_CONFIG_GFX_DEFAULT \
  42        ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
  43
  44MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  45MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  46MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  47MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  48MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  49MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  50MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  51MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  52MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  53
  54MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
  55MODULE_FIRMWARE("radeon/bonaire_me.bin");
  56MODULE_FIRMWARE("radeon/bonaire_ce.bin");
  57MODULE_FIRMWARE("radeon/bonaire_mec.bin");
  58MODULE_FIRMWARE("radeon/bonaire_mc.bin");
  59MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
  60MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
  61MODULE_FIRMWARE("radeon/bonaire_smc.bin");
  62MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
  63
  64MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  65MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  66MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  67MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  68MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  69MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  70MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  71MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  72MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  73
  74MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
  75MODULE_FIRMWARE("radeon/hawaii_me.bin");
  76MODULE_FIRMWARE("radeon/hawaii_ce.bin");
  77MODULE_FIRMWARE("radeon/hawaii_mec.bin");
  78MODULE_FIRMWARE("radeon/hawaii_mc.bin");
  79MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
  80MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
  81MODULE_FIRMWARE("radeon/hawaii_smc.bin");
  82MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
  83
  84MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  85MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  86MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  87MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  88MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  89MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  90
  91MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
  92MODULE_FIRMWARE("radeon/kaveri_me.bin");
  93MODULE_FIRMWARE("radeon/kaveri_ce.bin");
  94MODULE_FIRMWARE("radeon/kaveri_mec.bin");
  95MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
  96MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
  97MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
  98
  99MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
 100MODULE_FIRMWARE("radeon/KABINI_me.bin");
 101MODULE_FIRMWARE("radeon/KABINI_ce.bin");
 102MODULE_FIRMWARE("radeon/KABINI_mec.bin");
 103MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
 104MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
 105
 106MODULE_FIRMWARE("radeon/kabini_pfp.bin");
 107MODULE_FIRMWARE("radeon/kabini_me.bin");
 108MODULE_FIRMWARE("radeon/kabini_ce.bin");
 109MODULE_FIRMWARE("radeon/kabini_mec.bin");
 110MODULE_FIRMWARE("radeon/kabini_rlc.bin");
 111MODULE_FIRMWARE("radeon/kabini_sdma.bin");
 112
 113MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
 114MODULE_FIRMWARE("radeon/MULLINS_me.bin");
 115MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
 116MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
 117MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
 118MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
 119
 120MODULE_FIRMWARE("radeon/mullins_pfp.bin");
 121MODULE_FIRMWARE("radeon/mullins_me.bin");
 122MODULE_FIRMWARE("radeon/mullins_ce.bin");
 123MODULE_FIRMWARE("radeon/mullins_mec.bin");
 124MODULE_FIRMWARE("radeon/mullins_rlc.bin");
 125MODULE_FIRMWARE("radeon/mullins_sdma.bin");
 126
 127extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 128extern void r600_ih_ring_fini(struct radeon_device *rdev);
 129extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 130extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 131extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 132extern void sumo_rlc_fini(struct radeon_device *rdev);
 133extern int sumo_rlc_init(struct radeon_device *rdev);
 134extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
 135extern void si_rlc_reset(struct radeon_device *rdev);
 136extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
 137static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 138extern int cik_sdma_resume(struct radeon_device *rdev);
 139extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
 140extern void cik_sdma_fini(struct radeon_device *rdev);
 141extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
 142static void cik_rlc_stop(struct radeon_device *rdev);
 143static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 144static void cik_program_aspm(struct radeon_device *rdev);
 145static void cik_init_pg(struct radeon_device *rdev);
 146static void cik_init_cg(struct radeon_device *rdev);
 147static void cik_fini_pg(struct radeon_device *rdev);
 148static void cik_fini_cg(struct radeon_device *rdev);
 149static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
 150                                          bool enable);
 151
 152/**
 153 * cik_get_allowed_info_register - fetch the register for the info ioctl
 154 *
 155 * @rdev: radeon_device pointer
 156 * @reg: register offset in bytes
 157 * @val: register value
 158 *
 159 * Returns 0 for success or -EINVAL for an invalid register
 160 *
 161 */
 162int cik_get_allowed_info_register(struct radeon_device *rdev,
 163                                  u32 reg, u32 *val)
 164{
 165        switch (reg) {
 166        case GRBM_STATUS:
 167        case GRBM_STATUS2:
 168        case GRBM_STATUS_SE0:
 169        case GRBM_STATUS_SE1:
 170        case GRBM_STATUS_SE2:
 171        case GRBM_STATUS_SE3:
 172        case SRBM_STATUS:
 173        case SRBM_STATUS2:
 174        case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
 175        case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
 176        case UVD_STATUS:
 177        /* TODO VCE */
 178                *val = RREG32(reg);
 179                return 0;
 180        default:
 181                return -EINVAL;
 182        }
 183}
 184
 185/*
 186 * Indirect registers accessor
 187 */
 188u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
 189{
 190        unsigned long flags;
 191        u32 r;
 192
 193        spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 194        WREG32(CIK_DIDT_IND_INDEX, (reg));
 195        r = RREG32(CIK_DIDT_IND_DATA);
 196        spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 197        return r;
 198}
 199
 200void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 201{
 202        unsigned long flags;
 203
 204        spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 205        WREG32(CIK_DIDT_IND_INDEX, (reg));
 206        WREG32(CIK_DIDT_IND_DATA, (v));
 207        spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 208}
 209
 210/* get temperature in millidegrees */
 211int ci_get_temp(struct radeon_device *rdev)
 212{
 213        u32 temp;
 214        int actual_temp = 0;
 215
 216        temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 217                CTF_TEMP_SHIFT;
 218
 219        if (temp & 0x200)
 220                actual_temp = 255;
 221        else
 222                actual_temp = temp & 0x1ff;
 223
 224        actual_temp = actual_temp * 1000;
 225
 226        return actual_temp;
 227}
 228
 229/* get temperature in millidegrees */
 230int kv_get_temp(struct radeon_device *rdev)
 231{
 232        u32 temp;
 233        int actual_temp = 0;
 234
 235        temp = RREG32_SMC(0xC0300E0C);
 236
 237        if (temp)
 238                actual_temp = (temp / 8) - 49;
 239        else
 240                actual_temp = 0;
 241
 242        actual_temp = actual_temp * 1000;
 243
 244        return actual_temp;
 245}
 246
 247/*
 248 * Indirect registers accessor
 249 */
 250u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 251{
 252        unsigned long flags;
 253        u32 r;
 254
 255        spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 256        WREG32(PCIE_INDEX, reg);
 257        (void)RREG32(PCIE_INDEX);
 258        r = RREG32(PCIE_DATA);
 259        spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 260        return r;
 261}
 262
 263void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 264{
 265        unsigned long flags;
 266
 267        spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 268        WREG32(PCIE_INDEX, reg);
 269        (void)RREG32(PCIE_INDEX);
 270        WREG32(PCIE_DATA, v);
 271        (void)RREG32(PCIE_DATA);
 272        spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 273}
 274
 275static const u32 spectre_rlc_save_restore_register_list[] =
 276{
 277        (0x0e00 << 16) | (0xc12c >> 2),
 278        0x00000000,
 279        (0x0e00 << 16) | (0xc140 >> 2),
 280        0x00000000,
 281        (0x0e00 << 16) | (0xc150 >> 2),
 282        0x00000000,
 283        (0x0e00 << 16) | (0xc15c >> 2),
 284        0x00000000,
 285        (0x0e00 << 16) | (0xc168 >> 2),
 286        0x00000000,
 287        (0x0e00 << 16) | (0xc170 >> 2),
 288        0x00000000,
 289        (0x0e00 << 16) | (0xc178 >> 2),
 290        0x00000000,
 291        (0x0e00 << 16) | (0xc204 >> 2),
 292        0x00000000,
 293        (0x0e00 << 16) | (0xc2b4 >> 2),
 294        0x00000000,
 295        (0x0e00 << 16) | (0xc2b8 >> 2),
 296        0x00000000,
 297        (0x0e00 << 16) | (0xc2bc >> 2),
 298        0x00000000,
 299        (0x0e00 << 16) | (0xc2c0 >> 2),
 300        0x00000000,
 301        (0x0e00 << 16) | (0x8228 >> 2),
 302        0x00000000,
 303        (0x0e00 << 16) | (0x829c >> 2),
 304        0x00000000,
 305        (0x0e00 << 16) | (0x869c >> 2),
 306        0x00000000,
 307        (0x0600 << 16) | (0x98f4 >> 2),
 308        0x00000000,
 309        (0x0e00 << 16) | (0x98f8 >> 2),
 310        0x00000000,
 311        (0x0e00 << 16) | (0x9900 >> 2),
 312        0x00000000,
 313        (0x0e00 << 16) | (0xc260 >> 2),
 314        0x00000000,
 315        (0x0e00 << 16) | (0x90e8 >> 2),
 316        0x00000000,
 317        (0x0e00 << 16) | (0x3c000 >> 2),
 318        0x00000000,
 319        (0x0e00 << 16) | (0x3c00c >> 2),
 320        0x00000000,
 321        (0x0e00 << 16) | (0x8c1c >> 2),
 322        0x00000000,
 323        (0x0e00 << 16) | (0x9700 >> 2),
 324        0x00000000,
 325        (0x0e00 << 16) | (0xcd20 >> 2),
 326        0x00000000,
 327        (0x4e00 << 16) | (0xcd20 >> 2),
 328        0x00000000,
 329        (0x5e00 << 16) | (0xcd20 >> 2),
 330        0x00000000,
 331        (0x6e00 << 16) | (0xcd20 >> 2),
 332        0x00000000,
 333        (0x7e00 << 16) | (0xcd20 >> 2),
 334        0x00000000,
 335        (0x8e00 << 16) | (0xcd20 >> 2),
 336        0x00000000,
 337        (0x9e00 << 16) | (0xcd20 >> 2),
 338        0x00000000,
 339        (0xae00 << 16) | (0xcd20 >> 2),
 340        0x00000000,
 341        (0xbe00 << 16) | (0xcd20 >> 2),
 342        0x00000000,
 343        (0x0e00 << 16) | (0x89bc >> 2),
 344        0x00000000,
 345        (0x0e00 << 16) | (0x8900 >> 2),
 346        0x00000000,
 347        0x3,
 348        (0x0e00 << 16) | (0xc130 >> 2),
 349        0x00000000,
 350        (0x0e00 << 16) | (0xc134 >> 2),
 351        0x00000000,
 352        (0x0e00 << 16) | (0xc1fc >> 2),
 353        0x00000000,
 354        (0x0e00 << 16) | (0xc208 >> 2),
 355        0x00000000,
 356        (0x0e00 << 16) | (0xc264 >> 2),
 357        0x00000000,
 358        (0x0e00 << 16) | (0xc268 >> 2),
 359        0x00000000,
 360        (0x0e00 << 16) | (0xc26c >> 2),
 361        0x00000000,
 362        (0x0e00 << 16) | (0xc270 >> 2),
 363        0x00000000,
 364        (0x0e00 << 16) | (0xc274 >> 2),
 365        0x00000000,
 366        (0x0e00 << 16) | (0xc278 >> 2),
 367        0x00000000,
 368        (0x0e00 << 16) | (0xc27c >> 2),
 369        0x00000000,
 370        (0x0e00 << 16) | (0xc280 >> 2),
 371        0x00000000,
 372        (0x0e00 << 16) | (0xc284 >> 2),
 373        0x00000000,
 374        (0x0e00 << 16) | (0xc288 >> 2),
 375        0x00000000,
 376        (0x0e00 << 16) | (0xc28c >> 2),
 377        0x00000000,
 378        (0x0e00 << 16) | (0xc290 >> 2),
 379        0x00000000,
 380        (0x0e00 << 16) | (0xc294 >> 2),
 381        0x00000000,
 382        (0x0e00 << 16) | (0xc298 >> 2),
 383        0x00000000,
 384        (0x0e00 << 16) | (0xc29c >> 2),
 385        0x00000000,
 386        (0x0e00 << 16) | (0xc2a0 >> 2),
 387        0x00000000,
 388        (0x0e00 << 16) | (0xc2a4 >> 2),
 389        0x00000000,
 390        (0x0e00 << 16) | (0xc2a8 >> 2),
 391        0x00000000,
 392        (0x0e00 << 16) | (0xc2ac  >> 2),
 393        0x00000000,
 394        (0x0e00 << 16) | (0xc2b0 >> 2),
 395        0x00000000,
 396        (0x0e00 << 16) | (0x301d0 >> 2),
 397        0x00000000,
 398        (0x0e00 << 16) | (0x30238 >> 2),
 399        0x00000000,
 400        (0x0e00 << 16) | (0x30250 >> 2),
 401        0x00000000,
 402        (0x0e00 << 16) | (0x30254 >> 2),
 403        0x00000000,
 404        (0x0e00 << 16) | (0x30258 >> 2),
 405        0x00000000,
 406        (0x0e00 << 16) | (0x3025c >> 2),
 407        0x00000000,
 408        (0x4e00 << 16) | (0xc900 >> 2),
 409        0x00000000,
 410        (0x5e00 << 16) | (0xc900 >> 2),
 411        0x00000000,
 412        (0x6e00 << 16) | (0xc900 >> 2),
 413        0x00000000,
 414        (0x7e00 << 16) | (0xc900 >> 2),
 415        0x00000000,
 416        (0x8e00 << 16) | (0xc900 >> 2),
 417        0x00000000,
 418        (0x9e00 << 16) | (0xc900 >> 2),
 419        0x00000000,
 420        (0xae00 << 16) | (0xc900 >> 2),
 421        0x00000000,
 422        (0xbe00 << 16) | (0xc900 >> 2),
 423        0x00000000,
 424        (0x4e00 << 16) | (0xc904 >> 2),
 425        0x00000000,
 426        (0x5e00 << 16) | (0xc904 >> 2),
 427        0x00000000,
 428        (0x6e00 << 16) | (0xc904 >> 2),
 429        0x00000000,
 430        (0x7e00 << 16) | (0xc904 >> 2),
 431        0x00000000,
 432        (0x8e00 << 16) | (0xc904 >> 2),
 433        0x00000000,
 434        (0x9e00 << 16) | (0xc904 >> 2),
 435        0x00000000,
 436        (0xae00 << 16) | (0xc904 >> 2),
 437        0x00000000,
 438        (0xbe00 << 16) | (0xc904 >> 2),
 439        0x00000000,
 440        (0x4e00 << 16) | (0xc908 >> 2),
 441        0x00000000,
 442        (0x5e00 << 16) | (0xc908 >> 2),
 443        0x00000000,
 444        (0x6e00 << 16) | (0xc908 >> 2),
 445        0x00000000,
 446        (0x7e00 << 16) | (0xc908 >> 2),
 447        0x00000000,
 448        (0x8e00 << 16) | (0xc908 >> 2),
 449        0x00000000,
 450        (0x9e00 << 16) | (0xc908 >> 2),
 451        0x00000000,
 452        (0xae00 << 16) | (0xc908 >> 2),
 453        0x00000000,
 454        (0xbe00 << 16) | (0xc908 >> 2),
 455        0x00000000,
 456        (0x4e00 << 16) | (0xc90c >> 2),
 457        0x00000000,
 458        (0x5e00 << 16) | (0xc90c >> 2),
 459        0x00000000,
 460        (0x6e00 << 16) | (0xc90c >> 2),
 461        0x00000000,
 462        (0x7e00 << 16) | (0xc90c >> 2),
 463        0x00000000,
 464        (0x8e00 << 16) | (0xc90c >> 2),
 465        0x00000000,
 466        (0x9e00 << 16) | (0xc90c >> 2),
 467        0x00000000,
 468        (0xae00 << 16) | (0xc90c >> 2),
 469        0x00000000,
 470        (0xbe00 << 16) | (0xc90c >> 2),
 471        0x00000000,
 472        (0x4e00 << 16) | (0xc910 >> 2),
 473        0x00000000,
 474        (0x5e00 << 16) | (0xc910 >> 2),
 475        0x00000000,
 476        (0x6e00 << 16) | (0xc910 >> 2),
 477        0x00000000,
 478        (0x7e00 << 16) | (0xc910 >> 2),
 479        0x00000000,
 480        (0x8e00 << 16) | (0xc910 >> 2),
 481        0x00000000,
 482        (0x9e00 << 16) | (0xc910 >> 2),
 483        0x00000000,
 484        (0xae00 << 16) | (0xc910 >> 2),
 485        0x00000000,
 486        (0xbe00 << 16) | (0xc910 >> 2),
 487        0x00000000,
 488        (0x0e00 << 16) | (0xc99c >> 2),
 489        0x00000000,
 490        (0x0e00 << 16) | (0x9834 >> 2),
 491        0x00000000,
 492        (0x0000 << 16) | (0x30f00 >> 2),
 493        0x00000000,
 494        (0x0001 << 16) | (0x30f00 >> 2),
 495        0x00000000,
 496        (0x0000 << 16) | (0x30f04 >> 2),
 497        0x00000000,
 498        (0x0001 << 16) | (0x30f04 >> 2),
 499        0x00000000,
 500        (0x0000 << 16) | (0x30f08 >> 2),
 501        0x00000000,
 502        (0x0001 << 16) | (0x30f08 >> 2),
 503        0x00000000,
 504        (0x0000 << 16) | (0x30f0c >> 2),
 505        0x00000000,
 506        (0x0001 << 16) | (0x30f0c >> 2),
 507        0x00000000,
 508        (0x0600 << 16) | (0x9b7c >> 2),
 509        0x00000000,
 510        (0x0e00 << 16) | (0x8a14 >> 2),
 511        0x00000000,
 512        (0x0e00 << 16) | (0x8a18 >> 2),
 513        0x00000000,
 514        (0x0600 << 16) | (0x30a00 >> 2),
 515        0x00000000,
 516        (0x0e00 << 16) | (0x8bf0 >> 2),
 517        0x00000000,
 518        (0x0e00 << 16) | (0x8bcc >> 2),
 519        0x00000000,
 520        (0x0e00 << 16) | (0x8b24 >> 2),
 521        0x00000000,
 522        (0x0e00 << 16) | (0x30a04 >> 2),
 523        0x00000000,
 524        (0x0600 << 16) | (0x30a10 >> 2),
 525        0x00000000,
 526        (0x0600 << 16) | (0x30a14 >> 2),
 527        0x00000000,
 528        (0x0600 << 16) | (0x30a18 >> 2),
 529        0x00000000,
 530        (0x0600 << 16) | (0x30a2c >> 2),
 531        0x00000000,
 532        (0x0e00 << 16) | (0xc700 >> 2),
 533        0x00000000,
 534        (0x0e00 << 16) | (0xc704 >> 2),
 535        0x00000000,
 536        (0x0e00 << 16) | (0xc708 >> 2),
 537        0x00000000,
 538        (0x0e00 << 16) | (0xc768 >> 2),
 539        0x00000000,
 540        (0x0400 << 16) | (0xc770 >> 2),
 541        0x00000000,
 542        (0x0400 << 16) | (0xc774 >> 2),
 543        0x00000000,
 544        (0x0400 << 16) | (0xc778 >> 2),
 545        0x00000000,
 546        (0x0400 << 16) | (0xc77c >> 2),
 547        0x00000000,
 548        (0x0400 << 16) | (0xc780 >> 2),
 549        0x00000000,
 550        (0x0400 << 16) | (0xc784 >> 2),
 551        0x00000000,
 552        (0x0400 << 16) | (0xc788 >> 2),
 553        0x00000000,
 554        (0x0400 << 16) | (0xc78c >> 2),
 555        0x00000000,
 556        (0x0400 << 16) | (0xc798 >> 2),
 557        0x00000000,
 558        (0x0400 << 16) | (0xc79c >> 2),
 559        0x00000000,
 560        (0x0400 << 16) | (0xc7a0 >> 2),
 561        0x00000000,
 562        (0x0400 << 16) | (0xc7a4 >> 2),
 563        0x00000000,
 564        (0x0400 << 16) | (0xc7a8 >> 2),
 565        0x00000000,
 566        (0x0400 << 16) | (0xc7ac >> 2),
 567        0x00000000,
 568        (0x0400 << 16) | (0xc7b0 >> 2),
 569        0x00000000,
 570        (0x0400 << 16) | (0xc7b4 >> 2),
 571        0x00000000,
 572        (0x0e00 << 16) | (0x9100 >> 2),
 573        0x00000000,
 574        (0x0e00 << 16) | (0x3c010 >> 2),
 575        0x00000000,
 576        (0x0e00 << 16) | (0x92a8 >> 2),
 577        0x00000000,
 578        (0x0e00 << 16) | (0x92ac >> 2),
 579        0x00000000,
 580        (0x0e00 << 16) | (0x92b4 >> 2),
 581        0x00000000,
 582        (0x0e00 << 16) | (0x92b8 >> 2),
 583        0x00000000,
 584        (0x0e00 << 16) | (0x92bc >> 2),
 585        0x00000000,
 586        (0x0e00 << 16) | (0x92c0 >> 2),
 587        0x00000000,
 588        (0x0e00 << 16) | (0x92c4 >> 2),
 589        0x00000000,
 590        (0x0e00 << 16) | (0x92c8 >> 2),
 591        0x00000000,
 592        (0x0e00 << 16) | (0x92cc >> 2),
 593        0x00000000,
 594        (0x0e00 << 16) | (0x92d0 >> 2),
 595        0x00000000,
 596        (0x0e00 << 16) | (0x8c00 >> 2),
 597        0x00000000,
 598        (0x0e00 << 16) | (0x8c04 >> 2),
 599        0x00000000,
 600        (0x0e00 << 16) | (0x8c20 >> 2),
 601        0x00000000,
 602        (0x0e00 << 16) | (0x8c38 >> 2),
 603        0x00000000,
 604        (0x0e00 << 16) | (0x8c3c >> 2),
 605        0x00000000,
 606        (0x0e00 << 16) | (0xae00 >> 2),
 607        0x00000000,
 608        (0x0e00 << 16) | (0x9604 >> 2),
 609        0x00000000,
 610        (0x0e00 << 16) | (0xac08 >> 2),
 611        0x00000000,
 612        (0x0e00 << 16) | (0xac0c >> 2),
 613        0x00000000,
 614        (0x0e00 << 16) | (0xac10 >> 2),
 615        0x00000000,
 616        (0x0e00 << 16) | (0xac14 >> 2),
 617        0x00000000,
 618        (0x0e00 << 16) | (0xac58 >> 2),
 619        0x00000000,
 620        (0x0e00 << 16) | (0xac68 >> 2),
 621        0x00000000,
 622        (0x0e00 << 16) | (0xac6c >> 2),
 623        0x00000000,
 624        (0x0e00 << 16) | (0xac70 >> 2),
 625        0x00000000,
 626        (0x0e00 << 16) | (0xac74 >> 2),
 627        0x00000000,
 628        (0x0e00 << 16) | (0xac78 >> 2),
 629        0x00000000,
 630        (0x0e00 << 16) | (0xac7c >> 2),
 631        0x00000000,
 632        (0x0e00 << 16) | (0xac80 >> 2),
 633        0x00000000,
 634        (0x0e00 << 16) | (0xac84 >> 2),
 635        0x00000000,
 636        (0x0e00 << 16) | (0xac88 >> 2),
 637        0x00000000,
 638        (0x0e00 << 16) | (0xac8c >> 2),
 639        0x00000000,
 640        (0x0e00 << 16) | (0x970c >> 2),
 641        0x00000000,
 642        (0x0e00 << 16) | (0x9714 >> 2),
 643        0x00000000,
 644        (0x0e00 << 16) | (0x9718 >> 2),
 645        0x00000000,
 646        (0x0e00 << 16) | (0x971c >> 2),
 647        0x00000000,
 648        (0x0e00 << 16) | (0x31068 >> 2),
 649        0x00000000,
 650        (0x4e00 << 16) | (0x31068 >> 2),
 651        0x00000000,
 652        (0x5e00 << 16) | (0x31068 >> 2),
 653        0x00000000,
 654        (0x6e00 << 16) | (0x31068 >> 2),
 655        0x00000000,
 656        (0x7e00 << 16) | (0x31068 >> 2),
 657        0x00000000,
 658        (0x8e00 << 16) | (0x31068 >> 2),
 659        0x00000000,
 660        (0x9e00 << 16) | (0x31068 >> 2),
 661        0x00000000,
 662        (0xae00 << 16) | (0x31068 >> 2),
 663        0x00000000,
 664        (0xbe00 << 16) | (0x31068 >> 2),
 665        0x00000000,
 666        (0x0e00 << 16) | (0xcd10 >> 2),
 667        0x00000000,
 668        (0x0e00 << 16) | (0xcd14 >> 2),
 669        0x00000000,
 670        (0x0e00 << 16) | (0x88b0 >> 2),
 671        0x00000000,
 672        (0x0e00 << 16) | (0x88b4 >> 2),
 673        0x00000000,
 674        (0x0e00 << 16) | (0x88b8 >> 2),
 675        0x00000000,
 676        (0x0e00 << 16) | (0x88bc >> 2),
 677        0x00000000,
 678        (0x0400 << 16) | (0x89c0 >> 2),
 679        0x00000000,
 680        (0x0e00 << 16) | (0x88c4 >> 2),
 681        0x00000000,
 682        (0x0e00 << 16) | (0x88c8 >> 2),
 683        0x00000000,
 684        (0x0e00 << 16) | (0x88d0 >> 2),
 685        0x00000000,
 686        (0x0e00 << 16) | (0x88d4 >> 2),
 687        0x00000000,
 688        (0x0e00 << 16) | (0x88d8 >> 2),
 689        0x00000000,
 690        (0x0e00 << 16) | (0x8980 >> 2),
 691        0x00000000,
 692        (0x0e00 << 16) | (0x30938 >> 2),
 693        0x00000000,
 694        (0x0e00 << 16) | (0x3093c >> 2),
 695        0x00000000,
 696        (0x0e00 << 16) | (0x30940 >> 2),
 697        0x00000000,
 698        (0x0e00 << 16) | (0x89a0 >> 2),
 699        0x00000000,
 700        (0x0e00 << 16) | (0x30900 >> 2),
 701        0x00000000,
 702        (0x0e00 << 16) | (0x30904 >> 2),
 703        0x00000000,
 704        (0x0e00 << 16) | (0x89b4 >> 2),
 705        0x00000000,
 706        (0x0e00 << 16) | (0x3c210 >> 2),
 707        0x00000000,
 708        (0x0e00 << 16) | (0x3c214 >> 2),
 709        0x00000000,
 710        (0x0e00 << 16) | (0x3c218 >> 2),
 711        0x00000000,
 712        (0x0e00 << 16) | (0x8904 >> 2),
 713        0x00000000,
 714        0x5,
 715        (0x0e00 << 16) | (0x8c28 >> 2),
 716        (0x0e00 << 16) | (0x8c2c >> 2),
 717        (0x0e00 << 16) | (0x8c30 >> 2),
 718        (0x0e00 << 16) | (0x8c34 >> 2),
 719        (0x0e00 << 16) | (0x9600 >> 2),
 720};
 721
 722static const u32 kalindi_rlc_save_restore_register_list[] =
 723{
 724        (0x0e00 << 16) | (0xc12c >> 2),
 725        0x00000000,
 726        (0x0e00 << 16) | (0xc140 >> 2),
 727        0x00000000,
 728        (0x0e00 << 16) | (0xc150 >> 2),
 729        0x00000000,
 730        (0x0e00 << 16) | (0xc15c >> 2),
 731        0x00000000,
 732        (0x0e00 << 16) | (0xc168 >> 2),
 733        0x00000000,
 734        (0x0e00 << 16) | (0xc170 >> 2),
 735        0x00000000,
 736        (0x0e00 << 16) | (0xc204 >> 2),
 737        0x00000000,
 738        (0x0e00 << 16) | (0xc2b4 >> 2),
 739        0x00000000,
 740        (0x0e00 << 16) | (0xc2b8 >> 2),
 741        0x00000000,
 742        (0x0e00 << 16) | (0xc2bc >> 2),
 743        0x00000000,
 744        (0x0e00 << 16) | (0xc2c0 >> 2),
 745        0x00000000,
 746        (0x0e00 << 16) | (0x8228 >> 2),
 747        0x00000000,
 748        (0x0e00 << 16) | (0x829c >> 2),
 749        0x00000000,
 750        (0x0e00 << 16) | (0x869c >> 2),
 751        0x00000000,
 752        (0x0600 << 16) | (0x98f4 >> 2),
 753        0x00000000,
 754        (0x0e00 << 16) | (0x98f8 >> 2),
 755        0x00000000,
 756        (0x0e00 << 16) | (0x9900 >> 2),
 757        0x00000000,
 758        (0x0e00 << 16) | (0xc260 >> 2),
 759        0x00000000,
 760        (0x0e00 << 16) | (0x90e8 >> 2),
 761        0x00000000,
 762        (0x0e00 << 16) | (0x3c000 >> 2),
 763        0x00000000,
 764        (0x0e00 << 16) | (0x3c00c >> 2),
 765        0x00000000,
 766        (0x0e00 << 16) | (0x8c1c >> 2),
 767        0x00000000,
 768        (0x0e00 << 16) | (0x9700 >> 2),
 769        0x00000000,
 770        (0x0e00 << 16) | (0xcd20 >> 2),
 771        0x00000000,
 772        (0x4e00 << 16) | (0xcd20 >> 2),
 773        0x00000000,
 774        (0x5e00 << 16) | (0xcd20 >> 2),
 775        0x00000000,
 776        (0x6e00 << 16) | (0xcd20 >> 2),
 777        0x00000000,
 778        (0x7e00 << 16) | (0xcd20 >> 2),
 779        0x00000000,
 780        (0x0e00 << 16) | (0x89bc >> 2),
 781        0x00000000,
 782        (0x0e00 << 16) | (0x8900 >> 2),
 783        0x00000000,
 784        0x3,
 785        (0x0e00 << 16) | (0xc130 >> 2),
 786        0x00000000,
 787        (0x0e00 << 16) | (0xc134 >> 2),
 788        0x00000000,
 789        (0x0e00 << 16) | (0xc1fc >> 2),
 790        0x00000000,
 791        (0x0e00 << 16) | (0xc208 >> 2),
 792        0x00000000,
 793        (0x0e00 << 16) | (0xc264 >> 2),
 794        0x00000000,
 795        (0x0e00 << 16) | (0xc268 >> 2),
 796        0x00000000,
 797        (0x0e00 << 16) | (0xc26c >> 2),
 798        0x00000000,
 799        (0x0e00 << 16) | (0xc270 >> 2),
 800        0x00000000,
 801        (0x0e00 << 16) | (0xc274 >> 2),
 802        0x00000000,
 803        (0x0e00 << 16) | (0xc28c >> 2),
 804        0x00000000,
 805        (0x0e00 << 16) | (0xc290 >> 2),
 806        0x00000000,
 807        (0x0e00 << 16) | (0xc294 >> 2),
 808        0x00000000,
 809        (0x0e00 << 16) | (0xc298 >> 2),
 810        0x00000000,
 811        (0x0e00 << 16) | (0xc2a0 >> 2),
 812        0x00000000,
 813        (0x0e00 << 16) | (0xc2a4 >> 2),
 814        0x00000000,
 815        (0x0e00 << 16) | (0xc2a8 >> 2),
 816        0x00000000,
 817        (0x0e00 << 16) | (0xc2ac >> 2),
 818        0x00000000,
 819        (0x0e00 << 16) | (0x301d0 >> 2),
 820        0x00000000,
 821        (0x0e00 << 16) | (0x30238 >> 2),
 822        0x00000000,
 823        (0x0e00 << 16) | (0x30250 >> 2),
 824        0x00000000,
 825        (0x0e00 << 16) | (0x30254 >> 2),
 826        0x00000000,
 827        (0x0e00 << 16) | (0x30258 >> 2),
 828        0x00000000,
 829        (0x0e00 << 16) | (0x3025c >> 2),
 830        0x00000000,
 831        (0x4e00 << 16) | (0xc900 >> 2),
 832        0x00000000,
 833        (0x5e00 << 16) | (0xc900 >> 2),
 834        0x00000000,
 835        (0x6e00 << 16) | (0xc900 >> 2),
 836        0x00000000,
 837        (0x7e00 << 16) | (0xc900 >> 2),
 838        0x00000000,
 839        (0x4e00 << 16) | (0xc904 >> 2),
 840        0x00000000,
 841        (0x5e00 << 16) | (0xc904 >> 2),
 842        0x00000000,
 843        (0x6e00 << 16) | (0xc904 >> 2),
 844        0x00000000,
 845        (0x7e00 << 16) | (0xc904 >> 2),
 846        0x00000000,
 847        (0x4e00 << 16) | (0xc908 >> 2),
 848        0x00000000,
 849        (0x5e00 << 16) | (0xc908 >> 2),
 850        0x00000000,
 851        (0x6e00 << 16) | (0xc908 >> 2),
 852        0x00000000,
 853        (0x7e00 << 16) | (0xc908 >> 2),
 854        0x00000000,
 855        (0x4e00 << 16) | (0xc90c >> 2),
 856        0x00000000,
 857        (0x5e00 << 16) | (0xc90c >> 2),
 858        0x00000000,
 859        (0x6e00 << 16) | (0xc90c >> 2),
 860        0x00000000,
 861        (0x7e00 << 16) | (0xc90c >> 2),
 862        0x00000000,
 863        (0x4e00 << 16) | (0xc910 >> 2),
 864        0x00000000,
 865        (0x5e00 << 16) | (0xc910 >> 2),
 866        0x00000000,
 867        (0x6e00 << 16) | (0xc910 >> 2),
 868        0x00000000,
 869        (0x7e00 << 16) | (0xc910 >> 2),
 870        0x00000000,
 871        (0x0e00 << 16) | (0xc99c >> 2),
 872        0x00000000,
 873        (0x0e00 << 16) | (0x9834 >> 2),
 874        0x00000000,
 875        (0x0000 << 16) | (0x30f00 >> 2),
 876        0x00000000,
 877        (0x0000 << 16) | (0x30f04 >> 2),
 878        0x00000000,
 879        (0x0000 << 16) | (0x30f08 >> 2),
 880        0x00000000,
 881        (0x0000 << 16) | (0x30f0c >> 2),
 882        0x00000000,
 883        (0x0600 << 16) | (0x9b7c >> 2),
 884        0x00000000,
 885        (0x0e00 << 16) | (0x8a14 >> 2),
 886        0x00000000,
 887        (0x0e00 << 16) | (0x8a18 >> 2),
 888        0x00000000,
 889        (0x0600 << 16) | (0x30a00 >> 2),
 890        0x00000000,
 891        (0x0e00 << 16) | (0x8bf0 >> 2),
 892        0x00000000,
 893        (0x0e00 << 16) | (0x8bcc >> 2),
 894        0x00000000,
 895        (0x0e00 << 16) | (0x8b24 >> 2),
 896        0x00000000,
 897        (0x0e00 << 16) | (0x30a04 >> 2),
 898        0x00000000,
 899        (0x0600 << 16) | (0x30a10 >> 2),
 900        0x00000000,
 901        (0x0600 << 16) | (0x30a14 >> 2),
 902        0x00000000,
 903        (0x0600 << 16) | (0x30a18 >> 2),
 904        0x00000000,
 905        (0x0600 << 16) | (0x30a2c >> 2),
 906        0x00000000,
 907        (0x0e00 << 16) | (0xc700 >> 2),
 908        0x00000000,
 909        (0x0e00 << 16) | (0xc704 >> 2),
 910        0x00000000,
 911        (0x0e00 << 16) | (0xc708 >> 2),
 912        0x00000000,
 913        (0x0e00 << 16) | (0xc768 >> 2),
 914        0x00000000,
 915        (0x0400 << 16) | (0xc770 >> 2),
 916        0x00000000,
 917        (0x0400 << 16) | (0xc774 >> 2),
 918        0x00000000,
 919        (0x0400 << 16) | (0xc798 >> 2),
 920        0x00000000,
 921        (0x0400 << 16) | (0xc79c >> 2),
 922        0x00000000,
 923        (0x0e00 << 16) | (0x9100 >> 2),
 924        0x00000000,
 925        (0x0e00 << 16) | (0x3c010 >> 2),
 926        0x00000000,
 927        (0x0e00 << 16) | (0x8c00 >> 2),
 928        0x00000000,
 929        (0x0e00 << 16) | (0x8c04 >> 2),
 930        0x00000000,
 931        (0x0e00 << 16) | (0x8c20 >> 2),
 932        0x00000000,
 933        (0x0e00 << 16) | (0x8c38 >> 2),
 934        0x00000000,
 935        (0x0e00 << 16) | (0x8c3c >> 2),
 936        0x00000000,
 937        (0x0e00 << 16) | (0xae00 >> 2),
 938        0x00000000,
 939        (0x0e00 << 16) | (0x9604 >> 2),
 940        0x00000000,
 941        (0x0e00 << 16) | (0xac08 >> 2),
 942        0x00000000,
 943        (0x0e00 << 16) | (0xac0c >> 2),
 944        0x00000000,
 945        (0x0e00 << 16) | (0xac10 >> 2),
 946        0x00000000,
 947        (0x0e00 << 16) | (0xac14 >> 2),
 948        0x00000000,
 949        (0x0e00 << 16) | (0xac58 >> 2),
 950        0x00000000,
 951        (0x0e00 << 16) | (0xac68 >> 2),
 952        0x00000000,
 953        (0x0e00 << 16) | (0xac6c >> 2),
 954        0x00000000,
 955        (0x0e00 << 16) | (0xac70 >> 2),
 956        0x00000000,
 957        (0x0e00 << 16) | (0xac74 >> 2),
 958        0x00000000,
 959        (0x0e00 << 16) | (0xac78 >> 2),
 960        0x00000000,
 961        (0x0e00 << 16) | (0xac7c >> 2),
 962        0x00000000,
 963        (0x0e00 << 16) | (0xac80 >> 2),
 964        0x00000000,
 965        (0x0e00 << 16) | (0xac84 >> 2),
 966        0x00000000,
 967        (0x0e00 << 16) | (0xac88 >> 2),
 968        0x00000000,
 969        (0x0e00 << 16) | (0xac8c >> 2),
 970        0x00000000,
 971        (0x0e00 << 16) | (0x970c >> 2),
 972        0x00000000,
 973        (0x0e00 << 16) | (0x9714 >> 2),
 974        0x00000000,
 975        (0x0e00 << 16) | (0x9718 >> 2),
 976        0x00000000,
 977        (0x0e00 << 16) | (0x971c >> 2),
 978        0x00000000,
 979        (0x0e00 << 16) | (0x31068 >> 2),
 980        0x00000000,
 981        (0x4e00 << 16) | (0x31068 >> 2),
 982        0x00000000,
 983        (0x5e00 << 16) | (0x31068 >> 2),
 984        0x00000000,
 985        (0x6e00 << 16) | (0x31068 >> 2),
 986        0x00000000,
 987        (0x7e00 << 16) | (0x31068 >> 2),
 988        0x00000000,
 989        (0x0e00 << 16) | (0xcd10 >> 2),
 990        0x00000000,
 991        (0x0e00 << 16) | (0xcd14 >> 2),
 992        0x00000000,
 993        (0x0e00 << 16) | (0x88b0 >> 2),
 994        0x00000000,
 995        (0x0e00 << 16) | (0x88b4 >> 2),
 996        0x00000000,
 997        (0x0e00 << 16) | (0x88b8 >> 2),
 998        0x00000000,
 999        (0x0e00 << 16) | (0x88bc >> 2),
1000        0x00000000,
1001        (0x0400 << 16) | (0x89c0 >> 2),
1002        0x00000000,
1003        (0x0e00 << 16) | (0x88c4 >> 2),
1004        0x00000000,
1005        (0x0e00 << 16) | (0x88c8 >> 2),
1006        0x00000000,
1007        (0x0e00 << 16) | (0x88d0 >> 2),
1008        0x00000000,
1009        (0x0e00 << 16) | (0x88d4 >> 2),
1010        0x00000000,
1011        (0x0e00 << 16) | (0x88d8 >> 2),
1012        0x00000000,
1013        (0x0e00 << 16) | (0x8980 >> 2),
1014        0x00000000,
1015        (0x0e00 << 16) | (0x30938 >> 2),
1016        0x00000000,
1017        (0x0e00 << 16) | (0x3093c >> 2),
1018        0x00000000,
1019        (0x0e00 << 16) | (0x30940 >> 2),
1020        0x00000000,
1021        (0x0e00 << 16) | (0x89a0 >> 2),
1022        0x00000000,
1023        (0x0e00 << 16) | (0x30900 >> 2),
1024        0x00000000,
1025        (0x0e00 << 16) | (0x30904 >> 2),
1026        0x00000000,
1027        (0x0e00 << 16) | (0x89b4 >> 2),
1028        0x00000000,
1029        (0x0e00 << 16) | (0x3e1fc >> 2),
1030        0x00000000,
1031        (0x0e00 << 16) | (0x3c210 >> 2),
1032        0x00000000,
1033        (0x0e00 << 16) | (0x3c214 >> 2),
1034        0x00000000,
1035        (0x0e00 << 16) | (0x3c218 >> 2),
1036        0x00000000,
1037        (0x0e00 << 16) | (0x8904 >> 2),
1038        0x00000000,
1039        0x5,
1040        (0x0e00 << 16) | (0x8c28 >> 2),
1041        (0x0e00 << 16) | (0x8c2c >> 2),
1042        (0x0e00 << 16) | (0x8c30 >> 2),
1043        (0x0e00 << 16) | (0x8c34 >> 2),
1044        (0x0e00 << 16) | (0x9600 >> 2),
1045};
1046
1047static const u32 bonaire_golden_spm_registers[] =
1048{
1049        0x30800, 0xe0ffffff, 0xe0000000
1050};
1051
1052static const u32 bonaire_golden_common_registers[] =
1053{
1054        0xc770, 0xffffffff, 0x00000800,
1055        0xc774, 0xffffffff, 0x00000800,
1056        0xc798, 0xffffffff, 0x00007fbf,
1057        0xc79c, 0xffffffff, 0x00007faf
1058};
1059
1060static const u32 bonaire_golden_registers[] =
1061{
1062        0x3354, 0x00000333, 0x00000333,
1063        0x3350, 0x000c0fc0, 0x00040200,
1064        0x9a10, 0x00010000, 0x00058208,
1065        0x3c000, 0xffff1fff, 0x00140000,
1066        0x3c200, 0xfdfc0fff, 0x00000100,
1067        0x3c234, 0x40000000, 0x40000200,
1068        0x9830, 0xffffffff, 0x00000000,
1069        0x9834, 0xf00fffff, 0x00000400,
1070        0x9838, 0x0002021c, 0x00020200,
1071        0xc78, 0x00000080, 0x00000000,
1072        0x5bb0, 0x000000f0, 0x00000070,
1073        0x5bc0, 0xf0311fff, 0x80300000,
1074        0x98f8, 0x73773777, 0x12010001,
1075        0x350c, 0x00810000, 0x408af000,
1076        0x7030, 0x31000111, 0x00000011,
1077        0x2f48, 0x73773777, 0x12010001,
1078        0x220c, 0x00007fb6, 0x0021a1b1,
1079        0x2210, 0x00007fb6, 0x002021b1,
1080        0x2180, 0x00007fb6, 0x00002191,
1081        0x2218, 0x00007fb6, 0x002121b1,
1082        0x221c, 0x00007fb6, 0x002021b1,
1083        0x21dc, 0x00007fb6, 0x00002191,
1084        0x21e0, 0x00007fb6, 0x00002191,
1085        0x3628, 0x0000003f, 0x0000000a,
1086        0x362c, 0x0000003f, 0x0000000a,
1087        0x2ae4, 0x00073ffe, 0x000022a2,
1088        0x240c, 0x000007ff, 0x00000000,
1089        0x8a14, 0xf000003f, 0x00000007,
1090        0x8bf0, 0x00002001, 0x00000001,
1091        0x8b24, 0xffffffff, 0x00ffffff,
1092        0x30a04, 0x0000ff0f, 0x00000000,
1093        0x28a4c, 0x07ffffff, 0x06000000,
1094        0x4d8, 0x00000fff, 0x00000100,
1095        0x3e78, 0x00000001, 0x00000002,
1096        0x9100, 0x03000000, 0x0362c688,
1097        0x8c00, 0x000000ff, 0x00000001,
1098        0xe40, 0x00001fff, 0x00001fff,
1099        0x9060, 0x0000007f, 0x00000020,
1100        0x9508, 0x00010000, 0x00010000,
1101        0xac14, 0x000003ff, 0x000000f3,
1102        0xac0c, 0xffffffff, 0x00001032
1103};
1104
1105static const u32 bonaire_mgcg_cgcg_init[] =
1106{
1107        0xc420, 0xffffffff, 0xfffffffc,
1108        0x30800, 0xffffffff, 0xe0000000,
1109        0x3c2a0, 0xffffffff, 0x00000100,
1110        0x3c208, 0xffffffff, 0x00000100,
1111        0x3c2c0, 0xffffffff, 0xc0000100,
1112        0x3c2c8, 0xffffffff, 0xc0000100,
1113        0x3c2c4, 0xffffffff, 0xc0000100,
1114        0x55e4, 0xffffffff, 0x00600100,
1115        0x3c280, 0xffffffff, 0x00000100,
1116        0x3c214, 0xffffffff, 0x06000100,
1117        0x3c220, 0xffffffff, 0x00000100,
1118        0x3c218, 0xffffffff, 0x06000100,
1119        0x3c204, 0xffffffff, 0x00000100,
1120        0x3c2e0, 0xffffffff, 0x00000100,
1121        0x3c224, 0xffffffff, 0x00000100,
1122        0x3c200, 0xffffffff, 0x00000100,
1123        0x3c230, 0xffffffff, 0x00000100,
1124        0x3c234, 0xffffffff, 0x00000100,
1125        0x3c250, 0xffffffff, 0x00000100,
1126        0x3c254, 0xffffffff, 0x00000100,
1127        0x3c258, 0xffffffff, 0x00000100,
1128        0x3c25c, 0xffffffff, 0x00000100,
1129        0x3c260, 0xffffffff, 0x00000100,
1130        0x3c27c, 0xffffffff, 0x00000100,
1131        0x3c278, 0xffffffff, 0x00000100,
1132        0x3c210, 0xffffffff, 0x06000100,
1133        0x3c290, 0xffffffff, 0x00000100,
1134        0x3c274, 0xffffffff, 0x00000100,
1135        0x3c2b4, 0xffffffff, 0x00000100,
1136        0x3c2b0, 0xffffffff, 0x00000100,
1137        0x3c270, 0xffffffff, 0x00000100,
1138        0x30800, 0xffffffff, 0xe0000000,
1139        0x3c020, 0xffffffff, 0x00010000,
1140        0x3c024, 0xffffffff, 0x00030002,
1141        0x3c028, 0xffffffff, 0x00040007,
1142        0x3c02c, 0xffffffff, 0x00060005,
1143        0x3c030, 0xffffffff, 0x00090008,
1144        0x3c034, 0xffffffff, 0x00010000,
1145        0x3c038, 0xffffffff, 0x00030002,
1146        0x3c03c, 0xffffffff, 0x00040007,
1147        0x3c040, 0xffffffff, 0x00060005,
1148        0x3c044, 0xffffffff, 0x00090008,
1149        0x3c048, 0xffffffff, 0x00010000,
1150        0x3c04c, 0xffffffff, 0x00030002,
1151        0x3c050, 0xffffffff, 0x00040007,
1152        0x3c054, 0xffffffff, 0x00060005,
1153        0x3c058, 0xffffffff, 0x00090008,
1154        0x3c05c, 0xffffffff, 0x00010000,
1155        0x3c060, 0xffffffff, 0x00030002,
1156        0x3c064, 0xffffffff, 0x00040007,
1157        0x3c068, 0xffffffff, 0x00060005,
1158        0x3c06c, 0xffffffff, 0x00090008,
1159        0x3c070, 0xffffffff, 0x00010000,
1160        0x3c074, 0xffffffff, 0x00030002,
1161        0x3c078, 0xffffffff, 0x00040007,
1162        0x3c07c, 0xffffffff, 0x00060005,
1163        0x3c080, 0xffffffff, 0x00090008,
1164        0x3c084, 0xffffffff, 0x00010000,
1165        0x3c088, 0xffffffff, 0x00030002,
1166        0x3c08c, 0xffffffff, 0x00040007,
1167        0x3c090, 0xffffffff, 0x00060005,
1168        0x3c094, 0xffffffff, 0x00090008,
1169        0x3c098, 0xffffffff, 0x00010000,
1170        0x3c09c, 0xffffffff, 0x00030002,
1171        0x3c0a0, 0xffffffff, 0x00040007,
1172        0x3c0a4, 0xffffffff, 0x00060005,
1173        0x3c0a8, 0xffffffff, 0x00090008,
1174        0x3c000, 0xffffffff, 0x96e00200,
1175        0x8708, 0xffffffff, 0x00900100,
1176        0xc424, 0xffffffff, 0x0020003f,
1177        0x38, 0xffffffff, 0x0140001c,
1178        0x3c, 0x000f0000, 0x000f0000,
1179        0x220, 0xffffffff, 0xC060000C,
1180        0x224, 0xc0000fff, 0x00000100,
1181        0xf90, 0xffffffff, 0x00000100,
1182        0xf98, 0x00000101, 0x00000000,
1183        0x20a8, 0xffffffff, 0x00000104,
1184        0x55e4, 0xff000fff, 0x00000100,
1185        0x30cc, 0xc0000fff, 0x00000104,
1186        0xc1e4, 0x00000001, 0x00000001,
1187        0xd00c, 0xff000ff0, 0x00000100,
1188        0xd80c, 0xff000ff0, 0x00000100
1189};
1190
1191static const u32 spectre_golden_spm_registers[] =
1192{
1193        0x30800, 0xe0ffffff, 0xe0000000
1194};
1195
1196static const u32 spectre_golden_common_registers[] =
1197{
1198        0xc770, 0xffffffff, 0x00000800,
1199        0xc774, 0xffffffff, 0x00000800,
1200        0xc798, 0xffffffff, 0x00007fbf,
1201        0xc79c, 0xffffffff, 0x00007faf
1202};
1203
1204static const u32 spectre_golden_registers[] =
1205{
1206        0x3c000, 0xffff1fff, 0x96940200,
1207        0x3c00c, 0xffff0001, 0xff000000,
1208        0x3c200, 0xfffc0fff, 0x00000100,
1209        0x6ed8, 0x00010101, 0x00010000,
1210        0x9834, 0xf00fffff, 0x00000400,
1211        0x9838, 0xfffffffc, 0x00020200,
1212        0x5bb0, 0x000000f0, 0x00000070,
1213        0x5bc0, 0xf0311fff, 0x80300000,
1214        0x98f8, 0x73773777, 0x12010001,
1215        0x9b7c, 0x00ff0000, 0x00fc0000,
1216        0x2f48, 0x73773777, 0x12010001,
1217        0x8a14, 0xf000003f, 0x00000007,
1218        0x8b24, 0xffffffff, 0x00ffffff,
1219        0x28350, 0x3f3f3fff, 0x00000082,
1220        0x28354, 0x0000003f, 0x00000000,
1221        0x3e78, 0x00000001, 0x00000002,
1222        0x913c, 0xffff03df, 0x00000004,
1223        0xc768, 0x00000008, 0x00000008,
1224        0x8c00, 0x000008ff, 0x00000800,
1225        0x9508, 0x00010000, 0x00010000,
1226        0xac0c, 0xffffffff, 0x54763210,
1227        0x214f8, 0x01ff01ff, 0x00000002,
1228        0x21498, 0x007ff800, 0x00200000,
1229        0x2015c, 0xffffffff, 0x00000f40,
1230        0x30934, 0xffffffff, 0x00000001
1231};
1232
1233static const u32 spectre_mgcg_cgcg_init[] =
1234{
1235        0xc420, 0xffffffff, 0xfffffffc,
1236        0x30800, 0xffffffff, 0xe0000000,
1237        0x3c2a0, 0xffffffff, 0x00000100,
1238        0x3c208, 0xffffffff, 0x00000100,
1239        0x3c2c0, 0xffffffff, 0x00000100,
1240        0x3c2c8, 0xffffffff, 0x00000100,
1241        0x3c2c4, 0xffffffff, 0x00000100,
1242        0x55e4, 0xffffffff, 0x00600100,
1243        0x3c280, 0xffffffff, 0x00000100,
1244        0x3c214, 0xffffffff, 0x06000100,
1245        0x3c220, 0xffffffff, 0x00000100,
1246        0x3c218, 0xffffffff, 0x06000100,
1247        0x3c204, 0xffffffff, 0x00000100,
1248        0x3c2e0, 0xffffffff, 0x00000100,
1249        0x3c224, 0xffffffff, 0x00000100,
1250        0x3c200, 0xffffffff, 0x00000100,
1251        0x3c230, 0xffffffff, 0x00000100,
1252        0x3c234, 0xffffffff, 0x00000100,
1253        0x3c250, 0xffffffff, 0x00000100,
1254        0x3c254, 0xffffffff, 0x00000100,
1255        0x3c258, 0xffffffff, 0x00000100,
1256        0x3c25c, 0xffffffff, 0x00000100,
1257        0x3c260, 0xffffffff, 0x00000100,
1258        0x3c27c, 0xffffffff, 0x00000100,
1259        0x3c278, 0xffffffff, 0x00000100,
1260        0x3c210, 0xffffffff, 0x06000100,
1261        0x3c290, 0xffffffff, 0x00000100,
1262        0x3c274, 0xffffffff, 0x00000100,
1263        0x3c2b4, 0xffffffff, 0x00000100,
1264        0x3c2b0, 0xffffffff, 0x00000100,
1265        0x3c270, 0xffffffff, 0x00000100,
1266        0x30800, 0xffffffff, 0xe0000000,
1267        0x3c020, 0xffffffff, 0x00010000,
1268        0x3c024, 0xffffffff, 0x00030002,
1269        0x3c028, 0xffffffff, 0x00040007,
1270        0x3c02c, 0xffffffff, 0x00060005,
1271        0x3c030, 0xffffffff, 0x00090008,
1272        0x3c034, 0xffffffff, 0x00010000,
1273        0x3c038, 0xffffffff, 0x00030002,
1274        0x3c03c, 0xffffffff, 0x00040007,
1275        0x3c040, 0xffffffff, 0x00060005,
1276        0x3c044, 0xffffffff, 0x00090008,
1277        0x3c048, 0xffffffff, 0x00010000,
1278        0x3c04c, 0xffffffff, 0x00030002,
1279        0x3c050, 0xffffffff, 0x00040007,
1280        0x3c054, 0xffffffff, 0x00060005,
1281        0x3c058, 0xffffffff, 0x00090008,
1282        0x3c05c, 0xffffffff, 0x00010000,
1283        0x3c060, 0xffffffff, 0x00030002,
1284        0x3c064, 0xffffffff, 0x00040007,
1285        0x3c068, 0xffffffff, 0x00060005,
1286        0x3c06c, 0xffffffff, 0x00090008,
1287        0x3c070, 0xffffffff, 0x00010000,
1288        0x3c074, 0xffffffff, 0x00030002,
1289        0x3c078, 0xffffffff, 0x00040007,
1290        0x3c07c, 0xffffffff, 0x00060005,
1291        0x3c080, 0xffffffff, 0x00090008,
1292        0x3c084, 0xffffffff, 0x00010000,
1293        0x3c088, 0xffffffff, 0x00030002,
1294        0x3c08c, 0xffffffff, 0x00040007,
1295        0x3c090, 0xffffffff, 0x00060005,
1296        0x3c094, 0xffffffff, 0x00090008,
1297        0x3c098, 0xffffffff, 0x00010000,
1298        0x3c09c, 0xffffffff, 0x00030002,
1299        0x3c0a0, 0xffffffff, 0x00040007,
1300        0x3c0a4, 0xffffffff, 0x00060005,
1301        0x3c0a8, 0xffffffff, 0x00090008,
1302        0x3c0ac, 0xffffffff, 0x00010000,
1303        0x3c0b0, 0xffffffff, 0x00030002,
1304        0x3c0b4, 0xffffffff, 0x00040007,
1305        0x3c0b8, 0xffffffff, 0x00060005,
1306        0x3c0bc, 0xffffffff, 0x00090008,
1307        0x3c000, 0xffffffff, 0x96e00200,
1308        0x8708, 0xffffffff, 0x00900100,
1309        0xc424, 0xffffffff, 0x0020003f,
1310        0x38, 0xffffffff, 0x0140001c,
1311        0x3c, 0x000f0000, 0x000f0000,
1312        0x220, 0xffffffff, 0xC060000C,
1313        0x224, 0xc0000fff, 0x00000100,
1314        0xf90, 0xffffffff, 0x00000100,
1315        0xf98, 0x00000101, 0x00000000,
1316        0x20a8, 0xffffffff, 0x00000104,
1317        0x55e4, 0xff000fff, 0x00000100,
1318        0x30cc, 0xc0000fff, 0x00000104,
1319        0xc1e4, 0x00000001, 0x00000001,
1320        0xd00c, 0xff000ff0, 0x00000100,
1321        0xd80c, 0xff000ff0, 0x00000100
1322};
1323
1324static const u32 kalindi_golden_spm_registers[] =
1325{
1326        0x30800, 0xe0ffffff, 0xe0000000
1327};
1328
1329static const u32 kalindi_golden_common_registers[] =
1330{
1331        0xc770, 0xffffffff, 0x00000800,
1332        0xc774, 0xffffffff, 0x00000800,
1333        0xc798, 0xffffffff, 0x00007fbf,
1334        0xc79c, 0xffffffff, 0x00007faf
1335};
1336
1337static const u32 kalindi_golden_registers[] =
1338{
1339        0x3c000, 0xffffdfff, 0x6e944040,
1340        0x55e4, 0xff607fff, 0xfc000100,
1341        0x3c220, 0xff000fff, 0x00000100,
1342        0x3c224, 0xff000fff, 0x00000100,
1343        0x3c200, 0xfffc0fff, 0x00000100,
1344        0x6ed8, 0x00010101, 0x00010000,
1345        0x9830, 0xffffffff, 0x00000000,
1346        0x9834, 0xf00fffff, 0x00000400,
1347        0x5bb0, 0x000000f0, 0x00000070,
1348        0x5bc0, 0xf0311fff, 0x80300000,
1349        0x98f8, 0x73773777, 0x12010001,
1350        0x98fc, 0xffffffff, 0x00000010,
1351        0x9b7c, 0x00ff0000, 0x00fc0000,
1352        0x8030, 0x00001f0f, 0x0000100a,
1353        0x2f48, 0x73773777, 0x12010001,
1354        0x2408, 0x000fffff, 0x000c007f,
1355        0x8a14, 0xf000003f, 0x00000007,
1356        0x8b24, 0x3fff3fff, 0x00ffcfff,
1357        0x30a04, 0x0000ff0f, 0x00000000,
1358        0x28a4c, 0x07ffffff, 0x06000000,
1359        0x4d8, 0x00000fff, 0x00000100,
1360        0x3e78, 0x00000001, 0x00000002,
1361        0xc768, 0x00000008, 0x00000008,
1362        0x8c00, 0x000000ff, 0x00000003,
1363        0x214f8, 0x01ff01ff, 0x00000002,
1364        0x21498, 0x007ff800, 0x00200000,
1365        0x2015c, 0xffffffff, 0x00000f40,
1366        0x88c4, 0x001f3ae3, 0x00000082,
1367        0x88d4, 0x0000001f, 0x00000010,
1368        0x30934, 0xffffffff, 0x00000000
1369};
1370
1371static const u32 kalindi_mgcg_cgcg_init[] =
1372{
1373        0xc420, 0xffffffff, 0xfffffffc,
1374        0x30800, 0xffffffff, 0xe0000000,
1375        0x3c2a0, 0xffffffff, 0x00000100,
1376        0x3c208, 0xffffffff, 0x00000100,
1377        0x3c2c0, 0xffffffff, 0x00000100,
1378        0x3c2c8, 0xffffffff, 0x00000100,
1379        0x3c2c4, 0xffffffff, 0x00000100,
1380        0x55e4, 0xffffffff, 0x00600100,
1381        0x3c280, 0xffffffff, 0x00000100,
1382        0x3c214, 0xffffffff, 0x06000100,
1383        0x3c220, 0xffffffff, 0x00000100,
1384        0x3c218, 0xffffffff, 0x06000100,
1385        0x3c204, 0xffffffff, 0x00000100,
1386        0x3c2e0, 0xffffffff, 0x00000100,
1387        0x3c224, 0xffffffff, 0x00000100,
1388        0x3c200, 0xffffffff, 0x00000100,
1389        0x3c230, 0xffffffff, 0x00000100,
1390        0x3c234, 0xffffffff, 0x00000100,
1391        0x3c250, 0xffffffff, 0x00000100,
1392        0x3c254, 0xffffffff, 0x00000100,
1393        0x3c258, 0xffffffff, 0x00000100,
1394        0x3c25c, 0xffffffff, 0x00000100,
1395        0x3c260, 0xffffffff, 0x00000100,
1396        0x3c27c, 0xffffffff, 0x00000100,
1397        0x3c278, 0xffffffff, 0x00000100,
1398        0x3c210, 0xffffffff, 0x06000100,
1399        0x3c290, 0xffffffff, 0x00000100,
1400        0x3c274, 0xffffffff, 0x00000100,
1401        0x3c2b4, 0xffffffff, 0x00000100,
1402        0x3c2b0, 0xffffffff, 0x00000100,
1403        0x3c270, 0xffffffff, 0x00000100,
1404        0x30800, 0xffffffff, 0xe0000000,
1405        0x3c020, 0xffffffff, 0x00010000,
1406        0x3c024, 0xffffffff, 0x00030002,
1407        0x3c028, 0xffffffff, 0x00040007,
1408        0x3c02c, 0xffffffff, 0x00060005,
1409        0x3c030, 0xffffffff, 0x00090008,
1410        0x3c034, 0xffffffff, 0x00010000,
1411        0x3c038, 0xffffffff, 0x00030002,
1412        0x3c03c, 0xffffffff, 0x00040007,
1413        0x3c040, 0xffffffff, 0x00060005,
1414        0x3c044, 0xffffffff, 0x00090008,
1415        0x3c000, 0xffffffff, 0x96e00200,
1416        0x8708, 0xffffffff, 0x00900100,
1417        0xc424, 0xffffffff, 0x0020003f,
1418        0x38, 0xffffffff, 0x0140001c,
1419        0x3c, 0x000f0000, 0x000f0000,
1420        0x220, 0xffffffff, 0xC060000C,
1421        0x224, 0xc0000fff, 0x00000100,
1422        0x20a8, 0xffffffff, 0x00000104,
1423        0x55e4, 0xff000fff, 0x00000100,
1424        0x30cc, 0xc0000fff, 0x00000104,
1425        0xc1e4, 0x00000001, 0x00000001,
1426        0xd00c, 0xff000ff0, 0x00000100,
1427        0xd80c, 0xff000ff0, 0x00000100
1428};
1429
1430static const u32 hawaii_golden_spm_registers[] =
1431{
1432        0x30800, 0xe0ffffff, 0xe0000000
1433};
1434
1435static const u32 hawaii_golden_common_registers[] =
1436{
1437        0x30800, 0xffffffff, 0xe0000000,
1438        0x28350, 0xffffffff, 0x3a00161a,
1439        0x28354, 0xffffffff, 0x0000002e,
1440        0x9a10, 0xffffffff, 0x00018208,
1441        0x98f8, 0xffffffff, 0x12011003
1442};
1443
1444static const u32 hawaii_golden_registers[] =
1445{
1446        0x3354, 0x00000333, 0x00000333,
1447        0x9a10, 0x00010000, 0x00058208,
1448        0x9830, 0xffffffff, 0x00000000,
1449        0x9834, 0xf00fffff, 0x00000400,
1450        0x9838, 0x0002021c, 0x00020200,
1451        0xc78, 0x00000080, 0x00000000,
1452        0x5bb0, 0x000000f0, 0x00000070,
1453        0x5bc0, 0xf0311fff, 0x80300000,
1454        0x350c, 0x00810000, 0x408af000,
1455        0x7030, 0x31000111, 0x00000011,
1456        0x2f48, 0x73773777, 0x12010001,
1457        0x2120, 0x0000007f, 0x0000001b,
1458        0x21dc, 0x00007fb6, 0x00002191,
1459        0x3628, 0x0000003f, 0x0000000a,
1460        0x362c, 0x0000003f, 0x0000000a,
1461        0x2ae4, 0x00073ffe, 0x000022a2,
1462        0x240c, 0x000007ff, 0x00000000,
1463        0x8bf0, 0x00002001, 0x00000001,
1464        0x8b24, 0xffffffff, 0x00ffffff,
1465        0x30a04, 0x0000ff0f, 0x00000000,
1466        0x28a4c, 0x07ffffff, 0x06000000,
1467        0x3e78, 0x00000001, 0x00000002,
1468        0xc768, 0x00000008, 0x00000008,
1469        0xc770, 0x00000f00, 0x00000800,
1470        0xc774, 0x00000f00, 0x00000800,
1471        0xc798, 0x00ffffff, 0x00ff7fbf,
1472        0xc79c, 0x00ffffff, 0x00ff7faf,
1473        0x8c00, 0x000000ff, 0x00000800,
1474        0xe40, 0x00001fff, 0x00001fff,
1475        0x9060, 0x0000007f, 0x00000020,
1476        0x9508, 0x00010000, 0x00010000,
1477        0xae00, 0x00100000, 0x000ff07c,
1478        0xac14, 0x000003ff, 0x0000000f,
1479        0xac10, 0xffffffff, 0x7564fdec,
1480        0xac0c, 0xffffffff, 0x3120b9a8,
1481        0xac08, 0x20000000, 0x0f9c0000
1482};
1483
1484static const u32 hawaii_mgcg_cgcg_init[] =
1485{
1486        0xc420, 0xffffffff, 0xfffffffd,
1487        0x30800, 0xffffffff, 0xe0000000,
1488        0x3c2a0, 0xffffffff, 0x00000100,
1489        0x3c208, 0xffffffff, 0x00000100,
1490        0x3c2c0, 0xffffffff, 0x00000100,
1491        0x3c2c8, 0xffffffff, 0x00000100,
1492        0x3c2c4, 0xffffffff, 0x00000100,
1493        0x55e4, 0xffffffff, 0x00200100,
1494        0x3c280, 0xffffffff, 0x00000100,
1495        0x3c214, 0xffffffff, 0x06000100,
1496        0x3c220, 0xffffffff, 0x00000100,
1497        0x3c218, 0xffffffff, 0x06000100,
1498        0x3c204, 0xffffffff, 0x00000100,
1499        0x3c2e0, 0xffffffff, 0x00000100,
1500        0x3c224, 0xffffffff, 0x00000100,
1501        0x3c200, 0xffffffff, 0x00000100,
1502        0x3c230, 0xffffffff, 0x00000100,
1503        0x3c234, 0xffffffff, 0x00000100,
1504        0x3c250, 0xffffffff, 0x00000100,
1505        0x3c254, 0xffffffff, 0x00000100,
1506        0x3c258, 0xffffffff, 0x00000100,
1507        0x3c25c, 0xffffffff, 0x00000100,
1508        0x3c260, 0xffffffff, 0x00000100,
1509        0x3c27c, 0xffffffff, 0x00000100,
1510        0x3c278, 0xffffffff, 0x00000100,
1511        0x3c210, 0xffffffff, 0x06000100,
1512        0x3c290, 0xffffffff, 0x00000100,
1513        0x3c274, 0xffffffff, 0x00000100,
1514        0x3c2b4, 0xffffffff, 0x00000100,
1515        0x3c2b0, 0xffffffff, 0x00000100,
1516        0x3c270, 0xffffffff, 0x00000100,
1517        0x30800, 0xffffffff, 0xe0000000,
1518        0x3c020, 0xffffffff, 0x00010000,
1519        0x3c024, 0xffffffff, 0x00030002,
1520        0x3c028, 0xffffffff, 0x00040007,
1521        0x3c02c, 0xffffffff, 0x00060005,
1522        0x3c030, 0xffffffff, 0x00090008,
1523        0x3c034, 0xffffffff, 0x00010000,
1524        0x3c038, 0xffffffff, 0x00030002,
1525        0x3c03c, 0xffffffff, 0x00040007,
1526        0x3c040, 0xffffffff, 0x00060005,
1527        0x3c044, 0xffffffff, 0x00090008,
1528        0x3c048, 0xffffffff, 0x00010000,
1529        0x3c04c, 0xffffffff, 0x00030002,
1530        0x3c050, 0xffffffff, 0x00040007,
1531        0x3c054, 0xffffffff, 0x00060005,
1532        0x3c058, 0xffffffff, 0x00090008,
1533        0x3c05c, 0xffffffff, 0x00010000,
1534        0x3c060, 0xffffffff, 0x00030002,
1535        0x3c064, 0xffffffff, 0x00040007,
1536        0x3c068, 0xffffffff, 0x00060005,
1537        0x3c06c, 0xffffffff, 0x00090008,
1538        0x3c070, 0xffffffff, 0x00010000,
1539        0x3c074, 0xffffffff, 0x00030002,
1540        0x3c078, 0xffffffff, 0x00040007,
1541        0x3c07c, 0xffffffff, 0x00060005,
1542        0x3c080, 0xffffffff, 0x00090008,
1543        0x3c084, 0xffffffff, 0x00010000,
1544        0x3c088, 0xffffffff, 0x00030002,
1545        0x3c08c, 0xffffffff, 0x00040007,
1546        0x3c090, 0xffffffff, 0x00060005,
1547        0x3c094, 0xffffffff, 0x00090008,
1548        0x3c098, 0xffffffff, 0x00010000,
1549        0x3c09c, 0xffffffff, 0x00030002,
1550        0x3c0a0, 0xffffffff, 0x00040007,
1551        0x3c0a4, 0xffffffff, 0x00060005,
1552        0x3c0a8, 0xffffffff, 0x00090008,
1553        0x3c0ac, 0xffffffff, 0x00010000,
1554        0x3c0b0, 0xffffffff, 0x00030002,
1555        0x3c0b4, 0xffffffff, 0x00040007,
1556        0x3c0b8, 0xffffffff, 0x00060005,
1557        0x3c0bc, 0xffffffff, 0x00090008,
1558        0x3c0c0, 0xffffffff, 0x00010000,
1559        0x3c0c4, 0xffffffff, 0x00030002,
1560        0x3c0c8, 0xffffffff, 0x00040007,
1561        0x3c0cc, 0xffffffff, 0x00060005,
1562        0x3c0d0, 0xffffffff, 0x00090008,
1563        0x3c0d4, 0xffffffff, 0x00010000,
1564        0x3c0d8, 0xffffffff, 0x00030002,
1565        0x3c0dc, 0xffffffff, 0x00040007,
1566        0x3c0e0, 0xffffffff, 0x00060005,
1567        0x3c0e4, 0xffffffff, 0x00090008,
1568        0x3c0e8, 0xffffffff, 0x00010000,
1569        0x3c0ec, 0xffffffff, 0x00030002,
1570        0x3c0f0, 0xffffffff, 0x00040007,
1571        0x3c0f4, 0xffffffff, 0x00060005,
1572        0x3c0f8, 0xffffffff, 0x00090008,
1573        0xc318, 0xffffffff, 0x00020200,
1574        0x3350, 0xffffffff, 0x00000200,
1575        0x15c0, 0xffffffff, 0x00000400,
1576        0x55e8, 0xffffffff, 0x00000000,
1577        0x2f50, 0xffffffff, 0x00000902,
1578        0x3c000, 0xffffffff, 0x96940200,
1579        0x8708, 0xffffffff, 0x00900100,
1580        0xc424, 0xffffffff, 0x0020003f,
1581        0x38, 0xffffffff, 0x0140001c,
1582        0x3c, 0x000f0000, 0x000f0000,
1583        0x220, 0xffffffff, 0xc060000c,
1584        0x224, 0xc0000fff, 0x00000100,
1585        0xf90, 0xffffffff, 0x00000100,
1586        0xf98, 0x00000101, 0x00000000,
1587        0x20a8, 0xffffffff, 0x00000104,
1588        0x55e4, 0xff000fff, 0x00000100,
1589        0x30cc, 0xc0000fff, 0x00000104,
1590        0xc1e4, 0x00000001, 0x00000001,
1591        0xd00c, 0xff000ff0, 0x00000100,
1592        0xd80c, 0xff000ff0, 0x00000100
1593};
1594
1595static const u32 godavari_golden_registers[] =
1596{
1597        0x55e4, 0xff607fff, 0xfc000100,
1598        0x6ed8, 0x00010101, 0x00010000,
1599        0x9830, 0xffffffff, 0x00000000,
1600        0x98302, 0xf00fffff, 0x00000400,
1601        0x6130, 0xffffffff, 0x00010000,
1602        0x5bb0, 0x000000f0, 0x00000070,
1603        0x5bc0, 0xf0311fff, 0x80300000,
1604        0x98f8, 0x73773777, 0x12010001,
1605        0x98fc, 0xffffffff, 0x00000010,
1606        0x8030, 0x00001f0f, 0x0000100a,
1607        0x2f48, 0x73773777, 0x12010001,
1608        0x2408, 0x000fffff, 0x000c007f,
1609        0x8a14, 0xf000003f, 0x00000007,
1610        0x8b24, 0xffffffff, 0x00ff0fff,
1611        0x30a04, 0x0000ff0f, 0x00000000,
1612        0x28a4c, 0x07ffffff, 0x06000000,
1613        0x4d8, 0x00000fff, 0x00000100,
1614        0xd014, 0x00010000, 0x00810001,
1615        0xd814, 0x00010000, 0x00810001,
1616        0x3e78, 0x00000001, 0x00000002,
1617        0xc768, 0x00000008, 0x00000008,
1618        0xc770, 0x00000f00, 0x00000800,
1619        0xc774, 0x00000f00, 0x00000800,
1620        0xc798, 0x00ffffff, 0x00ff7fbf,
1621        0xc79c, 0x00ffffff, 0x00ff7faf,
1622        0x8c00, 0x000000ff, 0x00000001,
1623        0x214f8, 0x01ff01ff, 0x00000002,
1624        0x21498, 0x007ff800, 0x00200000,
1625        0x2015c, 0xffffffff, 0x00000f40,
1626        0x88c4, 0x001f3ae3, 0x00000082,
1627        0x88d4, 0x0000001f, 0x00000010,
1628        0x30934, 0xffffffff, 0x00000000
1629};
1630
1631
1632static void cik_init_golden_registers(struct radeon_device *rdev)
1633{
1634        switch (rdev->family) {
1635        case CHIP_BONAIRE:
1636                radeon_program_register_sequence(rdev,
1637                                                 bonaire_mgcg_cgcg_init,
1638                                                 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1639                radeon_program_register_sequence(rdev,
1640                                                 bonaire_golden_registers,
1641                                                 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1642                radeon_program_register_sequence(rdev,
1643                                                 bonaire_golden_common_registers,
1644                                                 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1645                radeon_program_register_sequence(rdev,
1646                                                 bonaire_golden_spm_registers,
1647                                                 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1648                break;
1649        case CHIP_KABINI:
1650                radeon_program_register_sequence(rdev,
1651                                                 kalindi_mgcg_cgcg_init,
1652                                                 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1653                radeon_program_register_sequence(rdev,
1654                                                 kalindi_golden_registers,
1655                                                 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1656                radeon_program_register_sequence(rdev,
1657                                                 kalindi_golden_common_registers,
1658                                                 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1659                radeon_program_register_sequence(rdev,
1660                                                 kalindi_golden_spm_registers,
1661                                                 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1662                break;
1663        case CHIP_MULLINS:
1664                radeon_program_register_sequence(rdev,
1665                                                 kalindi_mgcg_cgcg_init,
1666                                                 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1667                radeon_program_register_sequence(rdev,
1668                                                 godavari_golden_registers,
1669                                                 (const u32)ARRAY_SIZE(godavari_golden_registers));
1670                radeon_program_register_sequence(rdev,
1671                                                 kalindi_golden_common_registers,
1672                                                 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1673                radeon_program_register_sequence(rdev,
1674                                                 kalindi_golden_spm_registers,
1675                                                 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1676                break;
1677        case CHIP_KAVERI:
1678                radeon_program_register_sequence(rdev,
1679                                                 spectre_mgcg_cgcg_init,
1680                                                 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1681                radeon_program_register_sequence(rdev,
1682                                                 spectre_golden_registers,
1683                                                 (const u32)ARRAY_SIZE(spectre_golden_registers));
1684                radeon_program_register_sequence(rdev,
1685                                                 spectre_golden_common_registers,
1686                                                 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1687                radeon_program_register_sequence(rdev,
1688                                                 spectre_golden_spm_registers,
1689                                                 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1690                break;
1691        case CHIP_HAWAII:
1692                radeon_program_register_sequence(rdev,
1693                                                 hawaii_mgcg_cgcg_init,
1694                                                 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1695                radeon_program_register_sequence(rdev,
1696                                                 hawaii_golden_registers,
1697                                                 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1698                radeon_program_register_sequence(rdev,
1699                                                 hawaii_golden_common_registers,
1700                                                 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1701                radeon_program_register_sequence(rdev,
1702                                                 hawaii_golden_spm_registers,
1703                                                 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1704                break;
1705        default:
1706                break;
1707        }
1708}
1709
1710/**
1711 * cik_get_xclk - get the xclk
1712 *
1713 * @rdev: radeon_device pointer
1714 *
1715 * Returns the reference clock used by the gfx engine
1716 * (CIK).
1717 */
1718u32 cik_get_xclk(struct radeon_device *rdev)
1719{
1720        u32 reference_clock = rdev->clock.spll.reference_freq;
1721
1722        if (rdev->flags & RADEON_IS_IGP) {
1723                if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1724                        return reference_clock / 2;
1725        } else {
1726                if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1727                        return reference_clock / 4;
1728        }
1729        return reference_clock;
1730}
1731
1732/**
1733 * cik_mm_rdoorbell - read a doorbell dword
1734 *
1735 * @rdev: radeon_device pointer
1736 * @index: doorbell index
1737 *
1738 * Returns the value in the doorbell aperture at the
1739 * requested doorbell index (CIK).
1740 */
1741u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1742{
1743        if (index < rdev->doorbell.num_doorbells) {
1744                return readl(rdev->doorbell.ptr + index);
1745        } else {
1746                DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1747                return 0;
1748        }
1749}
1750
1751/**
1752 * cik_mm_wdoorbell - write a doorbell dword
1753 *
1754 * @rdev: radeon_device pointer
1755 * @index: doorbell index
1756 * @v: value to write
1757 *
1758 * Writes @v to the doorbell aperture at the
1759 * requested doorbell index (CIK).
1760 */
1761void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1762{
1763        if (index < rdev->doorbell.num_doorbells) {
1764                writel(v, rdev->doorbell.ptr + index);
1765        } else {
1766                DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1767        }
1768}
1769
1770#define BONAIRE_IO_MC_REGS_SIZE 36
1771
1772static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1773{
1774        {0x00000070, 0x04400000},
1775        {0x00000071, 0x80c01803},
1776        {0x00000072, 0x00004004},
1777        {0x00000073, 0x00000100},
1778        {0x00000074, 0x00ff0000},
1779        {0x00000075, 0x34000000},
1780        {0x00000076, 0x08000014},
1781        {0x00000077, 0x00cc08ec},
1782        {0x00000078, 0x00000400},
1783        {0x00000079, 0x00000000},
1784        {0x0000007a, 0x04090000},
1785        {0x0000007c, 0x00000000},
1786        {0x0000007e, 0x4408a8e8},
1787        {0x0000007f, 0x00000304},
1788        {0x00000080, 0x00000000},
1789        {0x00000082, 0x00000001},
1790        {0x00000083, 0x00000002},
1791        {0x00000084, 0xf3e4f400},
1792        {0x00000085, 0x052024e3},
1793        {0x00000087, 0x00000000},
1794        {0x00000088, 0x01000000},
1795        {0x0000008a, 0x1c0a0000},
1796        {0x0000008b, 0xff010000},
1797        {0x0000008d, 0xffffefff},
1798        {0x0000008e, 0xfff3efff},
1799        {0x0000008f, 0xfff3efbf},
1800        {0x00000092, 0xf7ffffff},
1801        {0x00000093, 0xffffff7f},
1802        {0x00000095, 0x00101101},
1803        {0x00000096, 0x00000fff},
1804        {0x00000097, 0x00116fff},
1805        {0x00000098, 0x60010000},
1806        {0x00000099, 0x10010000},
1807        {0x0000009a, 0x00006000},
1808        {0x0000009b, 0x00001000},
1809        {0x0000009f, 0x00b48000}
1810};
1811
1812#define HAWAII_IO_MC_REGS_SIZE 22
1813
1814static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1815{
1816        {0x0000007d, 0x40000000},
1817        {0x0000007e, 0x40180304},
1818        {0x0000007f, 0x0000ff00},
1819        {0x00000081, 0x00000000},
1820        {0x00000083, 0x00000800},
1821        {0x00000086, 0x00000000},
1822        {0x00000087, 0x00000100},
1823        {0x00000088, 0x00020100},
1824        {0x00000089, 0x00000000},
1825        {0x0000008b, 0x00040000},
1826        {0x0000008c, 0x00000100},
1827        {0x0000008e, 0xff010000},
1828        {0x00000090, 0xffffefff},
1829        {0x00000091, 0xfff3efff},
1830        {0x00000092, 0xfff3efbf},
1831        {0x00000093, 0xf7ffffff},
1832        {0x00000094, 0xffffff7f},
1833        {0x00000095, 0x00000fff},
1834        {0x00000096, 0x00116fff},
1835        {0x00000097, 0x60010000},
1836        {0x00000098, 0x10010000},
1837        {0x0000009f, 0x00c79000}
1838};
1839
1840
1841/**
1842 * cik_srbm_select - select specific register instances
1843 *
1844 * @rdev: radeon_device pointer
1845 * @me: selected ME (micro engine)
1846 * @pipe: pipe
1847 * @queue: queue
1848 * @vmid: VMID
1849 *
1850 * Switches the currently active registers instances.  Some
1851 * registers are instanced per VMID, others are instanced per
1852 * me/pipe/queue combination.
1853 */
1854static void cik_srbm_select(struct radeon_device *rdev,
1855                            u32 me, u32 pipe, u32 queue, u32 vmid)
1856{
1857        u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1858                             MEID(me & 0x3) |
1859                             VMID(vmid & 0xf) |
1860                             QUEUEID(queue & 0x7));
1861        WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1862}
1863
1864/* ucode loading */
1865/**
1866 * ci_mc_load_microcode - load MC ucode into the hw
1867 *
1868 * @rdev: radeon_device pointer
1869 *
1870 * Load the GDDR MC ucode into the hw (CIK).
1871 * Returns 0 on success, error on failure.
1872 */
1873int ci_mc_load_microcode(struct radeon_device *rdev)
1874{
1875        const __be32 *fw_data = NULL;
1876        const __le32 *new_fw_data = NULL;
1877        u32 running, tmp;
1878        u32 *io_mc_regs = NULL;
1879        const __le32 *new_io_mc_regs = NULL;
1880        int i, regs_size, ucode_size;
1881
1882        if (!rdev->mc_fw)
1883                return -EINVAL;
1884
1885        if (rdev->new_fw) {
1886                const struct mc_firmware_header_v1_0 *hdr =
1887                        (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1888
1889                radeon_ucode_print_mc_hdr(&hdr->header);
1890
1891                regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1892                new_io_mc_regs = (const __le32 *)
1893                        (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1894                ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1895                new_fw_data = (const __le32 *)
1896                        (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1897        } else {
1898                ucode_size = rdev->mc_fw->size / 4;
1899
1900                switch (rdev->family) {
1901                case CHIP_BONAIRE:
1902                        io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1903                        regs_size = BONAIRE_IO_MC_REGS_SIZE;
1904                        break;
1905                case CHIP_HAWAII:
1906                        io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1907                        regs_size = HAWAII_IO_MC_REGS_SIZE;
1908                        break;
1909                default:
1910                        return -EINVAL;
1911                }
1912                fw_data = (const __be32 *)rdev->mc_fw->data;
1913        }
1914
1915        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1916
1917        if (running == 0) {
1918                /* reset the engine and set to writable */
1919                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1920                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1921
1922                /* load mc io regs */
1923                for (i = 0; i < regs_size; i++) {
1924                        if (rdev->new_fw) {
1925                                WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1926                                WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1927                        } else {
1928                                WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1929                                WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1930                        }
1931                }
1932
1933                tmp = RREG32(MC_SEQ_MISC0);
1934                if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1935                        WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1936                        WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1937                        WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1938                        WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1939                }
1940
1941                /* load the MC ucode */
1942                for (i = 0; i < ucode_size; i++) {
1943                        if (rdev->new_fw)
1944                                WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1945                        else
1946                                WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1947                }
1948
1949                /* put the engine back into the active state */
1950                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1951                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1952                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1953
1954                /* wait for training to complete */
1955                for (i = 0; i < rdev->usec_timeout; i++) {
1956                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1957                                break;
1958                        udelay(1);
1959                }
1960                for (i = 0; i < rdev->usec_timeout; i++) {
1961                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1962                                break;
1963                        udelay(1);
1964                }
1965        }
1966
1967        return 0;
1968}
1969
1970/**
1971 * cik_init_microcode - load ucode images from disk
1972 *
1973 * @rdev: radeon_device pointer
1974 *
1975 * Use the firmware interface to load the ucode images into
1976 * the driver (not loaded into hw).
1977 * Returns 0 on success, error on failure.
1978 */
1979static int cik_init_microcode(struct radeon_device *rdev)
1980{
1981        const char *chip_name;
1982        const char *new_chip_name;
1983        size_t pfp_req_size, me_req_size, ce_req_size,
1984                mec_req_size, rlc_req_size, mc_req_size = 0,
1985                sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1986        char fw_name[30];
1987        int new_fw = 0;
1988        int err;
1989        int num_fw;
1990        bool new_smc = false;
1991
1992        DRM_DEBUG("\n");
1993
1994        switch (rdev->family) {
1995        case CHIP_BONAIRE:
1996                chip_name = "BONAIRE";
1997                if ((rdev->pdev->revision == 0x80) ||
1998                    (rdev->pdev->revision == 0x81) ||
1999                    (rdev->pdev->device == 0x665f))
2000                        new_smc = true;
2001                new_chip_name = "bonaire";
2002                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2003                me_req_size = CIK_ME_UCODE_SIZE * 4;
2004                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2005                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2006                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2007                mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2008                mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
2009                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2010                smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
2011                num_fw = 8;
2012                break;
2013        case CHIP_HAWAII:
2014                chip_name = "HAWAII";
2015                if (rdev->pdev->revision == 0x80)
2016                        new_smc = true;
2017                new_chip_name = "hawaii";
2018                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2019                me_req_size = CIK_ME_UCODE_SIZE * 4;
2020                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2021                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2022                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2023                mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2024                mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2025                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2026                smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2027                num_fw = 8;
2028                break;
2029        case CHIP_KAVERI:
2030                chip_name = "KAVERI";
2031                new_chip_name = "kaveri";
2032                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2033                me_req_size = CIK_ME_UCODE_SIZE * 4;
2034                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2035                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2036                rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2037                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2038                num_fw = 7;
2039                break;
2040        case CHIP_KABINI:
2041                chip_name = "KABINI";
2042                new_chip_name = "kabini";
2043                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2044                me_req_size = CIK_ME_UCODE_SIZE * 4;
2045                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2046                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2047                rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2048                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2049                num_fw = 6;
2050                break;
2051        case CHIP_MULLINS:
2052                chip_name = "MULLINS";
2053                new_chip_name = "mullins";
2054                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2055                me_req_size = CIK_ME_UCODE_SIZE * 4;
2056                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2057                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2058                rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2059                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2060                num_fw = 6;
2061                break;
2062        default: BUG();
2063        }
2064
2065        DRM_INFO("Loading %s Microcode\n", new_chip_name);
2066
2067        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2068        err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2069        if (err) {
2070                snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2071                err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2072                if (err)
2073                        goto out;
2074                if (rdev->pfp_fw->size != pfp_req_size) {
2075                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2076                               rdev->pfp_fw->size, fw_name);
2077                        err = -EINVAL;
2078                        goto out;
2079                }
2080        } else {
2081                err = radeon_ucode_validate(rdev->pfp_fw);
2082                if (err) {
2083                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2084                               fw_name);
2085                        goto out;
2086                } else {
2087                        new_fw++;
2088                }
2089        }
2090
2091        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2092        err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2093        if (err) {
2094                snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2095                err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2096                if (err)
2097                        goto out;
2098                if (rdev->me_fw->size != me_req_size) {
2099                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2100                               rdev->me_fw->size, fw_name);
2101                        err = -EINVAL;
2102                }
2103        } else {
2104                err = radeon_ucode_validate(rdev->me_fw);
2105                if (err) {
2106                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2107                               fw_name);
2108                        goto out;
2109                } else {
2110                        new_fw++;
2111                }
2112        }
2113
2114        snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2115        err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2116        if (err) {
2117                snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2118                err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2119                if (err)
2120                        goto out;
2121                if (rdev->ce_fw->size != ce_req_size) {
2122                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2123                               rdev->ce_fw->size, fw_name);
2124                        err = -EINVAL;
2125                }
2126        } else {
2127                err = radeon_ucode_validate(rdev->ce_fw);
2128                if (err) {
2129                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2130                               fw_name);
2131                        goto out;
2132                } else {
2133                        new_fw++;
2134                }
2135        }
2136
2137        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2138        err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2139        if (err) {
2140                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2141                err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2142                if (err)
2143                        goto out;
2144                if (rdev->mec_fw->size != mec_req_size) {
2145                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2146                               rdev->mec_fw->size, fw_name);
2147                        err = -EINVAL;
2148                }
2149        } else {
2150                err = radeon_ucode_validate(rdev->mec_fw);
2151                if (err) {
2152                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2153                               fw_name);
2154                        goto out;
2155                } else {
2156                        new_fw++;
2157                }
2158        }
2159
2160        if (rdev->family == CHIP_KAVERI) {
2161                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2162                err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2163                if (err) {
2164                        goto out;
2165                } else {
2166                        err = radeon_ucode_validate(rdev->mec2_fw);
2167                        if (err) {
2168                                goto out;
2169                        } else {
2170                                new_fw++;
2171                        }
2172                }
2173        }
2174
2175        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2176        err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2177        if (err) {
2178                snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2179                err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2180                if (err)
2181                        goto out;
2182                if (rdev->rlc_fw->size != rlc_req_size) {
2183                        pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2184                               rdev->rlc_fw->size, fw_name);
2185                        err = -EINVAL;
2186                }
2187        } else {
2188                err = radeon_ucode_validate(rdev->rlc_fw);
2189                if (err) {
2190                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2191                               fw_name);
2192                        goto out;
2193                } else {
2194                        new_fw++;
2195                }
2196        }
2197
2198        snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2199        err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2200        if (err) {
2201                snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2202                err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2203                if (err)
2204                        goto out;
2205                if (rdev->sdma_fw->size != sdma_req_size) {
2206                        pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2207                               rdev->sdma_fw->size, fw_name);
2208                        err = -EINVAL;
2209                }
2210        } else {
2211                err = radeon_ucode_validate(rdev->sdma_fw);
2212                if (err) {
2213                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2214                               fw_name);
2215                        goto out;
2216                } else {
2217                        new_fw++;
2218                }
2219        }
2220
2221        /* No SMC, MC ucode on APUs */
2222        if (!(rdev->flags & RADEON_IS_IGP)) {
2223                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2224                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2225                if (err) {
2226                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2227                        err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2228                        if (err) {
2229                                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2230                                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2231                                if (err)
2232                                        goto out;
2233                        }
2234                        if ((rdev->mc_fw->size != mc_req_size) &&
2235                            (rdev->mc_fw->size != mc2_req_size)){
2236                                pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
2237                                       rdev->mc_fw->size, fw_name);
2238                                err = -EINVAL;
2239                        }
2240                        DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2241                } else {
2242                        err = radeon_ucode_validate(rdev->mc_fw);
2243                        if (err) {
2244                                pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2245                                       fw_name);
2246                                goto out;
2247                        } else {
2248                                new_fw++;
2249                        }
2250                }
2251
2252                if (new_smc)
2253                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2254                else
2255                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2256                err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2257                if (err) {
2258                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2259                        err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2260                        if (err) {
2261                                pr_err("smc: error loading firmware \"%s\"\n",
2262                                       fw_name);
2263                                release_firmware(rdev->smc_fw);
2264                                rdev->smc_fw = NULL;
2265                                err = 0;
2266                        } else if (rdev->smc_fw->size != smc_req_size) {
2267                                pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
2268                                       rdev->smc_fw->size, fw_name);
2269                                err = -EINVAL;
2270                        }
2271                } else {
2272                        err = radeon_ucode_validate(rdev->smc_fw);
2273                        if (err) {
2274                                pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2275                                       fw_name);
2276                                goto out;
2277                        } else {
2278                                new_fw++;
2279                        }
2280                }
2281        }
2282
2283        if (new_fw == 0) {
2284                rdev->new_fw = false;
2285        } else if (new_fw < num_fw) {
2286                pr_err("ci_fw: mixing new and old firmware!\n");
2287                err = -EINVAL;
2288        } else {
2289                rdev->new_fw = true;
2290        }
2291
2292out:
2293        if (err) {
2294                if (err != -EINVAL)
2295                        pr_err("cik_cp: Failed to load firmware \"%s\"\n",
2296                               fw_name);
2297                release_firmware(rdev->pfp_fw);
2298                rdev->pfp_fw = NULL;
2299                release_firmware(rdev->me_fw);
2300                rdev->me_fw = NULL;
2301                release_firmware(rdev->ce_fw);
2302                rdev->ce_fw = NULL;
2303                release_firmware(rdev->mec_fw);
2304                rdev->mec_fw = NULL;
2305                release_firmware(rdev->mec2_fw);
2306                rdev->mec2_fw = NULL;
2307                release_firmware(rdev->rlc_fw);
2308                rdev->rlc_fw = NULL;
2309                release_firmware(rdev->sdma_fw);
2310                rdev->sdma_fw = NULL;
2311                release_firmware(rdev->mc_fw);
2312                rdev->mc_fw = NULL;
2313                release_firmware(rdev->smc_fw);
2314                rdev->smc_fw = NULL;
2315        }
2316        return err;
2317}
2318
2319/*
2320 * Core functions
2321 */
2322/**
2323 * cik_tiling_mode_table_init - init the hw tiling table
2324 *
2325 * @rdev: radeon_device pointer
2326 *
2327 * Starting with SI, the tiling setup is done globally in a
2328 * set of 32 tiling modes.  Rather than selecting each set of
2329 * parameters per surface as on older asics, we just select
2330 * which index in the tiling table we want to use, and the
2331 * surface uses those parameters (CIK).
2332 */
2333static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2334{
2335        u32 *tile = rdev->config.cik.tile_mode_array;
2336        u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2337        const u32 num_tile_mode_states =
2338                        ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2339        const u32 num_secondary_tile_mode_states =
2340                        ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2341        u32 reg_offset, split_equal_to_row_size;
2342        u32 num_pipe_configs;
2343        u32 num_rbs = rdev->config.cik.max_backends_per_se *
2344                rdev->config.cik.max_shader_engines;
2345
2346        switch (rdev->config.cik.mem_row_size_in_kb) {
2347        case 1:
2348                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2349                break;
2350        case 2:
2351        default:
2352                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2353                break;
2354        case 4:
2355                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2356                break;
2357        }
2358
2359        num_pipe_configs = rdev->config.cik.max_tile_pipes;
2360        if (num_pipe_configs > 8)
2361                num_pipe_configs = 16;
2362
2363        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2364                tile[reg_offset] = 0;
2365        for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2366                macrotile[reg_offset] = 0;
2367
2368        switch(num_pipe_configs) {
2369        case 16:
2370                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2372                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2374                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2376                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2377                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2378                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2379                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2380                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2381                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2382                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2383                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2384                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2385                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2386                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2387                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2388                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2389                           TILE_SPLIT(split_equal_to_row_size));
2390                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2391                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2393                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2395                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2397                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2398                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2399                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2400                           TILE_SPLIT(split_equal_to_row_size));
2401                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2402                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2403                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2404                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2405                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2406                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2407                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2408                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2409                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2410                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2412                            PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2415                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2416                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2417                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2418                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2419                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2420                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2421                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2422                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2423                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2424                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2426                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2427                            PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2428                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2429                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2430                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2431                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2432                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2433                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2434                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2435                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2436                tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2437                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2438                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2439                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2441                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2442                            PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2443                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2444                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2445                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2446                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2447                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2448
2449                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2451                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2452                           NUM_BANKS(ADDR_SURF_16_BANK));
2453                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2455                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2456                           NUM_BANKS(ADDR_SURF_16_BANK));
2457                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460                           NUM_BANKS(ADDR_SURF_16_BANK));
2461                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464                           NUM_BANKS(ADDR_SURF_16_BANK));
2465                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2467                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2468                           NUM_BANKS(ADDR_SURF_8_BANK));
2469                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2471                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2472                           NUM_BANKS(ADDR_SURF_4_BANK));
2473                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2475                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2476                           NUM_BANKS(ADDR_SURF_2_BANK));
2477                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2479                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2480                           NUM_BANKS(ADDR_SURF_16_BANK));
2481                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2483                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2484                           NUM_BANKS(ADDR_SURF_16_BANK));
2485                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488                            NUM_BANKS(ADDR_SURF_16_BANK));
2489                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2490                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2491                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2492                            NUM_BANKS(ADDR_SURF_8_BANK));
2493                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2495                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2496                            NUM_BANKS(ADDR_SURF_4_BANK));
2497                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500                            NUM_BANKS(ADDR_SURF_2_BANK));
2501                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2503                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2504                            NUM_BANKS(ADDR_SURF_2_BANK));
2505
2506                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2507                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2508                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2509                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2510                break;
2511
2512        case 8:
2513                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2515                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2516                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2517                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2518                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2519                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2520                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2521                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2523                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2524                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2525                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2529                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2530                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2531                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2532                           TILE_SPLIT(split_equal_to_row_size));
2533                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2534                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2535                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2536                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2538                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2540                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2541                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2542                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2543                           TILE_SPLIT(split_equal_to_row_size));
2544                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2545                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2546                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2547                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2548                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2549                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2551                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2553                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2554                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2555                            PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2556                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2557                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2558                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2559                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2560                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2561                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2562                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2564                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2566                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2567                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2568                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2569                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2570                            PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2571                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2572                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2573                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2574                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2575                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2576                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2579                tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2580                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2581                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2582                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2583                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2584                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2585                            PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2586                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2587                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2588                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2589                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2590                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591
2592                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2594                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2595                                NUM_BANKS(ADDR_SURF_16_BANK));
2596                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2597                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2598                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2599                                NUM_BANKS(ADDR_SURF_16_BANK));
2600                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2602                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2603                                NUM_BANKS(ADDR_SURF_16_BANK));
2604                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2605                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2606                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2607                                NUM_BANKS(ADDR_SURF_16_BANK));
2608                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2610                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2611                                NUM_BANKS(ADDR_SURF_8_BANK));
2612                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2613                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2614                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2615                                NUM_BANKS(ADDR_SURF_4_BANK));
2616                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2617                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2618                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2619                                NUM_BANKS(ADDR_SURF_2_BANK));
2620                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2621                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2622                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2623                                NUM_BANKS(ADDR_SURF_16_BANK));
2624                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2625                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2626                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2627                                NUM_BANKS(ADDR_SURF_16_BANK));
2628                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2630                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2631                                NUM_BANKS(ADDR_SURF_16_BANK));
2632                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2633                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2634                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2635                                NUM_BANKS(ADDR_SURF_16_BANK));
2636                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2638                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2639                                NUM_BANKS(ADDR_SURF_8_BANK));
2640                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2641                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2642                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2643                                NUM_BANKS(ADDR_SURF_4_BANK));
2644                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2645                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2646                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2647                                NUM_BANKS(ADDR_SURF_2_BANK));
2648
2649                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2650                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2651                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2652                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2653                break;
2654
2655        case 4:
2656                if (num_rbs == 4) {
2657                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2658                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2659                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2660                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2661                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2663                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2664                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2665                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2667                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2668                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2669                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2670                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2671                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2672                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2673                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2675                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2676                           TILE_SPLIT(split_equal_to_row_size));
2677                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2678                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2679                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2680                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2681                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2682                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2684                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2685                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2686                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2687                           TILE_SPLIT(split_equal_to_row_size));
2688                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2689                           PIPE_CONFIG(ADDR_SURF_P4_16x16));
2690                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2691                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2692                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2693                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2694                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2695                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2696                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2697                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2698                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2699                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2700                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2701                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2702                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2703                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2704                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2705                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2706                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2707                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2708                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2709                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2710                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2711                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2712                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2713                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2714                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2715                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2716                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2717                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2718                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2719                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2720                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2721                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2722                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2723                tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2724                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2725                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2726                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2727                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2728                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2729                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2730                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2731                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2732                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2733                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2734                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2735
2736                } else if (num_rbs < 4) {
2737                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2738                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2739                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2740                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2741                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2742                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2743                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2744                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2745                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2746                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2747                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2749                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2752                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2753                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2754                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2755                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2756                           TILE_SPLIT(split_equal_to_row_size));
2757                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2758                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2760                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2761                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2762                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2763                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2764                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2765                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2766                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2767                           TILE_SPLIT(split_equal_to_row_size));
2768                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2769                           PIPE_CONFIG(ADDR_SURF_P4_8x16));
2770                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2771                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2772                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2773                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2775                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2776                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2777                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2778                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2779                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2780                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2781                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2782                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2783                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2784                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2785                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2786                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2788                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2789                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2790                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2791                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2792                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2793                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2794                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2796                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2797                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2798                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2799                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2800                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2802                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2803                tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2804                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2805                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2806                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2807                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2808                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2809                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2810                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2811                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2812                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2813                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815                }
2816
2817                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2818                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2819                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2820                                NUM_BANKS(ADDR_SURF_16_BANK));
2821                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2822                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2823                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2824                                NUM_BANKS(ADDR_SURF_16_BANK));
2825                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2827                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2828                                NUM_BANKS(ADDR_SURF_16_BANK));
2829                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2830                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2831                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2832                                NUM_BANKS(ADDR_SURF_16_BANK));
2833                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2835                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2836                                NUM_BANKS(ADDR_SURF_16_BANK));
2837                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2838                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2839                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2840                                NUM_BANKS(ADDR_SURF_8_BANK));
2841                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2842                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2843                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2844                                NUM_BANKS(ADDR_SURF_4_BANK));
2845                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2846                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2847                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2848                                NUM_BANKS(ADDR_SURF_16_BANK));
2849                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2850                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2851                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2852                                NUM_BANKS(ADDR_SURF_16_BANK));
2853                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2854                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2855                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2856                                NUM_BANKS(ADDR_SURF_16_BANK));
2857                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2858                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2859                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2860                                NUM_BANKS(ADDR_SURF_16_BANK));
2861                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2863                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2864                                NUM_BANKS(ADDR_SURF_16_BANK));
2865                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2866                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2867                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2868                                NUM_BANKS(ADDR_SURF_8_BANK));
2869                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2870                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2871                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2872                                NUM_BANKS(ADDR_SURF_4_BANK));
2873
2874                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2875                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2876                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2877                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2878                break;
2879
2880        case 2:
2881                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2882                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2883                           PIPE_CONFIG(ADDR_SURF_P2) |
2884                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2885                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887                           PIPE_CONFIG(ADDR_SURF_P2) |
2888                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2890                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2891                           PIPE_CONFIG(ADDR_SURF_P2) |
2892                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2893                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2894                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2895                           PIPE_CONFIG(ADDR_SURF_P2) |
2896                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2897                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899                           PIPE_CONFIG(ADDR_SURF_P2) |
2900                           TILE_SPLIT(split_equal_to_row_size));
2901                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2902                           PIPE_CONFIG(ADDR_SURF_P2) |
2903                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2904                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2905                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2906                           PIPE_CONFIG(ADDR_SURF_P2) |
2907                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2908                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2909                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2910                           PIPE_CONFIG(ADDR_SURF_P2) |
2911                           TILE_SPLIT(split_equal_to_row_size));
2912                tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2913                           PIPE_CONFIG(ADDR_SURF_P2);
2914                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2915                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2916                           PIPE_CONFIG(ADDR_SURF_P2));
2917                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2918                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2919                            PIPE_CONFIG(ADDR_SURF_P2) |
2920                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2921                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2922                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2923                            PIPE_CONFIG(ADDR_SURF_P2) |
2924                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2925                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2926                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2927                            PIPE_CONFIG(ADDR_SURF_P2) |
2928                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2929                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2930                            PIPE_CONFIG(ADDR_SURF_P2) |
2931                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2932                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2933                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2934                            PIPE_CONFIG(ADDR_SURF_P2) |
2935                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2936                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2937                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2938                            PIPE_CONFIG(ADDR_SURF_P2) |
2939                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2940                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2941                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2942                            PIPE_CONFIG(ADDR_SURF_P2) |
2943                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2944                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2945                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2946                            PIPE_CONFIG(ADDR_SURF_P2));
2947                tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2949                            PIPE_CONFIG(ADDR_SURF_P2) |
2950                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2952                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2953                            PIPE_CONFIG(ADDR_SURF_P2) |
2954                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2955                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2956                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2957                            PIPE_CONFIG(ADDR_SURF_P2) |
2958                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2959
2960                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2961                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2962                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2963                                NUM_BANKS(ADDR_SURF_16_BANK));
2964                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2965                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2966                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2967                                NUM_BANKS(ADDR_SURF_16_BANK));
2968                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2969                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971                                NUM_BANKS(ADDR_SURF_16_BANK));
2972                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2974                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975                                NUM_BANKS(ADDR_SURF_16_BANK));
2976                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2977                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2978                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2979                                NUM_BANKS(ADDR_SURF_16_BANK));
2980                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983                                NUM_BANKS(ADDR_SURF_16_BANK));
2984                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2985                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2986                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2987                                NUM_BANKS(ADDR_SURF_8_BANK));
2988                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2989                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2990                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2991                                NUM_BANKS(ADDR_SURF_16_BANK));
2992                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2993                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2994                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2995                                NUM_BANKS(ADDR_SURF_16_BANK));
2996                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2997                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2998                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2999                                NUM_BANKS(ADDR_SURF_16_BANK));
3000                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3001                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3002                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003                                NUM_BANKS(ADDR_SURF_16_BANK));
3004                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3005                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3006                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3007                                NUM_BANKS(ADDR_SURF_16_BANK));
3008                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3009                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3010                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3011                                NUM_BANKS(ADDR_SURF_16_BANK));
3012                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3013                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3014                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3015                                NUM_BANKS(ADDR_SURF_8_BANK));
3016
3017                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3018                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3019                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3020                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3021                break;
3022
3023        default:
3024                DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3025        }
3026}
3027
3028/**
3029 * cik_select_se_sh - select which SE, SH to address
3030 *
3031 * @rdev: radeon_device pointer
3032 * @se_num: shader engine to address
3033 * @sh_num: sh block to address
3034 *
3035 * Select which SE, SH combinations to address. Certain
3036 * registers are instanced per SE or SH.  0xffffffff means
3037 * broadcast to all SEs or SHs (CIK).
3038 */
3039static void cik_select_se_sh(struct radeon_device *rdev,
3040                             u32 se_num, u32 sh_num)
3041{
3042        u32 data = INSTANCE_BROADCAST_WRITES;
3043
3044        if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3045                data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3046        else if (se_num == 0xffffffff)
3047                data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3048        else if (sh_num == 0xffffffff)
3049                data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3050        else
3051                data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3052        WREG32(GRBM_GFX_INDEX, data);
3053}
3054
3055/**
3056 * cik_create_bitmask - create a bitmask
3057 *
3058 * @bit_width: length of the mask
3059 *
3060 * create a variable length bit mask (CIK).
3061 * Returns the bitmask.
3062 */
3063static u32 cik_create_bitmask(u32 bit_width)
3064{
3065        u32 i, mask = 0;
3066
3067        for (i = 0; i < bit_width; i++) {
3068                mask <<= 1;
3069                mask |= 1;
3070        }
3071        return mask;
3072}
3073
3074/**
3075 * cik_get_rb_disabled - computes the mask of disabled RBs
3076 *
3077 * @rdev: radeon_device pointer
3078 * @max_rb_num: max RBs (render backends) for the asic
3079 * @se_num: number of SEs (shader engines) for the asic
3080 * @sh_per_se: number of SH blocks per SE for the asic
3081 *
3082 * Calculates the bitmask of disabled RBs (CIK).
3083 * Returns the disabled RB bitmask.
3084 */
3085static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3086                              u32 max_rb_num_per_se,
3087                              u32 sh_per_se)
3088{
3089        u32 data, mask;
3090
3091        data = RREG32(CC_RB_BACKEND_DISABLE);
3092        if (data & 1)
3093                data &= BACKEND_DISABLE_MASK;
3094        else
3095                data = 0;
3096        data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3097
3098        data >>= BACKEND_DISABLE_SHIFT;
3099
3100        mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3101
3102        return data & mask;
3103}
3104
3105/**
3106 * cik_setup_rb - setup the RBs on the asic
3107 *
3108 * @rdev: radeon_device pointer
3109 * @se_num: number of SEs (shader engines) for the asic
3110 * @sh_per_se: number of SH blocks per SE for the asic
3111 * @max_rb_num: max RBs (render backends) for the asic
3112 *
3113 * Configures per-SE/SH RB registers (CIK).
3114 */
3115static void cik_setup_rb(struct radeon_device *rdev,
3116                         u32 se_num, u32 sh_per_se,
3117                         u32 max_rb_num_per_se)
3118{
3119        int i, j;
3120        u32 data, mask;
3121        u32 disabled_rbs = 0;
3122        u32 enabled_rbs = 0;
3123
3124        for (i = 0; i < se_num; i++) {
3125                for (j = 0; j < sh_per_se; j++) {
3126                        cik_select_se_sh(rdev, i, j);
3127                        data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3128                        if (rdev->family == CHIP_HAWAII)
3129                                disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3130                        else
3131                                disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3132                }
3133        }
3134        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3135
3136        mask = 1;
3137        for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3138                if (!(disabled_rbs & mask))
3139                        enabled_rbs |= mask;
3140                mask <<= 1;
3141        }
3142
3143        rdev->config.cik.backend_enable_mask = enabled_rbs;
3144
3145        for (i = 0; i < se_num; i++) {
3146                cik_select_se_sh(rdev, i, 0xffffffff);
3147                data = 0;
3148                for (j = 0; j < sh_per_se; j++) {
3149                        switch (enabled_rbs & 3) {
3150                        case 0:
3151                                if (j == 0)
3152                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3153                                else
3154                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3155                                break;
3156                        case 1:
3157                                data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3158                                break;
3159                        case 2:
3160                                data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3161                                break;
3162                        case 3:
3163                        default:
3164                                data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3165                                break;
3166                        }
3167                        enabled_rbs >>= 2;
3168                }
3169                WREG32(PA_SC_RASTER_CONFIG, data);
3170        }
3171        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3172}
3173
3174/**
3175 * cik_gpu_init - setup the 3D engine
3176 *
3177 * @rdev: radeon_device pointer
3178 *
3179 * Configures the 3D engine and tiling configuration
3180 * registers so that the 3D engine is usable.
3181 */
3182static void cik_gpu_init(struct radeon_device *rdev)
3183{
3184        u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3185        u32 mc_shared_chmap, mc_arb_ramcfg;
3186        u32 hdp_host_path_cntl;
3187        u32 tmp;
3188        int i, j;
3189
3190        switch (rdev->family) {
3191        case CHIP_BONAIRE:
3192                rdev->config.cik.max_shader_engines = 2;
3193                rdev->config.cik.max_tile_pipes = 4;
3194                rdev->config.cik.max_cu_per_sh = 7;
3195                rdev->config.cik.max_sh_per_se = 1;
3196                rdev->config.cik.max_backends_per_se = 2;
3197                rdev->config.cik.max_texture_channel_caches = 4;
3198                rdev->config.cik.max_gprs = 256;
3199                rdev->config.cik.max_gs_threads = 32;
3200                rdev->config.cik.max_hw_contexts = 8;
3201
3202                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3203                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3204                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3205                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3206                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3207                break;
3208        case CHIP_HAWAII:
3209                rdev->config.cik.max_shader_engines = 4;
3210                rdev->config.cik.max_tile_pipes = 16;
3211                rdev->config.cik.max_cu_per_sh = 11;
3212                rdev->config.cik.max_sh_per_se = 1;
3213                rdev->config.cik.max_backends_per_se = 4;
3214                rdev->config.cik.max_texture_channel_caches = 16;
3215                rdev->config.cik.max_gprs = 256;
3216                rdev->config.cik.max_gs_threads = 32;
3217                rdev->config.cik.max_hw_contexts = 8;
3218
3219                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3220                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3221                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3222                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3223                gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3224                break;
3225        case CHIP_KAVERI:
3226                rdev->config.cik.max_shader_engines = 1;
3227                rdev->config.cik.max_tile_pipes = 4;
3228                rdev->config.cik.max_cu_per_sh = 8;
3229                rdev->config.cik.max_backends_per_se = 2;
3230                rdev->config.cik.max_sh_per_se = 1;
3231                rdev->config.cik.max_texture_channel_caches = 4;
3232                rdev->config.cik.max_gprs = 256;
3233                rdev->config.cik.max_gs_threads = 16;
3234                rdev->config.cik.max_hw_contexts = 8;
3235
3236                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3237                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3238                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3239                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3240                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3241                break;
3242        case CHIP_KABINI:
3243        case CHIP_MULLINS:
3244        default:
3245                rdev->config.cik.max_shader_engines = 1;
3246                rdev->config.cik.max_tile_pipes = 2;
3247                rdev->config.cik.max_cu_per_sh = 2;
3248                rdev->config.cik.max_sh_per_se = 1;
3249                rdev->config.cik.max_backends_per_se = 1;
3250                rdev->config.cik.max_texture_channel_caches = 2;
3251                rdev->config.cik.max_gprs = 256;
3252                rdev->config.cik.max_gs_threads = 16;
3253                rdev->config.cik.max_hw_contexts = 8;
3254
3255                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3256                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3257                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3258                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3259                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3260                break;
3261        }
3262
3263        /* Initialize HDP */
3264        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3265                WREG32((0x2c14 + j), 0x00000000);
3266                WREG32((0x2c18 + j), 0x00000000);
3267                WREG32((0x2c1c + j), 0x00000000);
3268                WREG32((0x2c20 + j), 0x00000000);
3269                WREG32((0x2c24 + j), 0x00000000);
3270        }
3271
3272        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3273        WREG32(SRBM_INT_CNTL, 0x1);
3274        WREG32(SRBM_INT_ACK, 0x1);
3275
3276        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3277
3278        mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3279        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3280
3281        rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3282        rdev->config.cik.mem_max_burst_length_bytes = 256;
3283        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3284        rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3285        if (rdev->config.cik.mem_row_size_in_kb > 4)
3286                rdev->config.cik.mem_row_size_in_kb = 4;
3287        /* XXX use MC settings? */
3288        rdev->config.cik.shader_engine_tile_size = 32;
3289        rdev->config.cik.num_gpus = 1;
3290        rdev->config.cik.multi_gpu_tile_size = 64;
3291
3292        /* fix up row size */
3293        gb_addr_config &= ~ROW_SIZE_MASK;
3294        switch (rdev->config.cik.mem_row_size_in_kb) {
3295        case 1:
3296        default:
3297                gb_addr_config |= ROW_SIZE(0);
3298                break;
3299        case 2:
3300                gb_addr_config |= ROW_SIZE(1);
3301                break;
3302        case 4:
3303                gb_addr_config |= ROW_SIZE(2);
3304                break;
3305        }
3306
3307        /* setup tiling info dword.  gb_addr_config is not adequate since it does
3308         * not have bank info, so create a custom tiling dword.
3309         * bits 3:0   num_pipes
3310         * bits 7:4   num_banks
3311         * bits 11:8  group_size
3312         * bits 15:12 row_size
3313         */
3314        rdev->config.cik.tile_config = 0;
3315        switch (rdev->config.cik.num_tile_pipes) {
3316        case 1:
3317                rdev->config.cik.tile_config |= (0 << 0);
3318                break;
3319        case 2:
3320                rdev->config.cik.tile_config |= (1 << 0);
3321                break;
3322        case 4:
3323                rdev->config.cik.tile_config |= (2 << 0);
3324                break;
3325        case 8:
3326        default:
3327                /* XXX what about 12? */
3328                rdev->config.cik.tile_config |= (3 << 0);
3329                break;
3330        }
3331        rdev->config.cik.tile_config |=
3332                ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3333        rdev->config.cik.tile_config |=
3334                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3335        rdev->config.cik.tile_config |=
3336                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3337
3338        WREG32(GB_ADDR_CONFIG, gb_addr_config);
3339        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3340        WREG32(DMIF_ADDR_CALC, gb_addr_config);
3341        WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3342        WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3343        WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3344        WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3345        WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3346
3347        cik_tiling_mode_table_init(rdev);
3348
3349        cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3350                     rdev->config.cik.max_sh_per_se,
3351                     rdev->config.cik.max_backends_per_se);
3352
3353        rdev->config.cik.active_cus = 0;
3354        for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3355                for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3356                        rdev->config.cik.active_cus +=
3357                                hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3358                }
3359        }
3360
3361        /* set HW defaults for 3D engine */
3362        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3363
3364        WREG32(SX_DEBUG_1, 0x20);
3365
3366        WREG32(TA_CNTL_AUX, 0x00010000);
3367
3368        tmp = RREG32(SPI_CONFIG_CNTL);
3369        tmp |= 0x03000000;
3370        WREG32(SPI_CONFIG_CNTL, tmp);
3371
3372        WREG32(SQ_CONFIG, 1);
3373
3374        WREG32(DB_DEBUG, 0);
3375
3376        tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3377        tmp |= 0x00000400;
3378        WREG32(DB_DEBUG2, tmp);
3379
3380        tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3381        tmp |= 0x00020200;
3382        WREG32(DB_DEBUG3, tmp);
3383
3384        tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3385        tmp |= 0x00018208;
3386        WREG32(CB_HW_CONTROL, tmp);
3387
3388        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3389
3390        WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3391                                 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3392                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3393                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3394
3395        WREG32(VGT_NUM_INSTANCES, 1);
3396
3397        WREG32(CP_PERFMON_CNTL, 0);
3398
3399        WREG32(SQ_CONFIG, 0);
3400
3401        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3402                                          FORCE_EOV_MAX_REZ_CNT(255)));
3403
3404        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3405               AUTO_INVLD_EN(ES_AND_GS_AUTO));
3406
3407        WREG32(VGT_GS_VERTEX_REUSE, 16);
3408        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3409
3410        tmp = RREG32(HDP_MISC_CNTL);
3411        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3412        WREG32(HDP_MISC_CNTL, tmp);
3413
3414        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3415        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3416
3417        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3418        WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3419
3420        udelay(50);
3421}
3422
3423/*
3424 * GPU scratch registers helpers function.
3425 */
3426/**
3427 * cik_scratch_init - setup driver info for CP scratch regs
3428 *
3429 * @rdev: radeon_device pointer
3430 *
3431 * Set up the number and offset of the CP scratch registers.
3432 * NOTE: use of CP scratch registers is a legacy inferface and
3433 * is not used by default on newer asics (r6xx+).  On newer asics,
3434 * memory buffers are used for fences rather than scratch regs.
3435 */
3436static void cik_scratch_init(struct radeon_device *rdev)
3437{
3438        int i;
3439
3440        rdev->scratch.num_reg = 7;
3441        rdev->scratch.reg_base = SCRATCH_REG0;
3442        for (i = 0; i < rdev->scratch.num_reg; i++) {
3443                rdev->scratch.free[i] = true;
3444                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3445        }
3446}
3447
3448/**
3449 * cik_ring_test - basic gfx ring test
3450 *
3451 * @rdev: radeon_device pointer
3452 * @ring: radeon_ring structure holding ring information
3453 *
3454 * Allocate a scratch register and write to it using the gfx ring (CIK).
3455 * Provides a basic gfx ring test to verify that the ring is working.
3456 * Used by cik_cp_gfx_resume();
3457 * Returns 0 on success, error on failure.
3458 */
3459int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3460{
3461        uint32_t scratch;
3462        uint32_t tmp = 0;
3463        unsigned i;
3464        int r;
3465
3466        r = radeon_scratch_get(rdev, &scratch);
3467        if (r) {
3468                DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3469                return r;
3470        }
3471        WREG32(scratch, 0xCAFEDEAD);
3472        r = radeon_ring_lock(rdev, ring, 3);
3473        if (r) {
3474                DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3475                radeon_scratch_free(rdev, scratch);
3476                return r;
3477        }
3478        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3479        radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3480        radeon_ring_write(ring, 0xDEADBEEF);
3481        radeon_ring_unlock_commit(rdev, ring, false);
3482
3483        for (i = 0; i < rdev->usec_timeout; i++) {
3484                tmp = RREG32(scratch);
3485                if (tmp == 0xDEADBEEF)
3486                        break;
3487                udelay(1);
3488        }
3489        if (i < rdev->usec_timeout) {
3490                DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3491        } else {
3492                DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3493                          ring->idx, scratch, tmp);
3494                r = -EINVAL;
3495        }
3496        radeon_scratch_free(rdev, scratch);
3497        return r;
3498}
3499
3500/**
3501 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3502 *
3503 * @rdev: radeon_device pointer
3504 * @ridx: radeon ring index
3505 *
3506 * Emits an hdp flush on the cp.
3507 */
3508static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3509                                       int ridx)
3510{
3511        struct radeon_ring *ring = &rdev->ring[ridx];
3512        u32 ref_and_mask;
3513
3514        switch (ring->idx) {
3515        case CAYMAN_RING_TYPE_CP1_INDEX:
3516        case CAYMAN_RING_TYPE_CP2_INDEX:
3517        default:
3518                switch (ring->me) {
3519                case 0:
3520                        ref_and_mask = CP2 << ring->pipe;
3521                        break;
3522                case 1:
3523                        ref_and_mask = CP6 << ring->pipe;
3524                        break;
3525                default:
3526                        return;
3527                }
3528                break;
3529        case RADEON_RING_TYPE_GFX_INDEX:
3530                ref_and_mask = CP0;
3531                break;
3532        }
3533
3534        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3535        radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3536                                 WAIT_REG_MEM_FUNCTION(3) |  /* == */
3537                                 WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3538        radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3539        radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3540        radeon_ring_write(ring, ref_and_mask);
3541        radeon_ring_write(ring, ref_and_mask);
3542        radeon_ring_write(ring, 0x20); /* poll interval */
3543}
3544
3545/**
3546 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3547 *
3548 * @rdev: radeon_device pointer
3549 * @fence: radeon fence object
3550 *
3551 * Emits a fence sequnce number on the gfx ring and flushes
3552 * GPU caches.
3553 */
3554void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3555                             struct radeon_fence *fence)
3556{
3557        struct radeon_ring *ring = &rdev->ring[fence->ring];
3558        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3559
3560        /* Workaround for cache flush problems. First send a dummy EOP
3561         * event down the pipe with seq one below.
3562         */
3563        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3564        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3565                                 EOP_TC_ACTION_EN |
3566                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3567                                 EVENT_INDEX(5)));
3568        radeon_ring_write(ring, addr & 0xfffffffc);
3569        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3570                                DATA_SEL(1) | INT_SEL(0));
3571        radeon_ring_write(ring, fence->seq - 1);
3572        radeon_ring_write(ring, 0);
3573
3574        /* Then send the real EOP event down the pipe. */
3575        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3576        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3577                                 EOP_TC_ACTION_EN |
3578                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3579                                 EVENT_INDEX(5)));
3580        radeon_ring_write(ring, addr & 0xfffffffc);
3581        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3582        radeon_ring_write(ring, fence->seq);
3583        radeon_ring_write(ring, 0);
3584}
3585
3586/**
3587 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3588 *
3589 * @rdev: radeon_device pointer
3590 * @fence: radeon fence object
3591 *
3592 * Emits a fence sequnce number on the compute ring and flushes
3593 * GPU caches.
3594 */
3595void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3596                                 struct radeon_fence *fence)
3597{
3598        struct radeon_ring *ring = &rdev->ring[fence->ring];
3599        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3600
3601        /* RELEASE_MEM - flush caches, send int */
3602        radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3603        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3604                                 EOP_TC_ACTION_EN |
3605                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3606                                 EVENT_INDEX(5)));
3607        radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3608        radeon_ring_write(ring, addr & 0xfffffffc);
3609        radeon_ring_write(ring, upper_32_bits(addr));
3610        radeon_ring_write(ring, fence->seq);
3611        radeon_ring_write(ring, 0);
3612}
3613
3614/**
3615 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3616 *
3617 * @rdev: radeon_device pointer
3618 * @ring: radeon ring buffer object
3619 * @semaphore: radeon semaphore object
3620 * @emit_wait: Is this a sempahore wait?
3621 *
3622 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3623 * from running ahead of semaphore waits.
3624 */
3625bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3626                             struct radeon_ring *ring,
3627                             struct radeon_semaphore *semaphore,
3628                             bool emit_wait)
3629{
3630        uint64_t addr = semaphore->gpu_addr;
3631        unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3632
3633        radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3634        radeon_ring_write(ring, lower_32_bits(addr));
3635        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3636
3637        if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3638                /* Prevent the PFP from running ahead of the semaphore wait */
3639                radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3640                radeon_ring_write(ring, 0x0);
3641        }
3642
3643        return true;
3644}
3645
3646/**
3647 * cik_copy_cpdma - copy pages using the CP DMA engine
3648 *
3649 * @rdev: radeon_device pointer
3650 * @src_offset: src GPU address
3651 * @dst_offset: dst GPU address
3652 * @num_gpu_pages: number of GPU pages to xfer
3653 * @resv: reservation object to sync to
3654 *
3655 * Copy GPU paging using the CP DMA engine (CIK+).
3656 * Used by the radeon ttm implementation to move pages if
3657 * registered as the asic copy callback.
3658 */
3659struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3660                                    uint64_t src_offset, uint64_t dst_offset,
3661                                    unsigned num_gpu_pages,
3662                                    struct dma_resv *resv)
3663{
3664        struct radeon_fence *fence;
3665        struct radeon_sync sync;
3666        int ring_index = rdev->asic->copy.blit_ring_index;
3667        struct radeon_ring *ring = &rdev->ring[ring_index];
3668        u32 size_in_bytes, cur_size_in_bytes, control;
3669        int i, num_loops;
3670        int r = 0;
3671
3672        radeon_sync_create(&sync);
3673
3674        size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3675        num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3676        r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3677        if (r) {
3678                DRM_ERROR("radeon: moving bo (%d).\n", r);
3679                radeon_sync_free(rdev, &sync, NULL);
3680                return ERR_PTR(r);
3681        }
3682
3683        radeon_sync_resv(rdev, &sync, resv, false);
3684        radeon_sync_rings(rdev, &sync, ring->idx);
3685
3686        for (i = 0; i < num_loops; i++) {
3687                cur_size_in_bytes = size_in_bytes;
3688                if (cur_size_in_bytes > 0x1fffff)
3689                        cur_size_in_bytes = 0x1fffff;
3690                size_in_bytes -= cur_size_in_bytes;
3691                control = 0;
3692                if (size_in_bytes == 0)
3693                        control |= PACKET3_DMA_DATA_CP_SYNC;
3694                radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3695                radeon_ring_write(ring, control);
3696                radeon_ring_write(ring, lower_32_bits(src_offset));
3697                radeon_ring_write(ring, upper_32_bits(src_offset));
3698                radeon_ring_write(ring, lower_32_bits(dst_offset));
3699                radeon_ring_write(ring, upper_32_bits(dst_offset));
3700                radeon_ring_write(ring, cur_size_in_bytes);
3701                src_offset += cur_size_in_bytes;
3702                dst_offset += cur_size_in_bytes;
3703        }
3704
3705        r = radeon_fence_emit(rdev, &fence, ring->idx);
3706        if (r) {
3707                radeon_ring_unlock_undo(rdev, ring);
3708                radeon_sync_free(rdev, &sync, NULL);
3709                return ERR_PTR(r);
3710        }
3711
3712        radeon_ring_unlock_commit(rdev, ring, false);
3713        radeon_sync_free(rdev, &sync, fence);
3714
3715        return fence;
3716}
3717
3718/*
3719 * IB stuff
3720 */
3721/**
3722 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3723 *
3724 * @rdev: radeon_device pointer
3725 * @ib: radeon indirect buffer object
3726 *
3727 * Emits a DE (drawing engine) or CE (constant engine) IB
3728 * on the gfx ring.  IBs are usually generated by userspace
3729 * acceleration drivers and submitted to the kernel for
3730 * scheduling on the ring.  This function schedules the IB
3731 * on the gfx ring for execution by the GPU.
3732 */
3733void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3734{
3735        struct radeon_ring *ring = &rdev->ring[ib->ring];
3736        unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3737        u32 header, control = INDIRECT_BUFFER_VALID;
3738
3739        if (ib->is_const_ib) {
3740                /* set switch buffer packet before const IB */
3741                radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3742                radeon_ring_write(ring, 0);
3743
3744                header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3745        } else {
3746                u32 next_rptr;
3747                if (ring->rptr_save_reg) {
3748                        next_rptr = ring->wptr + 3 + 4;
3749                        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3750                        radeon_ring_write(ring, ((ring->rptr_save_reg -
3751                                                  PACKET3_SET_UCONFIG_REG_START) >> 2));
3752                        radeon_ring_write(ring, next_rptr);
3753                } else if (rdev->wb.enabled) {
3754                        next_rptr = ring->wptr + 5 + 4;
3755                        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3756                        radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3757                        radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3758                        radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3759                        radeon_ring_write(ring, next_rptr);
3760                }
3761
3762                header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3763        }
3764
3765        control |= ib->length_dw | (vm_id << 24);
3766
3767        radeon_ring_write(ring, header);
3768        radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
3769        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3770        radeon_ring_write(ring, control);
3771}
3772
3773/**
3774 * cik_ib_test - basic gfx ring IB test
3775 *
3776 * @rdev: radeon_device pointer
3777 * @ring: radeon_ring structure holding ring information
3778 *
3779 * Allocate an IB and execute it on the gfx ring (CIK).
3780 * Provides a basic gfx ring test to verify that IBs are working.
3781 * Returns 0 on success, error on failure.
3782 */
3783int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3784{
3785        struct radeon_ib ib;
3786        uint32_t scratch;
3787        uint32_t tmp = 0;
3788        unsigned i;
3789        int r;
3790
3791        r = radeon_scratch_get(rdev, &scratch);
3792        if (r) {
3793                DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3794                return r;
3795        }
3796        WREG32(scratch, 0xCAFEDEAD);
3797        r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3798        if (r) {
3799                DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3800                radeon_scratch_free(rdev, scratch);
3801                return r;
3802        }
3803        ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3804        ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3805        ib.ptr[2] = 0xDEADBEEF;
3806        ib.length_dw = 3;
3807        r = radeon_ib_schedule(rdev, &ib, NULL, false);
3808        if (r) {
3809                radeon_scratch_free(rdev, scratch);
3810                radeon_ib_free(rdev, &ib);
3811                DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3812                return r;
3813        }
3814        r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3815                RADEON_USEC_IB_TEST_TIMEOUT));
3816        if (r < 0) {
3817                DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3818                radeon_scratch_free(rdev, scratch);
3819                radeon_ib_free(rdev, &ib);
3820                return r;
3821        } else if (r == 0) {
3822                DRM_ERROR("radeon: fence wait timed out.\n");
3823                radeon_scratch_free(rdev, scratch);
3824                radeon_ib_free(rdev, &ib);
3825                return -ETIMEDOUT;
3826        }
3827        r = 0;
3828        for (i = 0; i < rdev->usec_timeout; i++) {
3829                tmp = RREG32(scratch);
3830                if (tmp == 0xDEADBEEF)
3831                        break;
3832                udelay(1);
3833        }
3834        if (i < rdev->usec_timeout) {
3835                DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3836        } else {
3837                DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3838                          scratch, tmp);
3839                r = -EINVAL;
3840        }
3841        radeon_scratch_free(rdev, scratch);
3842        radeon_ib_free(rdev, &ib);
3843        return r;
3844}
3845
3846/*
3847 * CP.
3848 * On CIK, gfx and compute now have independant command processors.
3849 *
3850 * GFX
3851 * Gfx consists of a single ring and can process both gfx jobs and
3852 * compute jobs.  The gfx CP consists of three microengines (ME):
3853 * PFP - Pre-Fetch Parser
3854 * ME - Micro Engine
3855 * CE - Constant Engine
3856 * The PFP and ME make up what is considered the Drawing Engine (DE).
3857 * The CE is an asynchronous engine used for updating buffer desciptors
3858 * used by the DE so that they can be loaded into cache in parallel
3859 * while the DE is processing state update packets.
3860 *
3861 * Compute
3862 * The compute CP consists of two microengines (ME):
3863 * MEC1 - Compute MicroEngine 1
3864 * MEC2 - Compute MicroEngine 2
3865 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3866 * The queues are exposed to userspace and are programmed directly
3867 * by the compute runtime.
3868 */
3869/**
3870 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3871 *
3872 * @rdev: radeon_device pointer
3873 * @enable: enable or disable the MEs
3874 *
3875 * Halts or unhalts the gfx MEs.
3876 */
3877static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3878{
3879        if (enable)
3880                WREG32(CP_ME_CNTL, 0);
3881        else {
3882                if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3883                        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3884                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3885                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3886        }
3887        udelay(50);
3888}
3889
3890/**
3891 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3892 *
3893 * @rdev: radeon_device pointer
3894 *
3895 * Loads the gfx PFP, ME, and CE ucode.
3896 * Returns 0 for success, -EINVAL if the ucode is not available.
3897 */
3898static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3899{
3900        int i;
3901
3902        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3903                return -EINVAL;
3904
3905        cik_cp_gfx_enable(rdev, false);
3906
3907        if (rdev->new_fw) {
3908                const struct gfx_firmware_header_v1_0 *pfp_hdr =
3909                        (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3910                const struct gfx_firmware_header_v1_0 *ce_hdr =
3911                        (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3912                const struct gfx_firmware_header_v1_0 *me_hdr =
3913                        (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3914                const __le32 *fw_data;
3915                u32 fw_size;
3916
3917                radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3918                radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3919                radeon_ucode_print_gfx_hdr(&me_hdr->header);
3920
3921                /* PFP */
3922                fw_data = (const __le32 *)
3923                        (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3924                fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3925                WREG32(CP_PFP_UCODE_ADDR, 0);
3926                for (i = 0; i < fw_size; i++)
3927                        WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3928                WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
3929
3930                /* CE */
3931                fw_data = (const __le32 *)
3932                        (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3933                fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3934                WREG32(CP_CE_UCODE_ADDR, 0);
3935                for (i = 0; i < fw_size; i++)
3936                        WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3937                WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
3938
3939                /* ME */
3940                fw_data = (const __be32 *)
3941                        (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3942                fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3943                WREG32(CP_ME_RAM_WADDR, 0);
3944                for (i = 0; i < fw_size; i++)
3945                        WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3946                WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3947                WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
3948        } else {
3949                const __be32 *fw_data;
3950
3951                /* PFP */
3952                fw_data = (const __be32 *)rdev->pfp_fw->data;
3953                WREG32(CP_PFP_UCODE_ADDR, 0);
3954                for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3955                        WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3956                WREG32(CP_PFP_UCODE_ADDR, 0);
3957
3958                /* CE */
3959                fw_data = (const __be32 *)rdev->ce_fw->data;
3960                WREG32(CP_CE_UCODE_ADDR, 0);
3961                for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3962                        WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3963                WREG32(CP_CE_UCODE_ADDR, 0);
3964
3965                /* ME */
3966                fw_data = (const __be32 *)rdev->me_fw->data;
3967                WREG32(CP_ME_RAM_WADDR, 0);
3968                for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3969                        WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3970                WREG32(CP_ME_RAM_WADDR, 0);
3971        }
3972
3973        return 0;
3974}
3975
3976/**
3977 * cik_cp_gfx_start - start the gfx ring
3978 *
3979 * @rdev: radeon_device pointer
3980 *
3981 * Enables the ring and loads the clear state context and other
3982 * packets required to init the ring.
3983 * Returns 0 for success, error for failure.
3984 */
3985static int cik_cp_gfx_start(struct radeon_device *rdev)
3986{
3987        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3988        int r, i;
3989
3990        /* init the CP */
3991        WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3992        WREG32(CP_ENDIAN_SWAP, 0);
3993        WREG32(CP_DEVICE_ID, 1);
3994
3995        cik_cp_gfx_enable(rdev, true);
3996
3997        r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3998        if (r) {
3999                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4000                return r;
4001        }
4002
4003        /* init the CE partitions.  CE only used for gfx on CIK */
4004        radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4005        radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4006        radeon_ring_write(ring, 0x8000);
4007        radeon_ring_write(ring, 0x8000);
4008
4009        /* setup clear context state */
4010        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4011        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4012
4013        radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4014        radeon_ring_write(ring, 0x80000000);
4015        radeon_ring_write(ring, 0x80000000);
4016
4017        for (i = 0; i < cik_default_size; i++)
4018                radeon_ring_write(ring, cik_default_state[i]);
4019
4020        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4021        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4022
4023        /* set clear context state */
4024        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4025        radeon_ring_write(ring, 0);
4026
4027        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4028        radeon_ring_write(ring, 0x00000316);
4029        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4030        radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4031
4032        radeon_ring_unlock_commit(rdev, ring, false);
4033
4034        return 0;
4035}
4036
4037/**
4038 * cik_cp_gfx_fini - stop the gfx ring
4039 *
4040 * @rdev: radeon_device pointer
4041 *
4042 * Stop the gfx ring and tear down the driver ring
4043 * info.
4044 */
4045static void cik_cp_gfx_fini(struct radeon_device *rdev)
4046{
4047        cik_cp_gfx_enable(rdev, false);
4048        radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4049}
4050
4051/**
4052 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4053 *
4054 * @rdev: radeon_device pointer
4055 *
4056 * Program the location and size of the gfx ring buffer
4057 * and test it to make sure it's working.
4058 * Returns 0 for success, error for failure.
4059 */
4060static int cik_cp_gfx_resume(struct radeon_device *rdev)
4061{
4062        struct radeon_ring *ring;
4063        u32 tmp;
4064        u32 rb_bufsz;
4065        u64 rb_addr;
4066        int r;
4067
4068        WREG32(CP_SEM_WAIT_TIMER, 0x0);
4069        if (rdev->family != CHIP_HAWAII)
4070                WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4071
4072        /* Set the write pointer delay */
4073        WREG32(CP_RB_WPTR_DELAY, 0);
4074
4075        /* set the RB to use vmid 0 */
4076        WREG32(CP_RB_VMID, 0);
4077
4078        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4079
4080        /* ring 0 - compute and gfx */
4081        /* Set ring buffer size */
4082        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4083        rb_bufsz = order_base_2(ring->ring_size / 8);
4084        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4085#ifdef __BIG_ENDIAN
4086        tmp |= BUF_SWAP_32BIT;
4087#endif
4088        WREG32(CP_RB0_CNTL, tmp);
4089
4090        /* Initialize the ring buffer's read and write pointers */
4091        WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4092        ring->wptr = 0;
4093        WREG32(CP_RB0_WPTR, ring->wptr);
4094
4095        /* set the wb address wether it's enabled or not */
4096        WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4097        WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4098
4099        /* scratch register shadowing is no longer supported */
4100        WREG32(SCRATCH_UMSK, 0);
4101
4102        if (!rdev->wb.enabled)
4103                tmp |= RB_NO_UPDATE;
4104
4105        mdelay(1);
4106        WREG32(CP_RB0_CNTL, tmp);
4107
4108        rb_addr = ring->gpu_addr >> 8;
4109        WREG32(CP_RB0_BASE, rb_addr);
4110        WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4111
4112        /* start the ring */
4113        cik_cp_gfx_start(rdev);
4114        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4115        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4116        if (r) {
4117                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4118                return r;
4119        }
4120
4121        if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4122                radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4123
4124        return 0;
4125}
4126
4127u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4128                     struct radeon_ring *ring)
4129{
4130        u32 rptr;
4131
4132        if (rdev->wb.enabled)
4133                rptr = rdev->wb.wb[ring->rptr_offs/4];
4134        else
4135                rptr = RREG32(CP_RB0_RPTR);
4136
4137        return rptr;
4138}
4139
4140u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4141                     struct radeon_ring *ring)
4142{
4143        return RREG32(CP_RB0_WPTR);
4144}
4145
4146void cik_gfx_set_wptr(struct radeon_device *rdev,
4147                      struct radeon_ring *ring)
4148{
4149        WREG32(CP_RB0_WPTR, ring->wptr);
4150        (void)RREG32(CP_RB0_WPTR);
4151}
4152
4153u32 cik_compute_get_rptr(struct radeon_device *rdev,
4154                         struct radeon_ring *ring)
4155{
4156        u32 rptr;
4157
4158        if (rdev->wb.enabled) {
4159                rptr = rdev->wb.wb[ring->rptr_offs/4];
4160        } else {
4161                mutex_lock(&rdev->srbm_mutex);
4162                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4163                rptr = RREG32(CP_HQD_PQ_RPTR);
4164                cik_srbm_select(rdev, 0, 0, 0, 0);
4165                mutex_unlock(&rdev->srbm_mutex);
4166        }
4167
4168        return rptr;
4169}
4170
4171u32 cik_compute_get_wptr(struct radeon_device *rdev,
4172                         struct radeon_ring *ring)
4173{
4174        u32 wptr;
4175
4176        if (rdev->wb.enabled) {
4177                /* XXX check if swapping is necessary on BE */
4178                wptr = rdev->wb.wb[ring->wptr_offs/4];
4179        } else {
4180                mutex_lock(&rdev->srbm_mutex);
4181                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4182                wptr = RREG32(CP_HQD_PQ_WPTR);
4183                cik_srbm_select(rdev, 0, 0, 0, 0);
4184                mutex_unlock(&rdev->srbm_mutex);
4185        }
4186
4187        return wptr;
4188}
4189
4190void cik_compute_set_wptr(struct radeon_device *rdev,
4191                          struct radeon_ring *ring)
4192{
4193        /* XXX check if swapping is necessary on BE */
4194        rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4195        WDOORBELL32(ring->doorbell_index, ring->wptr);
4196}
4197
4198static void cik_compute_stop(struct radeon_device *rdev,
4199                             struct radeon_ring *ring)
4200{
4201        u32 j, tmp;
4202
4203        cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4204        /* Disable wptr polling. */
4205        tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4206        tmp &= ~WPTR_POLL_EN;
4207        WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4208        /* Disable HQD. */
4209        if (RREG32(CP_HQD_ACTIVE) & 1) {
4210                WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4211                for (j = 0; j < rdev->usec_timeout; j++) {
4212                        if (!(RREG32(CP_HQD_ACTIVE) & 1))
4213                                break;
4214                        udelay(1);
4215                }
4216                WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4217                WREG32(CP_HQD_PQ_RPTR, 0);
4218                WREG32(CP_HQD_PQ_WPTR, 0);
4219        }
4220        cik_srbm_select(rdev, 0, 0, 0, 0);
4221}
4222
4223/**
4224 * cik_cp_compute_enable - enable/disable the compute CP MEs
4225 *
4226 * @rdev: radeon_device pointer
4227 * @enable: enable or disable the MEs
4228 *
4229 * Halts or unhalts the compute MEs.
4230 */
4231static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4232{
4233        if (enable)
4234                WREG32(CP_MEC_CNTL, 0);
4235        else {
4236                /*
4237                 * To make hibernation reliable we need to clear compute ring
4238                 * configuration before halting the compute ring.
4239                 */
4240                mutex_lock(&rdev->srbm_mutex);
4241                cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4242                cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4243                mutex_unlock(&rdev->srbm_mutex);
4244
4245                WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4246                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4247                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4248        }
4249        udelay(50);
4250}
4251
4252/**
4253 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4254 *
4255 * @rdev: radeon_device pointer
4256 *
4257 * Loads the compute MEC1&2 ucode.
4258 * Returns 0 for success, -EINVAL if the ucode is not available.
4259 */
4260static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4261{
4262        int i;
4263
4264        if (!rdev->mec_fw)
4265                return -EINVAL;
4266
4267        cik_cp_compute_enable(rdev, false);
4268
4269        if (rdev->new_fw) {
4270                const struct gfx_firmware_header_v1_0 *mec_hdr =
4271                        (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4272                const __le32 *fw_data;
4273                u32 fw_size;
4274
4275                radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4276
4277                /* MEC1 */
4278                fw_data = (const __le32 *)
4279                        (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4280                fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4281                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4282                for (i = 0; i < fw_size; i++)
4283                        WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4284                WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4285
4286                /* MEC2 */
4287                if (rdev->family == CHIP_KAVERI) {
4288                        const struct gfx_firmware_header_v1_0 *mec2_hdr =
4289                                (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4290
4291                        fw_data = (const __le32 *)
4292                                (rdev->mec2_fw->data +
4293                                 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4294                        fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4295                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4296                        for (i = 0; i < fw_size; i++)
4297                                WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4298                        WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4299                }
4300        } else {
4301                const __be32 *fw_data;
4302
4303                /* MEC1 */
4304                fw_data = (const __be32 *)rdev->mec_fw->data;
4305                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4306                for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4307                        WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4308                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4309
4310                if (rdev->family == CHIP_KAVERI) {
4311                        /* MEC2 */
4312                        fw_data = (const __be32 *)rdev->mec_fw->data;
4313                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4314                        for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4315                                WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4316                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4317                }
4318        }
4319
4320        return 0;
4321}
4322
4323/**
4324 * cik_cp_compute_start - start the compute queues
4325 *
4326 * @rdev: radeon_device pointer
4327 *
4328 * Enable the compute queues.
4329 * Returns 0 for success, error for failure.
4330 */
4331static int cik_cp_compute_start(struct radeon_device *rdev)
4332{
4333        cik_cp_compute_enable(rdev, true);
4334
4335        return 0;
4336}
4337
4338/**
4339 * cik_cp_compute_fini - stop the compute queues
4340 *
4341 * @rdev: radeon_device pointer
4342 *
4343 * Stop the compute queues and tear down the driver queue
4344 * info.
4345 */
4346static void cik_cp_compute_fini(struct radeon_device *rdev)
4347{
4348        int i, idx, r;
4349
4350        cik_cp_compute_enable(rdev, false);
4351
4352        for (i = 0; i < 2; i++) {
4353                if (i == 0)
4354                        idx = CAYMAN_RING_TYPE_CP1_INDEX;
4355                else
4356                        idx = CAYMAN_RING_TYPE_CP2_INDEX;
4357
4358                if (rdev->ring[idx].mqd_obj) {
4359                        r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4360                        if (unlikely(r != 0))
4361                                dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4362
4363                        radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4364                        radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4365
4366                        radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4367                        rdev->ring[idx].mqd_obj = NULL;
4368                }
4369        }
4370}
4371
4372static void cik_mec_fini(struct radeon_device *rdev)
4373{
4374        int r;
4375
4376        if (rdev->mec.hpd_eop_obj) {
4377                r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4378                if (unlikely(r != 0))
4379                        dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4380                radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4381                radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4382
4383                radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4384                rdev->mec.hpd_eop_obj = NULL;
4385        }
4386}
4387
4388#define MEC_HPD_SIZE 2048
4389
4390static int cik_mec_init(struct radeon_device *rdev)
4391{
4392        int r;
4393        u32 *hpd;
4394
4395        /*
4396         * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4397         * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4398         */
4399        if (rdev->family == CHIP_KAVERI)
4400                rdev->mec.num_mec = 2;
4401        else
4402                rdev->mec.num_mec = 1;
4403        rdev->mec.num_pipe = 4;
4404        rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4405
4406        if (rdev->mec.hpd_eop_obj == NULL) {
4407                r = radeon_bo_create(rdev,
4408                                     rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4409                                     PAGE_SIZE, true,
4410                                     RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4411                                     &rdev->mec.hpd_eop_obj);
4412                if (r) {
4413                        dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4414                        return r;
4415                }
4416        }
4417
4418        r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4419        if (unlikely(r != 0)) {
4420                cik_mec_fini(rdev);
4421                return r;
4422        }
4423        r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4424                          &rdev->mec.hpd_eop_gpu_addr);
4425        if (r) {
4426                dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4427                cik_mec_fini(rdev);
4428                return r;
4429        }
4430        r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4431        if (r) {
4432                dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4433                cik_mec_fini(rdev);
4434                return r;
4435        }
4436
4437        /* clear memory.  Not sure if this is required or not */
4438        memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4439
4440        radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4441        radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4442
4443        return 0;
4444}
4445
4446struct hqd_registers
4447{
4448        u32 cp_mqd_base_addr;
4449        u32 cp_mqd_base_addr_hi;
4450        u32 cp_hqd_active;
4451        u32 cp_hqd_vmid;
4452        u32 cp_hqd_persistent_state;
4453        u32 cp_hqd_pipe_priority;
4454        u32 cp_hqd_queue_priority;
4455        u32 cp_hqd_quantum;
4456        u32 cp_hqd_pq_base;
4457        u32 cp_hqd_pq_base_hi;
4458        u32 cp_hqd_pq_rptr;
4459        u32 cp_hqd_pq_rptr_report_addr;
4460        u32 cp_hqd_pq_rptr_report_addr_hi;
4461        u32 cp_hqd_pq_wptr_poll_addr;
4462        u32 cp_hqd_pq_wptr_poll_addr_hi;
4463        u32 cp_hqd_pq_doorbell_control;
4464        u32 cp_hqd_pq_wptr;
4465        u32 cp_hqd_pq_control;
4466        u32 cp_hqd_ib_base_addr;
4467        u32 cp_hqd_ib_base_addr_hi;
4468        u32 cp_hqd_ib_rptr;
4469        u32 cp_hqd_ib_control;
4470        u32 cp_hqd_iq_timer;
4471        u32 cp_hqd_iq_rptr;
4472        u32 cp_hqd_dequeue_request;
4473        u32 cp_hqd_dma_offload;
4474        u32 cp_hqd_sema_cmd;
4475        u32 cp_hqd_msg_type;
4476        u32 cp_hqd_atomic0_preop_lo;
4477        u32 cp_hqd_atomic0_preop_hi;
4478        u32 cp_hqd_atomic1_preop_lo;
4479        u32 cp_hqd_atomic1_preop_hi;
4480        u32 cp_hqd_hq_scheduler0;
4481        u32 cp_hqd_hq_scheduler1;
4482        u32 cp_mqd_control;
4483};
4484
4485struct bonaire_mqd
4486{
4487        u32 header;
4488        u32 dispatch_initiator;
4489        u32 dimensions[3];
4490        u32 start_idx[3];
4491        u32 num_threads[3];
4492        u32 pipeline_stat_enable;
4493        u32 perf_counter_enable;
4494        u32 pgm[2];
4495        u32 tba[2];
4496        u32 tma[2];
4497        u32 pgm_rsrc[2];
4498        u32 vmid;
4499        u32 resource_limits;
4500        u32 static_thread_mgmt01[2];
4501        u32 tmp_ring_size;
4502        u32 static_thread_mgmt23[2];
4503        u32 restart[3];
4504        u32 thread_trace_enable;
4505        u32 reserved1;
4506        u32 user_data[16];
4507        u32 vgtcs_invoke_count[2];
4508        struct hqd_registers queue_state;
4509        u32 dequeue_cntr;
4510        u32 interrupt_queue[64];
4511};
4512
4513/**
4514 * cik_cp_compute_resume - setup the compute queue registers
4515 *
4516 * @rdev: radeon_device pointer
4517 *
4518 * Program the compute queues and test them to make sure they
4519 * are working.
4520 * Returns 0 for success, error for failure.
4521 */
4522static int cik_cp_compute_resume(struct radeon_device *rdev)
4523{
4524        int r, i, j, idx;
4525        u32 tmp;
4526        bool use_doorbell = true;
4527        u64 hqd_gpu_addr;
4528        u64 mqd_gpu_addr;
4529        u64 eop_gpu_addr;
4530        u64 wb_gpu_addr;
4531        u32 *buf;
4532        struct bonaire_mqd *mqd;
4533
4534        r = cik_cp_compute_start(rdev);
4535        if (r)
4536                return r;
4537
4538        /* fix up chicken bits */
4539        tmp = RREG32(CP_CPF_DEBUG);
4540        tmp |= (1 << 23);
4541        WREG32(CP_CPF_DEBUG, tmp);
4542
4543        /* init the pipes */
4544        mutex_lock(&rdev->srbm_mutex);
4545
4546        for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) {
4547                int me = (i < 4) ? 1 : 2;
4548                int pipe = (i < 4) ? i : (i - 4);
4549
4550                cik_srbm_select(rdev, me, pipe, 0, 0);
4551
4552                eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
4553                /* write the EOP addr */
4554                WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4555                WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4556
4557                /* set the VMID assigned */
4558                WREG32(CP_HPD_EOP_VMID, 0);
4559
4560                /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4561                tmp = RREG32(CP_HPD_EOP_CONTROL);
4562                tmp &= ~EOP_SIZE_MASK;
4563                tmp |= order_base_2(MEC_HPD_SIZE / 8);
4564                WREG32(CP_HPD_EOP_CONTROL, tmp);
4565
4566        }
4567        cik_srbm_select(rdev, 0, 0, 0, 0);
4568        mutex_unlock(&rdev->srbm_mutex);
4569
4570        /* init the queues.  Just two for now. */
4571        for (i = 0; i < 2; i++) {
4572                if (i == 0)
4573                        idx = CAYMAN_RING_TYPE_CP1_INDEX;
4574                else
4575                        idx = CAYMAN_RING_TYPE_CP2_INDEX;
4576
4577                if (rdev->ring[idx].mqd_obj == NULL) {
4578                        r = radeon_bo_create(rdev,
4579                                             sizeof(struct bonaire_mqd),
4580                                             PAGE_SIZE, true,
4581                                             RADEON_GEM_DOMAIN_GTT, 0, NULL,
4582                                             NULL, &rdev->ring[idx].mqd_obj);
4583                        if (r) {
4584                                dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4585                                return r;
4586                        }
4587                }
4588
4589                r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4590                if (unlikely(r != 0)) {
4591                        cik_cp_compute_fini(rdev);
4592                        return r;
4593                }
4594                r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4595                                  &mqd_gpu_addr);
4596                if (r) {
4597                        dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4598                        cik_cp_compute_fini(rdev);
4599                        return r;
4600                }
4601                r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4602                if (r) {
4603                        dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4604                        cik_cp_compute_fini(rdev);
4605                        return r;
4606                }
4607
4608                /* init the mqd struct */
4609                memset(buf, 0, sizeof(struct bonaire_mqd));
4610
4611                mqd = (struct bonaire_mqd *)buf;
4612                mqd->header = 0xC0310800;
4613                mqd->static_thread_mgmt01[0] = 0xffffffff;
4614                mqd->static_thread_mgmt01[1] = 0xffffffff;
4615                mqd->static_thread_mgmt23[0] = 0xffffffff;
4616                mqd->static_thread_mgmt23[1] = 0xffffffff;
4617
4618                mutex_lock(&rdev->srbm_mutex);
4619                cik_srbm_select(rdev, rdev->ring[idx].me,
4620                                rdev->ring[idx].pipe,
4621                                rdev->ring[idx].queue, 0);
4622
4623                /* disable wptr polling */
4624                tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4625                tmp &= ~WPTR_POLL_EN;
4626                WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4627
4628                /* enable doorbell? */
4629                mqd->queue_state.cp_hqd_pq_doorbell_control =
4630                        RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4631                if (use_doorbell)
4632                        mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4633                else
4634                        mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4635                WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4636                       mqd->queue_state.cp_hqd_pq_doorbell_control);
4637
4638                /* disable the queue if it's active */
4639                mqd->queue_state.cp_hqd_dequeue_request = 0;
4640                mqd->queue_state.cp_hqd_pq_rptr = 0;
4641                mqd->queue_state.cp_hqd_pq_wptr= 0;
4642                if (RREG32(CP_HQD_ACTIVE) & 1) {
4643                        WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4644                        for (j = 0; j < rdev->usec_timeout; j++) {
4645                                if (!(RREG32(CP_HQD_ACTIVE) & 1))
4646                                        break;
4647                                udelay(1);
4648                        }
4649                        WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4650                        WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4651                        WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4652                }
4653
4654                /* set the pointer to the MQD */
4655                mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4656                mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4657                WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4658                WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4659                /* set MQD vmid to 0 */
4660                mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4661                mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4662                WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4663
4664                /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4665                hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4666                mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4667                mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4668                WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4669                WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4670
4671                /* set up the HQD, this is similar to CP_RB0_CNTL */
4672                mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4673                mqd->queue_state.cp_hqd_pq_control &=
4674                        ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4675
4676                mqd->queue_state.cp_hqd_pq_control |=
4677                        order_base_2(rdev->ring[idx].ring_size / 8);
4678                mqd->queue_state.cp_hqd_pq_control |=
4679                        (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4680#ifdef __BIG_ENDIAN
4681                mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4682#endif
4683                mqd->queue_state.cp_hqd_pq_control &=
4684                        ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4685                mqd->queue_state.cp_hqd_pq_control |=
4686                        PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4687                WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4688
4689                /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4690                if (i == 0)
4691                        wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4692                else
4693                        wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4694                mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4695                mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4696                WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4697                WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4698                       mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4699
4700                /* set the wb address wether it's enabled or not */
4701                if (i == 0)
4702                        wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4703                else
4704                        wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4705                mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4706                mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4707                        upper_32_bits(wb_gpu_addr) & 0xffff;
4708                WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4709                       mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4710                WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4711                       mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4712
4713                /* enable the doorbell if requested */
4714                if (use_doorbell) {
4715                        mqd->queue_state.cp_hqd_pq_doorbell_control =
4716                                RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4717                        mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4718                        mqd->queue_state.cp_hqd_pq_doorbell_control |=
4719                                DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4720                        mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4721                        mqd->queue_state.cp_hqd_pq_doorbell_control &=
4722                                ~(DOORBELL_SOURCE | DOORBELL_HIT);
4723
4724                } else {
4725                        mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4726                }
4727                WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4728                       mqd->queue_state.cp_hqd_pq_doorbell_control);
4729
4730                /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4731                rdev->ring[idx].wptr = 0;
4732                mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4733                WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4734                mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4735
4736                /* set the vmid for the queue */
4737                mqd->queue_state.cp_hqd_vmid = 0;
4738                WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4739
4740                /* activate the queue */
4741                mqd->queue_state.cp_hqd_active = 1;
4742                WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4743
4744                cik_srbm_select(rdev, 0, 0, 0, 0);
4745                mutex_unlock(&rdev->srbm_mutex);
4746
4747                radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4748                radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4749
4750                rdev->ring[idx].ready = true;
4751                r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4752                if (r)
4753                        rdev->ring[idx].ready = false;
4754        }
4755
4756        return 0;
4757}
4758
4759static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4760{
4761        cik_cp_gfx_enable(rdev, enable);
4762        cik_cp_compute_enable(rdev, enable);
4763}
4764
4765static int cik_cp_load_microcode(struct radeon_device *rdev)
4766{
4767        int r;
4768
4769        r = cik_cp_gfx_load_microcode(rdev);
4770        if (r)
4771                return r;
4772        r = cik_cp_compute_load_microcode(rdev);
4773        if (r)
4774                return r;
4775
4776        return 0;
4777}
4778
4779static void cik_cp_fini(struct radeon_device *rdev)
4780{
4781        cik_cp_gfx_fini(rdev);
4782        cik_cp_compute_fini(rdev);
4783}
4784
4785static int cik_cp_resume(struct radeon_device *rdev)
4786{
4787        int r;
4788
4789        cik_enable_gui_idle_interrupt(rdev, false);
4790
4791        r = cik_cp_load_microcode(rdev);
4792        if (r)
4793                return r;
4794
4795        r = cik_cp_gfx_resume(rdev);
4796        if (r)
4797                return r;
4798        r = cik_cp_compute_resume(rdev);
4799        if (r)
4800                return r;
4801
4802        cik_enable_gui_idle_interrupt(rdev, true);
4803
4804        return 0;
4805}
4806
4807static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4808{
4809        dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4810                RREG32(GRBM_STATUS));
4811        dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4812                RREG32(GRBM_STATUS2));
4813        dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4814                RREG32(GRBM_STATUS_SE0));
4815        dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4816                RREG32(GRBM_STATUS_SE1));
4817        dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4818                RREG32(GRBM_STATUS_SE2));
4819        dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4820                RREG32(GRBM_STATUS_SE3));
4821        dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4822                RREG32(SRBM_STATUS));
4823        dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4824                RREG32(SRBM_STATUS2));
4825        dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4826                RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4827        dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4828                 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4829        dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4830        dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4831                 RREG32(CP_STALLED_STAT1));
4832        dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4833                 RREG32(CP_STALLED_STAT2));
4834        dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4835                 RREG32(CP_STALLED_STAT3));
4836        dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4837                 RREG32(CP_CPF_BUSY_STAT));
4838        dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4839                 RREG32(CP_CPF_STALLED_STAT1));
4840        dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4841        dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4842        dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4843                 RREG32(CP_CPC_STALLED_STAT1));
4844        dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4845}
4846
4847/**
4848 * cik_gpu_check_soft_reset - check which blocks are busy
4849 *
4850 * @rdev: radeon_device pointer
4851 *
4852 * Check which blocks are busy and return the relevant reset
4853 * mask to be used by cik_gpu_soft_reset().
4854 * Returns a mask of the blocks to be reset.
4855 */
4856u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4857{
4858        u32 reset_mask = 0;
4859        u32 tmp;
4860
4861        /* GRBM_STATUS */
4862        tmp = RREG32(GRBM_STATUS);
4863        if (tmp & (PA_BUSY | SC_BUSY |
4864                   BCI_BUSY | SX_BUSY |
4865                   TA_BUSY | VGT_BUSY |
4866                   DB_BUSY | CB_BUSY |
4867                   GDS_BUSY | SPI_BUSY |
4868                   IA_BUSY | IA_BUSY_NO_DMA))
4869                reset_mask |= RADEON_RESET_GFX;
4870
4871        if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4872                reset_mask |= RADEON_RESET_CP;
4873
4874        /* GRBM_STATUS2 */
4875        tmp = RREG32(GRBM_STATUS2);
4876        if (tmp & RLC_BUSY)
4877                reset_mask |= RADEON_RESET_RLC;
4878
4879        /* SDMA0_STATUS_REG */
4880        tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4881        if (!(tmp & SDMA_IDLE))
4882                reset_mask |= RADEON_RESET_DMA;
4883
4884        /* SDMA1_STATUS_REG */
4885        tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4886        if (!(tmp & SDMA_IDLE))
4887                reset_mask |= RADEON_RESET_DMA1;
4888
4889        /* SRBM_STATUS2 */
4890        tmp = RREG32(SRBM_STATUS2);
4891        if (tmp & SDMA_BUSY)
4892                reset_mask |= RADEON_RESET_DMA;
4893
4894        if (tmp & SDMA1_BUSY)
4895                reset_mask |= RADEON_RESET_DMA1;
4896
4897        /* SRBM_STATUS */
4898        tmp = RREG32(SRBM_STATUS);
4899
4900        if (tmp & IH_BUSY)
4901                reset_mask |= RADEON_RESET_IH;
4902
4903        if (tmp & SEM_BUSY)
4904                reset_mask |= RADEON_RESET_SEM;
4905
4906        if (tmp & GRBM_RQ_PENDING)
4907                reset_mask |= RADEON_RESET_GRBM;
4908
4909        if (tmp & VMC_BUSY)
4910                reset_mask |= RADEON_RESET_VMC;
4911
4912        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4913                   MCC_BUSY | MCD_BUSY))
4914                reset_mask |= RADEON_RESET_MC;
4915
4916        if (evergreen_is_display_hung(rdev))
4917                reset_mask |= RADEON_RESET_DISPLAY;
4918
4919        /* Skip MC reset as it's mostly likely not hung, just busy */
4920        if (reset_mask & RADEON_RESET_MC) {
4921                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4922                reset_mask &= ~RADEON_RESET_MC;
4923        }
4924
4925        return reset_mask;
4926}
4927
4928/**
4929 * cik_gpu_soft_reset - soft reset GPU
4930 *
4931 * @rdev: radeon_device pointer
4932 * @reset_mask: mask of which blocks to reset
4933 *
4934 * Soft reset the blocks specified in @reset_mask.
4935 */
4936static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4937{
4938        struct evergreen_mc_save save;
4939        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4940        u32 tmp;
4941
4942        if (reset_mask == 0)
4943                return;
4944
4945        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4946
4947        cik_print_gpu_status_regs(rdev);
4948        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4949                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4950        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4951                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4952
4953        /* disable CG/PG */
4954        cik_fini_pg(rdev);
4955        cik_fini_cg(rdev);
4956
4957        /* stop the rlc */
4958        cik_rlc_stop(rdev);
4959
4960        /* Disable GFX parsing/prefetching */
4961        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4962
4963        /* Disable MEC parsing/prefetching */
4964        WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4965
4966        if (reset_mask & RADEON_RESET_DMA) {
4967                /* sdma0 */
4968                tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4969                tmp |= SDMA_HALT;
4970                WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4971        }
4972        if (reset_mask & RADEON_RESET_DMA1) {
4973                /* sdma1 */
4974                tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4975                tmp |= SDMA_HALT;
4976                WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4977        }
4978
4979        evergreen_mc_stop(rdev, &save);
4980        if (evergreen_mc_wait_for_idle(rdev)) {
4981                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4982        }
4983
4984        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4985                grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4986
4987        if (reset_mask & RADEON_RESET_CP) {
4988                grbm_soft_reset |= SOFT_RESET_CP;
4989
4990                srbm_soft_reset |= SOFT_RESET_GRBM;
4991        }
4992
4993        if (reset_mask & RADEON_RESET_DMA)
4994                srbm_soft_reset |= SOFT_RESET_SDMA;
4995
4996        if (reset_mask & RADEON_RESET_DMA1)
4997                srbm_soft_reset |= SOFT_RESET_SDMA1;
4998
4999        if (reset_mask & RADEON_RESET_DISPLAY)
5000                srbm_soft_reset |= SOFT_RESET_DC;
5001
5002        if (reset_mask & RADEON_RESET_RLC)
5003                grbm_soft_reset |= SOFT_RESET_RLC;
5004
5005        if (reset_mask & RADEON_RESET_SEM)
5006                srbm_soft_reset |= SOFT_RESET_SEM;
5007
5008        if (reset_mask & RADEON_RESET_IH)
5009                srbm_soft_reset |= SOFT_RESET_IH;
5010
5011        if (reset_mask & RADEON_RESET_GRBM)
5012                srbm_soft_reset |= SOFT_RESET_GRBM;
5013
5014        if (reset_mask & RADEON_RESET_VMC)
5015                srbm_soft_reset |= SOFT_RESET_VMC;
5016
5017        if (!(rdev->flags & RADEON_IS_IGP)) {
5018                if (reset_mask & RADEON_RESET_MC)
5019                        srbm_soft_reset |= SOFT_RESET_MC;
5020        }
5021
5022        if (grbm_soft_reset) {
5023                tmp = RREG32(GRBM_SOFT_RESET);
5024                tmp |= grbm_soft_reset;
5025                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5026                WREG32(GRBM_SOFT_RESET, tmp);
5027                tmp = RREG32(GRBM_SOFT_RESET);
5028
5029                udelay(50);
5030
5031                tmp &= ~grbm_soft_reset;
5032                WREG32(GRBM_SOFT_RESET, tmp);
5033                tmp = RREG32(GRBM_SOFT_RESET);
5034        }
5035
5036        if (srbm_soft_reset) {
5037                tmp = RREG32(SRBM_SOFT_RESET);
5038                tmp |= srbm_soft_reset;
5039                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5040                WREG32(SRBM_SOFT_RESET, tmp);
5041                tmp = RREG32(SRBM_SOFT_RESET);
5042
5043                udelay(50);
5044
5045                tmp &= ~srbm_soft_reset;
5046                WREG32(SRBM_SOFT_RESET, tmp);
5047                tmp = RREG32(SRBM_SOFT_RESET);
5048        }
5049
5050        /* Wait a little for things to settle down */
5051        udelay(50);
5052
5053        evergreen_mc_resume(rdev, &save);
5054        udelay(50);
5055
5056        cik_print_gpu_status_regs(rdev);
5057}
5058
5059struct kv_reset_save_regs {
5060        u32 gmcon_reng_execute;
5061        u32 gmcon_misc;
5062        u32 gmcon_misc3;
5063};
5064
5065static void kv_save_regs_for_reset(struct radeon_device *rdev,
5066                                   struct kv_reset_save_regs *save)
5067{
5068        save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5069        save->gmcon_misc = RREG32(GMCON_MISC);
5070        save->gmcon_misc3 = RREG32(GMCON_MISC3);
5071
5072        WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5073        WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5074                                                STCTRL_STUTTER_EN));
5075}
5076
5077static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5078                                      struct kv_reset_save_regs *save)
5079{
5080        int i;
5081
5082        WREG32(GMCON_PGFSM_WRITE, 0);
5083        WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5084
5085        for (i = 0; i < 5; i++)
5086                WREG32(GMCON_PGFSM_WRITE, 0);
5087
5088        WREG32(GMCON_PGFSM_WRITE, 0);
5089        WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5090
5091        for (i = 0; i < 5; i++)
5092                WREG32(GMCON_PGFSM_WRITE, 0);
5093
5094        WREG32(GMCON_PGFSM_WRITE, 0x210000);
5095        WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5096
5097        for (i = 0; i < 5; i++)
5098                WREG32(GMCON_PGFSM_WRITE, 0);
5099
5100        WREG32(GMCON_PGFSM_WRITE, 0x21003);
5101        WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5102
5103        for (i = 0; i < 5; i++)
5104                WREG32(GMCON_PGFSM_WRITE, 0);
5105
5106        WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5107        WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5108
5109        for (i = 0; i < 5; i++)
5110                WREG32(GMCON_PGFSM_WRITE, 0);
5111
5112        WREG32(GMCON_PGFSM_WRITE, 0);
5113        WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5114
5115        for (i = 0; i < 5; i++)
5116                WREG32(GMCON_PGFSM_WRITE, 0);
5117
5118        WREG32(GMCON_PGFSM_WRITE, 0x420000);
5119        WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5120
5121        for (i = 0; i < 5; i++)
5122                WREG32(GMCON_PGFSM_WRITE, 0);
5123
5124        WREG32(GMCON_PGFSM_WRITE, 0x120202);
5125        WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5126
5127        for (i = 0; i < 5; i++)
5128                WREG32(GMCON_PGFSM_WRITE, 0);
5129
5130        WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5131        WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5132
5133        for (i = 0; i < 5; i++)
5134                WREG32(GMCON_PGFSM_WRITE, 0);
5135
5136        WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5137        WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5138
5139        for (i = 0; i < 5; i++)
5140                WREG32(GMCON_PGFSM_WRITE, 0);
5141
5142        WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5143        WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5144
5145        WREG32(GMCON_MISC3, save->gmcon_misc3);
5146        WREG32(GMCON_MISC, save->gmcon_misc);
5147        WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5148}
5149
5150static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5151{
5152        struct evergreen_mc_save save;
5153        struct kv_reset_save_regs kv_save = { 0 };
5154        u32 tmp, i;
5155
5156        dev_info(rdev->dev, "GPU pci config reset\n");
5157
5158        /* disable dpm? */
5159
5160        /* disable cg/pg */
5161        cik_fini_pg(rdev);
5162        cik_fini_cg(rdev);
5163
5164        /* Disable GFX parsing/prefetching */
5165        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5166
5167        /* Disable MEC parsing/prefetching */
5168        WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5169
5170        /* sdma0 */
5171        tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5172        tmp |= SDMA_HALT;
5173        WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5174        /* sdma1 */
5175        tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5176        tmp |= SDMA_HALT;
5177        WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5178        /* XXX other engines? */
5179
5180        /* halt the rlc, disable cp internal ints */
5181        cik_rlc_stop(rdev);
5182
5183        udelay(50);
5184
5185        /* disable mem access */
5186        evergreen_mc_stop(rdev, &save);
5187        if (evergreen_mc_wait_for_idle(rdev)) {
5188                dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5189        }
5190
5191        if (rdev->flags & RADEON_IS_IGP)
5192                kv_save_regs_for_reset(rdev, &kv_save);
5193
5194        /* disable BM */
5195        pci_clear_master(rdev->pdev);
5196        /* reset */
5197        radeon_pci_config_reset(rdev);
5198
5199        udelay(100);
5200
5201        /* wait for asic to come out of reset */
5202        for (i = 0; i < rdev->usec_timeout; i++) {
5203                if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5204                        break;
5205                udelay(1);
5206        }
5207
5208        /* does asic init need to be run first??? */
5209        if (rdev->flags & RADEON_IS_IGP)
5210                kv_restore_regs_for_reset(rdev, &kv_save);
5211}
5212
5213/**
5214 * cik_asic_reset - soft reset GPU
5215 *
5216 * @rdev: radeon_device pointer
5217 * @hard: force hard reset
5218 *
5219 * Look up which blocks are hung and attempt
5220 * to reset them.
5221 * Returns 0 for success.
5222 */
5223int cik_asic_reset(struct radeon_device *rdev, bool hard)
5224{
5225        u32 reset_mask;
5226
5227        if (hard) {
5228                cik_gpu_pci_config_reset(rdev);
5229                return 0;
5230        }
5231
5232        reset_mask = cik_gpu_check_soft_reset(rdev);
5233
5234        if (reset_mask)
5235                r600_set_bios_scratch_engine_hung(rdev, true);
5236
5237        /* try soft reset */
5238        cik_gpu_soft_reset(rdev, reset_mask);
5239
5240        reset_mask = cik_gpu_check_soft_reset(rdev);
5241
5242        /* try pci config reset */
5243        if (reset_mask && radeon_hard_reset)
5244                cik_gpu_pci_config_reset(rdev);
5245
5246        reset_mask = cik_gpu_check_soft_reset(rdev);
5247
5248        if (!reset_mask)
5249                r600_set_bios_scratch_engine_hung(rdev, false);
5250
5251        return 0;
5252}
5253
5254/**
5255 * cik_gfx_is_lockup - check if the 3D engine is locked up
5256 *
5257 * @rdev: radeon_device pointer
5258 * @ring: radeon_ring structure holding ring information
5259 *
5260 * Check if the 3D engine is locked up (CIK).
5261 * Returns true if the engine is locked, false if not.
5262 */
5263bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5264{
5265        u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5266
5267        if (!(reset_mask & (RADEON_RESET_GFX |
5268                            RADEON_RESET_COMPUTE |
5269                            RADEON_RESET_CP))) {
5270                radeon_ring_lockup_update(rdev, ring);
5271                return false;
5272        }
5273        return radeon_ring_test_lockup(rdev, ring);
5274}
5275
5276/* MC */
5277/**
5278 * cik_mc_program - program the GPU memory controller
5279 *
5280 * @rdev: radeon_device pointer
5281 *
5282 * Set the location of vram, gart, and AGP in the GPU's
5283 * physical address space (CIK).
5284 */
5285static void cik_mc_program(struct radeon_device *rdev)
5286{
5287        struct evergreen_mc_save save;
5288        u32 tmp;
5289        int i, j;
5290
5291        /* Initialize HDP */
5292        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5293                WREG32((0x2c14 + j), 0x00000000);
5294                WREG32((0x2c18 + j), 0x00000000);
5295                WREG32((0x2c1c + j), 0x00000000);
5296                WREG32((0x2c20 + j), 0x00000000);
5297                WREG32((0x2c24 + j), 0x00000000);
5298        }
5299        WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5300
5301        evergreen_mc_stop(rdev, &save);
5302        if (radeon_mc_wait_for_idle(rdev)) {
5303                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5304        }
5305        /* Lockout access through VGA aperture*/
5306        WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5307        /* Update configuration */
5308        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5309               rdev->mc.vram_start >> 12);
5310        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5311               rdev->mc.vram_end >> 12);
5312        WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5313               rdev->vram_scratch.gpu_addr >> 12);
5314        tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5315        tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5316        WREG32(MC_VM_FB_LOCATION, tmp);
5317        /* XXX double check these! */
5318        WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5319        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5320        WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5321        WREG32(MC_VM_AGP_BASE, 0);
5322        WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5323        WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5324        if (radeon_mc_wait_for_idle(rdev)) {
5325                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5326        }
5327        evergreen_mc_resume(rdev, &save);
5328        /* we need to own VRAM, so turn off the VGA renderer here
5329         * to stop it overwriting our objects */
5330        rv515_vga_render_disable(rdev);
5331}
5332
5333/**
5334 * cik_mc_init - initialize the memory controller driver params
5335 *
5336 * @rdev: radeon_device pointer
5337 *
5338 * Look up the amount of vram, vram width, and decide how to place
5339 * vram and gart within the GPU's physical address space (CIK).
5340 * Returns 0 for success.
5341 */
5342static int cik_mc_init(struct radeon_device *rdev)
5343{
5344        u32 tmp;
5345        int chansize, numchan;
5346
5347        /* Get VRAM informations */
5348        rdev->mc.vram_is_ddr = true;
5349        tmp = RREG32(MC_ARB_RAMCFG);
5350        if (tmp & CHANSIZE_MASK) {
5351                chansize = 64;
5352        } else {
5353                chansize = 32;
5354        }
5355        tmp = RREG32(MC_SHARED_CHMAP);
5356        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5357        case 0:
5358        default:
5359                numchan = 1;
5360                break;
5361        case 1:
5362                numchan = 2;
5363                break;
5364        case 2:
5365                numchan = 4;
5366                break;
5367        case 3:
5368                numchan = 8;
5369                break;
5370        case 4:
5371                numchan = 3;
5372                break;
5373        case 5:
5374                numchan = 6;
5375                break;
5376        case 6:
5377                numchan = 10;
5378                break;
5379        case 7:
5380                numchan = 12;
5381                break;
5382        case 8:
5383                numchan = 16;
5384                break;
5385        }
5386        rdev->mc.vram_width = numchan * chansize;
5387        /* Could aper size report 0 ? */
5388        rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5389        rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5390        /* size in MB on si */
5391        rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5392        rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5393        rdev->mc.visible_vram_size = rdev->mc.aper_size;
5394        si_vram_gtt_location(rdev, &rdev->mc);
5395        radeon_update_bandwidth_info(rdev);
5396
5397        return 0;
5398}
5399
5400/*
5401 * GART
5402 * VMID 0 is the physical GPU addresses as used by the kernel.
5403 * VMIDs 1-15 are used for userspace clients and are handled
5404 * by the radeon vm/hsa code.
5405 */
5406/**
5407 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5408 *
5409 * @rdev: radeon_device pointer
5410 *
5411 * Flush the TLB for the VMID 0 page table (CIK).
5412 */
5413void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5414{
5415        /* flush hdp cache */
5416        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5417
5418        /* bits 0-15 are the VM contexts0-15 */
5419        WREG32(VM_INVALIDATE_REQUEST, 0x1);
5420}
5421
5422