linux/drivers/gpu/drm/radeon/evergreen.c
<<
>>
Prefs
   1/*
   2 * Copyright 2010 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/pci.h>
  27#include <linux/slab.h>
  28
  29#include <drm/drm_vblank.h>
  30#include <drm/radeon_drm.h>
  31#include <drm/drm_fourcc.h>
  32
  33#include "atom.h"
  34#include "avivod.h"
  35#include "cik.h"
  36#include "ni.h"
  37#include "rv770.h"
  38#include "evergreen.h"
  39#include "evergreen_blit_shaders.h"
  40#include "evergreen_reg.h"
  41#include "evergreend.h"
  42#include "radeon.h"
  43#include "radeon_asic.h"
  44#include "radeon_audio.h"
  45#include "radeon_ucode.h"
  46#include "si.h"
  47
  48#define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  49#define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  50#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  51
  52/*
  53 * Indirect registers accessor
  54 */
  55u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  56{
  57        unsigned long flags;
  58        u32 r;
  59
  60        spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  61        WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  62        r = RREG32(EVERGREEN_CG_IND_DATA);
  63        spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  64        return r;
  65}
  66
  67void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  68{
  69        unsigned long flags;
  70
  71        spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  72        WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  73        WREG32(EVERGREEN_CG_IND_DATA, (v));
  74        spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  75}
  76
  77u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  78{
  79        unsigned long flags;
  80        u32 r;
  81
  82        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  83        WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  84        r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  85        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  86        return r;
  87}
  88
  89void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  90{
  91        unsigned long flags;
  92
  93        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  94        WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  95        WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  96        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  97}
  98
  99u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
 100{
 101        unsigned long flags;
 102        u32 r;
 103
 104        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 105        WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 106        r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 107        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 108        return r;
 109}
 110
 111void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 112{
 113        unsigned long flags;
 114
 115        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 116        WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 117        WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 118        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 119}
 120
 121static const u32 crtc_offsets[6] =
 122{
 123        EVERGREEN_CRTC0_REGISTER_OFFSET,
 124        EVERGREEN_CRTC1_REGISTER_OFFSET,
 125        EVERGREEN_CRTC2_REGISTER_OFFSET,
 126        EVERGREEN_CRTC3_REGISTER_OFFSET,
 127        EVERGREEN_CRTC4_REGISTER_OFFSET,
 128        EVERGREEN_CRTC5_REGISTER_OFFSET
 129};
 130
 131#include "clearstate_evergreen.h"
 132
 133static const u32 sumo_rlc_save_restore_register_list[] =
 134{
 135        0x98fc,
 136        0x9830,
 137        0x9834,
 138        0x9838,
 139        0x9870,
 140        0x9874,
 141        0x8a14,
 142        0x8b24,
 143        0x8bcc,
 144        0x8b10,
 145        0x8d00,
 146        0x8d04,
 147        0x8c00,
 148        0x8c04,
 149        0x8c08,
 150        0x8c0c,
 151        0x8d8c,
 152        0x8c20,
 153        0x8c24,
 154        0x8c28,
 155        0x8c18,
 156        0x8c1c,
 157        0x8cf0,
 158        0x8e2c,
 159        0x8e38,
 160        0x8c30,
 161        0x9508,
 162        0x9688,
 163        0x9608,
 164        0x960c,
 165        0x9610,
 166        0x9614,
 167        0x88c4,
 168        0x88d4,
 169        0xa008,
 170        0x900c,
 171        0x9100,
 172        0x913c,
 173        0x98f8,
 174        0x98f4,
 175        0x9b7c,
 176        0x3f8c,
 177        0x8950,
 178        0x8954,
 179        0x8a18,
 180        0x8b28,
 181        0x9144,
 182        0x9148,
 183        0x914c,
 184        0x3f90,
 185        0x3f94,
 186        0x915c,
 187        0x9160,
 188        0x9178,
 189        0x917c,
 190        0x9180,
 191        0x918c,
 192        0x9190,
 193        0x9194,
 194        0x9198,
 195        0x919c,
 196        0x91a8,
 197        0x91ac,
 198        0x91b0,
 199        0x91b4,
 200        0x91b8,
 201        0x91c4,
 202        0x91c8,
 203        0x91cc,
 204        0x91d0,
 205        0x91d4,
 206        0x91e0,
 207        0x91e4,
 208        0x91ec,
 209        0x91f0,
 210        0x91f4,
 211        0x9200,
 212        0x9204,
 213        0x929c,
 214        0x9150,
 215        0x802c,
 216};
 217
 218static void evergreen_gpu_init(struct radeon_device *rdev);
 219void evergreen_fini(struct radeon_device *rdev);
 220void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 221void evergreen_program_aspm(struct radeon_device *rdev);
 222
 223static const u32 evergreen_golden_registers[] =
 224{
 225        0x3f90, 0xffff0000, 0xff000000,
 226        0x9148, 0xffff0000, 0xff000000,
 227        0x3f94, 0xffff0000, 0xff000000,
 228        0x914c, 0xffff0000, 0xff000000,
 229        0x9b7c, 0xffffffff, 0x00000000,
 230        0x8a14, 0xffffffff, 0x00000007,
 231        0x8b10, 0xffffffff, 0x00000000,
 232        0x960c, 0xffffffff, 0x54763210,
 233        0x88c4, 0xffffffff, 0x000000c2,
 234        0x88d4, 0xffffffff, 0x00000010,
 235        0x8974, 0xffffffff, 0x00000000,
 236        0xc78, 0x00000080, 0x00000080,
 237        0x5eb4, 0xffffffff, 0x00000002,
 238        0x5e78, 0xffffffff, 0x001000f0,
 239        0x6104, 0x01000300, 0x00000000,
 240        0x5bc0, 0x00300000, 0x00000000,
 241        0x7030, 0xffffffff, 0x00000011,
 242        0x7c30, 0xffffffff, 0x00000011,
 243        0x10830, 0xffffffff, 0x00000011,
 244        0x11430, 0xffffffff, 0x00000011,
 245        0x12030, 0xffffffff, 0x00000011,
 246        0x12c30, 0xffffffff, 0x00000011,
 247        0xd02c, 0xffffffff, 0x08421000,
 248        0x240c, 0xffffffff, 0x00000380,
 249        0x8b24, 0xffffffff, 0x00ff0fff,
 250        0x28a4c, 0x06000000, 0x06000000,
 251        0x10c, 0x00000001, 0x00000001,
 252        0x8d00, 0xffffffff, 0x100e4848,
 253        0x8d04, 0xffffffff, 0x00164745,
 254        0x8c00, 0xffffffff, 0xe4000003,
 255        0x8c04, 0xffffffff, 0x40600060,
 256        0x8c08, 0xffffffff, 0x001c001c,
 257        0x8cf0, 0xffffffff, 0x08e00620,
 258        0x8c20, 0xffffffff, 0x00800080,
 259        0x8c24, 0xffffffff, 0x00800080,
 260        0x8c18, 0xffffffff, 0x20202078,
 261        0x8c1c, 0xffffffff, 0x00001010,
 262        0x28350, 0xffffffff, 0x00000000,
 263        0xa008, 0xffffffff, 0x00010000,
 264        0x5c4, 0xffffffff, 0x00000001,
 265        0x9508, 0xffffffff, 0x00000002,
 266        0x913c, 0x0000000f, 0x0000000a
 267};
 268
 269static const u32 evergreen_golden_registers2[] =
 270{
 271        0x2f4c, 0xffffffff, 0x00000000,
 272        0x54f4, 0xffffffff, 0x00000000,
 273        0x54f0, 0xffffffff, 0x00000000,
 274        0x5498, 0xffffffff, 0x00000000,
 275        0x549c, 0xffffffff, 0x00000000,
 276        0x5494, 0xffffffff, 0x00000000,
 277        0x53cc, 0xffffffff, 0x00000000,
 278        0x53c8, 0xffffffff, 0x00000000,
 279        0x53c4, 0xffffffff, 0x00000000,
 280        0x53c0, 0xffffffff, 0x00000000,
 281        0x53bc, 0xffffffff, 0x00000000,
 282        0x53b8, 0xffffffff, 0x00000000,
 283        0x53b4, 0xffffffff, 0x00000000,
 284        0x53b0, 0xffffffff, 0x00000000
 285};
 286
 287static const u32 cypress_mgcg_init[] =
 288{
 289        0x802c, 0xffffffff, 0xc0000000,
 290        0x5448, 0xffffffff, 0x00000100,
 291        0x55e4, 0xffffffff, 0x00000100,
 292        0x160c, 0xffffffff, 0x00000100,
 293        0x5644, 0xffffffff, 0x00000100,
 294        0xc164, 0xffffffff, 0x00000100,
 295        0x8a18, 0xffffffff, 0x00000100,
 296        0x897c, 0xffffffff, 0x06000100,
 297        0x8b28, 0xffffffff, 0x00000100,
 298        0x9144, 0xffffffff, 0x00000100,
 299        0x9a60, 0xffffffff, 0x00000100,
 300        0x9868, 0xffffffff, 0x00000100,
 301        0x8d58, 0xffffffff, 0x00000100,
 302        0x9510, 0xffffffff, 0x00000100,
 303        0x949c, 0xffffffff, 0x00000100,
 304        0x9654, 0xffffffff, 0x00000100,
 305        0x9030, 0xffffffff, 0x00000100,
 306        0x9034, 0xffffffff, 0x00000100,
 307        0x9038, 0xffffffff, 0x00000100,
 308        0x903c, 0xffffffff, 0x00000100,
 309        0x9040, 0xffffffff, 0x00000100,
 310        0xa200, 0xffffffff, 0x00000100,
 311        0xa204, 0xffffffff, 0x00000100,
 312        0xa208, 0xffffffff, 0x00000100,
 313        0xa20c, 0xffffffff, 0x00000100,
 314        0x971c, 0xffffffff, 0x00000100,
 315        0x977c, 0xffffffff, 0x00000100,
 316        0x3f80, 0xffffffff, 0x00000100,
 317        0xa210, 0xffffffff, 0x00000100,
 318        0xa214, 0xffffffff, 0x00000100,
 319        0x4d8, 0xffffffff, 0x00000100,
 320        0x9784, 0xffffffff, 0x00000100,
 321        0x9698, 0xffffffff, 0x00000100,
 322        0x4d4, 0xffffffff, 0x00000200,
 323        0x30cc, 0xffffffff, 0x00000100,
 324        0xd0c0, 0xffffffff, 0xff000100,
 325        0x802c, 0xffffffff, 0x40000000,
 326        0x915c, 0xffffffff, 0x00010000,
 327        0x9160, 0xffffffff, 0x00030002,
 328        0x9178, 0xffffffff, 0x00070000,
 329        0x917c, 0xffffffff, 0x00030002,
 330        0x9180, 0xffffffff, 0x00050004,
 331        0x918c, 0xffffffff, 0x00010006,
 332        0x9190, 0xffffffff, 0x00090008,
 333        0x9194, 0xffffffff, 0x00070000,
 334        0x9198, 0xffffffff, 0x00030002,
 335        0x919c, 0xffffffff, 0x00050004,
 336        0x91a8, 0xffffffff, 0x00010006,
 337        0x91ac, 0xffffffff, 0x00090008,
 338        0x91b0, 0xffffffff, 0x00070000,
 339        0x91b4, 0xffffffff, 0x00030002,
 340        0x91b8, 0xffffffff, 0x00050004,
 341        0x91c4, 0xffffffff, 0x00010006,
 342        0x91c8, 0xffffffff, 0x00090008,
 343        0x91cc, 0xffffffff, 0x00070000,
 344        0x91d0, 0xffffffff, 0x00030002,
 345        0x91d4, 0xffffffff, 0x00050004,
 346        0x91e0, 0xffffffff, 0x00010006,
 347        0x91e4, 0xffffffff, 0x00090008,
 348        0x91e8, 0xffffffff, 0x00000000,
 349        0x91ec, 0xffffffff, 0x00070000,
 350        0x91f0, 0xffffffff, 0x00030002,
 351        0x91f4, 0xffffffff, 0x00050004,
 352        0x9200, 0xffffffff, 0x00010006,
 353        0x9204, 0xffffffff, 0x00090008,
 354        0x9208, 0xffffffff, 0x00070000,
 355        0x920c, 0xffffffff, 0x00030002,
 356        0x9210, 0xffffffff, 0x00050004,
 357        0x921c, 0xffffffff, 0x00010006,
 358        0x9220, 0xffffffff, 0x00090008,
 359        0x9224, 0xffffffff, 0x00070000,
 360        0x9228, 0xffffffff, 0x00030002,
 361        0x922c, 0xffffffff, 0x00050004,
 362        0x9238, 0xffffffff, 0x00010006,
 363        0x923c, 0xffffffff, 0x00090008,
 364        0x9240, 0xffffffff, 0x00070000,
 365        0x9244, 0xffffffff, 0x00030002,
 366        0x9248, 0xffffffff, 0x00050004,
 367        0x9254, 0xffffffff, 0x00010006,
 368        0x9258, 0xffffffff, 0x00090008,
 369        0x925c, 0xffffffff, 0x00070000,
 370        0x9260, 0xffffffff, 0x00030002,
 371        0x9264, 0xffffffff, 0x00050004,
 372        0x9270, 0xffffffff, 0x00010006,
 373        0x9274, 0xffffffff, 0x00090008,
 374        0x9278, 0xffffffff, 0x00070000,
 375        0x927c, 0xffffffff, 0x00030002,
 376        0x9280, 0xffffffff, 0x00050004,
 377        0x928c, 0xffffffff, 0x00010006,
 378        0x9290, 0xffffffff, 0x00090008,
 379        0x9294, 0xffffffff, 0x00000000,
 380        0x929c, 0xffffffff, 0x00000001,
 381        0x802c, 0xffffffff, 0x40010000,
 382        0x915c, 0xffffffff, 0x00010000,
 383        0x9160, 0xffffffff, 0x00030002,
 384        0x9178, 0xffffffff, 0x00070000,
 385        0x917c, 0xffffffff, 0x00030002,
 386        0x9180, 0xffffffff, 0x00050004,
 387        0x918c, 0xffffffff, 0x00010006,
 388        0x9190, 0xffffffff, 0x00090008,
 389        0x9194, 0xffffffff, 0x00070000,
 390        0x9198, 0xffffffff, 0x00030002,
 391        0x919c, 0xffffffff, 0x00050004,
 392        0x91a8, 0xffffffff, 0x00010006,
 393        0x91ac, 0xffffffff, 0x00090008,
 394        0x91b0, 0xffffffff, 0x00070000,
 395        0x91b4, 0xffffffff, 0x00030002,
 396        0x91b8, 0xffffffff, 0x00050004,
 397        0x91c4, 0xffffffff, 0x00010006,
 398        0x91c8, 0xffffffff, 0x00090008,
 399        0x91cc, 0xffffffff, 0x00070000,
 400        0x91d0, 0xffffffff, 0x00030002,
 401        0x91d4, 0xffffffff, 0x00050004,
 402        0x91e0, 0xffffffff, 0x00010006,
 403        0x91e4, 0xffffffff, 0x00090008,
 404        0x91e8, 0xffffffff, 0x00000000,
 405        0x91ec, 0xffffffff, 0x00070000,
 406        0x91f0, 0xffffffff, 0x00030002,
 407        0x91f4, 0xffffffff, 0x00050004,
 408        0x9200, 0xffffffff, 0x00010006,
 409        0x9204, 0xffffffff, 0x00090008,
 410        0x9208, 0xffffffff, 0x00070000,
 411        0x920c, 0xffffffff, 0x00030002,
 412        0x9210, 0xffffffff, 0x00050004,
 413        0x921c, 0xffffffff, 0x00010006,
 414        0x9220, 0xffffffff, 0x00090008,
 415        0x9224, 0xffffffff, 0x00070000,
 416        0x9228, 0xffffffff, 0x00030002,
 417        0x922c, 0xffffffff, 0x00050004,
 418        0x9238, 0xffffffff, 0x00010006,
 419        0x923c, 0xffffffff, 0x00090008,
 420        0x9240, 0xffffffff, 0x00070000,
 421        0x9244, 0xffffffff, 0x00030002,
 422        0x9248, 0xffffffff, 0x00050004,
 423        0x9254, 0xffffffff, 0x00010006,
 424        0x9258, 0xffffffff, 0x00090008,
 425        0x925c, 0xffffffff, 0x00070000,
 426        0x9260, 0xffffffff, 0x00030002,
 427        0x9264, 0xffffffff, 0x00050004,
 428        0x9270, 0xffffffff, 0x00010006,
 429        0x9274, 0xffffffff, 0x00090008,
 430        0x9278, 0xffffffff, 0x00070000,
 431        0x927c, 0xffffffff, 0x00030002,
 432        0x9280, 0xffffffff, 0x00050004,
 433        0x928c, 0xffffffff, 0x00010006,
 434        0x9290, 0xffffffff, 0x00090008,
 435        0x9294, 0xffffffff, 0x00000000,
 436        0x929c, 0xffffffff, 0x00000001,
 437        0x802c, 0xffffffff, 0xc0000000
 438};
 439
 440static const u32 redwood_mgcg_init[] =
 441{
 442        0x802c, 0xffffffff, 0xc0000000,
 443        0x5448, 0xffffffff, 0x00000100,
 444        0x55e4, 0xffffffff, 0x00000100,
 445        0x160c, 0xffffffff, 0x00000100,
 446        0x5644, 0xffffffff, 0x00000100,
 447        0xc164, 0xffffffff, 0x00000100,
 448        0x8a18, 0xffffffff, 0x00000100,
 449        0x897c, 0xffffffff, 0x06000100,
 450        0x8b28, 0xffffffff, 0x00000100,
 451        0x9144, 0xffffffff, 0x00000100,
 452        0x9a60, 0xffffffff, 0x00000100,
 453        0x9868, 0xffffffff, 0x00000100,
 454        0x8d58, 0xffffffff, 0x00000100,
 455        0x9510, 0xffffffff, 0x00000100,
 456        0x949c, 0xffffffff, 0x00000100,
 457        0x9654, 0xffffffff, 0x00000100,
 458        0x9030, 0xffffffff, 0x00000100,
 459        0x9034, 0xffffffff, 0x00000100,
 460        0x9038, 0xffffffff, 0x00000100,
 461        0x903c, 0xffffffff, 0x00000100,
 462        0x9040, 0xffffffff, 0x00000100,
 463        0xa200, 0xffffffff, 0x00000100,
 464        0xa204, 0xffffffff, 0x00000100,
 465        0xa208, 0xffffffff, 0x00000100,
 466        0xa20c, 0xffffffff, 0x00000100,
 467        0x971c, 0xffffffff, 0x00000100,
 468        0x977c, 0xffffffff, 0x00000100,
 469        0x3f80, 0xffffffff, 0x00000100,
 470        0xa210, 0xffffffff, 0x00000100,
 471        0xa214, 0xffffffff, 0x00000100,
 472        0x4d8, 0xffffffff, 0x00000100,
 473        0x9784, 0xffffffff, 0x00000100,
 474        0x9698, 0xffffffff, 0x00000100,
 475        0x4d4, 0xffffffff, 0x00000200,
 476        0x30cc, 0xffffffff, 0x00000100,
 477        0xd0c0, 0xffffffff, 0xff000100,
 478        0x802c, 0xffffffff, 0x40000000,
 479        0x915c, 0xffffffff, 0x00010000,
 480        0x9160, 0xffffffff, 0x00030002,
 481        0x9178, 0xffffffff, 0x00070000,
 482        0x917c, 0xffffffff, 0x00030002,
 483        0x9180, 0xffffffff, 0x00050004,
 484        0x918c, 0xffffffff, 0x00010006,
 485        0x9190, 0xffffffff, 0x00090008,
 486        0x9194, 0xffffffff, 0x00070000,
 487        0x9198, 0xffffffff, 0x00030002,
 488        0x919c, 0xffffffff, 0x00050004,
 489        0x91a8, 0xffffffff, 0x00010006,
 490        0x91ac, 0xffffffff, 0x00090008,
 491        0x91b0, 0xffffffff, 0x00070000,
 492        0x91b4, 0xffffffff, 0x00030002,
 493        0x91b8, 0xffffffff, 0x00050004,
 494        0x91c4, 0xffffffff, 0x00010006,
 495        0x91c8, 0xffffffff, 0x00090008,
 496        0x91cc, 0xffffffff, 0x00070000,
 497        0x91d0, 0xffffffff, 0x00030002,
 498        0x91d4, 0xffffffff, 0x00050004,
 499        0x91e0, 0xffffffff, 0x00010006,
 500        0x91e4, 0xffffffff, 0x00090008,
 501        0x91e8, 0xffffffff, 0x00000000,
 502        0x91ec, 0xffffffff, 0x00070000,
 503        0x91f0, 0xffffffff, 0x00030002,
 504        0x91f4, 0xffffffff, 0x00050004,
 505        0x9200, 0xffffffff, 0x00010006,
 506        0x9204, 0xffffffff, 0x00090008,
 507        0x9294, 0xffffffff, 0x00000000,
 508        0x929c, 0xffffffff, 0x00000001,
 509        0x802c, 0xffffffff, 0xc0000000
 510};
 511
 512static const u32 cedar_golden_registers[] =
 513{
 514        0x3f90, 0xffff0000, 0xff000000,
 515        0x9148, 0xffff0000, 0xff000000,
 516        0x3f94, 0xffff0000, 0xff000000,
 517        0x914c, 0xffff0000, 0xff000000,
 518        0x9b7c, 0xffffffff, 0x00000000,
 519        0x8a14, 0xffffffff, 0x00000007,
 520        0x8b10, 0xffffffff, 0x00000000,
 521        0x960c, 0xffffffff, 0x54763210,
 522        0x88c4, 0xffffffff, 0x000000c2,
 523        0x88d4, 0xffffffff, 0x00000000,
 524        0x8974, 0xffffffff, 0x00000000,
 525        0xc78, 0x00000080, 0x00000080,
 526        0x5eb4, 0xffffffff, 0x00000002,
 527        0x5e78, 0xffffffff, 0x001000f0,
 528        0x6104, 0x01000300, 0x00000000,
 529        0x5bc0, 0x00300000, 0x00000000,
 530        0x7030, 0xffffffff, 0x00000011,
 531        0x7c30, 0xffffffff, 0x00000011,
 532        0x10830, 0xffffffff, 0x00000011,
 533        0x11430, 0xffffffff, 0x00000011,
 534        0xd02c, 0xffffffff, 0x08421000,
 535        0x240c, 0xffffffff, 0x00000380,
 536        0x8b24, 0xffffffff, 0x00ff0fff,
 537        0x28a4c, 0x06000000, 0x06000000,
 538        0x10c, 0x00000001, 0x00000001,
 539        0x8d00, 0xffffffff, 0x100e4848,
 540        0x8d04, 0xffffffff, 0x00164745,
 541        0x8c00, 0xffffffff, 0xe4000003,
 542        0x8c04, 0xffffffff, 0x40600060,
 543        0x8c08, 0xffffffff, 0x001c001c,
 544        0x8cf0, 0xffffffff, 0x08e00410,
 545        0x8c20, 0xffffffff, 0x00800080,
 546        0x8c24, 0xffffffff, 0x00800080,
 547        0x8c18, 0xffffffff, 0x20202078,
 548        0x8c1c, 0xffffffff, 0x00001010,
 549        0x28350, 0xffffffff, 0x00000000,
 550        0xa008, 0xffffffff, 0x00010000,
 551        0x5c4, 0xffffffff, 0x00000001,
 552        0x9508, 0xffffffff, 0x00000002
 553};
 554
 555static const u32 cedar_mgcg_init[] =
 556{
 557        0x802c, 0xffffffff, 0xc0000000,
 558        0x5448, 0xffffffff, 0x00000100,
 559        0x55e4, 0xffffffff, 0x00000100,
 560        0x160c, 0xffffffff, 0x00000100,
 561        0x5644, 0xffffffff, 0x00000100,
 562        0xc164, 0xffffffff, 0x00000100,
 563        0x8a18, 0xffffffff, 0x00000100,
 564        0x897c, 0xffffffff, 0x06000100,
 565        0x8b28, 0xffffffff, 0x00000100,
 566        0x9144, 0xffffffff, 0x00000100,
 567        0x9a60, 0xffffffff, 0x00000100,
 568        0x9868, 0xffffffff, 0x00000100,
 569        0x8d58, 0xffffffff, 0x00000100,
 570        0x9510, 0xffffffff, 0x00000100,
 571        0x949c, 0xffffffff, 0x00000100,
 572        0x9654, 0xffffffff, 0x00000100,
 573        0x9030, 0xffffffff, 0x00000100,
 574        0x9034, 0xffffffff, 0x00000100,
 575        0x9038, 0xffffffff, 0x00000100,
 576        0x903c, 0xffffffff, 0x00000100,
 577        0x9040, 0xffffffff, 0x00000100,
 578        0xa200, 0xffffffff, 0x00000100,
 579        0xa204, 0xffffffff, 0x00000100,
 580        0xa208, 0xffffffff, 0x00000100,
 581        0xa20c, 0xffffffff, 0x00000100,
 582        0x971c, 0xffffffff, 0x00000100,
 583        0x977c, 0xffffffff, 0x00000100,
 584        0x3f80, 0xffffffff, 0x00000100,
 585        0xa210, 0xffffffff, 0x00000100,
 586        0xa214, 0xffffffff, 0x00000100,
 587        0x4d8, 0xffffffff, 0x00000100,
 588        0x9784, 0xffffffff, 0x00000100,
 589        0x9698, 0xffffffff, 0x00000100,
 590        0x4d4, 0xffffffff, 0x00000200,
 591        0x30cc, 0xffffffff, 0x00000100,
 592        0xd0c0, 0xffffffff, 0xff000100,
 593        0x802c, 0xffffffff, 0x40000000,
 594        0x915c, 0xffffffff, 0x00010000,
 595        0x9178, 0xffffffff, 0x00050000,
 596        0x917c, 0xffffffff, 0x00030002,
 597        0x918c, 0xffffffff, 0x00010004,
 598        0x9190, 0xffffffff, 0x00070006,
 599        0x9194, 0xffffffff, 0x00050000,
 600        0x9198, 0xffffffff, 0x00030002,
 601        0x91a8, 0xffffffff, 0x00010004,
 602        0x91ac, 0xffffffff, 0x00070006,
 603        0x91e8, 0xffffffff, 0x00000000,
 604        0x9294, 0xffffffff, 0x00000000,
 605        0x929c, 0xffffffff, 0x00000001,
 606        0x802c, 0xffffffff, 0xc0000000
 607};
 608
 609static const u32 juniper_mgcg_init[] =
 610{
 611        0x802c, 0xffffffff, 0xc0000000,
 612        0x5448, 0xffffffff, 0x00000100,
 613        0x55e4, 0xffffffff, 0x00000100,
 614        0x160c, 0xffffffff, 0x00000100,
 615        0x5644, 0xffffffff, 0x00000100,
 616        0xc164, 0xffffffff, 0x00000100,
 617        0x8a18, 0xffffffff, 0x00000100,
 618        0x897c, 0xffffffff, 0x06000100,
 619        0x8b28, 0xffffffff, 0x00000100,
 620        0x9144, 0xffffffff, 0x00000100,
 621        0x9a60, 0xffffffff, 0x00000100,
 622        0x9868, 0xffffffff, 0x00000100,
 623        0x8d58, 0xffffffff, 0x00000100,
 624        0x9510, 0xffffffff, 0x00000100,
 625        0x949c, 0xffffffff, 0x00000100,
 626        0x9654, 0xffffffff, 0x00000100,
 627        0x9030, 0xffffffff, 0x00000100,
 628        0x9034, 0xffffffff, 0x00000100,
 629        0x9038, 0xffffffff, 0x00000100,
 630        0x903c, 0xffffffff, 0x00000100,
 631        0x9040, 0xffffffff, 0x00000100,
 632        0xa200, 0xffffffff, 0x00000100,
 633        0xa204, 0xffffffff, 0x00000100,
 634        0xa208, 0xffffffff, 0x00000100,
 635        0xa20c, 0xffffffff, 0x00000100,
 636        0x971c, 0xffffffff, 0x00000100,
 637        0xd0c0, 0xffffffff, 0xff000100,
 638        0x802c, 0xffffffff, 0x40000000,
 639        0x915c, 0xffffffff, 0x00010000,
 640        0x9160, 0xffffffff, 0x00030002,
 641        0x9178, 0xffffffff, 0x00070000,
 642        0x917c, 0xffffffff, 0x00030002,
 643        0x9180, 0xffffffff, 0x00050004,
 644        0x918c, 0xffffffff, 0x00010006,
 645        0x9190, 0xffffffff, 0x00090008,
 646        0x9194, 0xffffffff, 0x00070000,
 647        0x9198, 0xffffffff, 0x00030002,
 648        0x919c, 0xffffffff, 0x00050004,
 649        0x91a8, 0xffffffff, 0x00010006,
 650        0x91ac, 0xffffffff, 0x00090008,
 651        0x91b0, 0xffffffff, 0x00070000,
 652        0x91b4, 0xffffffff, 0x00030002,
 653        0x91b8, 0xffffffff, 0x00050004,
 654        0x91c4, 0xffffffff, 0x00010006,
 655        0x91c8, 0xffffffff, 0x00090008,
 656        0x91cc, 0xffffffff, 0x00070000,
 657        0x91d0, 0xffffffff, 0x00030002,
 658        0x91d4, 0xffffffff, 0x00050004,
 659        0x91e0, 0xffffffff, 0x00010006,
 660        0x91e4, 0xffffffff, 0x00090008,
 661        0x91e8, 0xffffffff, 0x00000000,
 662        0x91ec, 0xffffffff, 0x00070000,
 663        0x91f0, 0xffffffff, 0x00030002,
 664        0x91f4, 0xffffffff, 0x00050004,
 665        0x9200, 0xffffffff, 0x00010006,
 666        0x9204, 0xffffffff, 0x00090008,
 667        0x9208, 0xffffffff, 0x00070000,
 668        0x920c, 0xffffffff, 0x00030002,
 669        0x9210, 0xffffffff, 0x00050004,
 670        0x921c, 0xffffffff, 0x00010006,
 671        0x9220, 0xffffffff, 0x00090008,
 672        0x9224, 0xffffffff, 0x00070000,
 673        0x9228, 0xffffffff, 0x00030002,
 674        0x922c, 0xffffffff, 0x00050004,
 675        0x9238, 0xffffffff, 0x00010006,
 676        0x923c, 0xffffffff, 0x00090008,
 677        0x9240, 0xffffffff, 0x00070000,
 678        0x9244, 0xffffffff, 0x00030002,
 679        0x9248, 0xffffffff, 0x00050004,
 680        0x9254, 0xffffffff, 0x00010006,
 681        0x9258, 0xffffffff, 0x00090008,
 682        0x925c, 0xffffffff, 0x00070000,
 683        0x9260, 0xffffffff, 0x00030002,
 684        0x9264, 0xffffffff, 0x00050004,
 685        0x9270, 0xffffffff, 0x00010006,
 686        0x9274, 0xffffffff, 0x00090008,
 687        0x9278, 0xffffffff, 0x00070000,
 688        0x927c, 0xffffffff, 0x00030002,
 689        0x9280, 0xffffffff, 0x00050004,
 690        0x928c, 0xffffffff, 0x00010006,
 691        0x9290, 0xffffffff, 0x00090008,
 692        0x9294, 0xffffffff, 0x00000000,
 693        0x929c, 0xffffffff, 0x00000001,
 694        0x802c, 0xffffffff, 0xc0000000,
 695        0x977c, 0xffffffff, 0x00000100,
 696        0x3f80, 0xffffffff, 0x00000100,
 697        0xa210, 0xffffffff, 0x00000100,
 698        0xa214, 0xffffffff, 0x00000100,
 699        0x4d8, 0xffffffff, 0x00000100,
 700        0x9784, 0xffffffff, 0x00000100,
 701        0x9698, 0xffffffff, 0x00000100,
 702        0x4d4, 0xffffffff, 0x00000200,
 703        0x30cc, 0xffffffff, 0x00000100,
 704        0x802c, 0xffffffff, 0xc0000000
 705};
 706
 707static const u32 supersumo_golden_registers[] =
 708{
 709        0x5eb4, 0xffffffff, 0x00000002,
 710        0x5c4, 0xffffffff, 0x00000001,
 711        0x7030, 0xffffffff, 0x00000011,
 712        0x7c30, 0xffffffff, 0x00000011,
 713        0x6104, 0x01000300, 0x00000000,
 714        0x5bc0, 0x00300000, 0x00000000,
 715        0x8c04, 0xffffffff, 0x40600060,
 716        0x8c08, 0xffffffff, 0x001c001c,
 717        0x8c20, 0xffffffff, 0x00800080,
 718        0x8c24, 0xffffffff, 0x00800080,
 719        0x8c18, 0xffffffff, 0x20202078,
 720        0x8c1c, 0xffffffff, 0x00001010,
 721        0x918c, 0xffffffff, 0x00010006,
 722        0x91a8, 0xffffffff, 0x00010006,
 723        0x91c4, 0xffffffff, 0x00010006,
 724        0x91e0, 0xffffffff, 0x00010006,
 725        0x9200, 0xffffffff, 0x00010006,
 726        0x9150, 0xffffffff, 0x6e944040,
 727        0x917c, 0xffffffff, 0x00030002,
 728        0x9180, 0xffffffff, 0x00050004,
 729        0x9198, 0xffffffff, 0x00030002,
 730        0x919c, 0xffffffff, 0x00050004,
 731        0x91b4, 0xffffffff, 0x00030002,
 732        0x91b8, 0xffffffff, 0x00050004,
 733        0x91d0, 0xffffffff, 0x00030002,
 734        0x91d4, 0xffffffff, 0x00050004,
 735        0x91f0, 0xffffffff, 0x00030002,
 736        0x91f4, 0xffffffff, 0x00050004,
 737        0x915c, 0xffffffff, 0x00010000,
 738        0x9160, 0xffffffff, 0x00030002,
 739        0x3f90, 0xffff0000, 0xff000000,
 740        0x9178, 0xffffffff, 0x00070000,
 741        0x9194, 0xffffffff, 0x00070000,
 742        0x91b0, 0xffffffff, 0x00070000,
 743        0x91cc, 0xffffffff, 0x00070000,
 744        0x91ec, 0xffffffff, 0x00070000,
 745        0x9148, 0xffff0000, 0xff000000,
 746        0x9190, 0xffffffff, 0x00090008,
 747        0x91ac, 0xffffffff, 0x00090008,
 748        0x91c8, 0xffffffff, 0x00090008,
 749        0x91e4, 0xffffffff, 0x00090008,
 750        0x9204, 0xffffffff, 0x00090008,
 751        0x3f94, 0xffff0000, 0xff000000,
 752        0x914c, 0xffff0000, 0xff000000,
 753        0x929c, 0xffffffff, 0x00000001,
 754        0x8a18, 0xffffffff, 0x00000100,
 755        0x8b28, 0xffffffff, 0x00000100,
 756        0x9144, 0xffffffff, 0x00000100,
 757        0x5644, 0xffffffff, 0x00000100,
 758        0x9b7c, 0xffffffff, 0x00000000,
 759        0x8030, 0xffffffff, 0x0000100a,
 760        0x8a14, 0xffffffff, 0x00000007,
 761        0x8b24, 0xffffffff, 0x00ff0fff,
 762        0x8b10, 0xffffffff, 0x00000000,
 763        0x28a4c, 0x06000000, 0x06000000,
 764        0x4d8, 0xffffffff, 0x00000100,
 765        0x913c, 0xffff000f, 0x0100000a,
 766        0x960c, 0xffffffff, 0x54763210,
 767        0x88c4, 0xffffffff, 0x000000c2,
 768        0x88d4, 0xffffffff, 0x00000010,
 769        0x8974, 0xffffffff, 0x00000000,
 770        0xc78, 0x00000080, 0x00000080,
 771        0x5e78, 0xffffffff, 0x001000f0,
 772        0xd02c, 0xffffffff, 0x08421000,
 773        0xa008, 0xffffffff, 0x00010000,
 774        0x8d00, 0xffffffff, 0x100e4848,
 775        0x8d04, 0xffffffff, 0x00164745,
 776        0x8c00, 0xffffffff, 0xe4000003,
 777        0x8cf0, 0x1fffffff, 0x08e00620,
 778        0x28350, 0xffffffff, 0x00000000,
 779        0x9508, 0xffffffff, 0x00000002
 780};
 781
 782static const u32 sumo_golden_registers[] =
 783{
 784        0x900c, 0x00ffffff, 0x0017071f,
 785        0x8c18, 0xffffffff, 0x10101060,
 786        0x8c1c, 0xffffffff, 0x00001010,
 787        0x8c30, 0x0000000f, 0x00000005,
 788        0x9688, 0x0000000f, 0x00000007
 789};
 790
 791static const u32 wrestler_golden_registers[] =
 792{
 793        0x5eb4, 0xffffffff, 0x00000002,
 794        0x5c4, 0xffffffff, 0x00000001,
 795        0x7030, 0xffffffff, 0x00000011,
 796        0x7c30, 0xffffffff, 0x00000011,
 797        0x6104, 0x01000300, 0x00000000,
 798        0x5bc0, 0x00300000, 0x00000000,
 799        0x918c, 0xffffffff, 0x00010006,
 800        0x91a8, 0xffffffff, 0x00010006,
 801        0x9150, 0xffffffff, 0x6e944040,
 802        0x917c, 0xffffffff, 0x00030002,
 803        0x9198, 0xffffffff, 0x00030002,
 804        0x915c, 0xffffffff, 0x00010000,
 805        0x3f90, 0xffff0000, 0xff000000,
 806        0x9178, 0xffffffff, 0x00070000,
 807        0x9194, 0xffffffff, 0x00070000,
 808        0x9148, 0xffff0000, 0xff000000,
 809        0x9190, 0xffffffff, 0x00090008,
 810        0x91ac, 0xffffffff, 0x00090008,
 811        0x3f94, 0xffff0000, 0xff000000,
 812        0x914c, 0xffff0000, 0xff000000,
 813        0x929c, 0xffffffff, 0x00000001,
 814        0x8a18, 0xffffffff, 0x00000100,
 815        0x8b28, 0xffffffff, 0x00000100,
 816        0x9144, 0xffffffff, 0x00000100,
 817        0x9b7c, 0xffffffff, 0x00000000,
 818        0x8030, 0xffffffff, 0x0000100a,
 819        0x8a14, 0xffffffff, 0x00000001,
 820        0x8b24, 0xffffffff, 0x00ff0fff,
 821        0x8b10, 0xffffffff, 0x00000000,
 822        0x28a4c, 0x06000000, 0x06000000,
 823        0x4d8, 0xffffffff, 0x00000100,
 824        0x913c, 0xffff000f, 0x0100000a,
 825        0x960c, 0xffffffff, 0x54763210,
 826        0x88c4, 0xffffffff, 0x000000c2,
 827        0x88d4, 0xffffffff, 0x00000010,
 828        0x8974, 0xffffffff, 0x00000000,
 829        0xc78, 0x00000080, 0x00000080,
 830        0x5e78, 0xffffffff, 0x001000f0,
 831        0xd02c, 0xffffffff, 0x08421000,
 832        0xa008, 0xffffffff, 0x00010000,
 833        0x8d00, 0xffffffff, 0x100e4848,
 834        0x8d04, 0xffffffff, 0x00164745,
 835        0x8c00, 0xffffffff, 0xe4000003,
 836        0x8cf0, 0x1fffffff, 0x08e00410,
 837        0x28350, 0xffffffff, 0x00000000,
 838        0x9508, 0xffffffff, 0x00000002,
 839        0x900c, 0xffffffff, 0x0017071f,
 840        0x8c18, 0xffffffff, 0x10101060,
 841        0x8c1c, 0xffffffff, 0x00001010
 842};
 843
 844static const u32 barts_golden_registers[] =
 845{
 846        0x5eb4, 0xffffffff, 0x00000002,
 847        0x5e78, 0x8f311ff1, 0x001000f0,
 848        0x3f90, 0xffff0000, 0xff000000,
 849        0x9148, 0xffff0000, 0xff000000,
 850        0x3f94, 0xffff0000, 0xff000000,
 851        0x914c, 0xffff0000, 0xff000000,
 852        0xc78, 0x00000080, 0x00000080,
 853        0xbd4, 0x70073777, 0x00010001,
 854        0xd02c, 0xbfffff1f, 0x08421000,
 855        0xd0b8, 0x03773777, 0x02011003,
 856        0x5bc0, 0x00200000, 0x50100000,
 857        0x98f8, 0x33773777, 0x02011003,
 858        0x98fc, 0xffffffff, 0x76543210,
 859        0x7030, 0x31000311, 0x00000011,
 860        0x2f48, 0x00000007, 0x02011003,
 861        0x6b28, 0x00000010, 0x00000012,
 862        0x7728, 0x00000010, 0x00000012,
 863        0x10328, 0x00000010, 0x00000012,
 864        0x10f28, 0x00000010, 0x00000012,
 865        0x11b28, 0x00000010, 0x00000012,
 866        0x12728, 0x00000010, 0x00000012,
 867        0x240c, 0x000007ff, 0x00000380,
 868        0x8a14, 0xf000001f, 0x00000007,
 869        0x8b24, 0x3fff3fff, 0x00ff0fff,
 870        0x8b10, 0x0000ff0f, 0x00000000,
 871        0x28a4c, 0x07ffffff, 0x06000000,
 872        0x10c, 0x00000001, 0x00010003,
 873        0xa02c, 0xffffffff, 0x0000009b,
 874        0x913c, 0x0000000f, 0x0100000a,
 875        0x8d00, 0xffff7f7f, 0x100e4848,
 876        0x8d04, 0x00ffffff, 0x00164745,
 877        0x8c00, 0xfffc0003, 0xe4000003,
 878        0x8c04, 0xf8ff00ff, 0x40600060,
 879        0x8c08, 0x00ff00ff, 0x001c001c,
 880        0x8cf0, 0x1fff1fff, 0x08e00620,
 881        0x8c20, 0x0fff0fff, 0x00800080,
 882        0x8c24, 0x0fff0fff, 0x00800080,
 883        0x8c18, 0xffffffff, 0x20202078,
 884        0x8c1c, 0x0000ffff, 0x00001010,
 885        0x28350, 0x00000f01, 0x00000000,
 886        0x9508, 0x3700001f, 0x00000002,
 887        0x960c, 0xffffffff, 0x54763210,
 888        0x88c4, 0x001f3ae3, 0x000000c2,
 889        0x88d4, 0x0000001f, 0x00000010,
 890        0x8974, 0xffffffff, 0x00000000
 891};
 892
 893static const u32 turks_golden_registers[] =
 894{
 895        0x5eb4, 0xffffffff, 0x00000002,
 896        0x5e78, 0x8f311ff1, 0x001000f0,
 897        0x8c8, 0x00003000, 0x00001070,
 898        0x8cc, 0x000fffff, 0x00040035,
 899        0x3f90, 0xffff0000, 0xfff00000,
 900        0x9148, 0xffff0000, 0xfff00000,
 901        0x3f94, 0xffff0000, 0xfff00000,
 902        0x914c, 0xffff0000, 0xfff00000,
 903        0xc78, 0x00000080, 0x00000080,
 904        0xbd4, 0x00073007, 0x00010002,
 905        0xd02c, 0xbfffff1f, 0x08421000,
 906        0xd0b8, 0x03773777, 0x02010002,
 907        0x5bc0, 0x00200000, 0x50100000,
 908        0x98f8, 0x33773777, 0x00010002,
 909        0x98fc, 0xffffffff, 0x33221100,
 910        0x7030, 0x31000311, 0x00000011,
 911        0x2f48, 0x33773777, 0x00010002,
 912        0x6b28, 0x00000010, 0x00000012,
 913        0x7728, 0x00000010, 0x00000012,
 914        0x10328, 0x00000010, 0x00000012,
 915        0x10f28, 0x00000010, 0x00000012,
 916        0x11b28, 0x00000010, 0x00000012,
 917        0x12728, 0x00000010, 0x00000012,
 918        0x240c, 0x000007ff, 0x00000380,
 919        0x8a14, 0xf000001f, 0x00000007,
 920        0x8b24, 0x3fff3fff, 0x00ff0fff,
 921        0x8b10, 0x0000ff0f, 0x00000000,
 922        0x28a4c, 0x07ffffff, 0x06000000,
 923        0x10c, 0x00000001, 0x00010003,
 924        0xa02c, 0xffffffff, 0x0000009b,
 925        0x913c, 0x0000000f, 0x0100000a,
 926        0x8d00, 0xffff7f7f, 0x100e4848,
 927        0x8d04, 0x00ffffff, 0x00164745,
 928        0x8c00, 0xfffc0003, 0xe4000003,
 929        0x8c04, 0xf8ff00ff, 0x40600060,
 930        0x8c08, 0x00ff00ff, 0x001c001c,
 931        0x8cf0, 0x1fff1fff, 0x08e00410,
 932        0x8c20, 0x0fff0fff, 0x00800080,
 933        0x8c24, 0x0fff0fff, 0x00800080,
 934        0x8c18, 0xffffffff, 0x20202078,
 935        0x8c1c, 0x0000ffff, 0x00001010,
 936        0x28350, 0x00000f01, 0x00000000,
 937        0x9508, 0x3700001f, 0x00000002,
 938        0x960c, 0xffffffff, 0x54763210,
 939        0x88c4, 0x001f3ae3, 0x000000c2,
 940        0x88d4, 0x0000001f, 0x00000010,
 941        0x8974, 0xffffffff, 0x00000000
 942};
 943
 944static const u32 caicos_golden_registers[] =
 945{
 946        0x5eb4, 0xffffffff, 0x00000002,
 947        0x5e78, 0x8f311ff1, 0x001000f0,
 948        0x8c8, 0x00003420, 0x00001450,
 949        0x8cc, 0x000fffff, 0x00040035,
 950        0x3f90, 0xffff0000, 0xfffc0000,
 951        0x9148, 0xffff0000, 0xfffc0000,
 952        0x3f94, 0xffff0000, 0xfffc0000,
 953        0x914c, 0xffff0000, 0xfffc0000,
 954        0xc78, 0x00000080, 0x00000080,
 955        0xbd4, 0x00073007, 0x00010001,
 956        0xd02c, 0xbfffff1f, 0x08421000,
 957        0xd0b8, 0x03773777, 0x02010001,
 958        0x5bc0, 0x00200000, 0x50100000,
 959        0x98f8, 0x33773777, 0x02010001,
 960        0x98fc, 0xffffffff, 0x33221100,
 961        0x7030, 0x31000311, 0x00000011,
 962        0x2f48, 0x33773777, 0x02010001,
 963        0x6b28, 0x00000010, 0x00000012,
 964        0x7728, 0x00000010, 0x00000012,
 965        0x10328, 0x00000010, 0x00000012,
 966        0x10f28, 0x00000010, 0x00000012,
 967        0x11b28, 0x00000010, 0x00000012,
 968        0x12728, 0x00000010, 0x00000012,
 969        0x240c, 0x000007ff, 0x00000380,
 970        0x8a14, 0xf000001f, 0x00000001,
 971        0x8b24, 0x3fff3fff, 0x00ff0fff,
 972        0x8b10, 0x0000ff0f, 0x00000000,
 973        0x28a4c, 0x07ffffff, 0x06000000,
 974        0x10c, 0x00000001, 0x00010003,
 975        0xa02c, 0xffffffff, 0x0000009b,
 976        0x913c, 0x0000000f, 0x0100000a,
 977        0x8d00, 0xffff7f7f, 0x100e4848,
 978        0x8d04, 0x00ffffff, 0x00164745,
 979        0x8c00, 0xfffc0003, 0xe4000003,
 980        0x8c04, 0xf8ff00ff, 0x40600060,
 981        0x8c08, 0x00ff00ff, 0x001c001c,
 982        0x8cf0, 0x1fff1fff, 0x08e00410,
 983        0x8c20, 0x0fff0fff, 0x00800080,
 984        0x8c24, 0x0fff0fff, 0x00800080,
 985        0x8c18, 0xffffffff, 0x20202078,
 986        0x8c1c, 0x0000ffff, 0x00001010,
 987        0x28350, 0x00000f01, 0x00000000,
 988        0x9508, 0x3700001f, 0x00000002,
 989        0x960c, 0xffffffff, 0x54763210,
 990        0x88c4, 0x001f3ae3, 0x000000c2,
 991        0x88d4, 0x0000001f, 0x00000010,
 992        0x8974, 0xffffffff, 0x00000000
 993};
 994
 995static void evergreen_init_golden_registers(struct radeon_device *rdev)
 996{
 997        switch (rdev->family) {
 998        case CHIP_CYPRESS:
 999        case CHIP_HEMLOCK:
1000                radeon_program_register_sequence(rdev,
1001                                                 evergreen_golden_registers,
1002                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1003                radeon_program_register_sequence(rdev,
1004                                                 evergreen_golden_registers2,
1005                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1006                radeon_program_register_sequence(rdev,
1007                                                 cypress_mgcg_init,
1008                                                 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1009                break;
1010        case CHIP_JUNIPER:
1011                radeon_program_register_sequence(rdev,
1012                                                 evergreen_golden_registers,
1013                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1014                radeon_program_register_sequence(rdev,
1015                                                 evergreen_golden_registers2,
1016                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1017                radeon_program_register_sequence(rdev,
1018                                                 juniper_mgcg_init,
1019                                                 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1020                break;
1021        case CHIP_REDWOOD:
1022                radeon_program_register_sequence(rdev,
1023                                                 evergreen_golden_registers,
1024                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1025                radeon_program_register_sequence(rdev,
1026                                                 evergreen_golden_registers2,
1027                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1028                radeon_program_register_sequence(rdev,
1029                                                 redwood_mgcg_init,
1030                                                 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1031                break;
1032        case CHIP_CEDAR:
1033                radeon_program_register_sequence(rdev,
1034                                                 cedar_golden_registers,
1035                                                 (const u32)ARRAY_SIZE(cedar_golden_registers));
1036                radeon_program_register_sequence(rdev,
1037                                                 evergreen_golden_registers2,
1038                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1039                radeon_program_register_sequence(rdev,
1040                                                 cedar_mgcg_init,
1041                                                 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1042                break;
1043        case CHIP_PALM:
1044                radeon_program_register_sequence(rdev,
1045                                                 wrestler_golden_registers,
1046                                                 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1047                break;
1048        case CHIP_SUMO:
1049                radeon_program_register_sequence(rdev,
1050                                                 supersumo_golden_registers,
1051                                                 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1052                break;
1053        case CHIP_SUMO2:
1054                radeon_program_register_sequence(rdev,
1055                                                 supersumo_golden_registers,
1056                                                 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1057                radeon_program_register_sequence(rdev,
1058                                                 sumo_golden_registers,
1059                                                 (const u32)ARRAY_SIZE(sumo_golden_registers));
1060                break;
1061        case CHIP_BARTS:
1062                radeon_program_register_sequence(rdev,
1063                                                 barts_golden_registers,
1064                                                 (const u32)ARRAY_SIZE(barts_golden_registers));
1065                break;
1066        case CHIP_TURKS:
1067                radeon_program_register_sequence(rdev,
1068                                                 turks_golden_registers,
1069                                                 (const u32)ARRAY_SIZE(turks_golden_registers));
1070                break;
1071        case CHIP_CAICOS:
1072                radeon_program_register_sequence(rdev,
1073                                                 caicos_golden_registers,
1074                                                 (const u32)ARRAY_SIZE(caicos_golden_registers));
1075                break;
1076        default:
1077                break;
1078        }
1079}
1080
1081/**
1082 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1083 *
1084 * @rdev: radeon_device pointer
1085 * @reg: register offset in bytes
1086 * @val: register value
1087 *
1088 * Returns 0 for success or -EINVAL for an invalid register
1089 *
1090 */
1091int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1092                                        u32 reg, u32 *val)
1093{
1094        switch (reg) {
1095        case GRBM_STATUS:
1096        case GRBM_STATUS_SE0:
1097        case GRBM_STATUS_SE1:
1098        case SRBM_STATUS:
1099        case SRBM_STATUS2:
1100        case DMA_STATUS_REG:
1101        case UVD_STATUS:
1102                *val = RREG32(reg);
1103                return 0;
1104        default:
1105                return -EINVAL;
1106        }
1107}
1108
1109void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1110                             unsigned *bankh, unsigned *mtaspect,
1111                             unsigned *tile_split)
1112{
1113        *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1114        *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1115        *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1116        *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1117        switch (*bankw) {
1118        default:
1119        case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1120        case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1121        case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1122        case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1123        }
1124        switch (*bankh) {
1125        default:
1126        case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1127        case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1128        case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1129        case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1130        }
1131        switch (*mtaspect) {
1132        default:
1133        case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1134        case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1135        case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1136        case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1137        }
1138}
1139
1140static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1141                              u32 cntl_reg, u32 status_reg)
1142{
1143        int r, i;
1144        struct atom_clock_dividers dividers;
1145
1146        r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1147                                           clock, false, &dividers);
1148        if (r)
1149                return r;
1150
1151        WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1152
1153        for (i = 0; i < 100; i++) {
1154                if (RREG32(status_reg) & DCLK_STATUS)
1155                        break;
1156                mdelay(10);
1157        }
1158        if (i == 100)
1159                return -ETIMEDOUT;
1160
1161        return 0;
1162}
1163
1164int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1165{
1166        int r = 0;
1167        u32 cg_scratch = RREG32(CG_SCRATCH1);
1168
1169        r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1170        if (r)
1171                goto done;
1172        cg_scratch &= 0xffff0000;
1173        cg_scratch |= vclk / 100; /* Mhz */
1174
1175        r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1176        if (r)
1177                goto done;
1178        cg_scratch &= 0x0000ffff;
1179        cg_scratch |= (dclk / 100) << 16; /* Mhz */
1180
1181done:
1182        WREG32(CG_SCRATCH1, cg_scratch);
1183
1184        return r;
1185}
1186
1187int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1188{
1189        /* start off with something large */
1190        unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1191        int r;
1192
1193        /* bypass vclk and dclk with bclk */
1194        WREG32_P(CG_UPLL_FUNC_CNTL_2,
1195                VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1196                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1197
1198        /* put PLL in bypass mode */
1199        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1200
1201        if (!vclk || !dclk) {
1202                /* keep the Bypass mode, put PLL to sleep */
1203                WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1204                return 0;
1205        }
1206
1207        r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1208                                          16384, 0x03FFFFFF, 0, 128, 5,
1209                                          &fb_div, &vclk_div, &dclk_div);
1210        if (r)
1211                return r;
1212
1213        /* set VCO_MODE to 1 */
1214        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1215
1216        /* toggle UPLL_SLEEP to 1 then back to 0 */
1217        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1218        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1219
1220        /* deassert UPLL_RESET */
1221        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1222
1223        mdelay(1);
1224
1225        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1226        if (r)
1227                return r;
1228
1229        /* assert UPLL_RESET again */
1230        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1231
1232        /* disable spread spectrum. */
1233        WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1234
1235        /* set feedback divider */
1236        WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1237
1238        /* set ref divider to 0 */
1239        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1240
1241        if (fb_div < 307200)
1242                WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1243        else
1244                WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1245
1246        /* set PDIV_A and PDIV_B */
1247        WREG32_P(CG_UPLL_FUNC_CNTL_2,
1248                UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1249                ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1250
1251        /* give the PLL some time to settle */
1252        mdelay(15);
1253
1254        /* deassert PLL_RESET */
1255        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1256
1257        mdelay(15);
1258
1259        /* switch from bypass mode to normal mode */
1260        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1261
1262        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1263        if (r)
1264                return r;
1265
1266        /* switch VCLK and DCLK selection */
1267        WREG32_P(CG_UPLL_FUNC_CNTL_2,
1268                VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1269                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1270
1271        mdelay(100);
1272
1273        return 0;
1274}
1275
1276void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1277{
1278        int readrq;
1279        u16 v;
1280
1281        readrq = pcie_get_readrq(rdev->pdev);
1282        v = ffs(readrq) - 8;
1283        /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1284         * to avoid hangs or perfomance issues
1285         */
1286        if ((v == 0) || (v == 6) || (v == 7))
1287                pcie_set_readrq(rdev->pdev, 512);
1288}
1289
1290void dce4_program_fmt(struct drm_encoder *encoder)
1291{
1292        struct drm_device *dev = encoder->dev;
1293        struct radeon_device *rdev = dev->dev_private;
1294        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1295        struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1296        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1297        int bpc = 0;
1298        u32 tmp = 0;
1299        enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1300
1301        if (connector) {
1302                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1303                bpc = radeon_get_monitor_bpc(connector);
1304                dither = radeon_connector->dither;
1305        }
1306
1307        /* LVDS/eDP FMT is set up by atom */
1308        if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1309                return;
1310
1311        /* not needed for analog */
1312        if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1313            (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1314                return;
1315
1316        if (bpc == 0)
1317                return;
1318
1319        switch (bpc) {
1320        case 6:
1321                if (dither == RADEON_FMT_DITHER_ENABLE)
1322                        /* XXX sort out optimal dither settings */
1323                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1324                                FMT_SPATIAL_DITHER_EN);
1325                else
1326                        tmp |= FMT_TRUNCATE_EN;
1327                break;
1328        case 8:
1329                if (dither == RADEON_FMT_DITHER_ENABLE)
1330                        /* XXX sort out optimal dither settings */
1331                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1332                                FMT_RGB_RANDOM_ENABLE |
1333                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1334                else
1335                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1336                break;
1337        case 10:
1338        default:
1339                /* not needed */
1340                break;
1341        }
1342
1343        WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1344}
1345
1346static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1347{
1348        if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1349                return true;
1350        else
1351                return false;
1352}
1353
1354static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1355{
1356        u32 pos1, pos2;
1357
1358        pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1359        pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1360
1361        if (pos1 != pos2)
1362                return true;
1363        else
1364                return false;
1365}
1366
1367/**
1368 * dce4_wait_for_vblank - vblank wait asic callback.
1369 *
1370 * @rdev: radeon_device pointer
1371 * @crtc: crtc to wait for vblank on
1372 *
1373 * Wait for vblank on the requested crtc (evergreen+).
1374 */
1375void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1376{
1377        unsigned i = 0;
1378
1379        if (crtc >= rdev->num_crtc)
1380                return;
1381
1382        if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1383                return;
1384
1385        /* depending on when we hit vblank, we may be close to active; if so,
1386         * wait for another frame.
1387         */
1388        while (dce4_is_in_vblank(rdev, crtc)) {
1389                if (i++ % 100 == 0) {
1390                        if (!dce4_is_counter_moving(rdev, crtc))
1391                                break;
1392                }
1393        }
1394
1395        while (!dce4_is_in_vblank(rdev, crtc)) {
1396                if (i++ % 100 == 0) {
1397                        if (!dce4_is_counter_moving(rdev, crtc))
1398                                break;
1399                }
1400        }
1401}
1402
1403/**
1404 * evergreen_page_flip - pageflip callback.
1405 *
1406 * @rdev: radeon_device pointer
1407 * @crtc_id: crtc to cleanup pageflip on
1408 * @crtc_base: new address of the crtc (GPU MC address)
1409 * @async: asynchronous flip
1410 *
1411 * Triggers the actual pageflip by updating the primary
1412 * surface base address (evergreen+).
1413 */
1414void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1415                         bool async)
1416{
1417        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1418        struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1419
1420        /* flip at hsync for async, default is vsync */
1421        WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1422               async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1423        /* update pitch */
1424        WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1425               fb->pitches[0] / fb->format->cpp[0]);
1426        /* update the scanout addresses */
1427        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1428               upper_32_bits(crtc_base));
1429        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1430               (u32)crtc_base);
1431        /* post the write */
1432        RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1433}
1434
1435/**
1436 * evergreen_page_flip_pending - check if page flip is still pending
1437 *
1438 * @rdev: radeon_device pointer
1439 * @crtc_id: crtc to check
1440 *
1441 * Returns the current update pending status.
1442 */
1443bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1444{
1445        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1446
1447        /* Return current update_pending status: */
1448        return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1449                EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1450}
1451
1452/* get temperature in millidegrees */
1453int evergreen_get_temp(struct radeon_device *rdev)
1454{
1455        u32 temp, toffset;
1456        int actual_temp = 0;
1457
1458        if (rdev->family == CHIP_JUNIPER) {
1459                toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1460                        TOFFSET_SHIFT;
1461                temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1462                        TS0_ADC_DOUT_SHIFT;
1463
1464                if (toffset & 0x100)
1465                        actual_temp = temp / 2 - (0x200 - toffset);
1466                else
1467                        actual_temp = temp / 2 + toffset;
1468
1469                actual_temp = actual_temp * 1000;
1470
1471        } else {
1472                temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1473                        ASIC_T_SHIFT;
1474
1475                if (temp & 0x400)
1476                        actual_temp = -256;
1477                else if (temp & 0x200)
1478                        actual_temp = 255;
1479                else if (temp & 0x100) {
1480                        actual_temp = temp & 0x1ff;
1481                        actual_temp |= ~0x1ff;
1482                } else
1483                        actual_temp = temp & 0xff;
1484
1485                actual_temp = (actual_temp * 1000) / 2;
1486        }
1487
1488        return actual_temp;
1489}
1490
1491int sumo_get_temp(struct radeon_device *rdev)
1492{
1493        u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1494        int actual_temp = temp - 49;
1495
1496        return actual_temp * 1000;
1497}
1498
1499/**
1500 * sumo_pm_init_profile - Initialize power profiles callback.
1501 *
1502 * @rdev: radeon_device pointer
1503 *
1504 * Initialize the power states used in profile mode
1505 * (sumo, trinity, SI).
1506 * Used for profile mode only.
1507 */
1508void sumo_pm_init_profile(struct radeon_device *rdev)
1509{
1510        int idx;
1511
1512        /* default */
1513        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1514        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1515        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1516        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1517
1518        /* low,mid sh/mh */
1519        if (rdev->flags & RADEON_IS_MOBILITY)
1520                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1521        else
1522                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1523
1524        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1525        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1526        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1527        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1528
1529        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1530        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1531        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1532        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1533
1534        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1535        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1536        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1537        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1538
1539        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1540        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1541        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1542        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1543
1544        /* high sh/mh */
1545        idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1546        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1547        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1548        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1549        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1550                rdev->pm.power_state[idx].num_clock_modes - 1;
1551
1552        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1553        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1554        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1555        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1556                rdev->pm.power_state[idx].num_clock_modes - 1;
1557}
1558
1559/**
1560 * btc_pm_init_profile - Initialize power profiles callback.
1561 *
1562 * @rdev: radeon_device pointer
1563 *
1564 * Initialize the power states used in profile mode
1565 * (BTC, cayman).
1566 * Used for profile mode only.
1567 */
1568void btc_pm_init_profile(struct radeon_device *rdev)
1569{
1570        int idx;
1571
1572        /* default */
1573        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1574        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1575        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1576        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1577        /* starting with BTC, there is one state that is used for both
1578         * MH and SH.  Difference is that we always use the high clock index for
1579         * mclk.
1580         */
1581        if (rdev->flags & RADEON_IS_MOBILITY)
1582                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1583        else
1584                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1585        /* low sh */
1586        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1587        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1588        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1589        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1590        /* mid sh */
1591        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1592        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1593        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1594        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1595        /* high sh */
1596        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1597        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1598        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1599        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1600        /* low mh */
1601        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1602        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1603        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1604        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1605        /* mid mh */
1606        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1607        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1608        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1609        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1610        /* high mh */
1611        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1612        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1613        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1614        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1615}
1616
1617/**
1618 * evergreen_pm_misc - set additional pm hw parameters callback.
1619 *
1620 * @rdev: radeon_device pointer
1621 *
1622 * Set non-clock parameters associated with a power state
1623 * (voltage, etc.) (evergreen+).
1624 */
1625void evergreen_pm_misc(struct radeon_device *rdev)
1626{
1627        int req_ps_idx = rdev->pm.requested_power_state_index;
1628        int req_cm_idx = rdev->pm.requested_clock_mode_index;
1629        struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1630        struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1631
1632        if (voltage->type == VOLTAGE_SW) {
1633                /* 0xff0x are flags rather then an actual voltage */
1634                if ((voltage->voltage & 0xff00) == 0xff00)
1635                        return;
1636                if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1637                        radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1638                        rdev->pm.current_vddc = voltage->voltage;
1639                        DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1640                }
1641
1642                /* starting with BTC, there is one state that is used for both
1643                 * MH and SH.  Difference is that we always use the high clock index for
1644                 * mclk and vddci.
1645                 */
1646                if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1647                    (rdev->family >= CHIP_BARTS) &&
1648                    rdev->pm.active_crtc_count &&
1649                    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1650                     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1651                        voltage = &rdev->pm.power_state[req_ps_idx].
1652                                clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1653
1654                /* 0xff0x are flags rather then an actual voltage */
1655                if ((voltage->vddci & 0xff00) == 0xff00)
1656                        return;
1657                if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1658                        radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1659                        rdev->pm.current_vddci = voltage->vddci;
1660                        DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1661                }
1662        }
1663}
1664
1665/**
1666 * evergreen_pm_prepare - pre-power state change callback.
1667 *
1668 * @rdev: radeon_device pointer
1669 *
1670 * Prepare for a power state change (evergreen+).
1671 */
1672void evergreen_pm_prepare(struct radeon_device *rdev)
1673{
1674        struct drm_device *ddev = rdev->ddev;
1675        struct drm_crtc *crtc;
1676        struct radeon_crtc *radeon_crtc;
1677        u32 tmp;
1678
1679        /* disable any active CRTCs */
1680        list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1681                radeon_crtc = to_radeon_crtc(crtc);
1682                if (radeon_crtc->enabled) {
1683                        tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1684                        tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1685                        WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1686                }
1687        }
1688}
1689
1690/**
1691 * evergreen_pm_finish - post-power state change callback.
1692 *
1693 * @rdev: radeon_device pointer
1694 *
1695 * Clean up after a power state change (evergreen+).
1696 */
1697void evergreen_pm_finish(struct radeon_device *rdev)
1698{
1699        struct drm_device *ddev = rdev->ddev;
1700        struct drm_crtc *crtc;
1701        struct radeon_crtc *radeon_crtc;
1702        u32 tmp;
1703
1704        /* enable any active CRTCs */
1705        list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1706                radeon_crtc = to_radeon_crtc(crtc);
1707                if (radeon_crtc->enabled) {
1708                        tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1709                        tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1710                        WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1711                }
1712        }
1713}
1714
1715/**
1716 * evergreen_hpd_sense - hpd sense callback.
1717 *
1718 * @rdev: radeon_device pointer
1719 * @hpd: hpd (hotplug detect) pin
1720 *
1721 * Checks if a digital monitor is connected (evergreen+).
1722 * Returns true if connected, false if not connected.
1723 */
1724bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1725{
1726        if (hpd == RADEON_HPD_NONE)
1727                return false;
1728
1729        return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1730}
1731
1732/**
1733 * evergreen_hpd_set_polarity - hpd set polarity callback.
1734 *
1735 * @rdev: radeon_device pointer
1736 * @hpd: hpd (hotplug detect) pin
1737 *
1738 * Set the polarity of the hpd pin (evergreen+).
1739 */
1740void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1741                                enum radeon_hpd_id hpd)
1742{
1743        bool connected = evergreen_hpd_sense(rdev, hpd);
1744
1745        if (hpd == RADEON_HPD_NONE)
1746                return;
1747
1748        if (connected)
1749                WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1750        else
1751                WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1752}
1753
1754/**
1755 * evergreen_hpd_init - hpd setup callback.
1756 *
1757 * @rdev: radeon_device pointer
1758 *
1759 * Setup the hpd pins used by the card (evergreen+).
1760 * Enable the pin, set the polarity, and enable the hpd interrupts.
1761 */
1762void evergreen_hpd_init(struct radeon_device *rdev)
1763{
1764        struct drm_device *dev = rdev->ddev;
1765        struct drm_connector *connector;
1766        unsigned enabled = 0;
1767        u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1768                DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1769
1770        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1771                enum radeon_hpd_id hpd =
1772                        to_radeon_connector(connector)->hpd.hpd;
1773
1774                if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1775                    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1776                        /* don't try to enable hpd on eDP or LVDS avoid breaking the
1777                         * aux dp channel on imac and help (but not completely fix)
1778                         * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1779                         * also avoid interrupt storms during dpms.
1780                         */
1781                        continue;
1782                }
1783
1784                if (hpd == RADEON_HPD_NONE)
1785                        continue;
1786
1787                WREG32(DC_HPDx_CONTROL(hpd), tmp);
1788                enabled |= 1 << hpd;
1789
1790                radeon_hpd_set_polarity(rdev, hpd);
1791        }
1792        radeon_irq_kms_enable_hpd(rdev, enabled);
1793}
1794
1795/**
1796 * evergreen_hpd_fini - hpd tear down callback.
1797 *
1798 * @rdev: radeon_device pointer
1799 *
1800 * Tear down the hpd pins used by the card (evergreen+).
1801 * Disable the hpd interrupts.
1802 */
1803void evergreen_hpd_fini(struct radeon_device *rdev)
1804{
1805        struct drm_device *dev = rdev->ddev;
1806        struct drm_connector *connector;
1807        unsigned disabled = 0;
1808
1809        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1810                enum radeon_hpd_id hpd =
1811                        to_radeon_connector(connector)->hpd.hpd;
1812
1813                if (hpd == RADEON_HPD_NONE)
1814                        continue;
1815
1816                WREG32(DC_HPDx_CONTROL(hpd), 0);
1817                disabled |= 1 << hpd;
1818        }
1819        radeon_irq_kms_disable_hpd(rdev, disabled);
1820}
1821
1822/* watermark setup */
1823
1824static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1825                                        struct radeon_crtc *radeon_crtc,
1826                                        struct drm_display_mode *mode,
1827                                        struct drm_display_mode *other_mode)
1828{
1829        u32 tmp, buffer_alloc, i;
1830        u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1831        /*
1832         * Line Buffer Setup
1833         * There are 3 line buffers, each one shared by 2 display controllers.
1834         * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1835         * the display controllers.  The paritioning is done via one of four
1836         * preset allocations specified in bits 2:0:
1837         * first display controller
1838         *  0 - first half of lb (3840 * 2)
1839         *  1 - first 3/4 of lb (5760 * 2)
1840         *  2 - whole lb (7680 * 2), other crtc must be disabled
1841         *  3 - first 1/4 of lb (1920 * 2)
1842         * second display controller
1843         *  4 - second half of lb (3840 * 2)
1844         *  5 - second 3/4 of lb (5760 * 2)
1845         *  6 - whole lb (7680 * 2), other crtc must be disabled
1846         *  7 - last 1/4 of lb (1920 * 2)
1847         */
1848        /* this can get tricky if we have two large displays on a paired group
1849         * of crtcs.  Ideally for multiple large displays we'd assign them to
1850         * non-linked crtcs for maximum line buffer allocation.
1851         */
1852        if (radeon_crtc->base.enabled && mode) {
1853                if (other_mode) {
1854                        tmp = 0; /* 1/2 */
1855                        buffer_alloc = 1;
1856                } else {
1857                        tmp = 2; /* whole */
1858                        buffer_alloc = 2;
1859                }
1860        } else {
1861                tmp = 0;
1862                buffer_alloc = 0;
1863        }
1864
1865        /* second controller of the pair uses second half of the lb */
1866        if (radeon_crtc->crtc_id % 2)
1867                tmp += 4;
1868        WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1869
1870        if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1871                WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1872                       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1873                for (i = 0; i < rdev->usec_timeout; i++) {
1874                        if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1875                            DMIF_BUFFERS_ALLOCATED_COMPLETED)
1876                                break;
1877                        udelay(1);
1878                }
1879        }
1880
1881        if (radeon_crtc->base.enabled && mode) {
1882                switch (tmp) {
1883                case 0:
1884                case 4:
1885                default:
1886                        if (ASIC_IS_DCE5(rdev))
1887                                return 4096 * 2;
1888                        else
1889                                return 3840 * 2;
1890                case 1:
1891                case 5:
1892                        if (ASIC_IS_DCE5(rdev))
1893                                return 6144 * 2;
1894                        else
1895                                return 5760 * 2;
1896                case 2:
1897                case 6:
1898                        if (ASIC_IS_DCE5(rdev))
1899                                return 8192 * 2;
1900                        else
1901                                return 7680 * 2;
1902                case 3:
1903                case 7:
1904                        if (ASIC_IS_DCE5(rdev))
1905                                return 2048 * 2;
1906                        else
1907                                return 1920 * 2;
1908                }
1909        }
1910
1911        /* controller not enabled, so no lb used */
1912        return 0;
1913}
1914
1915u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1916{
1917        u32 tmp = RREG32(MC_SHARED_CHMAP);
1918
1919        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1920        case 0:
1921        default:
1922                return 1;
1923        case 1:
1924                return 2;
1925        case 2:
1926                return 4;
1927        case 3:
1928                return 8;
1929        }
1930}
1931
1932struct evergreen_wm_params {
1933        u32 dram_channels; /* number of dram channels */
1934        u32 yclk;          /* bandwidth per dram data pin in kHz */
1935        u32 sclk;          /* engine clock in kHz */
1936        u32 disp_clk;      /* display clock in kHz */
1937        u32 src_width;     /* viewport width */
1938        u32 active_time;   /* active display time in ns */
1939        u32 blank_time;    /* blank time in ns */
1940        bool interlaced;    /* mode is interlaced */
1941        fixed20_12 vsc;    /* vertical scale ratio */
1942        u32 num_heads;     /* number of active crtcs */
1943        u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1944        u32 lb_size;       /* line buffer allocated to pipe */
1945        u32 vtaps;         /* vertical scaler taps */
1946};
1947
1948static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1949{
1950        /* Calculate DRAM Bandwidth and the part allocated to display. */
1951        fixed20_12 dram_efficiency; /* 0.7 */
1952        fixed20_12 yclk, dram_channels, bandwidth;
1953        fixed20_12 a;
1954
1955        a.full = dfixed_const(1000);
1956        yclk.full = dfixed_const(wm->yclk);
1957        yclk.full = dfixed_div(yclk, a);
1958        dram_channels.full = dfixed_const(wm->dram_channels * 4);
1959        a.full = dfixed_const(10);
1960        dram_efficiency.full = dfixed_const(7);
1961        dram_efficiency.full = dfixed_div(dram_efficiency, a);
1962        bandwidth.full = dfixed_mul(dram_channels, yclk);
1963        bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1964
1965        return dfixed_trunc(bandwidth);
1966}
1967
1968static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1969{
1970        /* Calculate DRAM Bandwidth and the part allocated to display. */
1971        fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1972        fixed20_12 yclk, dram_channels, bandwidth;
1973        fixed20_12 a;
1974
1975        a.full = dfixed_const(1000);
1976        yclk.full = dfixed_const(wm->yclk);
1977        yclk.full = dfixed_div(yclk, a);
1978        dram_channels.full = dfixed_const(wm->dram_channels * 4);
1979        a.full = dfixed_const(10);
1980        disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1981        disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1982        bandwidth.full = dfixed_mul(dram_channels, yclk);
1983        bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1984
1985        return dfixed_trunc(bandwidth);
1986}
1987
1988static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1989{
1990        /* Calculate the display Data return Bandwidth */
1991        fixed20_12 return_efficiency; /* 0.8 */
1992        fixed20_12 sclk, bandwidth;
1993        fixed20_12 a;
1994
1995        a.full = dfixed_const(1000);
1996        sclk.full = dfixed_const(wm->sclk);
1997        sclk.full = dfixed_div(sclk, a);
1998        a.full = dfixed_const(10);
1999        return_efficiency.full = dfixed_const(8);
2000        return_efficiency.full = dfixed_div(return_efficiency, a);
2001        a.full = dfixed_const(32);
2002        bandwidth.full = dfixed_mul(a, sclk);
2003        bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2004
2005        return dfixed_trunc(bandwidth);
2006}
2007
2008static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2009{
2010        /* Calculate the DMIF Request Bandwidth */
2011        fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2012        fixed20_12 disp_clk, bandwidth;
2013        fixed20_12 a;
2014
2015        a.full = dfixed_const(1000);
2016        disp_clk.full = dfixed_const(wm->disp_clk);
2017        disp_clk.full = dfixed_div(disp_clk, a);
2018        a.full = dfixed_const(10);
2019        disp_clk_request_efficiency.full = dfixed_const(8);
2020        disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2021        a.full = dfixed_const(32);
2022        bandwidth.full = dfixed_mul(a, disp_clk);
2023        bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2024
2025        return dfixed_trunc(bandwidth);
2026}
2027
2028static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2029{
2030        /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2031        u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2032        u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2033        u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2034
2035        return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2036}
2037
2038static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2039{
2040        /* Calculate the display mode Average Bandwidth
2041         * DisplayMode should contain the source and destination dimensions,
2042         * timing, etc.
2043         */
2044        fixed20_12 bpp;
2045        fixed20_12 line_time;
2046        fixed20_12 src_width;
2047        fixed20_12 bandwidth;
2048        fixed20_12 a;
2049
2050        a.full = dfixed_const(1000);
2051        line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2052        line_time.full = dfixed_div(line_time, a);
2053        bpp.full = dfixed_const(wm->bytes_per_pixel);
2054        src_width.full = dfixed_const(wm->src_width);
2055        bandwidth.full = dfixed_mul(src_width, bpp);
2056        bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2057        bandwidth.full = dfixed_div(bandwidth, line_time);
2058
2059        return dfixed_trunc(bandwidth);
2060}
2061
2062static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2063{
2064        /* First calcualte the latency in ns */
2065        u32 mc_latency = 2000; /* 2000 ns. */
2066        u32 available_bandwidth = evergreen_available_bandwidth(wm);
2067        u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2068        u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2069        u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2070        u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2071                (wm->num_heads * cursor_line_pair_return_time);
2072        u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2073        u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2074        fixed20_12 a, b, c;
2075
2076        if (wm->num_heads == 0)
2077                return 0;
2078
2079        a.full = dfixed_const(2);
2080        b.full = dfixed_const(1);
2081        if ((wm->vsc.full > a.full) ||
2082            ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2083            (wm->vtaps >= 5) ||
2084            ((wm->vsc.full >= a.full) && wm->interlaced))
2085                max_src_lines_per_dst_line = 4;
2086        else
2087                max_src_lines_per_dst_line = 2;
2088
2089        a.full = dfixed_const(available_bandwidth);
2090        b.full = dfixed_const(wm->num_heads);
2091        a.full = dfixed_div(a, b);
2092
2093        lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2094
2095        a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2096        b.full = dfixed_const(1000);
2097        c.full = dfixed_const(lb_fill_bw);
2098        b.full = dfixed_div(c, b);
2099        a.full = dfixed_div(a, b);
2100        line_fill_time = dfixed_trunc(a);
2101
2102        if (line_fill_time < wm->active_time)
2103                return latency;
2104        else
2105                return latency + (line_fill_time - wm->active_time);
2106
2107}
2108
2109static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2110{
2111        if (evergreen_average_bandwidth(wm) <=
2112            (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2113                return true;
2114        else
2115                return false;
2116};
2117
2118static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2119{
2120        if (evergreen_average_bandwidth(wm) <=
2121            (evergreen_available_bandwidth(wm) / wm->num_heads))
2122                return true;
2123        else
2124                return false;
2125};
2126
2127static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2128{
2129        u32 lb_partitions = wm->lb_size / wm->src_width;
2130        u32 line_time = wm->active_time + wm->blank_time;
2131        u32 latency_tolerant_lines;
2132        u32 latency_hiding;
2133        fixed20_12 a;
2134
2135        a.full = dfixed_const(1);
2136        if (wm->vsc.full > a.full)
2137                latency_tolerant_lines = 1;
2138        else {
2139                if (lb_partitions <= (wm->vtaps + 1))
2140                        latency_tolerant_lines = 1;
2141                else
2142                        latency_tolerant_lines = 2;
2143        }
2144
2145        latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2146
2147        if (evergreen_latency_watermark(wm) <= latency_hiding)
2148                return true;
2149        else
2150                return false;
2151}
2152
2153static void evergreen_program_watermarks(struct radeon_device *rdev,
2154                                         struct radeon_crtc *radeon_crtc,
2155                                         u32 lb_size, u32 num_heads)
2156{
2157        struct drm_display_mode *mode = &radeon_crtc->base.mode;
2158        struct evergreen_wm_params wm_low, wm_high;
2159        u32 dram_channels;
2160        u32 active_time;
2161        u32 line_time = 0;
2162        u32 latency_watermark_a = 0, latency_watermark_b = 0;
2163        u32 priority_a_mark = 0, priority_b_mark = 0;
2164        u32 priority_a_cnt = PRIORITY_OFF;
2165        u32 priority_b_cnt = PRIORITY_OFF;
2166        u32 pipe_offset = radeon_crtc->crtc_id * 16;
2167        u32 tmp, arb_control3;
2168        fixed20_12 a, b, c;
2169
2170        if (radeon_crtc->base.enabled && num_heads && mode) {
2171                active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2172                                            (u32)mode->clock);
2173                line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2174                                          (u32)mode->clock);
2175                line_time = min(line_time, (u32)65535);
2176                priority_a_cnt = 0;
2177                priority_b_cnt = 0;
2178                dram_channels = evergreen_get_number_of_dram_channels(rdev);
2179
2180                /* watermark for high clocks */
2181                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2182                        wm_high.yclk =
2183                                radeon_dpm_get_mclk(rdev, false) * 10;
2184                        wm_high.sclk =
2185                                radeon_dpm_get_sclk(rdev, false) * 10;
2186                } else {
2187                        wm_high.yclk = rdev->pm.current_mclk * 10;
2188                        wm_high.sclk = rdev->pm.current_sclk * 10;
2189                }
2190
2191                wm_high.disp_clk = mode->clock;
2192                wm_high.src_width = mode->crtc_hdisplay;
2193                wm_high.active_time = active_time;
2194                wm_high.blank_time = line_time - wm_high.active_time;
2195                wm_high.interlaced = false;
2196                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2197                        wm_high.interlaced = true;
2198                wm_high.vsc = radeon_crtc->vsc;
2199                wm_high.vtaps = 1;
2200                if (radeon_crtc->rmx_type != RMX_OFF)
2201                        wm_high.vtaps = 2;
2202                wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2203                wm_high.lb_size = lb_size;
2204                wm_high.dram_channels = dram_channels;
2205                wm_high.num_heads = num_heads;
2206
2207                /* watermark for low clocks */
2208                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2209                        wm_low.yclk =
2210                                radeon_dpm_get_mclk(rdev, true) * 10;
2211                        wm_low.sclk =
2212                                radeon_dpm_get_sclk(rdev, true) * 10;
2213                } else {
2214                        wm_low.yclk = rdev->pm.current_mclk * 10;
2215                        wm_low.sclk = rdev->pm.current_sclk * 10;
2216                }
2217
2218                wm_low.disp_clk = mode->clock;
2219                wm_low.src_width = mode->crtc_hdisplay;
2220                wm_low.active_time = active_time;
2221                wm_low.blank_time = line_time - wm_low.active_time;
2222                wm_low.interlaced = false;
2223                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2224                        wm_low.interlaced = true;
2225                wm_low.vsc = radeon_crtc->vsc;
2226                wm_low.vtaps = 1;
2227                if (radeon_crtc->rmx_type != RMX_OFF)
2228                        wm_low.vtaps = 2;
2229                wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2230                wm_low.lb_size = lb_size;
2231                wm_low.dram_channels = dram_channels;
2232                wm_low.num_heads = num_heads;
2233
2234                /* set for high clocks */
2235                latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2236                /* set for low clocks */
2237                latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2238
2239                /* possibly force display priority to high */
2240                /* should really do this at mode validation time... */
2241                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2242                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2243                    !evergreen_check_latency_hiding(&wm_high) ||
2244                    (rdev->disp_priority == 2)) {
2245                        DRM_DEBUG_KMS("force priority a to high\n");
2246                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
2247                }
2248                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2249                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2250                    !evergreen_check_latency_hiding(&wm_low) ||
2251                    (rdev->disp_priority == 2)) {
2252                        DRM_DEBUG_KMS("force priority b to high\n");
2253                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
2254                }
2255
2256                a.full = dfixed_const(1000);
2257                b.full = dfixed_const(mode->clock);
2258                b.full = dfixed_div(b, a);
2259                c.full = dfixed_const(latency_watermark_a);
2260                c.full = dfixed_mul(c, b);
2261                c.full = dfixed_mul(c, radeon_crtc->hsc);
2262                c.full = dfixed_div(c, a);
2263                a.full = dfixed_const(16);
2264                c.full = dfixed_div(c, a);
2265                priority_a_mark = dfixed_trunc(c);
2266                priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2267
2268                a.full = dfixed_const(1000);
2269                b.full = dfixed_const(mode->clock);
2270                b.full = dfixed_div(b, a);
2271                c.full = dfixed_const(latency_watermark_b);
2272                c.full = dfixed_mul(c, b);
2273                c.full = dfixed_mul(c, radeon_crtc->hsc);
2274                c.full = dfixed_div(c, a);
2275                a.full = dfixed_const(16);
2276                c.full = dfixed_div(c, a);
2277                priority_b_mark = dfixed_trunc(c);
2278                priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2279
2280                /* Save number of lines the linebuffer leads before the scanout */
2281                radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2282        }
2283
2284        /* select wm A */
2285        arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2286        tmp = arb_control3;
2287        tmp &= ~LATENCY_WATERMARK_MASK(3);
2288        tmp |= LATENCY_WATERMARK_MASK(1);
2289        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2290        WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2291               (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2292                LATENCY_HIGH_WATERMARK(line_time)));
2293        /* select wm B */
2294        tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2295        tmp &= ~LATENCY_WATERMARK_MASK(3);
2296        tmp |= LATENCY_WATERMARK_MASK(2);
2297        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2298        WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2299               (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2300                LATENCY_HIGH_WATERMARK(line_time)));
2301        /* restore original selection */
2302        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2303
2304        /* write the priority marks */
2305        WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2306        WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2307
2308        /* save values for DPM */
2309        radeon_crtc->line_time = line_time;
2310        radeon_crtc->wm_high = latency_watermark_a;
2311        radeon_crtc->wm_low = latency_watermark_b;
2312}
2313
2314/**
2315 * evergreen_bandwidth_update - update display watermarks callback.
2316 *
2317 * @rdev: radeon_device pointer
2318 *
2319 * Update the display watermarks based on the requested mode(s)
2320 * (evergreen+).
2321 */
2322void evergreen_bandwidth_update(struct radeon_device *rdev)
2323{
2324        struct drm_display_mode *mode0 = NULL;
2325        struct drm_display_mode *mode1 = NULL;
2326        u32 num_heads = 0, lb_size;
2327        int i;
2328
2329        if (!rdev->mode_info.mode_config_initialized)
2330                return;
2331
2332        radeon_update_display_priority(rdev);
2333
2334        for (i = 0; i < rdev->num_crtc; i++) {
2335                if (rdev->mode_info.crtcs[i]->base.enabled)
2336                        num_heads++;
2337        }
2338        for (i = 0; i < rdev->num_crtc; i += 2) {
2339                mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2340                mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2341                lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2342                evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2343                lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2344                evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2345        }
2346}
2347
2348/**
2349 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2350 *
2351 * @rdev: radeon_device pointer
2352 *
2353 * Wait for the MC (memory controller) to be idle.
2354 * (evergreen+).
2355 * Returns 0 if the MC is idle, -1 if not.
2356 */
2357int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2358{
2359        unsigned i;
2360        u32 tmp;
2361
2362        for (i = 0; i < rdev->usec_timeout; i++) {
2363                /* read MC_STATUS */
2364                tmp = RREG32(SRBM_STATUS) & 0x1F00;
2365                if (!tmp)
2366                        return 0;
2367                udelay(1);
2368        }
2369        return -1;
2370}
2371
2372/*
2373 * GART
2374 */
2375void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2376{
2377        unsigned i;
2378        u32 tmp;
2379
2380        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2381
2382        WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2383        for (i = 0; i < rdev->usec_timeout; i++) {
2384                /* read MC_STATUS */
2385                tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2386                tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2387                if (tmp == 2) {
2388                        pr_warn("[drm] r600 flush TLB failed\n");
2389                        return;
2390                }
2391                if (tmp) {
2392                        return;
2393                }
2394                udelay(1);
2395        }
2396}
2397
2398static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2399{
2400        u32 tmp;
2401        int r;
2402
2403        if (rdev->gart.robj == NULL) {
2404                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2405                return -EINVAL;
2406        }
2407        r = radeon_gart_table_vram_pin(rdev);
2408        if (r)
2409                return r;
2410        /* Setup L2 cache */
2411        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2412                                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2413                                EFFECTIVE_L2_QUEUE_SIZE(7));
2414        WREG32(VM_L2_CNTL2, 0);
2415        WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2416        /* Setup TLB control */
2417        tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2418                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2419                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2420                EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2421        if (rdev->flags & RADEON_IS_IGP) {
2422                WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2423                WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2424                WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2425        } else {
2426                WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2427                WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2428                WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2429                if ((rdev->family == CHIP_JUNIPER) ||
2430                    (rdev->family == CHIP_CYPRESS) ||
2431                    (rdev->family == CHIP_HEMLOCK) ||
2432                    (rdev->family == CHIP_BARTS))
2433                        WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2434        }
2435        WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2436        WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2437        WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2438        WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2439        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2440        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2441        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2442        WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2443                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2444        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2445                        (u32)(rdev->dummy_page.addr >> 12));
2446        WREG32(VM_CONTEXT1_CNTL, 0);
2447
2448        evergreen_pcie_gart_tlb_flush(rdev);
2449        DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2450                 (unsigned)(rdev->mc.gtt_size >> 20),
2451                 (unsigned long long)rdev->gart.table_addr);
2452        rdev->gart.ready = true;
2453        return 0;
2454}
2455
2456static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2457{
2458        u32 tmp;
2459
2460        /* Disable all tables */
2461        WREG32(VM_CONTEXT0_CNTL, 0);
2462        WREG32(VM_CONTEXT1_CNTL, 0);
2463
2464        /* Setup L2 cache */
2465        WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2466                                EFFECTIVE_L2_QUEUE_SIZE(7));
2467        WREG32(VM_L2_CNTL2, 0);
2468        WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2469        /* Setup TLB control */
2470        tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2471        WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2472        WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2473        WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2474        WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2475        WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2476        WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2477        WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2478        radeon_gart_table_vram_unpin(rdev);
2479}
2480
2481static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2482{
2483        evergreen_pcie_gart_disable(rdev);
2484        radeon_gart_table_vram_free(rdev);
2485        radeon_gart_fini(rdev);
2486}
2487
2488
2489static void evergreen_agp_enable(struct radeon_device *rdev)
2490{
2491        u32 tmp;
2492
2493        /* Setup L2 cache */
2494        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2495                                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2496                                EFFECTIVE_L2_QUEUE_SIZE(7));
2497        WREG32(VM_L2_CNTL2, 0);
2498        WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2499        /* Setup TLB control */
2500        tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2501                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2502                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2503                EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2504        WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2505        WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2506        WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2507        WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2508        WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2509        WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2510        WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2511        WREG32(VM_CONTEXT0_CNTL, 0);
2512        WREG32(VM_CONTEXT1_CNTL, 0);
2513}
2514
2515static const unsigned ni_dig_offsets[] =
2516{
2517        NI_DIG0_REGISTER_OFFSET,
2518        NI_DIG1_REGISTER_OFFSET,
2519        NI_DIG2_REGISTER_OFFSET,
2520        NI_DIG3_REGISTER_OFFSET,
2521        NI_DIG4_REGISTER_OFFSET,
2522        NI_DIG5_REGISTER_OFFSET
2523};
2524
2525static const unsigned ni_tx_offsets[] =
2526{
2527        NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2528        NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2529        NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2530        NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2531        NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2532        NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2533};
2534
2535static const unsigned evergreen_dp_offsets[] =
2536{
2537        EVERGREEN_DP0_REGISTER_OFFSET,
2538        EVERGREEN_DP1_REGISTER_OFFSET,
2539        EVERGREEN_DP2_REGISTER_OFFSET,
2540        EVERGREEN_DP3_REGISTER_OFFSET,
2541        EVERGREEN_DP4_REGISTER_OFFSET,
2542        EVERGREEN_DP5_REGISTER_OFFSET
2543};
2544
2545static const unsigned evergreen_disp_int_status[] =
2546{
2547        DISP_INTERRUPT_STATUS,
2548        DISP_INTERRUPT_STATUS_CONTINUE,
2549        DISP_INTERRUPT_STATUS_CONTINUE2,
2550        DISP_INTERRUPT_STATUS_CONTINUE3,
2551        DISP_INTERRUPT_STATUS_CONTINUE4,
2552        DISP_INTERRUPT_STATUS_CONTINUE5
2553};
2554
2555/*
2556 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2557 * We go from crtc to connector and it is not relible  since it
2558 * should be an opposite direction .If crtc is enable then
2559 * find the dig_fe which selects this crtc and insure that it enable.
2560 * if such dig_fe is found then find dig_be which selects found dig_be and
2561 * insure that it enable and in DP_SST mode.
2562 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2563 * from dp symbols clocks .
2564 */
2565static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2566                                               unsigned crtc_id, unsigned *ret_dig_fe)
2567{
2568        unsigned i;
2569        unsigned dig_fe;
2570        unsigned dig_be;
2571        unsigned dig_en_be;
2572        unsigned uniphy_pll;
2573        unsigned digs_fe_selected;
2574        unsigned dig_be_mode;
2575        unsigned dig_fe_mask;
2576        bool is_enabled = false;
2577        bool found_crtc = false;
2578
2579        /* loop through all running dig_fe to find selected crtc */
2580        for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2581                dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2582                if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2583                    crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2584                        /* found running pipe */
2585                        found_crtc = true;
2586                        dig_fe_mask = 1 << i;
2587                        dig_fe = i;
2588                        break;
2589                }
2590        }
2591
2592        if (found_crtc) {
2593                /* loop through all running dig_be to find selected dig_fe */
2594                for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2595                        dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2596                        /* if dig_fe_selected by dig_be? */
2597                        digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2598                        dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2599                        if (dig_fe_mask &  digs_fe_selected &&
2600                            /* if dig_be in sst mode? */
2601                            dig_be_mode == NI_DIG_BE_DPSST) {
2602                                dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2603                                                   ni_dig_offsets[i]);
2604                                uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2605                                                    ni_tx_offsets[i]);
2606                                /* dig_be enable and tx is running */
2607                                if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2608                                    dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2609                                    uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2610                                        is_enabled = true;
2611                                        *ret_dig_fe = dig_fe;
2612                                        break;
2613                                }
2614                        }
2615                }
2616        }
2617
2618        return is_enabled;
2619}
2620
2621/*
2622 * Blank dig when in dp sst mode
2623 * Dig ignores crtc timing
2624 */
2625static void evergreen_blank_dp_output(struct radeon_device *rdev,
2626                                      unsigned dig_fe)
2627{
2628        unsigned stream_ctrl;
2629        unsigned fifo_ctrl;
2630        unsigned counter = 0;
2631
2632        if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2633                DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2634                return;
2635        }
2636
2637        stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2638                             evergreen_dp_offsets[dig_fe]);
2639        if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2640                DRM_ERROR("dig %d , should be enable\n", dig_fe);
2641                return;
2642        }
2643
2644        stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2645        WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2646               evergreen_dp_offsets[dig_fe], stream_ctrl);
2647
2648        stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2649                             evergreen_dp_offsets[dig_fe]);
2650        while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2651                msleep(1);
2652                counter++;
2653                stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2654                                     evergreen_dp_offsets[dig_fe]);
2655        }
2656        if (counter >= 32 )
2657                DRM_ERROR("counter exceeds %d\n", counter);
2658
2659        fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2660        fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2661        WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2662
2663}
2664
2665void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2666{
2667        u32 crtc_enabled, tmp, frame_count, blackout;
2668        int i, j;
2669        unsigned dig_fe;
2670
2671        if (!ASIC_IS_NODCE(rdev)) {
2672                save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2673                save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2674
2675                /* disable VGA render */
2676                WREG32(VGA_RENDER_CONTROL, 0);
2677        }
2678        /* blank the display controllers */
2679        for (i = 0; i < rdev->num_crtc; i++) {
2680                crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2681                if (crtc_enabled) {
2682                        save->crtc_enabled[i] = true;
2683                        if (ASIC_IS_DCE6(rdev)) {
2684                                tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2685                                if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2686                                        radeon_wait_for_vblank(rdev, i);
2687                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2688                                        tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2689                                        WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2690                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2691                                }
2692                        } else {
2693                                tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2694                                if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2695                                        radeon_wait_for_vblank(rdev, i);
2696                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2697                                        tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2698                                        WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2699                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2700                                }
2701                        }
2702                        /* wait for the next frame */
2703                        frame_count = radeon_get_vblank_counter(rdev, i);
2704                        for (j = 0; j < rdev->usec_timeout; j++) {
2705                                if (radeon_get_vblank_counter(rdev, i) != frame_count)
2706                                        break;
2707                                udelay(1);
2708                        }
2709                        /*we should disable dig if it drives dp sst*/
2710                        /*but we are in radeon_device_init and the topology is unknown*/
2711                        /*and it is available after radeon_modeset_init*/
2712                        /*the following method radeon_atom_encoder_dpms_dig*/
2713                        /*does the job if we initialize it properly*/
2714                        /*for now we do it this manually*/
2715                        /**/
2716                        if (ASIC_IS_DCE5(rdev) &&
2717                            evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2718                                evergreen_blank_dp_output(rdev, dig_fe);
2719                        /*we could remove 6 lines below*/
2720                        /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2721                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2722                        tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2723                        tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2724                        WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2725                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2726                        save->crtc_enabled[i] = false;
2727                        /* ***** */
2728                } else {
2729                        save->crtc_enabled[i] = false;
2730                }
2731        }
2732
2733        radeon_mc_wait_for_idle(rdev);
2734
2735        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2736        if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2737                /* Block CPU access */
2738                WREG32(BIF_FB_EN, 0);
2739                /* blackout the MC */
2740                blackout &= ~BLACKOUT_MODE_MASK;
2741                WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2742        }
2743        /* wait for the MC to settle */
2744        udelay(100);
2745
2746        /* lock double buffered regs */
2747        for (i = 0; i < rdev->num_crtc; i++) {
2748                if (save->crtc_enabled[i]) {
2749                        tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2750                        if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2751                                tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2752                                WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2753                        }
2754                        tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2755                        if (!(tmp & 1)) {
2756                                tmp |= 1;
2757                                WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2758                        }
2759                }
2760        }
2761}
2762
2763void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2764{
2765        u32 tmp, frame_count;
2766        int i, j;
2767
2768        /* update crtc base addresses */
2769        for (i = 0; i < rdev->num_crtc; i++) {
2770                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2771                       upper_32_bits(rdev->mc.vram_start));
2772                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2773                       upper_32_bits(rdev->mc.vram_start));
2774                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2775                       (u32)rdev->mc.vram_start);
2776                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2777                       (u32)rdev->mc.vram_start);
2778        }
2779
2780        if (!ASIC_IS_NODCE(rdev)) {
2781                WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2782                WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2783        }
2784
2785        /* unlock regs and wait for update */
2786        for (i = 0; i < rdev->num_crtc; i++) {
2787                if (save->crtc_enabled[i]) {
2788                        tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2789                        if ((tmp & 0x7) != 0) {
2790                                tmp &= ~0x7;
2791                                WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2792                        }
2793                        tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2794                        if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2795                                tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2796                                WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2797                        }
2798                        tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2799                        if (tmp & 1) {
2800                                tmp &= ~1;
2801                                WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2802                        }
2803                        for (j = 0; j < rdev->usec_timeout; j++) {
2804                                tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2805                                if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2806                                        break;
2807                                udelay(1);
2808                        }
2809                }
2810        }
2811
2812        /* unblackout the MC */
2813        tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2814        tmp &= ~BLACKOUT_MODE_MASK;
2815        WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2816        /* allow CPU access */
2817        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2818
2819        for (i = 0; i < rdev->num_crtc; i++) {
2820                if (save->crtc_enabled[i]) {
2821                        if (ASIC_IS_DCE6(rdev)) {
2822                                tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2823                                tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2824                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2825                                WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2826                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2827                        } else {
2828                                tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2829                                tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2830                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2831                                WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2832                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2833                        }
2834                        /* wait for the next frame */
2835                        frame_count = radeon_get_vblank_counter(rdev, i);
2836                        for (j = 0; j < rdev->usec_timeout; j++) {
2837                                if (radeon_get_vblank_counter(rdev, i) != frame_count)
2838                                        break;
2839                                udelay(1);
2840                        }
2841                }
2842        }
2843        if (!ASIC_IS_NODCE(rdev)) {
2844                /* Unlock vga access */
2845                WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2846                mdelay(1);
2847                WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2848        }
2849}
2850
2851void evergreen_mc_program(struct radeon_device *rdev)
2852{
2853        struct evergreen_mc_save save;
2854        u32 tmp;
2855        int i, j;
2856
2857        /* Initialize HDP */
2858        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2859                WREG32((0x2c14 + j), 0x00000000);
2860                WREG32((0x2c18 + j), 0x00000000);
2861                WREG32((0x2c1c + j), 0x00000000);
2862                WREG32((0x2c20 + j), 0x00000000);
2863                WREG32((0x2c24 + j), 0x00000000);
2864        }
2865        WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2866
2867        evergreen_mc_stop(rdev, &save);
2868        if (evergreen_mc_wait_for_idle(rdev)) {
2869                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2870        }
2871        /* Lockout access through VGA aperture*/
2872        WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2873        /* Update configuration */
2874        if (rdev->flags & RADEON_IS_AGP) {
2875                if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2876                        /* VRAM before AGP */
2877                        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2878                                rdev->mc.vram_start >> 12);
2879                        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2880                                rdev->mc.gtt_end >> 12);
2881                } else {
2882                        /* VRAM after AGP */
2883                        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2884                                rdev->mc.gtt_start >> 12);
2885                        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2886                                rdev->mc.vram_end >> 12);
2887                }
2888        } else {
2889                WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2890                        rdev->mc.vram_start >> 12);
2891                WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2892                        rdev->mc.vram_end >> 12);
2893        }
2894        WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2895        /* llano/ontario only */
2896        if ((rdev->family == CHIP_PALM) ||
2897            (rdev->family == CHIP_SUMO) ||
2898            (rdev->family == CHIP_SUMO2)) {
2899                tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2900                tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2901                tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2902                WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2903        }
2904        tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2905        tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2906        WREG32(MC_VM_FB_LOCATION, tmp);
2907        WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2908        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2909        WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2910        if (rdev->flags & RADEON_IS_AGP) {
2911                WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2912                WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2913                WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2914        } else {
2915                WREG32(MC_VM_AGP_BASE, 0);
2916                WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2917                WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2918        }
2919        if (evergreen_mc_wait_for_idle(rdev)) {
2920                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2921        }
2922        evergreen_mc_resume(rdev, &save);
2923        /* we need to own VRAM, so turn off the VGA renderer here
2924         * to stop it overwriting our objects */
2925        rv515_vga_render_disable(rdev);
2926}
2927
2928/*
2929 * CP.
2930 */
2931void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2932{
2933        struct radeon_ring *ring = &rdev->ring[ib->ring];
2934        u32 next_rptr;
2935
2936        /* set to DX10/11 mode */
2937        radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2938        radeon_ring_write(ring, 1);
2939
2940        if (ring->rptr_save_reg) {
2941                next_rptr = ring->wptr + 3 + 4;
2942                radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2943                radeon_ring_write(ring, ((ring->rptr_save_reg - 
2944                                          PACKET3_SET_CONFIG_REG_START) >> 2));
2945                radeon_ring_write(ring, next_rptr);
2946        } else if (rdev->wb.enabled) {
2947                next_rptr = ring->wptr + 5 + 4;
2948                radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2949                radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2950                radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2951                radeon_ring_write(ring, next_rptr);
2952                radeon_ring_write(ring, 0);
2953        }
2954
2955        radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2956        radeon_ring_write(ring,
2957#ifdef __BIG_ENDIAN
2958                          (2 << 0) |
2959#endif
2960                          (ib->gpu_addr & 0xFFFFFFFC));
2961        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2962        radeon_ring_write(ring, ib->length_dw);
2963}
2964
2965
2966static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2967{
2968        const __be32 *fw_data;
2969        int i;
2970
2971        if (!rdev->me_fw || !rdev->pfp_fw)
2972                return -EINVAL;
2973
2974        r700_cp_stop(rdev);
2975        WREG32(CP_RB_CNTL,
2976#ifdef __BIG_ENDIAN
2977               BUF_SWAP_32BIT |
2978#endif
2979               RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2980
2981        fw_data = (const __be32 *)rdev->pfp_fw->data;
2982        WREG32(CP_PFP_UCODE_ADDR, 0);
2983        for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2984                WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2985        WREG32(CP_PFP_UCODE_ADDR, 0);
2986
2987        fw_data = (const __be32 *)rdev->me_fw->data;
2988        WREG32(CP_ME_RAM_WADDR, 0);
2989        for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2990                WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2991
2992        WREG32(CP_PFP_UCODE_ADDR, 0);
2993        WREG32(CP_ME_RAM_WADDR, 0);
2994        WREG32(CP_ME_RAM_RADDR, 0);
2995        return 0;
2996}
2997
2998static int evergreen_cp_start(struct radeon_device *rdev)
2999{
3000        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3001        int r, i;
3002        uint32_t cp_me;
3003
3004        r = radeon_ring_lock(rdev, ring, 7);
3005        if (r) {
3006                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3007                return r;
3008        }
3009        radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3010        radeon_ring_write(ring, 0x1);
3011        radeon_ring_write(ring, 0x0);
3012        radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3013        radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3014        radeon_ring_write(ring, 0);
3015        radeon_ring_write(ring, 0);
3016        radeon_ring_unlock_commit(rdev, ring, false);
3017
3018        cp_me = 0xff;
3019        WREG32(CP_ME_CNTL, cp_me);
3020
3021        r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3022        if (r) {
3023                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3024                return r;
3025        }
3026
3027        /* setup clear context state */
3028        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3029        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3030
3031        for (i = 0; i < evergreen_default_size; i++)
3032                radeon_ring_write(ring, evergreen_default_state[i]);
3033
3034        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3035        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3036
3037        /* set clear context state */
3038        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3039        radeon_ring_write(ring, 0);
3040
3041        /* SQ_VTX_BASE_VTX_LOC */
3042        radeon_ring_write(ring, 0xc0026f00);
3043        radeon_ring_write(ring, 0x00000000);
3044        radeon_ring_write(ring, 0x00000000);
3045        radeon_ring_write(ring, 0x00000000);
3046
3047        /* Clear consts */
3048        radeon_ring_write(ring, 0xc0036f00);
3049        radeon_ring_write(ring, 0x00000bc4);
3050        radeon_ring_write(ring, 0xffffffff);
3051        radeon_ring_write(ring, 0xffffffff);
3052        radeon_ring_write(ring, 0xffffffff);
3053
3054        radeon_ring_write(ring, 0xc0026900);
3055        radeon_ring_write(ring, 0x00000316);
3056        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3057        radeon_ring_write(ring, 0x00000010); /*  */
3058
3059        radeon_ring_unlock_commit(rdev, ring, false);
3060
3061        return 0;
3062}
3063
3064static int evergreen_cp_resume(struct radeon_device *rdev)
3065{
3066        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3067        u32 tmp;
3068        u32 rb_bufsz;
3069        int r;
3070
3071        /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3072        WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3073                                 SOFT_RESET_PA |
3074                                 SOFT_RESET_SH |
3075                                 SOFT_RESET_VGT |
3076                                 SOFT_RESET_SPI |
3077                                 SOFT_RESET_SX));
3078        RREG32(GRBM_SOFT_RESET);
3079        mdelay(15);
3080        WREG32(GRBM_SOFT_RESET, 0);
3081        RREG32(GRBM_SOFT_RESET);
3082
3083        /* Set ring buffer size */
3084        rb_bufsz = order_base_2(ring->ring_size / 8);
3085        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3086#ifdef __BIG_ENDIAN
3087        tmp |= BUF_SWAP_32BIT;
3088#endif
3089        WREG32(CP_RB_CNTL, tmp);
3090        WREG32(CP_SEM_WAIT_TIMER, 0x0);
3091        WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3092
3093        /* Set the write pointer delay */
3094        WREG32(CP_RB_WPTR_DELAY, 0);
3095
3096        /* Initialize the ring buffer's read and write pointers */
3097        WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3098        WREG32(CP_RB_RPTR_WR, 0);
3099        ring->wptr = 0;
3100        WREG32(CP_RB_WPTR, ring->wptr);
3101
3102        /* set the wb address whether it's enabled or not */
3103        WREG32(CP_RB_RPTR_ADDR,
3104               ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3105        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3106        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3107
3108        if (rdev->wb.enabled)
3109                WREG32(SCRATCH_UMSK, 0xff);
3110        else {
3111                tmp |= RB_NO_UPDATE;
3112                WREG32(SCRATCH_UMSK, 0);
3113        }
3114
3115        mdelay(1);
3116        WREG32(CP_RB_CNTL, tmp);
3117
3118        WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3119        WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3120
3121        evergreen_cp_start(rdev);
3122        ring->ready = true;
3123        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3124        if (r) {
3125                ring->ready = false;
3126                return r;
3127        }
3128        return 0;
3129}
3130
3131/*
3132 * Core functions
3133 */
3134static void evergreen_gpu_init(struct radeon_device *rdev)
3135{
3136        u32 gb_addr_config;
3137        u32 mc_arb_ramcfg;
3138        u32 sx_debug_1;
3139        u32 smx_dc_ctl0;
3140        u32 sq_config;
3141        u32 sq_lds_resource_mgmt;
3142        u32 sq_gpr_resource_mgmt_1;
3143        u32 sq_gpr_resource_mgmt_2;
3144        u32 sq_gpr_resource_mgmt_3;
3145        u32 sq_thread_resource_mgmt;
3146        u32 sq_thread_resource_mgmt_2;
3147        u32 sq_stack_resource_mgmt_1;
3148        u32 sq_stack_resource_mgmt_2;
3149        u32 sq_stack_resource_mgmt_3;
3150        u32 vgt_cache_invalidation;
3151        u32 hdp_host_path_cntl, tmp;
3152        u32 disabled_rb_mask;
3153        int i, j, ps_thread_count;
3154
3155        switch (rdev->family) {
3156        case CHIP_CYPRESS:
3157        case CHIP_HEMLOCK:
3158                rdev->config.evergreen.num_ses = 2;
3159                rdev->config.evergreen.max_pipes = 4;
3160                rdev->config.evergreen.max_tile_pipes = 8;
3161                rdev->config.evergreen.max_simds = 10;
3162                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3163                rdev->config.evergreen.max_gprs = 256;
3164                rdev->config.evergreen.max_threads = 248;
3165                rdev->config.evergreen.max_gs_threads = 32;
3166                rdev->config.evergreen.max_stack_entries = 512;
3167                rdev->config.evergreen.sx_num_of_sets = 4;
3168                rdev->config.evergreen.sx_max_export_size = 256;
3169                rdev->config.evergreen.sx_max_export_pos_size = 64;
3170                rdev->config.evergreen.sx_max_export_smx_size = 192;
3171                rdev->config.evergreen.max_hw_contexts = 8;
3172                rdev->config.evergreen.sq_num_cf_insts = 2;
3173
3174                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3175                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3177                gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3178                break;
3179        case CHIP_JUNIPER:
3180                rdev->config.evergreen.num_ses = 1;
3181                rdev->config.evergreen.max_pipes = 4;
3182                rdev->config.evergreen.max_tile_pipes = 4;
3183                rdev->config.evergreen.max_simds = 10;
3184                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3185                rdev->config.evergreen.max_gprs = 256;
3186                rdev->config.evergreen.max_threads = 248;
3187                rdev->config.evergreen.max_gs_threads = 32;
3188                rdev->config.evergreen.max_stack_entries = 512;
3189                rdev->config.evergreen.sx_num_of_sets = 4;
3190                rdev->config.evergreen.sx_max_export_size = 256;
3191                rdev->config.evergreen.sx_max_export_pos_size = 64;
3192                rdev->config.evergreen.sx_max_export_smx_size = 192;
3193                rdev->config.evergreen.max_hw_contexts = 8;
3194                rdev->config.evergreen.sq_num_cf_insts = 2;
3195
3196                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3197                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3198                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3199                gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3200                break;
3201        case CHIP_REDWOOD:
3202                rdev->config.evergreen.num_ses = 1;
3203                rdev->config.evergreen.max_pipes = 4;
3204                rdev->config.evergreen.max_tile_pipes = 4;
3205                rdev->config.evergreen.max_simds = 5;
3206                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3207                rdev->config.evergreen.max_gprs = 256;
3208                rdev->config.evergreen.max_threads = 248;
3209                rdev->config.evergreen.max_gs_threads = 32;
3210                rdev->config.evergreen.max_stack_entries = 256;
3211                rdev->config.evergreen.sx_num_of_sets = 4;
3212                rdev->config.evergreen.sx_max_export_size = 256;
3213                rdev->config.evergreen.sx_max_export_pos_size = 64;
3214                rdev->config.evergreen.sx_max_export_smx_size = 192;
3215                rdev->config.evergreen.max_hw_contexts = 8;
3216                rdev->config.evergreen.sq_num_cf_insts = 2;
3217
3218                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3219                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3220                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3221                gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3222                break;
3223        case CHIP_CEDAR:
3224        default:
3225                rdev->config.evergreen.num_ses = 1;
3226                rdev->config.evergreen.max_pipes = 2;
3227                rdev->config.evergreen.max_tile_pipes = 2;
3228                rdev->config.evergreen.max_simds = 2;
3229                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3230                rdev->config.evergreen.max_gprs = 256;
3231                rdev->config.evergreen.max_threads = 192;
3232                rdev->config.evergreen.max_gs_threads = 16;
3233                rdev->config.evergreen.max_stack_entries = 256;
3234                rdev->config.evergreen.sx_num_of_sets = 4;
3235                rdev->config.evergreen.sx_max_export_size = 128;
3236                rdev->config.evergreen.sx_max_export_pos_size = 32;
3237                rdev->config.evergreen.sx_max_export_smx_size = 96;
3238                rdev->config.evergreen.max_hw_contexts = 4;
3239                rdev->config.evergreen.sq_num_cf_insts = 1;
3240
3241                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3242                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3243                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3244                gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3245                break;
3246        case CHIP_PALM:
3247                rdev->config.evergreen.num_ses = 1;
3248                rdev->config.evergreen.max_pipes = 2;
3249                rdev->config.evergreen.max_tile_pipes = 2;
3250                rdev->config.evergreen.max_simds = 2;
3251                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3252                rdev->config.evergreen.max_gprs = 256;
3253                rdev->config.evergreen.max_threads = 192;
3254                rdev->config.evergreen.max_gs_threads = 16;
3255                rdev->config.evergreen.max_stack_entries = 256;
3256                rdev->config.evergreen.sx_num_of_sets = 4;
3257                rdev->config.evergreen.sx_max_export_size = 128;
3258                rdev->config.evergreen.sx_max_export_pos_size = 32;
3259                rdev->config.evergreen.sx_max_export_smx_size = 96;
3260                rdev->config.evergreen.max_hw_contexts = 4;
3261                rdev->config.evergreen.sq_num_cf_insts = 1;
3262
3263                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3264                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3265                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3266                gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3267                break;
3268        case CHIP_SUMO:
3269                rdev->config.evergreen.num_ses = 1;
3270                rdev->config.evergreen.max_pipes = 4;
3271                rdev->config.evergreen.max_tile_pipes = 4;
3272                if (rdev->pdev->device == 0x9648)
3273                        rdev->config.evergreen.max_simds = 3;
3274                else if ((rdev->pdev->device == 0x9647) ||
3275                         (rdev->pdev->device == 0x964a))
3276                        rdev->config.evergreen.max_simds = 4;
3277                else
3278                        rdev->config.evergreen.max_simds = 5;
3279                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3280                rdev->config.evergreen.max_gprs = 256;
3281                rdev->config.evergreen.max_threads = 248;
3282                rdev->config.evergreen.max_gs_threads = 32;
3283                rdev->config.evergreen.max_stack_entries = 256;
3284                rdev->config.evergreen.sx_num_of_sets = 4;
3285                rdev->config.evergreen.sx_max_export_size = 256;
3286                rdev->config.evergreen.sx_max_export_pos_size = 64;
3287                rdev->config.evergreen.sx_max_export_smx_size = 192;
3288                rdev->config.evergreen.max_hw_contexts = 8;
3289                rdev->config.evergreen.sq_num_cf_insts = 2;
3290
3291                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3292                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3293                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3294                gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3295                break;
3296        case CHIP_SUMO2:
3297                rdev->config.evergreen.num_ses = 1;
3298                rdev->config.evergreen.max_pipes = 4;
3299                rdev->config.evergreen.max_tile_pipes = 4;
3300                rdev->config.evergreen.max_simds = 2;
3301                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3302                rdev->config.evergreen.max_gprs = 256;
3303                rdev->config.evergreen.max_threads = 248;
3304                rdev->config.evergreen.max_gs_threads = 32;
3305                rdev->config.evergreen.max_stack_entries = 512;
3306                rdev->config.evergreen.sx_num_of_sets = 4;
3307                rdev->config.evergreen.sx_max_export_size = 256;
3308                rdev->config.evergreen.sx_max_export_pos_size = 64;
3309                rdev->config.evergreen.sx_max_export_smx_size = 192;
3310                rdev->config.evergreen.max_hw_contexts = 4;
3311                rdev->config.evergreen.sq_num_cf_insts = 2;
3312
3313                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3314                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3315                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3316                gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3317                break;
3318        case CHIP_BARTS:
3319                rdev->config.evergreen.num_ses = 2;
3320                rdev->config.evergreen.max_pipes = 4;
3321                rdev->config.evergreen.max_tile_pipes = 8;
3322                rdev->config.evergreen.max_simds = 7;
3323                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3324                rdev->config.evergreen.max_gprs = 256;
3325                rdev->config.evergreen.max_threads = 248;
3326                rdev->config.evergreen.max_gs_threads = 32;
3327                rdev->config.evergreen.max_stack_entries = 512;
3328                rdev->config.evergreen.sx_num_of_sets = 4;
3329                rdev->config.evergreen.sx_max_export_size = 256;
3330                rdev->config.evergreen.sx_max_export_pos_size = 64;
3331                rdev->config.evergreen.sx_max_export_smx_size = 192;
3332                rdev->config.evergreen.max_hw_contexts = 8;
3333                rdev->config.evergreen.sq_num_cf_insts = 2;
3334
3335                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3336                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3337                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3338                gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3339                break;
3340        case CHIP_TURKS:
3341                rdev->config.evergreen.num_ses = 1;
3342                rdev->config.evergreen.max_pipes = 4;
3343                rdev->config.evergreen.max_tile_pipes = 4;
3344                rdev->config.evergreen.max_simds = 6;
3345                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3346                rdev->config.evergreen.max_gprs = 256;
3347                rdev->config.evergreen.max_threads = 248;
3348                rdev->config.evergreen.max_gs_threads = 32;
3349                rdev->config.evergreen.max_stack_entries = 256;
3350                rdev->config.evergreen.sx_num_of_sets = 4;
3351                rdev->config.evergreen.sx_max_export_size = 256;
3352                rdev->config.evergreen.sx_max_export_pos_size = 64;
3353                rdev->config.evergreen.sx_max_export_smx_size = 192;
3354                rdev->config.evergreen.max_hw_contexts = 8;
3355                rdev->config.evergreen.sq_num_cf_insts = 2;
3356
3357                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3358                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3359                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3360                gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3361                break;
3362        case CHIP_CAICOS:
3363                rdev->config.evergreen.num_ses = 1;
3364                rdev->config.evergreen.max_pipes = 2;
3365                rdev->config.evergreen.max_tile_pipes = 2;
3366                rdev->config.evergreen.max_simds = 2;
3367                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3368                rdev->config.evergreen.max_gprs = 256;
3369                rdev->config.evergreen.max_threads = 192;
3370                rdev->config.evergreen.max_gs_threads = 16;
3371                rdev->config.evergreen.max_stack_entries = 256;
3372                rdev->config.evergreen.sx_num_of_sets = 4;
3373                rdev->config.evergreen.sx_max_export_size = 128;
3374                rdev->config.evergreen.sx_max_export_pos_size = 32;
3375                rdev->config.evergreen.sx_max_export_smx_size = 96;
3376                rdev->config.evergreen.max_hw_contexts = 4;
3377                rdev->config.evergreen.sq_num_cf_insts = 1;
3378
3379                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3380                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3381                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3382                gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3383                break;
3384        }
3385
3386        /* Initialize HDP */
3387        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3388                WREG32((0x2c14 + j), 0x00000000);
3389                WREG32((0x2c18 + j), 0x00000000);
3390                WREG32((0x2c1c + j), 0x00000000);
3391                WREG32((0x2c20 + j), 0x00000000);
3392                WREG32((0x2c24 + j), 0x00000000);
3393        }
3394
3395        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3396        WREG32(SRBM_INT_CNTL, 0x1);
3397        WREG32(SRBM_INT_ACK, 0x1);
3398
3399        evergreen_fix_pci_max_read_req_size(rdev);
3400
3401        RREG32(MC_SHARED_CHMAP);
3402        if ((rdev->family == CHIP_PALM) ||
3403            (rdev->family == CHIP_SUMO) ||
3404            (rdev->family == CHIP_SUMO2))
3405                mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3406        else
3407                mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3408
3409        /* setup tiling info dword.  gb_addr_config is not adequate since it does
3410         * not have bank info, so create a custom tiling dword.
3411         * bits 3:0   num_pipes
3412         * bits 7:4   num_banks
3413         * bits 11:8  group_size
3414         * bits 15:12 row_size
3415         */
3416        rdev->config.evergreen.tile_config = 0;
3417        switch (rdev->config.evergreen.max_tile_pipes) {
3418        case 1:
3419        default:
3420                rdev->config.evergreen.tile_config |= (0 << 0);
3421                break;
3422        case 2:
3423                rdev->config.evergreen.tile_config |= (1 << 0);
3424                break;
3425        case 4:
3426                rdev->config.evergreen.tile_config |= (2 << 0);
3427                break;
3428        case 8:
3429                rdev->config.evergreen.tile_config |= (3 << 0);
3430                break;
3431        }
3432        /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3433        if (rdev->flags & RADEON_IS_IGP)
3434                rdev->config.evergreen.tile_config |= 1 << 4;
3435        else {
3436                switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3437                case 0: /* four banks */
3438                        rdev->config.evergreen.tile_config |= 0 << 4;
3439                        break;
3440                case 1: /* eight banks */
3441                        rdev->config.evergreen.tile_config |= 1 << 4;
3442                        break;
3443                case 2: /* sixteen banks */
3444                default:
3445                        rdev->config.evergreen.tile_config |= 2 << 4;
3446                        break;
3447                }
3448        }
3449        rdev->config.evergreen.tile_config |= 0 << 8;
3450        rdev->config.evergreen.tile_config |=
3451                ((gb_addr_config & 0x30000000) >> 28) << 12;
3452
3453        if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3454                u32 efuse_straps_4;
3455                u32 efuse_straps_3;
3456
3457                efuse_straps_4 = RREG32_RCU(0x204);
3458                efuse_straps_3 = RREG32_RCU(0x203);
3459                tmp = (((efuse_straps_4 & 0xf) << 4) |
3460                      ((efuse_straps_3 & 0xf0000000) >> 28));
3461        } else {
3462                tmp = 0;
3463                for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3464                        u32 rb_disable_bitmap;
3465
3466                        WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3467                        WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3468                        rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3469                        tmp <<= 4;
3470                        tmp |= rb_disable_bitmap;
3471                }
3472        }
3473        /* enabled rb are just the one not disabled :) */
3474        disabled_rb_mask = tmp;
3475        tmp = 0;
3476        for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3477                tmp |= (1 << i);
3478        /* if all the backends are disabled, fix it up here */
3479        if ((disabled_rb_mask & tmp) == tmp) {
3480                for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3481                        disabled_rb_mask &= ~(1 << i);
3482        }
3483
3484        for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3485                u32 simd_disable_bitmap;
3486
3487                WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3488                WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3489                simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3490                simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3491                tmp <<= 16;
3492                tmp |= simd_disable_bitmap;
3493        }
3494        rdev->config.evergreen.active_simds = hweight32(~tmp);
3495
3496        WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497        WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3498
3499        WREG32(GB_ADDR_CONFIG, gb_addr_config);
3500        WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3501        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3502        WREG32(DMA_TILING_CONFIG, gb_addr_config);
3503        WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3504        WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3505        WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3506
3507        if ((rdev->config.evergreen.max_backends == 1) &&
3508            (rdev->flags & RADEON_IS_IGP)) {
3509                if ((disabled_rb_mask & 3) == 1) {
3510                        /* RB0 disabled, RB1 enabled */
3511                        tmp = 0x11111111;
3512                } else {
3513                        /* RB1 disabled, RB0 enabled */
3514                        tmp = 0x00000000;
3515                }
3516        } else {
3517                tmp = gb_addr_config & NUM_PIPES_MASK;
3518                tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3519                                                EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3520        }
3521        rdev->config.evergreen.backend_map = tmp;
3522        WREG32(GB_BACKEND_MAP, tmp);
3523
3524        WREG32(CGTS_SYS_TCC_DISABLE, 0);
3525        WREG32(CGTS_TCC_DISABLE, 0);
3526        WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3527        WREG32(CGTS_USER_TCC_DISABLE, 0);
3528
3529        /* set HW defaults for 3D engine */
3530        WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3531                                     ROQ_IB2_START(0x2b)));
3532
3533        WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3534
3535        WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3536                             SYNC_GRADIENT |
3537                             SYNC_WALKER |
3538                             SYNC_ALIGNER));
3539
3540        sx_debug_1 = RREG32(SX_DEBUG_1);
3541        sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3542        WREG32(SX_DEBUG_1, sx_debug_1);
3543
3544
3545        smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3546        smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3547        smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3548        WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3549
3550        if (rdev->family <= CHIP_SUMO2)
3551                WREG32(SMX_SAR_CTL0, 0x00010000);
3552
3553        WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3554                                        POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3555                                        SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3556
3557        WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3558                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3559                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3560
3561        WREG32(VGT_NUM_INSTANCES, 1);
3562        WREG32(SPI_CONFIG_CNTL, 0);
3563        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3564        WREG32(CP_PERFMON_CNTL, 0);
3565
3566        WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3567                                  FETCH_FIFO_HIWATER(0x4) |
3568                                  DONE_FIFO_HIWATER(0xe0) |
3569                                  ALU_UPDATE_FIFO_HIWATER(0x8)));
3570
3571        sq_config = RREG32(SQ_CONFIG);
3572        sq_config &= ~(PS_PRIO(3) |
3573                       VS_PRIO(3) |
3574                       GS_PRIO(3) |
3575                       ES_PRIO(3));
3576        sq_config |= (VC_ENABLE |
3577                      EXPORT_SRC_C |
3578                      PS_PRIO(0) |
3579                      VS_PRIO(1) |
3580                      GS_PRIO(2) |
3581                      ES_PRIO(3));
3582
3583        switch (rdev->family) {
3584        case CHIP_CEDAR:
3585        case CHIP_PALM:
3586        case CHIP_SUMO:
3587        case CHIP_SUMO2:
3588        case CHIP_CAICOS:
3589                /* no vertex cache */
3590                sq_config &= ~VC_ENABLE;
3591                break;
3592        default:
3593                break;
3594        }
3595
3596        sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3597
3598        sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3599        sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3600        sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3601        sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3602        sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3603        sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604        sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3605
3606        switch (rdev->family) {
3607        case CHIP_CEDAR:
3608        case CHIP_PALM:
3609        case CHIP_SUMO:
3610        case CHIP_SUMO2:
3611                ps_thread_count = 96;
3612                break;
3613        default:
3614                ps_thread_count = 128;
3615                break;
3616        }
3617
3618        sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3619        sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620        sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621        sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622        sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623        sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3624
3625        sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626        sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627        sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628        sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629        sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630        sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3631
3632        WREG32(SQ_CONFIG, sq_config);
3633        WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3634        WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3635        WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3636        WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3637        WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3638        WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3639        WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3640        WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3641        WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3642        WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3643
3644        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3645                                          FORCE_EOV_MAX_REZ_CNT(255)));
3646
3647        switch (rdev->family) {
3648        case CHIP_CEDAR:
3649        case CHIP_PALM:
3650        case CHIP_SUMO:
3651        case CHIP_SUMO2:
3652        case CHIP_CAICOS:
3653                vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3654                break;
3655        default:
3656                vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3657                break;
3658        }
3659        vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3660        WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3661
3662        WREG32(VGT_GS_VERTEX_REUSE, 16);
3663        WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3664        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3665
3666        WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3667        WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3668
3669        WREG32(CB_PERF_CTR0_SEL_0, 0);
3670        WREG32(CB_PERF_CTR0_SEL_1, 0);
3671        WREG32(CB_PERF_CTR1_SEL_0, 0);
3672        WREG32(CB_PERF_CTR1_SEL_1, 0);
3673        WREG32(CB_PERF_CTR2_SEL_0, 0);
3674        WREG32(CB_PERF_CTR2_SEL_1, 0);
3675        WREG32(CB_PERF_CTR3_SEL_0, 0);
3676        WREG32(CB_PERF_CTR3_SEL_1, 0);
3677
3678        /* clear render buffer base addresses */
3679        WREG32(CB_COLOR0_BASE, 0);
3680        WREG32(CB_COLOR1_BASE, 0);
3681        WREG32(CB_COLOR2_BASE, 0);
3682        WREG32(CB_COLOR3_BASE, 0);
3683        WREG32(CB_COLOR4_BASE, 0);
3684        WREG32(CB_COLOR5_BASE, 0);
3685        WREG32(CB_COLOR6_BASE, 0);
3686        WREG32(CB_COLOR7_BASE, 0);
3687        WREG32(CB_COLOR8_BASE, 0);
3688        WREG32(CB_COLOR9_BASE, 0);
3689        WREG32(CB_COLOR10_BASE, 0);
3690        WREG32(CB_COLOR11_BASE, 0);
3691
3692        /* set the shader const cache sizes to 0 */
3693        for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3694                WREG32(i, 0);
3695        for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3696                WREG32(i, 0);
3697
3698        tmp = RREG32(HDP_MISC_CNTL);
3699        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3700        WREG32(HDP_MISC_CNTL, tmp);
3701
3702        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3703        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3704
3705        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3706
3707        udelay(50);
3708
3709}
3710
3711int evergreen_mc_init(struct radeon_device *rdev)
3712{
3713        u32 tmp;
3714        int chansize, numchan;
3715
3716        /* Get VRAM informations */
3717        rdev->mc.vram_is_ddr = true;
3718        if ((rdev->family == CHIP_PALM) ||
3719            (rdev->family == CHIP_SUMO) ||
3720            (rdev->family == CHIP_SUMO2))
3721                tmp = RREG32(FUS_MC_ARB_RAMCFG);
3722        else
3723                tmp = RREG32(MC_ARB_RAMCFG);
3724        if (tmp & CHANSIZE_OVERRIDE) {
3725                chansize = 16;
3726        } else if (tmp & CHANSIZE_MASK) {
3727                chansize = 64;
3728        } else {
3729                chansize = 32;
3730        }
3731        tmp = RREG32(MC_SHARED_CHMAP);
3732        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3733        case 0:
3734        default:
3735                numchan = 1;
3736                break;
3737        case 1:
3738                numchan = 2;
3739                break;
3740        case 2:
3741                numchan = 4;
3742                break;
3743        case 3:
3744                numchan = 8;
3745                break;
3746        }
3747        rdev->mc.vram_width = numchan * chansize;
3748        /* Could aper size report 0 ? */
3749        rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3750        rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3751        /* Setup GPU memory space */
3752        if ((rdev->family == CHIP_PALM) ||
3753            (rdev->family == CHIP_SUMO) ||
3754            (rdev->family == CHIP_SUMO2)) {
3755                /* size in bytes on fusion */
3756                rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3757                rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3758        } else {
3759                /* size in MB on evergreen/cayman/tn */
3760                rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761                rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3762        }
3763        rdev->mc.visible_vram_size = rdev->mc.aper_size;
3764        r700_vram_gtt_location(rdev, &rdev->mc);
3765        radeon_update_bandwidth_info(rdev);
3766
3767        return 0;
3768}
3769
3770void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3771{
3772        dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3773                RREG32(GRBM_STATUS));
3774        dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3775                RREG32(GRBM_STATUS_SE0));
3776        dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3777                RREG32(GRBM_STATUS_SE1));
3778        dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3779                RREG32(SRBM_STATUS));
3780        dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3781                RREG32(SRBM_STATUS2));
3782        dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3783                RREG32(CP_STALLED_STAT1));
3784        dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3785                RREG32(CP_STALLED_STAT2));
3786        dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3787                RREG32(CP_BUSY_STAT));
3788        dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3789                RREG32(CP_STAT));
3790        dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3791                RREG32(DMA_STATUS_REG));
3792        if (rdev->family >= CHIP_CAYMAN) {
3793                dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3794                         RREG32(DMA_STATUS_REG + 0x800));
3795        }
3796}
3797
3798bool evergreen_is_display_hung(struct radeon_device *rdev)
3799{
3800        u32 crtc_hung = 0;
3801        u32 crtc_status[6];
3802        u32 i, j, tmp;
3803
3804        for (i = 0; i < rdev->num_crtc; i++) {
3805                if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3806                        crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3807                        crtc_hung |= (1 << i);
3808                }
3809        }
3810
3811        for (j = 0; j < 10; j++) {
3812                for (i = 0; i < rdev->num_crtc; i++) {
3813                        if (crtc_hung & (1 << i)) {
3814                                tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3815                                if (tmp != crtc_status[i])
3816                                        crtc_hung &= ~(1 << i);
3817                        }
3818                }
3819                if (crtc_hung == 0)
3820                        return false;
3821                udelay(100);
3822        }
3823
3824        return true;
3825}
3826
3827u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3828{
3829        u32 reset_mask = 0;
3830        u32 tmp;
3831
3832        /* GRBM_STATUS */
3833        tmp = RREG32(GRBM_STATUS);
3834        if (tmp & (PA_BUSY | SC_BUSY |
3835                   SH_BUSY | SX_BUSY |
3836                   TA_BUSY | VGT_BUSY |
3837                   DB_BUSY | CB_BUSY |
3838                   SPI_BUSY | VGT_BUSY_NO_DMA))
3839                reset_mask |= RADEON_RESET_GFX;
3840
3841        if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3842                   CP_BUSY | CP_COHERENCY_BUSY))
3843                reset_mask |= RADEON_RESET_CP;
3844
3845        if (tmp & GRBM_EE_BUSY)
3846                reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3847
3848        /* DMA_STATUS_REG */
3849        tmp = RREG32(DMA_STATUS_REG);
3850        if (!(tmp & DMA_IDLE))
3851                reset_mask |= RADEON_RESET_DMA;
3852
3853        /* SRBM_STATUS2 */
3854        tmp = RREG32(SRBM_STATUS2);
3855        if (tmp & DMA_BUSY)
3856                reset_mask |= RADEON_RESET_DMA;
3857
3858        /* SRBM_STATUS */
3859        tmp = RREG32(SRBM_STATUS);
3860        if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3861                reset_mask |= RADEON_RESET_RLC;
3862
3863        if (tmp & IH_BUSY)
3864                reset_mask |= RADEON_RESET_IH;
3865
3866        if (tmp & SEM_BUSY)
3867                reset_mask |= RADEON_RESET_SEM;
3868
3869        if (tmp & GRBM_RQ_PENDING)
3870                reset_mask |= RADEON_RESET_GRBM;
3871
3872        if (tmp & VMC_BUSY)
3873                reset_mask |= RADEON_RESET_VMC;
3874
3875        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3876                   MCC_BUSY | MCD_BUSY))
3877                reset_mask |= RADEON_RESET_MC;
3878
3879        if (evergreen_is_display_hung(rdev))
3880                reset_mask |= RADEON_RESET_DISPLAY;
3881
3882        /* VM_L2_STATUS */
3883        tmp = RREG32(VM_L2_STATUS);
3884        if (tmp & L2_BUSY)
3885                reset_mask |= RADEON_RESET_VMC;
3886
3887        /* Skip MC reset as it's mostly likely not hung, just busy */
3888        if (reset_mask & RADEON_RESET_MC) {
3889                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3890                reset_mask &= ~RADEON_RESET_MC;
3891        }
3892
3893        return reset_mask;
3894}
3895
3896static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3897{
3898        struct evergreen_mc_save save;
3899        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3900        u32 tmp;
3901
3902        if (reset_mask == 0)
3903                return;
3904
3905        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3906
3907        evergreen_print_gpu_status_regs(rdev);
3908
3909        /* Disable CP parsing/prefetching */
3910        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3911
3912        if (reset_mask & RADEON_RESET_DMA) {
3913                /* Disable DMA */
3914                tmp = RREG32(DMA_RB_CNTL);
3915                tmp &= ~DMA_RB_ENABLE;
3916                WREG32(DMA_RB_CNTL, tmp);
3917        }
3918
3919        udelay(50);
3920
3921        evergreen_mc_stop(rdev, &save);
3922        if (evergreen_mc_wait_for_idle(rdev)) {
3923                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3924        }
3925
3926        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3927                grbm_soft_reset |= SOFT_RESET_DB |
3928                        SOFT_RESET_CB |
3929                        SOFT_RESET_PA |
3930                        SOFT_RESET_SC |
3931                        SOFT_RESET_SPI |
3932                        SOFT_RESET_SX |
3933                        SOFT_RESET_SH |
3934                        SOFT_RESET_TC |
3935                        SOFT_RESET_TA |
3936                        SOFT_RESET_VC |
3937                        SOFT_RESET_VGT;
3938        }
3939
3940        if (reset_mask & RADEON_RESET_CP) {
3941                grbm_soft_reset |= SOFT_RESET_CP |
3942                        SOFT_RESET_VGT;
3943
3944                srbm_soft_reset |= SOFT_RESET_GRBM;
3945        }
3946
3947        if (reset_mask & RADEON_RESET_DMA)
3948                srbm_soft_reset |= SOFT_RESET_DMA;
3949
3950        if (reset_mask & RADEON_RESET_DISPLAY)
3951                srbm_soft_reset |= SOFT_RESET_DC;
3952
3953        if (reset_mask & RADEON_RESET_RLC)
3954                srbm_soft_reset |= SOFT_RESET_RLC;
3955
3956        if (reset_mask & RADEON_RESET_SEM)
3957                srbm_soft_reset |= SOFT_RESET_SEM;
3958
3959        if (reset_mask & RADEON_RESET_IH)
3960                srbm_soft_reset |= SOFT_RESET_IH;
3961
3962        if (reset_mask & RADEON_RESET_GRBM)
3963                srbm_soft_reset |= SOFT_RESET_GRBM;
3964
3965        if (reset_mask & RADEON_RESET_VMC)
3966                srbm_soft_reset |= SOFT_RESET_VMC;
3967
3968        if (!(rdev->flags & RADEON_IS_IGP)) {
3969                if (reset_mask & RADEON_RESET_MC)
3970                        srbm_soft_reset |= SOFT_RESET_MC;
3971        }
3972
3973        if (grbm_soft_reset) {
3974                tmp = RREG32(GRBM_SOFT_RESET);
3975                tmp |= grbm_soft_reset;
3976                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3977                WREG32(GRBM_SOFT_RESET, tmp);
3978                tmp = RREG32(GRBM_SOFT_RESET);
3979
3980                udelay(50);
3981
3982                tmp &= ~grbm_soft_reset;
3983                WREG32(GRBM_SOFT_RESET, tmp);
3984                tmp = RREG32(GRBM_SOFT_RESET);
3985        }
3986
3987        if (srbm_soft_reset) {
3988                tmp = RREG32(SRBM_SOFT_RESET);
3989                tmp |= srbm_soft_reset;
3990                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3991                WREG32(SRBM_SOFT_RESET, tmp);
3992                tmp = RREG32(SRBM_SOFT_RESET);
3993
3994                udelay(50);
3995
3996                tmp &= ~srbm_soft_reset;
3997                WREG32(SRBM_SOFT_RESET, tmp);
3998                tmp = RREG32(SRBM_SOFT_RESET);
3999        }
4000
4001        /* Wait a little for things to settle down */
4002        udelay(50);
4003
4004        evergreen_mc_resume(rdev, &save);
4005        udelay(50);
4006
4007        evergreen_print_gpu_status_regs(rdev);
4008}
4009
4010void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4011{
4012        struct evergreen_mc_save save;
4013        u32 tmp, i;
4014
4015        dev_info(rdev->dev, "GPU pci config reset\n");
4016
4017        /* disable dpm? */
4018
4019        /* Disable CP parsing/prefetching */
4020        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4021        udelay(50);
4022        /* Disable DMA */
4023        tmp = RREG32(DMA_RB_CNTL);
4024        tmp &= ~DMA_RB_ENABLE;
4025        WREG32(DMA_RB_CNTL, tmp);
4026        /* XXX other engines? */
4027
4028        /* halt the rlc */
4029        r600_rlc_stop(rdev);
4030
4031        udelay(50);
4032
4033        /* set mclk/sclk to bypass */
4034        rv770_set_clk_bypass_mode(rdev);
4035        /* disable BM */
4036        pci_clear_master(rdev->pdev);
4037        /* disable mem access */
4038        evergreen_mc_stop(rdev, &save);
4039        if (evergreen_mc_wait_for_idle(rdev)) {
4040                dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4041        }
4042        /* reset */
4043        radeon_pci_config_reset(rdev);
4044        /* wait for asic to come out of reset */
4045        for (i = 0; i < rdev->usec_timeout; i++) {
4046                if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4047                        break;
4048                udelay(1);
4049        }
4050}
4051
4052int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4053{
4054        u32 reset_mask;
4055
4056        if (hard) {
4057                evergreen_gpu_pci_config_reset(rdev);
4058                return 0;
4059        }
4060
4061        reset_mask = evergreen_gpu_check_soft_reset(rdev);
4062
4063        if (reset_mask)
4064                r600_set_bios_scratch_engine_hung(rdev, true);
4065
4066        /* try soft reset */
4067        evergreen_gpu_soft_reset(rdev, reset_mask);
4068
4069        reset_mask = evergreen_gpu_check_soft_reset(rdev);
4070
4071        /* try pci config reset */
4072        if (reset_mask && radeon_hard_reset)
4073                evergreen_gpu_pci_config_reset(rdev);
4074
4075        reset_mask = evergreen_gpu_check_soft_reset(rdev);
4076
4077        if (!reset_mask)
4078                r600_set_bios_scratch_engine_hung(rdev, false);
4079
4080        return 0;
4081}
4082
4083/**
4084 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4085 *
4086 * @rdev: radeon_device pointer
4087 * @ring: radeon_ring structure holding ring information
4088 *
4089 * Check if the GFX engine is locked up.
4090 * Returns true if the engine appears to be locked up, false if not.
4091 */
4092bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4093{
4094        u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4095
4096        if (!(reset_mask & (RADEON_RESET_GFX |
4097                            RADEON_RESET_COMPUTE |
4098                            RADEON_RESET_CP))) {
4099                radeon_ring_lockup_update(rdev, ring);
4100                return false;
4101        }
4102        return radeon_ring_test_lockup(rdev, ring);
4103}
4104
4105/*
4106 * RLC
4107 */
4108#define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4109#define RLC_CLEAR_STATE_END_MARKER          0x00000001
4110
4111void sumo_rlc_fini(struct radeon_device *rdev)
4112{
4113        int r;
4114
4115        /* save restore block */
4116        if (rdev->rlc.save_restore_obj) {
4117                r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4118                if (unlikely(r != 0))
4119                        dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4120                radeon_bo_unpin(rdev->rlc.save_restore_obj);
4121                radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4122
4123                radeon_bo_unref(&rdev->rlc.save_restore_obj);
4124                rdev->rlc.save_restore_obj = NULL;
4125        }
4126
4127        /* clear state block */
4128        if (rdev->rlc.clear_state_obj) {
4129                r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4130                if (unlikely(r != 0))
4131                        dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4132                radeon_bo_unpin(rdev->rlc.clear_state_obj);
4133                radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4134
4135                radeon_bo_unref(&rdev->rlc.clear_state_obj);
4136                rdev->rlc.clear_state_obj = NULL;
4137        }
4138
4139        /* clear state block */
4140        if (rdev->rlc.cp_table_obj) {
4141                r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4142                if (unlikely(r != 0))
4143                        dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4144                radeon_bo_unpin(rdev->rlc.cp_table_obj);
4145                radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4146
4147                radeon_bo_unref(&rdev->rlc.cp_table_obj);
4148                rdev->rlc.cp_table_obj = NULL;
4149        }
4150}
4151
4152#define CP_ME_TABLE_SIZE    96
4153
4154int sumo_rlc_init(struct radeon_device *rdev)
4155{
4156        const u32 *src_ptr;
4157        volatile u32 *dst_ptr;
4158        u32 dws, data, i, j, k, reg_num;
4159        u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4160        u64 reg_list_mc_addr;
4161        const struct cs_section_def *cs_data;
4162        int r;
4163
4164        src_ptr = rdev->rlc.reg_list;
4165        dws = rdev->rlc.reg_list_size;
4166        if (rdev->family >= CHIP_BONAIRE) {
4167                dws += (5 * 16) + 48 + 48 + 64;
4168        }
4169        cs_data = rdev->rlc.cs_data;
4170
4171        if (src_ptr) {
4172                /* save restore block */
4173                if (rdev->rlc.save_restore_obj == NULL) {
4174                        r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4175                                             RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4176                                             NULL, &rdev->rlc.save_restore_obj);
4177                        if (r) {
4178                                dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4179                                return r;
4180                        }
4181                }
4182
4183                r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4184                if (unlikely(r != 0)) {
4185                        sumo_rlc_fini(rdev);
4186                        return r;
4187                }
4188                r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4189                                  &rdev->rlc.save_restore_gpu_addr);
4190                if (r) {
4191                        radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4192                        dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4193                        sumo_rlc_fini(rdev);
4194                        return r;
4195                }
4196
4197                r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4198                if (r) {
4199                        dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4200                        sumo_rlc_fini(rdev);
4201                        return r;
4202                }
4203                /* write the sr buffer */
4204                dst_ptr = rdev->rlc.sr_ptr;
4205                if (rdev->family >= CHIP_TAHITI) {
4206                        /* SI */
4207                        for (i = 0; i < rdev->rlc.reg_list_size; i++)
4208                                dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4209                } else {
4210                        /* ON/LN/TN */
4211                        /* format:
4212                         * dw0: (reg2 << 16) | reg1
4213                         * dw1: reg1 save space
4214                         * dw2: reg2 save space
4215                         */
4216                        for (i = 0; i < dws; i++) {
4217                                data = src_ptr[i] >> 2;
4218                                i++;
4219                                if (i < dws)
4220                                        data |= (src_ptr[i] >> 2) << 16;
4221                                j = (((i - 1) * 3) / 2);
4222                                dst_ptr[j] = cpu_to_le32(data);
4223                        }
4224                        j = ((i * 3) / 2);
4225                        dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4226                }
4227                radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4228                radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4229        }
4230
4231        if (cs_data) {
4232                /* clear state block */
4233                if (rdev->family >= CHIP_BONAIRE) {
4234                        rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4235                } else if (rdev->family >= CHIP_TAHITI) {
4236                        rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4237                        dws = rdev->rlc.clear_state_size + (256 / 4);
4238                } else {
4239                        reg_list_num = 0;
4240                        dws = 0;
4241                        for (i = 0; cs_data[i].section != NULL; i++) {
4242                                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4243                                        reg_list_num++;
4244                                        dws += cs_data[i].section[j].reg_count;
4245                                }
4246                        }
4247                        reg_list_blk_index = (3 * reg_list_num + 2);
4248                        dws += reg_list_blk_index;
4249                        rdev->rlc.clear_state_size = dws;
4250                }
4251
4252                if (rdev->rlc.clear_state_obj == NULL) {
4253                        r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4254                                             RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4255                                             NULL, &rdev->rlc.clear_state_obj);
4256                        if (r) {
4257                                dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4258                                sumo_rlc_fini(rdev);
4259                                return r;
4260                        }
4261                }
4262                r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4263                if (unlikely(r != 0)) {
4264                        sumo_rlc_fini(rdev);
4265                        return r;
4266                }
4267                r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4268                                  &rdev->rlc.clear_state_gpu_addr);
4269                if (r) {
4270                        radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4271                        dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4272                        sumo_rlc_fini(rdev);
4273                        return r;
4274                }
4275
4276                r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4277                if (r) {
4278                        dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4279                        sumo_rlc_fini(rdev);
4280                        return r;
4281                }
4282                /* set up the cs buffer */
4283                dst_ptr = rdev->rlc.cs_ptr;
4284                if (rdev->family >= CHIP_BONAIRE) {
4285                        cik_get_csb_buffer(rdev, dst_ptr);
4286                } else if (rdev->family >= CHIP_TAHITI) {
4287                        reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4288                        dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4289                        dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4290                        dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4291                        si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4292                } else {
4293                        reg_list_hdr_blk_index = 0;
4294                        reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4295                        data = upper_32_bits(reg_list_mc_addr);
4296                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4297                        reg_list_hdr_blk_index++;
4298                        for (i = 0; cs_data[i].section != NULL; i++) {
4299                                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4300                                        reg_num = cs_data[i].section[j].reg_count;
4301                                        data = reg_list_mc_addr & 0xffffffff;
4302                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4303                                        reg_list_hdr_blk_index++;
4304
4305                                        data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4306                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4307                                        reg_list_hdr_blk_index++;
4308
4309                                        data = 0x08000000 | (reg_num * 4);
4310                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4311                                        reg_list_hdr_blk_index++;
4312
4313                                        for (k = 0; k < reg_num; k++) {
4314                                                data = cs_data[i].section[j].extent[k];
4315                                                dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4316                                        }
4317                                        reg_list_mc_addr += reg_num * 4;
4318                                        reg_list_blk_index += reg_num;
4319                                }
4320                        }
4321                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4322                }
4323                radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4324                radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4325        }
4326
4327        if (rdev->rlc.cp_table_size) {
4328                if (rdev->rlc.cp_table_obj == NULL) {
4329                        r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4330                                             PAGE_SIZE, true,
4331                                             RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4332                                             NULL, &rdev->rlc.cp_table_obj);
4333                        if (r) {
4334                                dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4335                                sumo_rlc_fini(rdev);
4336                                return r;
4337                        }
4338                }
4339
4340                r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4341                if (unlikely(r != 0)) {
4342                        dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4343                        sumo_rlc_fini(rdev);
4344                        return r;
4345                }
4346                r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4347                                  &rdev->rlc.cp_table_gpu_addr);
4348                if (r) {
4349                        radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4350                        dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4351                        sumo_rlc_fini(rdev);
4352                        return r;
4353                }
4354                r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4355                if (r) {
4356                        dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4357                        sumo_rlc_fini(rdev);
4358                        return r;
4359                }
4360
4361                cik_init_cp_pg_table(rdev);
4362
4363                radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4364                radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4365
4366        }
4367
4368        return 0;
4369}
4370
4371static void evergreen_rlc_start(struct radeon_device *rdev)
4372{
4373        u32 mask = RLC_ENABLE;
4374
4375        if (rdev->flags & RADEON_IS_IGP) {
4376                mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4377        }
4378
4379        WREG32(RLC_CNTL, mask);
4380}
4381
4382int evergreen_rlc_resume(struct radeon_device *rdev)
4383{
4384        u32 i;
4385        const __be32 *fw_data;
4386
4387        if (!rdev->rlc_fw)
4388                return -EINVAL;
4389
4390        r600_rlc_stop(rdev);
4391
4392        WREG32(RLC_HB_CNTL, 0);
4393
4394        if (rdev->flags & RADEON_IS_IGP) {
4395                if (rdev->family == CHIP_ARUBA) {
4396                        u32 always_on_bitmap =
4397                                3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4398                        /* find out the number of active simds */
4399                        u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4400                        tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4401                        tmp = hweight32(~tmp);
4402                        if (tmp == rdev->config.cayman.max_simds_per_se) {
4403                                WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4404                                WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4405                                WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4406                                WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4407                                WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4408                        }
4409                } else {
4410                        WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4411                        WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4412                }
4413                WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4414                WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4415        } else {
4416                WREG32(RLC_HB_BASE, 0);
4417                WREG32(RLC_HB_RPTR, 0);
4418                WREG32(RLC_HB_WPTR, 0);
4419                WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4420                WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4421        }
4422        WREG32(RLC_MC_CNTL, 0);
4423        WREG32(RLC_UCODE_CNTL, 0);
4424
4425        fw_data = (const __be32 *)rdev->rlc_fw->data;
4426        if (rdev->family >= CHIP_ARUBA) {
4427                for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4428                        WREG32(RLC_UCODE_ADDR, i);
4429                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4430                }
4431        } else if (rdev->family >= CHIP_CAYMAN) {
4432                for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4433                        WREG32(RLC_UCODE_ADDR, i);
4434                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4435                }
4436        } else {
4437                for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4438                        WREG32(RLC_UCODE_ADDR, i);
4439                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4440                }
4441        }
4442        WREG32(RLC_UCODE_ADDR, 0);
4443
4444        evergreen_rlc_start(rdev);
4445
4446        return 0;
4447}
4448
4449/* Interrupts */
4450
4451u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4452{
4453        if (crtc >= rdev->num_crtc)
4454                return 0;
4455        else
4456                return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4457}
4458
4459void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4460{
4461        int i;
4462        u32 tmp;
4463
4464        if (rdev->family >= CHIP_CAYMAN) {
4465                cayman_cp_int_cntl_setup(rdev, 0,
4466                                         CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4467                cayman_cp_int_cntl_setup(rdev, 1, 0);
4468                cayman_cp_int_cntl_setup(rdev, 2, 0);
4469                tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4470                WREG32(CAYMAN_DMA1_CNTL, tmp);
4471        } else
4472                WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4473        tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4474        WREG32(DMA_CNTL, tmp);
4475        WREG32(GRBM_INT_CNTL, 0);
4476        WREG32(SRBM_INT_CNTL, 0);
4477        for (i = 0; i < rdev->num_crtc; i++)
4478                WREG32(INT_MASK + crtc_offsets[i], 0);
4479        for (i = 0; i < rdev->num_crtc; i++)
4480                WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4481
4482        /* only one DAC on DCE5 */
4483        if (!ASIC_IS_DCE5(rdev))
4484                WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4485        WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4486
4487        for (i = 0; i < 6; i++)
4488                WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4489}
4490
4491/* Note that the order we write back regs here is important */
4492int evergreen_irq_set(struct radeon_device *rdev)
4493{
4494        int i;
4495        u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4496        u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4497        u32 grbm_int_cntl = 0;
4498        u32 dma_cntl, dma_cntl1 = 0;
4499        u32 thermal_int = 0;
4500
4501        if (!rdev->irq.installed) {
4502                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4503                return -EINVAL;
4504        }
4505        /* don't enable anything if the ih is disabled */
4506        if (!rdev->ih.enabled) {
4507                r600_disable_interrupts(rdev);
4508                /* force the active interrupt state to all disabled */
4509                evergreen_disable_interrupt_state(rdev);
4510                return 0;
4511        }
4512
4513        if (rdev->family == CHIP_ARUBA)
4514                thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4515                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4516        else
4517                thermal_int = RREG32(CG_THERMAL_INT) &
4518                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4519
4520        dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4521
4522        if (rdev->family >= CHIP_CAYMAN) {
4523                /* enable CP interrupts on all rings */
4524                if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4525                        DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4526                        cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4527                }
4528                if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4529                        DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4530                        cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4531                }
4532                if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4533                        DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4534                        cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4535                }
4536        } else {
4537                if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4538                        DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4539                        cp_int_cntl |= RB_INT_ENABLE;
4540                        cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4541                }
4542        }
4543
4544        if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4545                DRM_DEBUG("r600_irq_set: sw int dma\n");
4546                dma_cntl |= TRAP_ENABLE;
4547        }
4548
4549        if (rdev->family >= CHIP_CAYMAN) {
4550                dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4551                if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4552                        DRM_DEBUG("r600_irq_set: sw int dma1\n");
4553                        dma_cntl1 |= TRAP_ENABLE;
4554                }
4555        }
4556
4557        if (rdev->irq.dpm_thermal) {
4558                DRM_DEBUG("dpm thermal\n");
4559                thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4560        }
4561
4562        if (rdev->family >= CHIP_CAYMAN) {
4563                cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4564                cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4565                cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4566        } else
4567                WREG32(CP_INT_CNTL, cp_int_cntl);
4568
4569        WREG32(DMA_CNTL, dma_cntl);
4570
4571        if (rdev->family >= CHIP_CAYMAN)
4572                WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4573
4574        WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4575
4576        for (i = 0; i < rdev->num_crtc; i++) {
4577                radeon_irq_kms_set_irq_n_enabled(
4578                    rdev, INT_MASK + crtc_offsets[i],
4579                    VBLANK_INT_MASK,
4580                    rdev->irq.crtc_vblank_int[i] ||
4581                    atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4582        }
4583
4584        for (i = 0; i < rdev->num_crtc; i++)
4585                WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4586
4587        for (i = 0; i < 6; i++) {
4588                radeon_irq_kms_set_irq_n_enabled(
4589                    rdev, DC_HPDx_INT_CONTROL(i),
4590                    DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4591                    rdev->irq.hpd[i], "HPD", i);
4592        }
4593
4594        if (rdev->family == CHIP_ARUBA)
4595                WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4596        else
4597                WREG32(CG_THERMAL_INT, thermal_int);
4598
4599        for (i = 0; i < 6; i++) {
4600                radeon_irq_kms_set_irq_n_enabled(
4601                    rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4602                    AFMT_AZ_FORMAT_WTRIG_MASK,
4603                    rdev->irq.afmt[i], "HDMI", i);
4604        }
4605
4606        /* posting read */
4607        RREG32(SRBM_STATUS);
4608
4609        return 0;
4610}
4611
4612/* Note that the order we write back regs here is important */
4613static void evergreen_irq_ack(struct radeon_device *rdev)
4614{
4615        int i, j;
4616        u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4617        u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4618        u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4619
4620        for (i = 0; i < 6; i++) {
4621                disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4622                afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4623                if (i < rdev->num_crtc)
4624                        grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4625        }
4626
4627        /* We write back each interrupt register in pairs of two */
4628        for (i = 0; i < rdev->num_crtc; i += 2) {
4629                for (j = i; j < (i + 2); j++) {
4630                        if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4631                                WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4632                                       GRPH_PFLIP_INT_CLEAR);
4633                }
4634
4635                for (j = i; j < (i + 2); j++) {
4636                        if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4637                                WREG32(VBLANK_STATUS + crtc_offsets[j],
4638                                       VBLANK_ACK);
4639                        if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4640                                WREG32(VLINE_STATUS + crtc_offsets[j],
4641                                       VLINE_ACK);
4642                }
4643        }
4644
4645        for (i = 0; i < 6; i++) {
4646                if (disp_int[i] & DC_HPD1_INTERRUPT)
4647                        WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4648        }
4649
4650        for (i = 0; i < 6; i++) {
4651                if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4652                        WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4653        }
4654
4655        for (i = 0; i < 6; i++) {
4656                if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4657                        WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4658                                  AFMT_AZ_FORMAT_WTRIG_ACK);
4659        }
4660}
4661
4662static void evergreen_irq_disable(struct radeon_device *rdev)
4663{
4664        r600_disable_interrupts(rdev);
4665        /* Wait and acknowledge irq */
4666        mdelay(1);
4667        evergreen_irq_ack(rdev);
4668        evergreen_disable_interrupt_state(rdev);
4669}
4670
4671void evergreen_irq_suspend(struct radeon_device *rdev)
4672{
4673        evergreen_irq_disable(rdev);
4674        r600_rlc_stop(rdev);
4675}
4676
4677static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4678{
4679        u32 wptr, tmp;
4680
4681        if (rdev->wb.enabled)
4682                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4683        else
4684                wptr = RREG32(IH_RB_WPTR);
4685
4686        if (wptr & RB_OVERFLOW) {
4687                wptr &= ~RB_OVERFLOW;
4688                /* When a ring buffer overflow happen start parsing interrupt
4689                 * from the last not overwritten vector (wptr + 16). Hopefully
4690                 * this should allow us to catchup.
4691                 */
4692                dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4693                         wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4694                rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4695                tmp = RREG32(IH_RB_CNTL);
4696                tmp |= IH_WPTR_OVERFLOW_CLEAR;
4697                WREG32(IH_RB_CNTL, tmp);
4698        }
4699        return (wptr & rdev->ih.ptr_mask);
4700}
4701
4702int evergreen_irq_process(struct radeon_device *rdev)
4703{
4704        u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4705        u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4706        u32 crtc_idx, hpd_idx, afmt_idx;
4707        u32 mask;
4708        u32 wptr;
4709        u32 rptr;
4710        u32 src_id, src_data;
4711        u32 ring_index;
4712        bool queue_hotplug = false;
4713        bool queue_hdmi = false;
4714        bool queue_dp = false;
4715        bool queue_thermal = false;
4716        u32 status, addr;
4717        const char *event_name;
4718
4719        if (!rdev->ih.enabled || rdev->shutdown)
4720                return IRQ_NONE;
4721
4722        wptr = evergreen_get_ih_wptr(rdev);
4723
4724restart_ih:
4725        /* is somebody else already processing irqs? */
4726        if (atomic_xchg(&rdev->ih.lock, 1))
4727                return IRQ_NONE;
4728
4729        rptr = rdev->ih.rptr;
4730        DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4731
4732        /* Order reading of wptr vs. reading of IH ring data */
4733        rmb();
4734
4735        /* display interrupts */
4736        evergreen_irq_ack(rdev);
4737
4738        while (rptr != wptr) {
4739                /* wptr/rptr are in bytes! */
4740                ring_index = rptr / 4;
4741                src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4742                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4743
4744                switch (src_id) {
4745                case 1: /* D1 vblank/vline */
4746                case 2: /* D2 vblank/vline */
4747                case 3: /* D3 vblank/vline */
4748                case 4: /* D4 vblank/vline */
4749                case 5: /* D5 vblank/vline */
4750                case 6: /* D6 vblank/vline */
4751                        crtc_idx = src_id - 1;
4752
4753                        if (src_data == 0) { /* vblank */
4754                                mask = LB_D1_VBLANK_INTERRUPT;
4755                                event_name = "vblank";
4756
4757                                if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4758                                        drm_handle_vblank(rdev->ddev, crtc_idx);
4759                                        rdev->pm.vblank_sync = true;
4760                                        wake_up(&rdev->irq.vblank_queue);
4761                                }
4762                                if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4763                                        radeon_crtc_handle_vblank(rdev,
4764                                                                  crtc_idx);
4765                                }
4766
4767                        } else if (src_data == 1) { /* vline */
4768                                mask = LB_D1_VLINE_INTERRUPT;
4769                                event_name = "vline";
4770                        } else {
4771                                DRM_DEBUG("Unhandled interrupt: %d %d\n",
4772                                          src_id, src_data);
4773                                break;
4774                        }
4775
4776                        if (!(disp_int[crtc_idx] & mask)) {
4777                                DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4778                                          crtc_idx + 1, event_name);
4779                        }
4780
4781                        disp_int[crtc_idx] &= ~mask;
4782                        DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4783
4784                        break;
4785                case 8: /* D1 page flip */
4786                case 10: /* D2 page flip */
4787                case 12: /* D3 page flip */
4788                case 14: /* D4 page flip */
4789                case 16: /* D5 page flip */
4790                case 18: /* D6 page flip */
4791                        DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4792                        if (radeon_use_pflipirq > 0)
4793                                radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4794                        break;
4795                case 42: /* HPD hotplug */
4796                        if (src_data <= 5) {
4797                                hpd_idx = src_data;
4798                                mask = DC_HPD1_INTERRUPT;
4799                                queue_hotplug = true;
4800                                event_name = "HPD";
4801
4802                        } else if (src_data <= 11) {
4803                                hpd_idx = src_data - 6;
4804                                mask = DC_HPD1_RX_INTERRUPT;
4805                                queue_dp = true;
4806                                event_name = "HPD_RX";
4807
4808                        } else {
4809                                DRM_DEBUG("Unhandled interrupt: %d %d\n",
4810                                          src_id, src_data);
4811                                break;
4812                        }
4813
4814                        if (!(disp_int[hpd_idx] & mask))
4815                                DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4816
4817                        disp_int[hpd_idx] &= ~mask;
4818                        DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4819
4820                        break;
4821                case 44: /* hdmi */
4822                        afmt_idx = src_data;
4823                        if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4824                                DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4825
4826                        if (afmt_idx > 5) {
4827                                DRM_ERROR("Unhandled interrupt: %d %d\n",
4828                                          src_id, src_data);
4829                                break;
4830                        }
4831                        afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4832                        queue_hdmi = true;
4833                        DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4834                        break;
4835                case 96:
4836                        DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4837                        WREG32(SRBM_INT_ACK, 0x1);
4838                        break;
4839                case 124: /* UVD */
4840                        DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4841                        radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4842                        break;
4843                case 146:
4844                case 147:
4845                        addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4846                        status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4847                        /* reset addr and status */
4848                        WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4849                        if (addr == 0x0 && status == 0x0)
4850                                break;
4851                        dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4852                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4853                                addr);
4854                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4855                                status);
4856                        cayman_vm_decode_fault(rdev, status, addr);
4857                        break;
4858                case 176: /* CP_INT in ring buffer */
4859                case 177: /* CP_INT in IB1 */
4860                case 178: /* CP_INT in IB2 */
4861                        DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4862                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4863                        break;
4864                case 181: /* CP EOP event */
4865                        DRM_DEBUG("IH: CP EOP\n");
4866                        if (rdev->family >= CHIP_CAYMAN) {
4867                                switch (src_data) {
4868                                case 0:
4869                                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4870                                        break;
4871                                case 1:
4872                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4873                                        break;
4874                                case 2:
4875                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4876                                        break;
4877                                }
4878                        } else
4879                                radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4880                        break;
4881                case 224: /* DMA trap event */
4882                        DRM_DEBUG("IH: DMA trap\n");
4883                        radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4884                        break;
4885                case 230: /* thermal low to high */
4886                        DRM_DEBUG("IH: thermal low to high\n");
4887                        rdev->pm.dpm.thermal.high_to_low = false;
4888                        queue_thermal = true;
4889                        break;
4890                case 231: /* thermal high to low */
4891                        DRM_DEBUG("IH: thermal high to low\n");
4892                        rdev->pm.dpm.thermal.high_to_low = true;
4893                        queue_thermal = true;
4894                        break;
4895                case 233: /* GUI IDLE */
4896                        DRM_DEBUG("IH: GUI idle\n");
4897                        break;
4898                case 244: /* DMA trap event */
4899                        if (rdev->family >= CHIP_CAYMAN) {
4900                                DRM_DEBUG("IH: DMA1 trap\n");
4901                                radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4902                        }
4903                        break;
4904                default:
4905                        DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4906                        break;
4907                }
4908
4909                /* wptr/rptr are in bytes! */
4910                rptr += 16;
4911                rptr &= rdev->ih.ptr_mask;
4912                WREG32(IH_RB_RPTR, rptr);
4913        }
4914        if (queue_dp)
4915                schedule_work(&rdev->dp_work);
4916        if (queue_hotplug)
4917                schedule_delayed_work(&rdev->hotplug_work, 0);
4918        if (queue_hdmi)
4919                schedule_work(&rdev->audio_work);
4920        if (queue_thermal && rdev->pm.dpm_enabled)
4921                schedule_work(&rdev->pm.dpm.thermal.work);
4922        rdev->ih.rptr = rptr;
4923        atomic_set(&rdev->ih.lock, 0);
4924
4925        /* make sure wptr hasn't changed while processing */
4926        wptr = evergreen_get_ih_wptr(rdev);
4927        if (wptr != rptr)
4928                goto restart_ih;
4929
4930        return IRQ_HANDLED;
4931}
4932
4933static void evergreen_uvd_init(struct radeon_device *rdev)
4934{
4935        int r;
4936
4937        if (!rdev->has_uvd)
4938                return;
4939
4940        r = radeon_uvd_init(rdev);
4941        if (r) {
4942                dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4943                /*
4944                 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4945                 * to early fails uvd_v2_2_resume() and thus nothing happens
4946                 * there. So it is pointless to try to go through that code
4947                 * hence why we disable uvd here.
4948                 */
4949                rdev->has_uvd = false;
4950                return;
4951        }
4952        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4953        r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4954}
4955
4956static void evergreen_uvd_start(struct radeon_device *rdev)
4957{
4958        int r;
4959
4960        if (!rdev->has_uvd)
4961                return;
4962
4963        r = uvd_v2_2_resume(rdev);
4964        if (r) {
4965                dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4966                goto error;
4967        }
4968        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4969        if (r) {
4970                dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4971                goto error;
4972        }
4973        return;
4974
4975error:
4976        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4977}
4978
4979static void evergreen_uvd_resume(struct radeon_device *rdev)
4980{
4981        struct radeon_ring *ring;
4982        int r;
4983
4984        if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4985                return;
4986
4987        ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4988        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4989        if (r) {
4990                dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4991                return;
4992        }
4993        r = uvd_v1_0_init(rdev);
4994        if (r) {
4995                dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4996                return;
4997        }
4998}
4999
5000static int evergreen_startup(struct radeon_device *rdev)
5001{
5002        struct radeon_ring *ring;
5003        int r;
5004
5005        /* enable pcie gen2 link */
5006        evergreen_pcie_gen2_enable(rdev);
5007        /* enable aspm */
5008        evergreen_program_aspm(rdev);
5009
5010        /* scratch needs to be initialized before MC */
5011        r = r600_vram_scratch_init(rdev);
5012        if (r)
5013                return r;
5014
5015        evergreen_mc_program(rdev);
5016
5017        if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5018                r = ni_mc_load_microcode(rdev);
5019                if (r) {
5020                        DRM_ERROR("Failed to load MC firmware!\n");
5021                        return r;
5022                }
5023        }
5024
5025        if (rdev->flags & RADEON_IS_AGP) {
5026                evergreen_agp_enable(rdev);
5027        } else {
5028                r = evergreen_pcie_gart_enable(rdev);
5029                if (r)
5030                        return r;
5031        }
5032        evergreen_gpu_init(rdev);
5033
5034        /* allocate rlc buffers */
5035        if (rdev->flags & RADEON_IS_IGP) {
5036                rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5037                rdev->rlc.reg_list_size =
5038                        (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5039                rdev->rlc.cs_data = evergreen_cs_data;
5040                r = sumo_rlc_init(rdev);
5041                if (r) {
5042                        DRM_ERROR("Failed to init rlc BOs!\n");
5043                        return r;
5044                }
5045        }
5046
5047        /* allocate wb buffer */
5048        r = radeon_wb_init(rdev);
5049        if (r)
5050                return r;
5051
5052        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5053        if (r) {
5054                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5055                return r;
5056        }
5057
5058        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5059        if (r) {
5060                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5061                return r;
5062        }
5063
5064        evergreen_uvd_start(rdev);
5065
5066        /* Enable IRQ */
5067        if (!rdev->irq.installed) {
5068                r = radeon_irq_kms_init(rdev);
5069                if (r)
5070                        return r;
5071        }
5072
5073        r = r600_irq_init(rdev);
5074        if (r) {
5075                DRM_ERROR("radeon: IH init failed (%d).\n", r);
5076                radeon_irq_kms_fini(rdev);
5077                return r;
5078        }
5079        evergreen_irq_set(rdev);
5080
5081        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5082        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5083                             RADEON_CP_PACKET2);
5084        if (r)
5085                return r;
5086
5087        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5088        r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5089                             DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5090        if (r)
5091                return r;
5092
5093        r = evergreen_cp_load_microcode(rdev);
5094        if (r)
5095                return r;
5096        r = evergreen_cp_resume(rdev);
5097        if (r)
5098                return r;
5099        r = r600_dma_resume(rdev);
5100        if (r)
5101                return r;
5102
5103        evergreen_uvd_resume(rdev);
5104
5105        r = radeon_ib_pool_init(rdev);
5106        if (r) {
5107                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5108                return r;
5109        }
5110
5111        r = radeon_audio_init(rdev);
5112        if (r) {
5113                DRM_ERROR("radeon: audio init failed\n");
5114                return r;
5115        }
5116
5117        return 0;
5118}
5119
5120int evergreen_resume(struct radeon_device *rdev)
5121{
5122        int r;
5123
5124        /* reset the asic, the gfx blocks are often in a bad state
5125         * after the driver is unloaded or after a resume
5126         */
5127        if (radeon_asic_reset(rdev))
5128                dev_warn(rdev->dev, "GPU reset failed !\n");
5129        /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5130         * posting will perform necessary task to bring back GPU into good
5131         * shape.
5132         */
5133        /* post card */
5134        atom_asic_init(rdev->mode_info.atom_context);
5135
5136        /* init golden registers */
5137        evergreen_init_golden_registers(rdev);
5138
5139        if (rdev->pm.pm_method == PM_METHOD_DPM)
5140                radeon_pm_resume(rdev);
5141
5142        rdev->accel_working = true;
5143        r = evergreen_startup(rdev);
5144        if (r) {
5145                DRM_ERROR("evergreen startup failed on resume\n");
5146                rdev->accel_working = false;
5147                return r;
5148        }
5149
5150        return r;
5151
5152}
5153
5154int evergreen_suspend(struct radeon_device *rdev)
5155{
5156        radeon_pm_suspend(rdev);
5157        radeon_audio_fini(rdev);
5158        if (rdev->has_uvd) {
5159                radeon_uvd_suspend(rdev);
5160                uvd_v1_0_fini(rdev);
5161        }
5162        r700_cp_stop(rdev);
5163        r600_dma_stop(rdev);
5164        evergreen_irq_suspend(rdev);
5165        radeon_wb_disable(rdev);
5166        evergreen_pcie_gart_disable(rdev);
5167
5168        return 0;
5169}
5170
5171/* Plan is to move initialization in that function and use
5172 * helper function so that radeon_device_init pretty much
5173 * do nothing more than calling asic specific function. This
5174 * should also allow to remove a bunch of callback function
5175 * like vram_info.
5176 */
5177int evergreen_init(struct radeon_device *rdev)
5178{
5179        int r;
5180
5181        /* Read BIOS */
5182        if (!radeon_get_bios(rdev)) {
5183                if (ASIC_IS_AVIVO(rdev))
5184                        return -EINVAL;
5185        }
5186        /* Must be an ATOMBIOS */
5187        if (!rdev->is_atom_bios) {
5188                dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5189                return -EINVAL;
5190        }
5191        r = radeon_atombios_init(rdev);
5192        if (r)
5193                return r;
5194        /* reset the asic, the gfx blocks are often in a bad state
5195         * after the driver is unloaded or after a resume
5196         */
5197        if (radeon_asic_reset(rdev))
5198                dev_warn(rdev->dev, "GPU reset failed !\n");
5199        /* Post card if necessary */
5200        if (!radeon_card_posted(rdev)) {
5201                if (!rdev->bios) {
5202                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5203                        return -EINVAL;
5204                }
5205                DRM_INFO("GPU not posted. posting now...\n");
5206                atom_asic_init(rdev->mode_info.atom_context);
5207        }
5208        /* init golden registers */
5209        evergreen_init_golden_registers(rdev);
5210        /* Initialize scratch registers */
5211        r600_scratch_init(rdev);
5212        /* Initialize surface registers */
5213        radeon_surface_init(rdev);
5214        /* Initialize clocks */
5215        radeon_get_clock_info(rdev->ddev);
5216        /* Fence driver */
5217        radeon_fence_driver_init(rdev);
5218        /* initialize AGP */
5219        if (rdev->flags & RADEON_IS_AGP) {
5220                r = radeon_agp_init(rdev);
5221                if (r)
5222                        radeon_agp_disable(rdev);
5223        }
5224        /* initialize memory controller */
5225        r = evergreen_mc_init(rdev);
5226        if (r)
5227                return r;
5228        /* Memory manager */
5229        r = radeon_bo_init(rdev);
5230        if (r)
5231                return r;
5232
5233        if (ASIC_IS_DCE5(rdev)) {
5234                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5235                        r = ni_init_microcode(rdev);
5236                        if (r) {
5237                                DRM_ERROR("Failed to load firmware!\n");
5238                                return r;
5239                        }
5240                }
5241        } else {
5242                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5243                        r = r600_init_microcode(rdev);
5244                        if (r) {
5245                                DRM_ERROR("Failed to load firmware!\n");
5246                                return r;
5247                        }
5248                }
5249        }
5250
5251        /* Initialize power management */
5252        radeon_pm_init(rdev);
5253
5254        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5255        r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5256
5257        rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5258        r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5259
5260        evergreen_uvd_init(rdev);
5261
5262        rdev->ih.ring_obj = NULL;
5263        r600_ih_ring_init(rdev, 64 * 1024);
5264
5265        r = r600_pcie_gart_init(rdev);
5266        if (r)
5267                return r;
5268
5269        rdev->accel_working = true;
5270        r = evergreen_startup(rdev);
5271        if (r) {
5272                dev_err(rdev->dev, "disabling GPU acceleration\n");
5273                r700_cp_fini(rdev);
5274                r600_dma_fini(rdev);
5275                r600_irq_fini(rdev);
5276                if (rdev->flags & RADEON_IS_IGP)
5277                        sumo_rlc_fini(rdev);
5278                radeon_wb_fini(rdev);
5279                radeon_ib_pool_fini(rdev);
5280                radeon_irq_kms_fini(rdev);
5281                evergreen_pcie_gart_fini(rdev);
5282                rdev->accel_working = false;
5283        }
5284
5285        /* Don't start up if the MC ucode is missing on BTC parts.
5286         * The default clocks and voltages before the MC ucode
5287         * is loaded are not suffient for advanced operations.
5288         */
5289        if (ASIC_IS_DCE5(rdev)) {
5290                if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5291                        DRM_ERROR("radeon: MC ucode required for NI+.\n");
5292                        return -EINVAL;
5293                }
5294        }
5295
5296        return 0;
5297}
5298
5299void evergreen_fini(struct radeon_device *rdev)
5300{
5301        radeon_pm_fini(rdev);
5302        radeon_audio_fini(rdev);
5303        r700_cp_fini(rdev);
5304        r600_dma_fini(rdev);
5305        r600_irq_fini(rdev);
5306        if (rdev->flags & RADEON_IS_IGP)
5307                sumo_rlc_fini(rdev);
5308        radeon_wb_fini(rdev);
5309        radeon_ib_pool_fini(rdev);
5310        radeon_irq_kms_fini(rdev);
5311        uvd_v1_0_fini(rdev);
5312        radeon_uvd_fini(rdev);
5313        evergreen_pcie_gart_fini(rdev);
5314        r600_vram_scratch_fini(rdev);
5315        radeon_gem_fini(rdev);
5316        radeon_fence_driver_fini(rdev);
5317        radeon_agp_fini(rdev);
5318        radeon_bo_fini(rdev);
5319        radeon_atombios_fini(rdev);
5320        kfree(rdev->bios);
5321        rdev->bios = NULL;
5322}
5323
5324void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5325{
5326        u32 link_width_cntl, speed_cntl;
5327
5328        if (radeon_pcie_gen2 == 0)
5329                return;
5330
5331        if (rdev->flags & RADEON_IS_IGP)
5332                return;
5333
5334        if (!(rdev->flags & RADEON_IS_PCIE))
5335                return;
5336
5337        /* x2 cards have a special sequence */
5338        if (ASIC_IS_X2(rdev))
5339                return;
5340
5341        if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5342                (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5343                return;
5344
5345        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5346        if (speed_cntl & LC_CURRENT_DATA_RATE) {
5347                DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5348                return;
5349        }
5350
5351        DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5352
5353        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5354            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5355
5356                link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5357                link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5358                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5359
5360                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5361                speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5362                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5363
5364                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5365                speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5366                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5367
5368                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5369                speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5370                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5371
5372                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5373                speed_cntl |= LC_GEN2_EN_STRAP;
5374                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5375
5376        } else {
5377                link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5378                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5379                if (1)
5380                        link_width_cntl |= LC_UPCONFIGURE_DIS;
5381                else
5382                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5383                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5384        }
5385}
5386
5387void evergreen_program_aspm(struct radeon_device *rdev)
5388{
5389        u32 data, orig;
5390        u32 pcie_lc_cntl, pcie_lc_cntl_old;
5391        bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5392        /* fusion_platform = true
5393         * if the system is a fusion system
5394         * (APU or DGPU in a fusion system).
5395         * todo: check if the system is a fusion platform.
5396         */
5397        bool fusion_platform = false;
5398
5399        if (radeon_aspm == 0)
5400                return;
5401
5402        if (!(rdev->flags & RADEON_IS_PCIE))
5403                return;
5404
5405        switch (rdev->family) {
5406        case CHIP_CYPRESS:
5407        case CHIP_HEMLOCK:
5408        case CHIP_JUNIPER:
5409        case CHIP_REDWOOD:
5410        case CHIP_CEDAR:
5411        case CHIP_SUMO:
5412        case CHIP_SUMO2:
5413        case CHIP_PALM:
5414        case CHIP_ARUBA:
5415                disable_l0s = true;
5416                break;
5417        default:
5418                disable_l0s = false;
5419                break;
5420        }
5421
5422        if (rdev->flags & RADEON_IS_IGP)
5423                fusion_platform = true; /* XXX also dGPUs in a fusion system */
5424
5425        data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5426        if (fusion_platform)
5427                data &= ~MULTI_PIF;
5428        else
5429                data |= MULTI_PIF;
5430        if (data != orig)
5431                WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5432
5433        data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5434        if (fusion_platform)
5435                data &= ~MULTI_PIF;
5436        else
5437                data |= MULTI_PIF;
5438        if (data != orig)
5439                WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5440
5441        pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5442        pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5443        if (!disable_l0s) {
5444                if (rdev->family >= CHIP_BARTS)
5445                        pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5446                else
5447                        pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5448        }
5449
5450        if (!disable_l1) {
5451                if (rdev->family >= CHIP_BARTS)
5452                        pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5453                else
5454                        pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5455
5456                if (!disable_plloff_in_l1) {
5457                        data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5458                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5459                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5460                        if (data != orig)
5461                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5462
5463                        data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5464                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5465                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5466                        if (data != orig)
5467                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5468
5469                        data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5470                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5471                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5472                        if (data != orig)
5473                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5474
5475                        data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5476                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5477                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5478                        if (data != orig)
5479                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5480
5481                        if (rdev->family >= CHIP_BARTS) {
5482                                data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5483                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
5484                                data |= PLL_RAMP_UP_TIME_0(4);
5485                                if (data != orig)
5486                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5487
5488                                data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5489                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
5490                                data |= PLL_RAMP_UP_TIME_1(4);
5491                                if (data != orig)
5492                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5493
5494                                data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5495                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
5496                                data |= PLL_RAMP_UP_TIME_0(4);
5497                                if (data != orig)
5498                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5499
5500                                data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5501                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
5502                                data |= PLL_RAMP_UP_TIME_1(4);
5503                                if (data != orig)
5504                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5505                        }
5506
5507                        data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5508                        data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5509                        data |= LC_DYN_LANES_PWR_STATE(3);
5510                        if (data != orig)
5511                                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5512
5513                        if (rdev->family >= CHIP_BARTS) {
5514                                data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5515                                data &= ~LS2_EXIT_TIME_MASK;
5516                                data |= LS2_EXIT_TIME(1);
5517                                if (data != orig)
5518                                        WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5519
5520                                data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5521                                data &= ~LS2_EXIT_TIME_MASK;
5522                                data |= LS2_EXIT_TIME(1);
5523                                if (data != orig)
5524                                        WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5525                        }
5526                }
5527        }
5528
5529        /* evergreen parts only */
5530        if (rdev->family < CHIP_BARTS)
5531                pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5532
5533        if (pcie_lc_cntl != pcie_lc_cntl_old)
5534                WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5535}
5536