LXR linux/drivers/gpu/drm/radeon/evergreen.c

   1/*
   2 * Copyright 2010 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/pci.h>
  27#include <linux/slab.h>
  28
  29#include <drm/drm_vblank.h>
  30#include <drm/radeon_drm.h>
  31
  32#include "atom.h"
  33#include "avivod.h"
  34#include "evergreen_blit_shaders.h"
  35#include "evergreen_reg.h"
  36#include "evergreend.h"
  37#include "radeon.h"
  38#include "radeon_asic.h"
  39#include "radeon_audio.h"
  40#include "radeon_ucode.h"
  41
  42#define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  43#define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  44#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  45
  46/*
  47 * Indirect registers accessor
  48 */
  49u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  50{
  51        unsigned long flags;
  52        u32 r;
  53
  54        spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  55        WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  56        r = RREG32(EVERGREEN_CG_IND_DATA);
  57        spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  58        return r;
  59}
  60
  61void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  62{
  63        unsigned long flags;
  64
  65        spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  66        WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  67        WREG32(EVERGREEN_CG_IND_DATA, (v));
  68        spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  69}
  70
  71u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  72{
  73        unsigned long flags;
  74        u32 r;
  75
  76        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  77        WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  78        r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  79        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  80        return r;
  81}
  82
  83void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  84{
  85        unsigned long flags;
  86
  87        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  88        WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  89        WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  90        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  91}
  92
  93u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  94{
  95        unsigned long flags;
  96        u32 r;
  97
  98        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  99        WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 100        r = RREG32(EVERGREEN_PIF_PHY1_DATA);
 101        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 102        return r;
 103}
 104
 105void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 106{
 107        unsigned long flags;
 108
 109        spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 110        WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 111        WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 112        spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 113}
 114
 115static const u32 crtc_offsets[6] =
 116{
 117        EVERGREEN_CRTC0_REGISTER_OFFSET,
 118        EVERGREEN_CRTC1_REGISTER_OFFSET,
 119        EVERGREEN_CRTC2_REGISTER_OFFSET,
 120        EVERGREEN_CRTC3_REGISTER_OFFSET,
 121        EVERGREEN_CRTC4_REGISTER_OFFSET,
 122        EVERGREEN_CRTC5_REGISTER_OFFSET
 123};
 124
 125#include "clearstate_evergreen.h"
 126
 127static const u32 sumo_rlc_save_restore_register_list[] =
 128{
 129        0x98fc,
 130        0x9830,
 131        0x9834,
 132        0x9838,
 133        0x9870,
 134        0x9874,
 135        0x8a14,
 136        0x8b24,
 137        0x8bcc,
 138        0x8b10,
 139        0x8d00,
 140        0x8d04,
 141        0x8c00,
 142        0x8c04,
 143        0x8c08,
 144        0x8c0c,
 145        0x8d8c,
 146        0x8c20,
 147        0x8c24,
 148        0x8c28,
 149        0x8c18,
 150        0x8c1c,
 151        0x8cf0,
 152        0x8e2c,
 153        0x8e38,
 154        0x8c30,
 155        0x9508,
 156        0x9688,
 157        0x9608,
 158        0x960c,
 159        0x9610,
 160        0x9614,
 161        0x88c4,
 162        0x88d4,
 163        0xa008,
 164        0x900c,
 165        0x9100,
 166        0x913c,
 167        0x98f8,
 168        0x98f4,
 169        0x9b7c,
 170        0x3f8c,
 171        0x8950,
 172        0x8954,
 173        0x8a18,
 174        0x8b28,
 175        0x9144,
 176        0x9148,
 177        0x914c,
 178        0x3f90,
 179        0x3f94,
 180        0x915c,
 181        0x9160,
 182        0x9178,
 183        0x917c,
 184        0x9180,
 185        0x918c,
 186        0x9190,
 187        0x9194,
 188        0x9198,
 189        0x919c,
 190        0x91a8,
 191        0x91ac,
 192        0x91b0,
 193        0x91b4,
 194        0x91b8,
 195        0x91c4,
 196        0x91c8,
 197        0x91cc,
 198        0x91d0,
 199        0x91d4,
 200        0x91e0,
 201        0x91e4,
 202        0x91ec,
 203        0x91f0,
 204        0x91f4,
 205        0x9200,
 206        0x9204,
 207        0x929c,
 208        0x9150,
 209        0x802c,
 210};
 211
 212static void evergreen_gpu_init(struct radeon_device *rdev);
 213void evergreen_fini(struct radeon_device *rdev);
 214void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 215void evergreen_program_aspm(struct radeon_device *rdev);
 216extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 217                                     int ring, u32 cp_int_cntl);
 218extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 219                                   u32 status, u32 addr);
 220void cik_init_cp_pg_table(struct radeon_device *rdev);
 221
 222extern u32 si_get_csb_size(struct radeon_device *rdev);
 223extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 224extern u32 cik_get_csb_size(struct radeon_device *rdev);
 225extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 226extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 227
 228static const u32 evergreen_golden_registers[] =
 229{
 230        0x3f90, 0xffff0000, 0xff000000,
 231        0x9148, 0xffff0000, 0xff000000,
 232        0x3f94, 0xffff0000, 0xff000000,
 233        0x914c, 0xffff0000, 0xff000000,
 234        0x9b7c, 0xffffffff, 0x00000000,
 235        0x8a14, 0xffffffff, 0x00000007,
 236        0x8b10, 0xffffffff, 0x00000000,
 237        0x960c, 0xffffffff, 0x54763210,
 238        0x88c4, 0xffffffff, 0x000000c2,
 239        0x88d4, 0xffffffff, 0x00000010,
 240        0x8974, 0xffffffff, 0x00000000,
 241        0xc78, 0x00000080, 0x00000080,
 242        0x5eb4, 0xffffffff, 0x00000002,
 243        0x5e78, 0xffffffff, 0x001000f0,
 244        0x6104, 0x01000300, 0x00000000,
 245        0x5bc0, 0x00300000, 0x00000000,
 246        0x7030, 0xffffffff, 0x00000011,
 247        0x7c30, 0xffffffff, 0x00000011,
 248        0x10830, 0xffffffff, 0x00000011,
 249        0x11430, 0xffffffff, 0x00000011,
 250        0x12030, 0xffffffff, 0x00000011,
 251        0x12c30, 0xffffffff, 0x00000011,
 252        0xd02c, 0xffffffff, 0x08421000,
 253        0x240c, 0xffffffff, 0x00000380,
 254        0x8b24, 0xffffffff, 0x00ff0fff,
 255        0x28a4c, 0x06000000, 0x06000000,
 256        0x10c, 0x00000001, 0x00000001,
 257        0x8d00, 0xffffffff, 0x100e4848,
 258        0x8d04, 0xffffffff, 0x00164745,
 259        0x8c00, 0xffffffff, 0xe4000003,
 260        0x8c04, 0xffffffff, 0x40600060,
 261        0x8c08, 0xffffffff, 0x001c001c,
 262        0x8cf0, 0xffffffff, 0x08e00620,
 263        0x8c20, 0xffffffff, 0x00800080,
 264        0x8c24, 0xffffffff, 0x00800080,
 265        0x8c18, 0xffffffff, 0x20202078,
 266        0x8c1c, 0xffffffff, 0x00001010,
 267        0x28350, 0xffffffff, 0x00000000,
 268        0xa008, 0xffffffff, 0x00010000,
 269        0x5c4, 0xffffffff, 0x00000001,
 270        0x9508, 0xffffffff, 0x00000002,
 271        0x913c, 0x0000000f, 0x0000000a
 272};
 273
 274static const u32 evergreen_golden_registers2[] =
 275{
 276        0x2f4c, 0xffffffff, 0x00000000,
 277        0x54f4, 0xffffffff, 0x00000000,
 278        0x54f0, 0xffffffff, 0x00000000,
 279        0x5498, 0xffffffff, 0x00000000,
 280        0x549c, 0xffffffff, 0x00000000,
 281        0x5494, 0xffffffff, 0x00000000,
 282        0x53cc, 0xffffffff, 0x00000000,
 283        0x53c8, 0xffffffff, 0x00000000,
 284        0x53c4, 0xffffffff, 0x00000000,
 285        0x53c0, 0xffffffff, 0x00000000,
 286        0x53bc, 0xffffffff, 0x00000000,
 287        0x53b8, 0xffffffff, 0x00000000,
 288        0x53b4, 0xffffffff, 0x00000000,
 289        0x53b0, 0xffffffff, 0x00000000
 290};
 291
 292static const u32 cypress_mgcg_init[] =
 293{
 294        0x802c, 0xffffffff, 0xc0000000,
 295        0x5448, 0xffffffff, 0x00000100,
 296        0x55e4, 0xffffffff, 0x00000100,
 297        0x160c, 0xffffffff, 0x00000100,
 298        0x5644, 0xffffffff, 0x00000100,
 299        0xc164, 0xffffffff, 0x00000100,
 300        0x8a18, 0xffffffff, 0x00000100,
 301        0x897c, 0xffffffff, 0x06000100,
 302        0x8b28, 0xffffffff, 0x00000100,
 303        0x9144, 0xffffffff, 0x00000100,
 304        0x9a60, 0xffffffff, 0x00000100,
 305        0x9868, 0xffffffff, 0x00000100,
 306        0x8d58, 0xffffffff, 0x00000100,
 307        0x9510, 0xffffffff, 0x00000100,
 308        0x949c, 0xffffffff, 0x00000100,
 309        0x9654, 0xffffffff, 0x00000100,
 310        0x9030, 0xffffffff, 0x00000100,
 311        0x9034, 0xffffffff, 0x00000100,
 312        0x9038, 0xffffffff, 0x00000100,
 313        0x903c, 0xffffffff, 0x00000100,
 314        0x9040, 0xffffffff, 0x00000100,
 315        0xa200, 0xffffffff, 0x00000100,
 316        0xa204, 0xffffffff, 0x00000100,
 317        0xa208, 0xffffffff, 0x00000100,
 318        0xa20c, 0xffffffff, 0x00000100,
 319        0x971c, 0xffffffff, 0x00000100,
 320        0x977c, 0xffffffff, 0x00000100,
 321        0x3f80, 0xffffffff, 0x00000100,
 322        0xa210, 0xffffffff, 0x00000100,
 323        0xa214, 0xffffffff, 0x00000100,
 324        0x4d8, 0xffffffff, 0x00000100,
 325        0x9784, 0xffffffff, 0x00000100,
 326        0x9698, 0xffffffff, 0x00000100,
 327        0x4d4, 0xffffffff, 0x00000200,
 328        0x30cc, 0xffffffff, 0x00000100,
 329        0xd0c0, 0xffffffff, 0xff000100,
 330        0x802c, 0xffffffff, 0x40000000,
 331        0x915c, 0xffffffff, 0x00010000,
 332        0x9160, 0xffffffff, 0x00030002,
 333        0x9178, 0xffffffff, 0x00070000,
 334        0x917c, 0xffffffff, 0x00030002,
 335        0x9180, 0xffffffff, 0x00050004,
 336        0x918c, 0xffffffff, 0x00010006,
 337        0x9190, 0xffffffff, 0x00090008,
 338        0x9194, 0xffffffff, 0x00070000,
 339        0x9198, 0xffffffff, 0x00030002,
 340        0x919c, 0xffffffff, 0x00050004,
 341        0x91a8, 0xffffffff, 0x00010006,
 342        0x91ac, 0xffffffff, 0x00090008,
 343        0x91b0, 0xffffffff, 0x00070000,
 344        0x91b4, 0xffffffff, 0x00030002,
 345        0x91b8, 0xffffffff, 0x00050004,
 346        0x91c4, 0xffffffff, 0x00010006,
 347        0x91c8, 0xffffffff, 0x00090008,
 348        0x91cc, 0xffffffff, 0x00070000,
 349        0x91d0, 0xffffffff, 0x00030002,
 350        0x91d4, 0xffffffff, 0x00050004,
 351        0x91e0, 0xffffffff, 0x00010006,
 352        0x91e4, 0xffffffff, 0x00090008,
 353        0x91e8, 0xffffffff, 0x00000000,
 354        0x91ec, 0xffffffff, 0x00070000,
 355        0x91f0, 0xffffffff, 0x00030002,
 356        0x91f4, 0xffffffff, 0x00050004,
 357        0x9200, 0xffffffff, 0x00010006,
 358        0x9204, 0xffffffff, 0x00090008,
 359        0x9208, 0xffffffff, 0x00070000,
 360        0x920c, 0xffffffff, 0x00030002,
 361        0x9210, 0xffffffff, 0x00050004,
 362        0x921c, 0xffffffff, 0x00010006,
 363        0x9220, 0xffffffff, 0x00090008,
 364        0x9224, 0xffffffff, 0x00070000,
 365        0x9228, 0xffffffff, 0x00030002,
 366        0x922c, 0xffffffff, 0x00050004,
 367        0x9238, 0xffffffff, 0x00010006,
 368        0x923c, 0xffffffff, 0x00090008,
 369        0x9240, 0xffffffff, 0x00070000,
 370        0x9244, 0xffffffff, 0x00030002,
 371        0x9248, 0xffffffff, 0x00050004,
 372        0x9254, 0xffffffff, 0x00010006,
 373        0x9258, 0xffffffff, 0x00090008,
 374        0x925c, 0xffffffff, 0x00070000,
 375        0x9260, 0xffffffff, 0x00030002,
 376        0x9264, 0xffffffff, 0x00050004,
 377        0x9270, 0xffffffff, 0x00010006,
 378        0x9274, 0xffffffff, 0x00090008,
 379        0x9278, 0xffffffff, 0x00070000,
 380        0x927c, 0xffffffff, 0x00030002,
 381        0x9280, 0xffffffff, 0x00050004,
 382        0x928c, 0xffffffff, 0x00010006,
 383        0x9290, 0xffffffff, 0x00090008,
 384        0x9294, 0xffffffff, 0x00000000,
 385        0x929c, 0xffffffff, 0x00000001,
 386        0x802c, 0xffffffff, 0x40010000,
 387        0x915c, 0xffffffff, 0x00010000,
 388        0x9160, 0xffffffff, 0x00030002,
 389        0x9178, 0xffffffff, 0x00070000,
 390        0x917c, 0xffffffff, 0x00030002,
 391        0x9180, 0xffffffff, 0x00050004,
 392        0x918c, 0xffffffff, 0x00010006,
 393        0x9190, 0xffffffff, 0x00090008,
 394        0x9194, 0xffffffff, 0x00070000,
 395        0x9198, 0xffffffff, 0x00030002,
 396        0x919c, 0xffffffff, 0x00050004,
 397        0x91a8, 0xffffffff, 0x00010006,
 398        0x91ac, 0xffffffff, 0x00090008,
 399        0x91b0, 0xffffffff, 0x00070000,
 400        0x91b4, 0xffffffff, 0x00030002,
 401        0x91b8, 0xffffffff, 0x00050004,
 402        0x91c4, 0xffffffff, 0x00010006,
 403        0x91c8, 0xffffffff, 0x00090008,
 404        0x91cc, 0xffffffff, 0x00070000,
 405        0x91d0, 0xffffffff, 0x00030002,
 406        0x91d4, 0xffffffff, 0x00050004,
 407        0x91e0, 0xffffffff, 0x00010006,
 408        0x91e4, 0xffffffff, 0x00090008,
 409        0x91e8, 0xffffffff, 0x00000000,
 410        0x91ec, 0xffffffff, 0x00070000,
 411        0x91f0, 0xffffffff, 0x00030002,
 412        0x91f4, 0xffffffff, 0x00050004,
 413        0x9200, 0xffffffff, 0x00010006,
 414        0x9204, 0xffffffff, 0x00090008,
 415        0x9208, 0xffffffff, 0x00070000,
 416        0x920c, 0xffffffff, 0x00030002,
 417        0x9210, 0xffffffff, 0x00050004,
 418        0x921c, 0xffffffff, 0x00010006,
 419        0x9220, 0xffffffff, 0x00090008,
 420        0x9224, 0xffffffff, 0x00070000,
 421        0x9228, 0xffffffff, 0x00030002,
 422        0x922c, 0xffffffff, 0x00050004,
 423        0x9238, 0xffffffff, 0x00010006,
 424        0x923c, 0xffffffff, 0x00090008,
 425        0x9240, 0xffffffff, 0x00070000,
 426        0x9244, 0xffffffff, 0x00030002,
 427        0x9248, 0xffffffff, 0x00050004,
 428        0x9254, 0xffffffff, 0x00010006,
 429        0x9258, 0xffffffff, 0x00090008,
 430        0x925c, 0xffffffff, 0x00070000,
 431        0x9260, 0xffffffff, 0x00030002,
 432        0x9264, 0xffffffff, 0x00050004,
 433        0x9270, 0xffffffff, 0x00010006,
 434        0x9274, 0xffffffff, 0x00090008,
 435        0x9278, 0xffffffff, 0x00070000,
 436        0x927c, 0xffffffff, 0x00030002,
 437        0x9280, 0xffffffff, 0x00050004,
 438        0x928c, 0xffffffff, 0x00010006,
 439        0x9290, 0xffffffff, 0x00090008,
 440        0x9294, 0xffffffff, 0x00000000,
 441        0x929c, 0xffffffff, 0x00000001,
 442        0x802c, 0xffffffff, 0xc0000000
 443};
 444
 445static const u32 redwood_mgcg_init[] =
 446{
 447        0x802c, 0xffffffff, 0xc0000000,
 448        0x5448, 0xffffffff, 0x00000100,
 449        0x55e4, 0xffffffff, 0x00000100,
 450        0x160c, 0xffffffff, 0x00000100,
 451        0x5644, 0xffffffff, 0x00000100,
 452        0xc164, 0xffffffff, 0x00000100,
 453        0x8a18, 0xffffffff, 0x00000100,
 454        0x897c, 0xffffffff, 0x06000100,
 455        0x8b28, 0xffffffff, 0x00000100,
 456        0x9144, 0xffffffff, 0x00000100,
 457        0x9a60, 0xffffffff, 0x00000100,
 458        0x9868, 0xffffffff, 0x00000100,
 459        0x8d58, 0xffffffff, 0x00000100,
 460        0x9510, 0xffffffff, 0x00000100,
 461        0x949c, 0xffffffff, 0x00000100,
 462        0x9654, 0xffffffff, 0x00000100,
 463        0x9030, 0xffffffff, 0x00000100,
 464        0x9034, 0xffffffff, 0x00000100,
 465        0x9038, 0xffffffff, 0x00000100,
 466        0x903c, 0xffffffff, 0x00000100,
 467        0x9040, 0xffffffff, 0x00000100,
 468        0xa200, 0xffffffff, 0x00000100,
 469        0xa204, 0xffffffff, 0x00000100,
 470        0xa208, 0xffffffff, 0x00000100,
 471        0xa20c, 0xffffffff, 0x00000100,
 472        0x971c, 0xffffffff, 0x00000100,
 473        0x977c, 0xffffffff, 0x00000100,
 474        0x3f80, 0xffffffff, 0x00000100,
 475        0xa210, 0xffffffff, 0x00000100,
 476        0xa214, 0xffffffff, 0x00000100,
 477        0x4d8, 0xffffffff, 0x00000100,
 478        0x9784, 0xffffffff, 0x00000100,
 479        0x9698, 0xffffffff, 0x00000100,
 480        0x4d4, 0xffffffff, 0x00000200,
 481        0x30cc, 0xffffffff, 0x00000100,
 482        0xd0c0, 0xffffffff, 0xff000100,
 483        0x802c, 0xffffffff, 0x40000000,
 484        0x915c, 0xffffffff, 0x00010000,
 485        0x9160, 0xffffffff, 0x00030002,
 486        0x9178, 0xffffffff, 0x00070000,
 487        0x917c, 0xffffffff, 0x00030002,
 488        0x9180, 0xffffffff, 0x00050004,
 489        0x918c, 0xffffffff, 0x00010006,
 490        0x9190, 0xffffffff, 0x00090008,
 491        0x9194, 0xffffffff, 0x00070000,
 492        0x9198, 0xffffffff, 0x00030002,
 493        0x919c, 0xffffffff, 0x00050004,
 494        0x91a8, 0xffffffff, 0x00010006,
 495        0x91ac, 0xffffffff, 0x00090008,
 496        0x91b0, 0xffffffff, 0x00070000,
 497        0x91b4, 0xffffffff, 0x00030002,
 498        0x91b8, 0xffffffff, 0x00050004,
 499        0x91c4, 0xffffffff, 0x00010006,
 500        0x91c8, 0xffffffff, 0x00090008,
 501        0x91cc, 0xffffffff, 0x00070000,
 502        0x91d0, 0xffffffff, 0x00030002,
 503        0x91d4, 0xffffffff, 0x00050004,
 504        0x91e0, 0xffffffff, 0x00010006,
 505        0x91e4, 0xffffffff, 0x00090008,
 506        0x91e8, 0xffffffff, 0x00000000,
 507        0x91ec, 0xffffffff, 0x00070000,
 508        0x91f0, 0xffffffff, 0x00030002,
 509        0x91f4, 0xffffffff, 0x00050004,
 510        0x9200, 0xffffffff, 0x00010006,
 511        0x9204, 0xffffffff, 0x00090008,
 512        0x9294, 0xffffffff, 0x00000000,
 513        0x929c, 0xffffffff, 0x00000001,
 514        0x802c, 0xffffffff, 0xc0000000
 515};
 516
 517static const u32 cedar_golden_registers[] =
 518{
 519        0x3f90, 0xffff0000, 0xff000000,
 520        0x9148, 0xffff0000, 0xff000000,
 521        0x3f94, 0xffff0000, 0xff000000,
 522        0x914c, 0xffff0000, 0xff000000,
 523        0x9b7c, 0xffffffff, 0x00000000,
 524        0x8a14, 0xffffffff, 0x00000007,
 525        0x8b10, 0xffffffff, 0x00000000,
 526        0x960c, 0xffffffff, 0x54763210,
 527        0x88c4, 0xffffffff, 0x000000c2,
 528        0x88d4, 0xffffffff, 0x00000000,
 529        0x8974, 0xffffffff, 0x00000000,
 530        0xc78, 0x00000080, 0x00000080,
 531        0x5eb4, 0xffffffff, 0x00000002,
 532        0x5e78, 0xffffffff, 0x001000f0,
 533        0x6104, 0x01000300, 0x00000000,
 534        0x5bc0, 0x00300000, 0x00000000,
 535        0x7030, 0xffffffff, 0x00000011,
 536        0x7c30, 0xffffffff, 0x00000011,
 537        0x10830, 0xffffffff, 0x00000011,
 538        0x11430, 0xffffffff, 0x00000011,
 539        0xd02c, 0xffffffff, 0x08421000,
 540        0x240c, 0xffffffff, 0x00000380,
 541        0x8b24, 0xffffffff, 0x00ff0fff,
 542        0x28a4c, 0x06000000, 0x06000000,
 543        0x10c, 0x00000001, 0x00000001,
 544        0x8d00, 0xffffffff, 0x100e4848,
 545        0x8d04, 0xffffffff, 0x00164745,
 546        0x8c00, 0xffffffff, 0xe4000003,
 547        0x8c04, 0xffffffff, 0x40600060,
 548        0x8c08, 0xffffffff, 0x001c001c,
 549        0x8cf0, 0xffffffff, 0x08e00410,
 550        0x8c20, 0xffffffff, 0x00800080,
 551        0x8c24, 0xffffffff, 0x00800080,
 552        0x8c18, 0xffffffff, 0x20202078,
 553        0x8c1c, 0xffffffff, 0x00001010,
 554        0x28350, 0xffffffff, 0x00000000,
 555        0xa008, 0xffffffff, 0x00010000,
 556        0x5c4, 0xffffffff, 0x00000001,
 557        0x9508, 0xffffffff, 0x00000002
 558};
 559
 560static const u32 cedar_mgcg_init[] =
 561{
 562        0x802c, 0xffffffff, 0xc0000000,
 563        0x5448, 0xffffffff, 0x00000100,
 564        0x55e4, 0xffffffff, 0x00000100,
 565        0x160c, 0xffffffff, 0x00000100,
 566        0x5644, 0xffffffff, 0x00000100,
 567        0xc164, 0xffffffff, 0x00000100,
 568        0x8a18, 0xffffffff, 0x00000100,
 569        0x897c, 0xffffffff, 0x06000100,
 570        0x8b28, 0xffffffff, 0x00000100,
 571        0x9144, 0xffffffff, 0x00000100,
 572        0x9a60, 0xffffffff, 0x00000100,
 573        0x9868, 0xffffffff, 0x00000100,
 574        0x8d58, 0xffffffff, 0x00000100,
 575        0x9510, 0xffffffff, 0x00000100,
 576        0x949c, 0xffffffff, 0x00000100,
 577        0x9654, 0xffffffff, 0x00000100,
 578        0x9030, 0xffffffff, 0x00000100,
 579        0x9034, 0xffffffff, 0x00000100,
 580        0x9038, 0xffffffff, 0x00000100,
 581        0x903c, 0xffffffff, 0x00000100,
 582        0x9040, 0xffffffff, 0x00000100,
 583        0xa200, 0xffffffff, 0x00000100,
 584        0xa204, 0xffffffff, 0x00000100,
 585        0xa208, 0xffffffff, 0x00000100,
 586        0xa20c, 0xffffffff, 0x00000100,
 587        0x971c, 0xffffffff, 0x00000100,
 588        0x977c, 0xffffffff, 0x00000100,
 589        0x3f80, 0xffffffff, 0x00000100,
 590        0xa210, 0xffffffff, 0x00000100,
 591        0xa214, 0xffffffff, 0x00000100,
 592        0x4d8, 0xffffffff, 0x00000100,
 593        0x9784, 0xffffffff, 0x00000100,
 594        0x9698, 0xffffffff, 0x00000100,
 595        0x4d4, 0xffffffff, 0x00000200,
 596        0x30cc, 0xffffffff, 0x00000100,
 597        0xd0c0, 0xffffffff, 0xff000100,
 598        0x802c, 0xffffffff, 0x40000000,
 599        0x915c, 0xffffffff, 0x00010000,
 600        0x9178, 0xffffffff, 0x00050000,
 601        0x917c, 0xffffffff, 0x00030002,
 602        0x918c, 0xffffffff, 0x00010004,
 603        0x9190, 0xffffffff, 0x00070006,
 604        0x9194, 0xffffffff, 0x00050000,
 605        0x9198, 0xffffffff, 0x00030002,
 606        0x91a8, 0xffffffff, 0x00010004,
 607        0x91ac, 0xffffffff, 0x00070006,
 608        0x91e8, 0xffffffff, 0x00000000,
 609        0x9294, 0xffffffff, 0x00000000,
 610        0x929c, 0xffffffff, 0x00000001,
 611        0x802c, 0xffffffff, 0xc0000000
 612};
 613
 614static const u32 juniper_mgcg_init[] =
 615{
 616        0x802c, 0xffffffff, 0xc0000000,
 617        0x5448, 0xffffffff, 0x00000100,
 618        0x55e4, 0xffffffff, 0x00000100,
 619        0x160c, 0xffffffff, 0x00000100,
 620        0x5644, 0xffffffff, 0x00000100,
 621        0xc164, 0xffffffff, 0x00000100,
 622        0x8a18, 0xffffffff, 0x00000100,
 623        0x897c, 0xffffffff, 0x06000100,
 624        0x8b28, 0xffffffff, 0x00000100,
 625        0x9144, 0xffffffff, 0x00000100,
 626        0x9a60, 0xffffffff, 0x00000100,
 627        0x9868, 0xffffffff, 0x00000100,
 628        0x8d58, 0xffffffff, 0x00000100,
 629        0x9510, 0xffffffff, 0x00000100,
 630        0x949c, 0xffffffff, 0x00000100,
 631        0x9654, 0xffffffff, 0x00000100,
 632        0x9030, 0xffffffff, 0x00000100,
 633        0x9034, 0xffffffff, 0x00000100,
 634        0x9038, 0xffffffff, 0x00000100,
 635        0x903c, 0xffffffff, 0x00000100,
 636        0x9040, 0xffffffff, 0x00000100,
 637        0xa200, 0xffffffff, 0x00000100,
 638        0xa204, 0xffffffff, 0x00000100,
 639        0xa208, 0xffffffff, 0x00000100,
 640        0xa20c, 0xffffffff, 0x00000100,
 641        0x971c, 0xffffffff, 0x00000100,
 642        0xd0c0, 0xffffffff, 0xff000100,
 643        0x802c, 0xffffffff, 0x40000000,
 644        0x915c, 0xffffffff, 0x00010000,
 645        0x9160, 0xffffffff, 0x00030002,
 646        0x9178, 0xffffffff, 0x00070000,
 647        0x917c, 0xffffffff, 0x00030002,
 648        0x9180, 0xffffffff, 0x00050004,
 649        0x918c, 0xffffffff, 0x00010006,
 650        0x9190, 0xffffffff, 0x00090008,
 651        0x9194, 0xffffffff, 0x00070000,
 652        0x9198, 0xffffffff, 0x00030002,
 653        0x919c, 0xffffffff, 0x00050004,
 654        0x91a8, 0xffffffff, 0x00010006,
 655        0x91ac, 0xffffffff, 0x00090008,
 656        0x91b0, 0xffffffff, 0x00070000,
 657        0x91b4, 0xffffffff, 0x00030002,
 658        0x91b8, 0xffffffff, 0x00050004,
 659        0x91c4, 0xffffffff, 0x00010006,
 660        0x91c8, 0xffffffff, 0x00090008,
 661        0x91cc, 0xffffffff, 0x00070000,
 662        0x91d0, 0xffffffff, 0x00030002,
 663        0x91d4, 0xffffffff, 0x00050004,
 664        0x91e0, 0xffffffff, 0x00010006,
 665        0x91e4, 0xffffffff, 0x00090008,
 666        0x91e8, 0xffffffff, 0x00000000,
 667        0x91ec, 0xffffffff, 0x00070000,
 668        0x91f0, 0xffffffff, 0x00030002,
 669        0x91f4, 0xffffffff, 0x00050004,
 670        0x9200, 0xffffffff, 0x00010006,
 671        0x9204, 0xffffffff, 0x00090008,
 672        0x9208, 0xffffffff, 0x00070000,
 673        0x920c, 0xffffffff, 0x00030002,
 674        0x9210, 0xffffffff, 0x00050004,
 675        0x921c, 0xffffffff, 0x00010006,
 676        0x9220, 0xffffffff, 0x00090008,
 677        0x9224, 0xffffffff, 0x00070000,
 678        0x9228, 0xffffffff, 0x00030002,
 679        0x922c, 0xffffffff, 0x00050004,
 680        0x9238, 0xffffffff, 0x00010006,
 681        0x923c, 0xffffffff, 0x00090008,
 682        0x9240, 0xffffffff, 0x00070000,
 683        0x9244, 0xffffffff, 0x00030002,
 684        0x9248, 0xffffffff, 0x00050004,
 685        0x9254, 0xffffffff, 0x00010006,
 686        0x9258, 0xffffffff, 0x00090008,
 687        0x925c, 0xffffffff, 0x00070000,
 688        0x9260, 0xffffffff, 0x00030002,
 689        0x9264, 0xffffffff, 0x00050004,
 690        0x9270, 0xffffffff, 0x00010006,
 691        0x9274, 0xffffffff, 0x00090008,
 692        0x9278, 0xffffffff, 0x00070000,
 693        0x927c, 0xffffffff, 0x00030002,
 694        0x9280, 0xffffffff, 0x00050004,
 695        0x928c, 0xffffffff, 0x00010006,
 696        0x9290, 0xffffffff, 0x00090008,
 697        0x9294, 0xffffffff, 0x00000000,
 698        0x929c, 0xffffffff, 0x00000001,
 699        0x802c, 0xffffffff, 0xc0000000,
 700        0x977c, 0xffffffff, 0x00000100,
 701        0x3f80, 0xffffffff, 0x00000100,
 702        0xa210, 0xffffffff, 0x00000100,
 703        0xa214, 0xffffffff, 0x00000100,
 704        0x4d8, 0xffffffff, 0x00000100,
 705        0x9784, 0xffffffff, 0x00000100,
 706        0x9698, 0xffffffff, 0x00000100,
 707        0x4d4, 0xffffffff, 0x00000200,
 708        0x30cc, 0xffffffff, 0x00000100,
 709        0x802c, 0xffffffff, 0xc0000000
 710};
 711
 712static const u32 supersumo_golden_registers[] =
 713{
 714        0x5eb4, 0xffffffff, 0x00000002,
 715        0x5c4, 0xffffffff, 0x00000001,
 716        0x7030, 0xffffffff, 0x00000011,
 717        0x7c30, 0xffffffff, 0x00000011,
 718        0x6104, 0x01000300, 0x00000000,
 719        0x5bc0, 0x00300000, 0x00000000,
 720        0x8c04, 0xffffffff, 0x40600060,
 721        0x8c08, 0xffffffff, 0x001c001c,
 722        0x8c20, 0xffffffff, 0x00800080,
 723        0x8c24, 0xffffffff, 0x00800080,
 724        0x8c18, 0xffffffff, 0x20202078,
 725        0x8c1c, 0xffffffff, 0x00001010,
 726        0x918c, 0xffffffff, 0x00010006,
 727        0x91a8, 0xffffffff, 0x00010006,
 728        0x91c4, 0xffffffff, 0x00010006,
 729        0x91e0, 0xffffffff, 0x00010006,
 730        0x9200, 0xffffffff, 0x00010006,
 731        0x9150, 0xffffffff, 0x6e944040,
 732        0x917c, 0xffffffff, 0x00030002,
 733        0x9180, 0xffffffff, 0x00050004,
 734        0x9198, 0xffffffff, 0x00030002,
 735        0x919c, 0xffffffff, 0x00050004,
 736        0x91b4, 0xffffffff, 0x00030002,
 737        0x91b8, 0xffffffff, 0x00050004,
 738        0x91d0, 0xffffffff, 0x00030002,
 739        0x91d4, 0xffffffff, 0x00050004,
 740        0x91f0, 0xffffffff, 0x00030002,
 741        0x91f4, 0xffffffff, 0x00050004,
 742        0x915c, 0xffffffff, 0x00010000,
 743        0x9160, 0xffffffff, 0x00030002,
 744        0x3f90, 0xffff0000, 0xff000000,
 745        0x9178, 0xffffffff, 0x00070000,
 746        0x9194, 0xffffffff, 0x00070000,
 747        0x91b0, 0xffffffff, 0x00070000,
 748        0x91cc, 0xffffffff, 0x00070000,
 749        0x91ec, 0xffffffff, 0x00070000,
 750        0x9148, 0xffff0000, 0xff000000,
 751        0x9190, 0xffffffff, 0x00090008,
 752        0x91ac, 0xffffffff, 0x00090008,
 753        0x91c8, 0xffffffff, 0x00090008,
 754        0x91e4, 0xffffffff, 0x00090008,
 755        0x9204, 0xffffffff, 0x00090008,
 756        0x3f94, 0xffff0000, 0xff000000,
 757        0x914c, 0xffff0000, 0xff000000,
 758        0x929c, 0xffffffff, 0x00000001,
 759        0x8a18, 0xffffffff, 0x00000100,
 760        0x8b28, 0xffffffff, 0x00000100,
 761        0x9144, 0xffffffff, 0x00000100,
 762        0x5644, 0xffffffff, 0x00000100,
 763        0x9b7c, 0xffffffff, 0x00000000,
 764        0x8030, 0xffffffff, 0x0000100a,
 765        0x8a14, 0xffffffff, 0x00000007,
 766        0x8b24, 0xffffffff, 0x00ff0fff,
 767        0x8b10, 0xffffffff, 0x00000000,
 768        0x28a4c, 0x06000000, 0x06000000,
 769        0x4d8, 0xffffffff, 0x00000100,
 770        0x913c, 0xffff000f, 0x0100000a,
 771        0x960c, 0xffffffff, 0x54763210,
 772        0x88c4, 0xffffffff, 0x000000c2,
 773        0x88d4, 0xffffffff, 0x00000010,
 774        0x8974, 0xffffffff, 0x00000000,
 775        0xc78, 0x00000080, 0x00000080,
 776        0x5e78, 0xffffffff, 0x001000f0,
 777        0xd02c, 0xffffffff, 0x08421000,
 778        0xa008, 0xffffffff, 0x00010000,
 779        0x8d00, 0xffffffff, 0x100e4848,
 780        0x8d04, 0xffffffff, 0x00164745,
 781        0x8c00, 0xffffffff, 0xe4000003,
 782        0x8cf0, 0x1fffffff, 0x08e00620,
 783        0x28350, 0xffffffff, 0x00000000,
 784        0x9508, 0xffffffff, 0x00000002
 785};
 786
 787static const u32 sumo_golden_registers[] =
 788{
 789        0x900c, 0x00ffffff, 0x0017071f,
 790        0x8c18, 0xffffffff, 0x10101060,
 791        0x8c1c, 0xffffffff, 0x00001010,
 792        0x8c30, 0x0000000f, 0x00000005,
 793        0x9688, 0x0000000f, 0x00000007
 794};
 795
 796static const u32 wrestler_golden_registers[] =
 797{
 798        0x5eb4, 0xffffffff, 0x00000002,
 799        0x5c4, 0xffffffff, 0x00000001,
 800        0x7030, 0xffffffff, 0x00000011,
 801        0x7c30, 0xffffffff, 0x00000011,
 802        0x6104, 0x01000300, 0x00000000,
 803        0x5bc0, 0x00300000, 0x00000000,
 804        0x918c, 0xffffffff, 0x00010006,
 805        0x91a8, 0xffffffff, 0x00010006,
 806        0x9150, 0xffffffff, 0x6e944040,
 807        0x917c, 0xffffffff, 0x00030002,
 808        0x9198, 0xffffffff, 0x00030002,
 809        0x915c, 0xffffffff, 0x00010000,
 810        0x3f90, 0xffff0000, 0xff000000,
 811        0x9178, 0xffffffff, 0x00070000,
 812        0x9194, 0xffffffff, 0x00070000,
 813        0x9148, 0xffff0000, 0xff000000,
 814        0x9190, 0xffffffff, 0x00090008,
 815        0x91ac, 0xffffffff, 0x00090008,
 816        0x3f94, 0xffff0000, 0xff000000,
 817        0x914c, 0xffff0000, 0xff000000,
 818        0x929c, 0xffffffff, 0x00000001,
 819        0x8a18, 0xffffffff, 0x00000100,
 820        0x8b28, 0xffffffff, 0x00000100,
 821        0x9144, 0xffffffff, 0x00000100,
 822        0x9b7c, 0xffffffff, 0x00000000,
 823        0x8030, 0xffffffff, 0x0000100a,
 824        0x8a14, 0xffffffff, 0x00000001,
 825        0x8b24, 0xffffffff, 0x00ff0fff,
 826        0x8b10, 0xffffffff, 0x00000000,
 827        0x28a4c, 0x06000000, 0x06000000,
 828        0x4d8, 0xffffffff, 0x00000100,
 829        0x913c, 0xffff000f, 0x0100000a,
 830        0x960c, 0xffffffff, 0x54763210,
 831        0x88c4, 0xffffffff, 0x000000c2,
 832        0x88d4, 0xffffffff, 0x00000010,
 833        0x8974, 0xffffffff, 0x00000000,
 834        0xc78, 0x00000080, 0x00000080,
 835        0x5e78, 0xffffffff, 0x001000f0,
 836        0xd02c, 0xffffffff, 0x08421000,
 837        0xa008, 0xffffffff, 0x00010000,
 838        0x8d00, 0xffffffff, 0x100e4848,
 839        0x8d04, 0xffffffff, 0x00164745,
 840        0x8c00, 0xffffffff, 0xe4000003,
 841        0x8cf0, 0x1fffffff, 0x08e00410,
 842        0x28350, 0xffffffff, 0x00000000,
 843        0x9508, 0xffffffff, 0x00000002,
 844        0x900c, 0xffffffff, 0x0017071f,
 845        0x8c18, 0xffffffff, 0x10101060,
 846        0x8c1c, 0xffffffff, 0x00001010
 847};
 848
 849static const u32 barts_golden_registers[] =
 850{
 851        0x5eb4, 0xffffffff, 0x00000002,
 852        0x5e78, 0x8f311ff1, 0x001000f0,
 853        0x3f90, 0xffff0000, 0xff000000,
 854        0x9148, 0xffff0000, 0xff000000,
 855        0x3f94, 0xffff0000, 0xff000000,
 856        0x914c, 0xffff0000, 0xff000000,
 857        0xc78, 0x00000080, 0x00000080,
 858        0xbd4, 0x70073777, 0x00010001,
 859        0xd02c, 0xbfffff1f, 0x08421000,
 860        0xd0b8, 0x03773777, 0x02011003,
 861        0x5bc0, 0x00200000, 0x50100000,
 862        0x98f8, 0x33773777, 0x02011003,
 863        0x98fc, 0xffffffff, 0x76543210,
 864        0x7030, 0x31000311, 0x00000011,
 865        0x2f48, 0x00000007, 0x02011003,
 866        0x6b28, 0x00000010, 0x00000012,
 867        0x7728, 0x00000010, 0x00000012,
 868        0x10328, 0x00000010, 0x00000012,
 869        0x10f28, 0x00000010, 0x00000012,
 870        0x11b28, 0x00000010, 0x00000012,
 871        0x12728, 0x00000010, 0x00000012,
 872        0x240c, 0x000007ff, 0x00000380,
 873        0x8a14, 0xf000001f, 0x00000007,
 874        0x8b24, 0x3fff3fff, 0x00ff0fff,
 875        0x8b10, 0x0000ff0f, 0x00000000,
 876        0x28a4c, 0x07ffffff, 0x06000000,
 877        0x10c, 0x00000001, 0x00010003,
 878        0xa02c, 0xffffffff, 0x0000009b,
 879        0x913c, 0x0000000f, 0x0100000a,
 880        0x8d00, 0xffff7f7f, 0x100e4848,
 881        0x8d04, 0x00ffffff, 0x00164745,
 882        0x8c00, 0xfffc0003, 0xe4000003,
 883        0x8c04, 0xf8ff00ff, 0x40600060,
 884        0x8c08, 0x00ff00ff, 0x001c001c,
 885        0x8cf0, 0x1fff1fff, 0x08e00620,
 886        0x8c20, 0x0fff0fff, 0x00800080,
 887        0x8c24, 0x0fff0fff, 0x00800080,
 888        0x8c18, 0xffffffff, 0x20202078,
 889        0x8c1c, 0x0000ffff, 0x00001010,
 890        0x28350, 0x00000f01, 0x00000000,
 891        0x9508, 0x3700001f, 0x00000002,
 892        0x960c, 0xffffffff, 0x54763210,
 893        0x88c4, 0x001f3ae3, 0x000000c2,
 894        0x88d4, 0x0000001f, 0x00000010,
 895        0x8974, 0xffffffff, 0x00000000
 896};
 897
 898static const u32 turks_golden_registers[] =
 899{
 900        0x5eb4, 0xffffffff, 0x00000002,
 901        0x5e78, 0x8f311ff1, 0x001000f0,
 902        0x8c8, 0x00003000, 0x00001070,
 903        0x8cc, 0x000fffff, 0x00040035,
 904        0x3f90, 0xffff0000, 0xfff00000,
 905        0x9148, 0xffff0000, 0xfff00000,
 906        0x3f94, 0xffff0000, 0xfff00000,
 907        0x914c, 0xffff0000, 0xfff00000,
 908        0xc78, 0x00000080, 0x00000080,
 909        0xbd4, 0x00073007, 0x00010002,
 910        0xd02c, 0xbfffff1f, 0x08421000,
 911        0xd0b8, 0x03773777, 0x02010002,
 912        0x5bc0, 0x00200000, 0x50100000,
 913        0x98f8, 0x33773777, 0x00010002,
 914        0x98fc, 0xffffffff, 0x33221100,
 915        0x7030, 0x31000311, 0x00000011,
 916        0x2f48, 0x33773777, 0x00010002,
 917        0x6b28, 0x00000010, 0x00000012,
 918        0x7728, 0x00000010, 0x00000012,
 919        0x10328, 0x00000010, 0x00000012,
 920        0x10f28, 0x00000010, 0x00000012,
 921        0x11b28, 0x00000010, 0x00000012,
 922        0x12728, 0x00000010, 0x00000012,
 923        0x240c, 0x000007ff, 0x00000380,
 924        0x8a14, 0xf000001f, 0x00000007,
 925        0x8b24, 0x3fff3fff, 0x00ff0fff,
 926        0x8b10, 0x0000ff0f, 0x00000000,
 927        0x28a4c, 0x07ffffff, 0x06000000,
 928        0x10c, 0x00000001, 0x00010003,
 929        0xa02c, 0xffffffff, 0x0000009b,
 930        0x913c, 0x0000000f, 0x0100000a,
 931        0x8d00, 0xffff7f7f, 0x100e4848,
 932        0x8d04, 0x00ffffff, 0x00164745,
 933        0x8c00, 0xfffc0003, 0xe4000003,
 934        0x8c04, 0xf8ff00ff, 0x40600060,
 935        0x8c08, 0x00ff00ff, 0x001c001c,
 936        0x8cf0, 0x1fff1fff, 0x08e00410,
 937        0x8c20, 0x0fff0fff, 0x00800080,
 938        0x8c24, 0x0fff0fff, 0x00800080,
 939        0x8c18, 0xffffffff, 0x20202078,
 940        0x8c1c, 0x0000ffff, 0x00001010,
 941        0x28350, 0x00000f01, 0x00000000,
 942        0x9508, 0x3700001f, 0x00000002,
 943        0x960c, 0xffffffff, 0x54763210,
 944        0x88c4, 0x001f3ae3, 0x000000c2,
 945        0x88d4, 0x0000001f, 0x00000010,
 946        0x8974, 0xffffffff, 0x00000000
 947};
 948
 949static const u32 caicos_golden_registers[] =
 950{
 951        0x5eb4, 0xffffffff, 0x00000002,
 952        0x5e78, 0x8f311ff1, 0x001000f0,
 953        0x8c8, 0x00003420, 0x00001450,
 954        0x8cc, 0x000fffff, 0x00040035,
 955        0x3f90, 0xffff0000, 0xfffc0000,
 956        0x9148, 0xffff0000, 0xfffc0000,
 957        0x3f94, 0xffff0000, 0xfffc0000,
 958        0x914c, 0xffff0000, 0xfffc0000,
 959        0xc78, 0x00000080, 0x00000080,
 960        0xbd4, 0x00073007, 0x00010001,
 961        0xd02c, 0xbfffff1f, 0x08421000,
 962        0xd0b8, 0x03773777, 0x02010001,
 963        0x5bc0, 0x00200000, 0x50100000,
 964        0x98f8, 0x33773777, 0x02010001,
 965        0x98fc, 0xffffffff, 0x33221100,
 966        0x7030, 0x31000311, 0x00000011,
 967        0x2f48, 0x33773777, 0x02010001,
 968        0x6b28, 0x00000010, 0x00000012,
 969        0x7728, 0x00000010, 0x00000012,
 970        0x10328, 0x00000010, 0x00000012,
 971        0x10f28, 0x00000010, 0x00000012,
 972        0x11b28, 0x00000010, 0x00000012,
 973        0x12728, 0x00000010, 0x00000012,
 974        0x240c, 0x000007ff, 0x00000380,
 975        0x8a14, 0xf000001f, 0x00000001,
 976        0x8b24, 0x3fff3fff, 0x00ff0fff,
 977        0x8b10, 0x0000ff0f, 0x00000000,
 978        0x28a4c, 0x07ffffff, 0x06000000,
 979        0x10c, 0x00000001, 0x00010003,
 980        0xa02c, 0xffffffff, 0x0000009b,
 981        0x913c, 0x0000000f, 0x0100000a,
 982        0x8d00, 0xffff7f7f, 0x100e4848,
 983        0x8d04, 0x00ffffff, 0x00164745,
 984        0x8c00, 0xfffc0003, 0xe4000003,
 985        0x8c04, 0xf8ff00ff, 0x40600060,
 986        0x8c08, 0x00ff00ff, 0x001c001c,
 987        0x8cf0, 0x1fff1fff, 0x08e00410,
 988        0x8c20, 0x0fff0fff, 0x00800080,
 989        0x8c24, 0x0fff0fff, 0x00800080,
 990        0x8c18, 0xffffffff, 0x20202078,
 991        0x8c1c, 0x0000ffff, 0x00001010,
 992        0x28350, 0x00000f01, 0x00000000,
 993        0x9508, 0x3700001f, 0x00000002,
 994        0x960c, 0xffffffff, 0x54763210,
 995        0x88c4, 0x001f3ae3, 0x000000c2,
 996        0x88d4, 0x0000001f, 0x00000010,
 997        0x8974, 0xffffffff, 0x00000000
 998};
 999
1000static void evergreen_init_golden_registers(struct radeon_device *rdev)

1001{
1002        switch (rdev->family) {
1003        case CHIP_CYPRESS:
1004        case CHIP_HEMLOCK:
1005                radeon_program_register_sequence(rdev,
1006                                                 evergreen_golden_registers,
1007                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1008                radeon_program_register_sequence(rdev,
1009                                                 evergreen_golden_registers2,
1010                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1011                radeon_program_register_sequence(rdev,
1012                                                 cypress_mgcg_init,
1013                                                 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1014                break;
1015        case CHIP_JUNIPER:
1016                radeon_program_register_sequence(rdev,
1017                                                 evergreen_golden_registers,
1018                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1019                radeon_program_register_sequence(rdev,
1020                                                 evergreen_golden_registers2,
1021                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1022                radeon_program_register_sequence(rdev,
1023                                                 juniper_mgcg_init,
1024                                                 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1025                break;
1026        case CHIP_REDWOOD:
1027                radeon_program_register_sequence(rdev,
1028                                                 evergreen_golden_registers,
1029                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1030                radeon_program_register_sequence(rdev,
1031                                                 evergreen_golden_registers2,
1032                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1033                radeon_program_register_sequence(rdev,
1034                                                 redwood_mgcg_init,
1035                                                 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1036                break;
1037        case CHIP_CEDAR:
1038                radeon_program_register_sequence(rdev,
1039                                                 cedar_golden_registers,
1040                                                 (const u32)ARRAY_SIZE(cedar_golden_registers));
1041                radeon_program_register_sequence(rdev,
1042                                                 evergreen_golden_registers2,
1043                                                 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1044                radeon_program_register_sequence(rdev,
1045                                                 cedar_mgcg_init,
1046                                                 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1047                break;
1048        case CHIP_PALM:
1049                radeon_program_register_sequence(rdev,
1050                                                 wrestler_golden_registers,
1051                                                 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1052                break;
1053        case CHIP_SUMO:
1054                radeon_program_register_sequence(rdev,
1055                                                 supersumo_golden_registers,
1056                                                 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1057                break;
1058        case CHIP_SUMO2:
1059                radeon_program_register_sequence(rdev,
1060                                                 supersumo_golden_registers,
1061                                                 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1062                radeon_program_register_sequence(rdev,
1063                                                 sumo_golden_registers,
1064                                                 (const u32)ARRAY_SIZE(sumo_golden_registers));
1065                break;
1066        case CHIP_BARTS:
1067                radeon_program_register_sequence(rdev,
1068                                                 barts_golden_registers,
1069                                                 (const u32)ARRAY_SIZE(barts_golden_registers));
1070                break;
1071        case CHIP_TURKS:
1072                radeon_program_register_sequence(rdev,
1073                                                 turks_golden_registers,
1074                                                 (const u32)ARRAY_SIZE(turks_golden_registers));
1075                break;
1076        case CHIP_CAICOS:
1077                radeon_program_register_sequence(rdev,
1078                                                 caicos_golden_registers,
1079                                                 (const u32)ARRAY_SIZE(caicos_golden_registers));
1080                break;
1081        default:
1082                break;
1083        }
1084}
1085
1086/**
1087 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1088 *
1089 * @rdev: radeon_device pointer
1090 * @reg: register offset in bytes
1091 * @val: register value
1092 *
1093 * Returns 0 for success or -EINVAL for an invalid register
1094 *
1095 */
1096int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1097                                        u32 reg, u32 *val)
1098{
1099        switch (reg) {
1100        case GRBM_STATUS:
1101        case GRBM_STATUS_SE0:
1102        case GRBM_STATUS_SE1:
1103        case SRBM_STATUS:
1104        case SRBM_STATUS2:
1105        case DMA_STATUS_REG:
1106        case UVD_STATUS:
1107                *val = RREG32(reg);
1108                return 0;
1109        default:
1110                return -EINVAL;
1111        }
1112}
1113
1114void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1115                             unsigned *bankh, unsigned *mtaspect,
1116                             unsigned *tile_split)
1117{
1118        *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1119        *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1120        *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1121        *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1122        switch (*bankw) {
1123        default:
1124        case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1125        case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1126        case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1127        case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1128        }
1129        switch (*bankh) {
1130        default:
1131        case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1132        case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1133        case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1134        case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1135        }
1136        switch (*mtaspect) {
1137        default:
1138        case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1139        case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1140        case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1141        case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1142        }
1143}
1144
1145static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1146                              u32 cntl_reg, u32 status_reg)
1147{
1148        int r, i;
1149        struct atom_clock_dividers dividers;
1150
1151        r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1152                                           clock, false, &dividers);
1153        if (r)
1154                return r;
1155
1156        WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1157
1158        for (i = 0; i < 100; i++) {
1159                if (RREG32(status_reg) & DCLK_STATUS)
1160                        break;
1161                mdelay(10);
1162        }
1163        if (i == 100)
1164                return -ETIMEDOUT;
1165
1166        return 0;
1167}
1168
1169int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1170{
1171        int r = 0;
1172        u32 cg_scratch = RREG32(CG_SCRATCH1);
1173
1174        r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1175        if (r)
1176                goto done;
1177        cg_scratch &= 0xffff0000;
1178        cg_scratch |= vclk / 100; /* Mhz */
1179
1180        r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1181        if (r)
1182                goto done;
1183        cg_scratch &= 0x0000ffff;
1184        cg_scratch |= (dclk / 100) << 16; /* Mhz */
1185
1186done:
1187        WREG32(CG_SCRATCH1, cg_scratch);
1188
1189        return r;
1190}
1191
1192int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1193{
1194        /* start off with something large */
1195        unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1196        int r;
1197
1198        /* bypass vclk and dclk with bclk */
1199        WREG32_P(CG_UPLL_FUNC_CNTL_2,
1200                VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1201                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1202
1203        /* put PLL in bypass mode */
1204        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1205
1206        if (!vclk || !dclk) {
1207                /* keep the Bypass mode, put PLL to sleep */
1208                WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1209                return 0;
1210        }
1211
1212        r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1213                                          16384, 0x03FFFFFF, 0, 128, 5,
1214                                          &fb_div, &vclk_div, &dclk_div);
1215        if (r)
1216                return r;
1217
1218        /* set VCO_MODE to 1 */
1219        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1220
1221        /* toggle UPLL_SLEEP to 1 then back to 0 */
1222        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1223        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1224
1225        /* deassert UPLL_RESET */
1226        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1227
1228        mdelay(1);
1229
1230        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1231        if (r)
1232                return r;
1233
1234        /* assert UPLL_RESET again */
1235        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1236
1237        /* disable spread spectrum. */
1238        WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1239
1240        /* set feedback divider */
1241        WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1242
1243        /* set ref divider to 0 */
1244        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1245
1246        if (fb_div < 307200)
1247                WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1248        else
1249                WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1250
1251        /* set PDIV_A and PDIV_B */
1252        WREG32_P(CG_UPLL_FUNC_CNTL_2,
1253                UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1254                ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1255
1256        /* give the PLL some time to settle */
1257        mdelay(15);
1258
1259        /* deassert PLL_RESET */
1260        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1261
1262        mdelay(15);
1263
1264        /* switch from bypass mode to normal mode */
1265        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1266
1267        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1268        if (r)
1269                return r;
1270
1271        /* switch VCLK and DCLK selection */
1272        WREG32_P(CG_UPLL_FUNC_CNTL_2,
1273                VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1274                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1275
1276        mdelay(100);
1277
1278        return 0;
1279}
1280
1281void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1282{
1283        int readrq;
1284        u16 v;
1285
1286        readrq = pcie_get_readrq(rdev->pdev);
1287        v = ffs(readrq) - 8;
1288        /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1289         * to avoid hangs or perfomance issues
1290         */
1291        if ((v == 0) || (v == 6) || (v == 7))
1292                pcie_set_readrq(rdev->pdev, 512);
1293}
1294
1295void dce4_program_fmt(struct drm_encoder *encoder)
1296{
1297        struct drm_device *dev = encoder->dev;
1298        struct radeon_device *rdev = dev->dev_private;
1299        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1300        struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1301        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1302        int bpc = 0;
1303        u32 tmp = 0;
1304        enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1305
1306        if (connector) {
1307                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1308                bpc = radeon_get_monitor_bpc(connector);
1309                dither = radeon_connector->dither;
1310        }
1311
1312        /* LVDS/eDP FMT is set up by atom */
1313        if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1314                return;
1315
1316        /* not needed for analog */
1317        if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1318            (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1319                return;
1320
1321        if (bpc == 0)
1322                return;
1323
1324        switch (bpc) {
1325        case 6:
1326                if (dither == RADEON_FMT_DITHER_ENABLE)
1327                        /* XXX sort out optimal dither settings */
1328                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329                                FMT_SPATIAL_DITHER_EN);
1330                else
1331                        tmp |= FMT_TRUNCATE_EN;
1332                break;
1333        case 8:
1334                if (dither == RADEON_FMT_DITHER_ENABLE)
1335                        /* XXX sort out optimal dither settings */
1336                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1337                                FMT_RGB_RANDOM_ENABLE |
1338                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1339                else
1340                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1341                break;
1342        case 10:
1343        default:
1344                /* not needed */
1345                break;
1346        }
1347
1348        WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1349}
1350
1351static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1352{
1353        if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1354                return true;
1355        else
1356                return false;
1357}
1358
1359static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1360{
1361        u32 pos1, pos2;
1362
1363        pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1364        pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1365
1366        if (pos1 != pos2)
1367                return true;
1368        else
1369                return false;
1370}
1371
1372/**
1373 * dce4_wait_for_vblank - vblank wait asic callback.
1374 *
1375 * @rdev: radeon_device pointer
1376 * @crtc: crtc to wait for vblank on
1377 *
1378 * Wait for vblank on the requested crtc (evergreen+).
1379 */
1380void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1381{
1382        unsigned i = 0;
1383
1384        if (crtc >= rdev->num_crtc)
1385                return;
1386
1387        if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1388                return;
1389
1390        /* depending on when we hit vblank, we may be close to active; if so,
1391         * wait for another frame.
1392         */
1393        while (dce4_is_in_vblank(rdev, crtc)) {
1394                if (i++ % 100 == 0) {
1395                        if (!dce4_is_counter_moving(rdev, crtc))
1396                                break;
1397                }
1398        }
1399
1400        while (!dce4_is_in_vblank(rdev, crtc)) {
1401                if (i++ % 100 == 0) {
1402                        if (!dce4_is_counter_moving(rdev, crtc))
1403                                break;
1404                }
1405        }
1406}
1407
1408/**
1409 * evergreen_page_flip - pageflip callback.
1410 *
1411 * @rdev: radeon_device pointer
1412 * @crtc_id: crtc to cleanup pageflip on
1413 * @crtc_base: new address of the crtc (GPU MC address)
1414 *
1415 * Triggers the actual pageflip by updating the primary
1416 * surface base address (evergreen+).
1417 */
1418void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1419                         bool async)
1420{
1421        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1422
1423        /* update the scanout addresses */
1424        WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1425               async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1426        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1427               upper_32_bits(crtc_base));
1428        WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1429               (u32)crtc_base);
1430        /* post the write */
1431        RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1432}
1433
1434/**
1435 * evergreen_page_flip_pending - check if page flip is still pending
1436 *
1437 * @rdev: radeon_device pointer
1438 * @crtc_id: crtc to check
1439 *
1440 * Returns the current update pending status.
1441 */
1442bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1443{
1444        struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1445
1446        /* Return current update_pending status: */
1447        return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1448                EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1449}
1450
1451/* get temperature in millidegrees */
1452int evergreen_get_temp(struct radeon_device *rdev)
1453{
1454        u32 temp, toffset;
1455        int actual_temp = 0;
1456
1457        if (rdev->family == CHIP_JUNIPER) {
1458                toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1459                        TOFFSET_SHIFT;
1460                temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1461                        TS0_ADC_DOUT_SHIFT;
1462
1463                if (toffset & 0x100)
1464                        actual_temp = temp / 2 - (0x200 - toffset);
1465                else
1466                        actual_temp = temp / 2 + toffset;
1467
1468                actual_temp = actual_temp * 1000;
1469
1470        } else {
1471                temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1472                        ASIC_T_SHIFT;
1473
1474                if (temp & 0x400)
1475                        actual_temp = -256;
1476                else if (temp & 0x200)
1477                        actual_temp = 255;
1478                else if (temp & 0x100) {
1479                        actual_temp = temp & 0x1ff;
1480                        actual_temp |= ~0x1ff;
1481                } else
1482                        actual_temp = temp & 0xff;
1483
1484                actual_temp = (actual_temp * 1000) / 2;
1485        }
1486
1487        return actual_temp;
1488}
1489
1490int sumo_get_temp(struct radeon_device *rdev)
1491{
1492        u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1493        int actual_temp = temp - 49;
1494
1495        return actual_temp * 1000;
1496}
1497
1498/**
1499 * sumo_pm_init_profile - Initialize power profiles callback.
1500 *
1501 * @rdev: radeon_device pointer
1502 *
1503 * Initialize the power states used in profile mode
1504 * (sumo, trinity, SI).
1505 * Used for profile mode only.
1506 */
1507void sumo_pm_init_profile(struct radeon_device *rdev)
1508{
1509        int idx;
1510
1511        /* default */
1512        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1513        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1514        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1515        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1516
1517        /* low,mid sh/mh */
1518        if (rdev->flags & RADEON_IS_MOBILITY)
1519                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1520        else
1521                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1522
1523        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1524        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1525        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1526        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1527
1528        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1529        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1530        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1531        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1532
1533        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1534        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1535        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1536        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1537
1538        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1539        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1540        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1541        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1542
1543        /* high sh/mh */
1544        idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1545        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1546        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1547        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1548        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1549                rdev->pm.power_state[idx].num_clock_modes - 1;
1550
1551        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1552        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1553        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1554        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1555                rdev->pm.power_state[idx].num_clock_modes - 1;
1556}
1557
1558/**
1559 * btc_pm_init_profile - Initialize power profiles callback.
1560 *
1561 * @rdev: radeon_device pointer
1562 *
1563 * Initialize the power states used in profile mode
1564 * (BTC, cayman).
1565 * Used for profile mode only.
1566 */
1567void btc_pm_init_profile(struct radeon_device *rdev)
1568{
1569        int idx;
1570
1571        /* default */
1572        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1573        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1574        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1575        rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1576        /* starting with BTC, there is one state that is used for both
1577         * MH and SH.  Difference is that we always use the high clock index for
1578         * mclk.
1579         */
1580        if (rdev->flags & RADEON_IS_MOBILITY)
1581                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1582        else
1583                idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1584        /* low sh */
1585        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1586        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1587        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1588        rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1589        /* mid sh */
1590        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1591        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1592        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1593        rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1594        /* high sh */
1595        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1596        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1597        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1598        rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1599        /* low mh */
1600        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1601        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1602        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1603        rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1604        /* mid mh */
1605        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1606        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1607        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1608        rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1609        /* high mh */
1610        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1611        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1612        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1613        rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1614}
1615
1616/**
1617 * evergreen_pm_misc - set additional pm hw parameters callback.
1618 *
1619 * @rdev: radeon_device pointer
1620 *
1621 * Set non-clock parameters associated with a power state
1622 * (voltage, etc.) (evergreen+).
1623 */
1624void evergreen_pm_misc(struct radeon_device *rdev)
1625{
1626        int req_ps_idx = rdev->pm.requested_power_state_index;
1627        int req_cm_idx = rdev->pm.requested_clock_mode_index;
1628        struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1629        struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1630
1631        if (voltage->type == VOLTAGE_SW) {
1632                /* 0xff0x are flags rather then an actual voltage */
1633                if ((voltage->voltage & 0xff00) == 0xff00)
1634                        return;
1635                if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1636                        radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1637                        rdev->pm.current_vddc = voltage->voltage;
1638                        DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1639                }
1640
1641                /* starting with BTC, there is one state that is used for both
1642                 * MH and SH.  Difference is that we always use the high clock index for
1643                 * mclk and vddci.
1644                 */
1645                if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1646                    (rdev->family >= CHIP_BARTS) &&
1647                    rdev->pm.active_crtc_count &&
1648                    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1649                     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1650                        voltage = &rdev->pm.power_state[req_ps_idx].
1651                                clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1652
1653                /* 0xff0x are flags rather then an actual voltage */
1654                if ((voltage->vddci & 0xff00) == 0xff00)
1655                        return;
1656                if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1657                        radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1658                        rdev->pm.current_vddci = voltage->vddci;
1659                        DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1660                }
1661        }
1662}
1663
1664/**
1665 * evergreen_pm_prepare - pre-power state change callback.
1666 *
1667 * @rdev: radeon_device pointer
1668 *
1669 * Prepare for a power state change (evergreen+).
1670 */
1671void evergreen_pm_prepare(struct radeon_device *rdev)
1672{
1673        struct drm_device *ddev = rdev->ddev;
1674        struct drm_crtc *crtc;
1675        struct radeon_crtc *radeon_crtc;
1676        u32 tmp;
1677
1678        /* disable any active CRTCs */
1679        list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1680                radeon_crtc = to_radeon_crtc(crtc);
1681                if (radeon_crtc->enabled) {
1682                        tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1683                        tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1684                        WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1685                }
1686        }
1687}
1688
1689/**
1690 * evergreen_pm_finish - post-power state change callback.
1691 *
1692 * @rdev: radeon_device pointer
1693 *
1694 * Clean up after a power state change (evergreen+).
1695 */
1696void evergreen_pm_finish(struct radeon_device *rdev)
1697{
1698        struct drm_device *ddev = rdev->ddev;
1699        struct drm_crtc *crtc;
1700        struct radeon_crtc *radeon_crtc;
1701        u32 tmp;
1702
1703        /* enable any active CRTCs */
1704        list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1705                radeon_crtc = to_radeon_crtc(crtc);
1706                if (radeon_crtc->enabled) {
1707                        tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1708                        tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1709                        WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1710                }
1711        }
1712}
1713
1714/**
1715 * evergreen_hpd_sense - hpd sense callback.
1716 *
1717 * @rdev: radeon_device pointer
1718 * @hpd: hpd (hotplug detect) pin
1719 *
1720 * Checks if a digital monitor is connected (evergreen+).
1721 * Returns true if connected, false if not connected.
1722 */
1723bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1724{
1725        if (hpd == RADEON_HPD_NONE)
1726                return false;
1727
1728        return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1729}
1730
1731/**
1732 * evergreen_hpd_set_polarity - hpd set polarity callback.
1733 *
1734 * @rdev: radeon_device pointer
1735 * @hpd: hpd (hotplug detect) pin
1736 *
1737 * Set the polarity of the hpd pin (evergreen+).
1738 */
1739void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1740                                enum radeon_hpd_id hpd)
1741{
1742        bool connected = evergreen_hpd_sense(rdev, hpd);
1743
1744        if (hpd == RADEON_HPD_NONE)
1745                return;
1746
1747        if (connected)
1748                WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1749        else
1750                WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1751}
1752
1753/**
1754 * evergreen_hpd_init - hpd setup callback.
1755 *
1756 * @rdev: radeon_device pointer
1757 *
1758 * Setup the hpd pins used by the card (evergreen+).
1759 * Enable the pin, set the polarity, and enable the hpd interrupts.
1760 */
1761void evergreen_hpd_init(struct radeon_device *rdev)
1762{
1763        struct drm_device *dev = rdev->ddev;
1764        struct drm_connector *connector;
1765        unsigned enabled = 0;
1766        u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1767                DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1768
1769        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1770                enum radeon_hpd_id hpd =
1771                        to_radeon_connector(connector)->hpd.hpd;
1772
1773                if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1774                    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1775                        /* don't try to enable hpd on eDP or LVDS avoid breaking the
1776                         * aux dp channel on imac and help (but not completely fix)
1777                         * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1778                         * also avoid interrupt storms during dpms.
1779                         */
1780                        continue;
1781                }
1782
1783                if (hpd == RADEON_HPD_NONE)
1784                        continue;
1785
1786                WREG32(DC_HPDx_CONTROL(hpd), tmp);
1787                enabled |= 1 << hpd;
1788
1789                radeon_hpd_set_polarity(rdev, hpd);
1790        }
1791        radeon_irq_kms_enable_hpd(rdev, enabled);
1792}
1793
1794/**
1795 * evergreen_hpd_fini - hpd tear down callback.
1796 *
1797 * @rdev: radeon_device pointer
1798 *
1799 * Tear down the hpd pins used by the card (evergreen+).
1800 * Disable the hpd interrupts.
1801 */
1802void evergreen_hpd_fini(struct radeon_device *rdev)
1803{
1804        struct drm_device *dev = rdev->ddev;
1805        struct drm_connector *connector;
1806        unsigned disabled = 0;
1807
1808        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1809                enum radeon_hpd_id hpd =
1810                        to_radeon_connector(connector)->hpd.hpd;
1811
1812                if (hpd == RADEON_HPD_NONE)
1813                        continue;
1814
1815                WREG32(DC_HPDx_CONTROL(hpd), 0);
1816                disabled |= 1 << hpd;
1817        }
1818        radeon_irq_kms_disable_hpd(rdev, disabled);
1819}
1820
1821/* watermark setup */
1822
1823static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1824                                        struct radeon_crtc *radeon_crtc,
1825                                        struct drm_display_mode *mode,
1826                                        struct drm_display_mode *other_mode)
1827{
1828        u32 tmp, buffer_alloc, i;
1829        u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1830        /*
1831         * Line Buffer Setup
1832         * There are 3 line buffers, each one shared by 2 display controllers.
1833         * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1834         * the display controllers.  The paritioning is done via one of four
1835         * preset allocations specified in bits 2:0:
1836         * first display controller
1837         *  0 - first half of lb (3840 * 2)
1838         *  1 - first 3/4 of lb (5760 * 2)
1839         *  2 - whole lb (7680 * 2), other crtc must be disabled
1840         *  3 - first 1/4 of lb (1920 * 2)
1841         * second display controller
1842         *  4 - second half of lb (3840 * 2)
1843         *  5 - second 3/4 of lb (5760 * 2)
1844         *  6 - whole lb (7680 * 2), other crtc must be disabled
1845         *  7 - last 1/4 of lb (1920 * 2)
1846         */
1847        /* this can get tricky if we have two large displays on a paired group
1848         * of crtcs.  Ideally for multiple large displays we'd assign them to
1849         * non-linked crtcs for maximum line buffer allocation.
1850         */
1851        if (radeon_crtc->base.enabled && mode) {
1852                if (other_mode) {
1853                        tmp = 0; /* 1/2 */
1854                        buffer_alloc = 1;
1855                } else {
1856                        tmp = 2; /* whole */
1857                        buffer_alloc = 2;
1858                }
1859        } else {
1860                tmp = 0;
1861                buffer_alloc = 0;
1862        }
1863
1864        /* second controller of the pair uses second half of the lb */
1865        if (radeon_crtc->crtc_id % 2)
1866                tmp += 4;
1867        WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1868
1869        if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1870                WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1871                       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1872                for (i = 0; i < rdev->usec_timeout; i++) {
1873                        if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1874                            DMIF_BUFFERS_ALLOCATED_COMPLETED)
1875                                break;
1876                        udelay(1);
1877                }
1878        }
1879
1880        if (radeon_crtc->base.enabled && mode) {
1881                switch (tmp) {
1882                case 0:
1883                case 4:
1884                default:
1885                        if (ASIC_IS_DCE5(rdev))
1886                                return 4096 * 2;
1887                        else
1888                                return 3840 * 2;
1889                case 1:
1890                case 5:
1891                        if (ASIC_IS_DCE5(rdev))
1892                                return 6144 * 2;
1893                        else
1894                                return 5760 * 2;
1895                case 2:
1896                case 6:
1897                        if (ASIC_IS_DCE5(rdev))
1898                                return 8192 * 2;
1899                        else
1900                                return 7680 * 2;
1901                case 3:
1902                case 7:
1903                        if (ASIC_IS_DCE5(rdev))
1904                                return 2048 * 2;
1905                        else
1906                                return 1920 * 2;
1907                }
1908        }
1909
1910        /* controller not enabled, so no lb used */
1911        return 0;
1912}
1913
1914u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1915{
1916        u32 tmp = RREG32(MC_SHARED_CHMAP);
1917
1918        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1919        case 0:
1920        default:
1921                return 1;
1922        case 1:
1923                return 2;
1924        case 2:
1925                return 4;
1926        case 3:
1927                return 8;
1928        }
1929}
1930
1931struct evergreen_wm_params {
1932        u32 dram_channels; /* number of dram channels */
1933        u32 yclk;          /* bandwidth per dram data pin in kHz */
1934        u32 sclk;          /* engine clock in kHz */
1935        u32 disp_clk;      /* display clock in kHz */
1936        u32 src_width;     /* viewport width */
1937        u32 active_time;   /* active display time in ns */
1938        u32 blank_time;    /* blank time in ns */
1939        bool interlaced;    /* mode is interlaced */
1940        fixed20_12 vsc;    /* vertical scale ratio */
1941        u32 num_heads;     /* number of active crtcs */
1942        u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1943        u32 lb_size;       /* line buffer allocated to pipe */
1944        u32 vtaps;         /* vertical scaler taps */
1945};
1946
1947static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1948{
1949        /* Calculate DRAM Bandwidth and the part allocated to display. */
1950        fixed20_12 dram_efficiency; /* 0.7 */
1951        fixed20_12 yclk, dram_channels, bandwidth;
1952        fixed20_12 a;
1953
1954        a.full = dfixed_const(1000);
1955        yclk.full = dfixed_const(wm->yclk);
1956        yclk.full = dfixed_div(yclk, a);
1957        dram_channels.full = dfixed_const(wm->dram_channels * 4);
1958        a.full = dfixed_const(10);
1959        dram_efficiency.full = dfixed_const(7);
1960        dram_efficiency.full = dfixed_div(dram_efficiency, a);
1961        bandwidth.full = dfixed_mul(dram_channels, yclk);
1962        bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1963
1964        return dfixed_trunc(bandwidth);
1965}
1966
1967static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1968{
1969        /* Calculate DRAM Bandwidth and the part allocated to display. */
1970        fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1971        fixed20_12 yclk, dram_channels, bandwidth;
1972        fixed20_12 a;
1973
1974        a.full = dfixed_const(1000);
1975        yclk.full = dfixed_const(wm->yclk);
1976        yclk.full = dfixed_div(yclk, a);
1977        dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978        a.full = dfixed_const(10);
1979        disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1980        disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1981        bandwidth.full = dfixed_mul(dram_channels, yclk);
1982        bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1983
1984        return dfixed_trunc(bandwidth);
1985}
1986
1987static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1988{
1989        /* Calculate the display Data return Bandwidth */
1990        fixed20_12 return_efficiency; /* 0.8 */
1991        fixed20_12 sclk, bandwidth;
1992        fixed20_12 a;
1993
1994        a.full = dfixed_const(1000);
1995        sclk.full = dfixed_const(wm->sclk);
1996        sclk.full = dfixed_div(sclk, a);
1997        a.full = dfixed_const(10);
1998        return_efficiency.full = dfixed_const(8);
1999        return_efficiency.full = dfixed_div(return_efficiency, a);
2000        a.full = dfixed_const(32);

2001        bandwidth.full = dfixed_mul(a, sclk);
2002        bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2003
2004        return dfixed_trunc(bandwidth);
2005}
2006
2007static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2008{
2009        /* Calculate the DMIF Request Bandwidth */
2010        fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2011        fixed20_12 disp_clk, bandwidth;
2012        fixed20_12 a;
2013
2014        a.full = dfixed_const(1000);
2015        disp_clk.full = dfixed_const(wm->disp_clk);
2016        disp_clk.full = dfixed_div(disp_clk, a);
2017        a.full = dfixed_const(10);
2018        disp_clk_request_efficiency.full = dfixed_const(8);
2019        disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2020        a.full = dfixed_const(32);
2021        bandwidth.full = dfixed_mul(a, disp_clk);
2022        bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2023
2024        return dfixed_trunc(bandwidth);
2025}
2026
2027static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2028{
2029        /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2030        u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2031        u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2032        u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2033
2034        return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2035}
2036
2037static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2038{
2039        /* Calculate the display mode Average Bandwidth
2040         * DisplayMode should contain the source and destination dimensions,
2041         * timing, etc.
2042         */
2043        fixed20_12 bpp;
2044        fixed20_12 line_time;
2045        fixed20_12 src_width;
2046        fixed20_12 bandwidth;
2047        fixed20_12 a;
2048
2049        a.full = dfixed_const(1000);
2050        line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2051        line_time.full = dfixed_div(line_time, a);
2052        bpp.full = dfixed_const(wm->bytes_per_pixel);
2053        src_width.full = dfixed_const(wm->src_width);
2054        bandwidth.full = dfixed_mul(src_width, bpp);
2055        bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2056        bandwidth.full = dfixed_div(bandwidth, line_time);
2057
2058        return dfixed_trunc(bandwidth);
2059}
2060
2061static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2062{
2063        /* First calcualte the latency in ns */
2064        u32 mc_latency = 2000; /* 2000 ns. */
2065        u32 available_bandwidth = evergreen_available_bandwidth(wm);
2066        u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2067        u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2068        u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2069        u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2070                (wm->num_heads * cursor_line_pair_return_time);
2071        u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2072        u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2073        fixed20_12 a, b, c;
2074
2075        if (wm->num_heads == 0)
2076                return 0;
2077
2078        a.full = dfixed_const(2);
2079        b.full = dfixed_const(1);
2080        if ((wm->vsc.full > a.full) ||
2081            ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2082            (wm->vtaps >= 5) ||
2083            ((wm->vsc.full >= a.full) && wm->interlaced))
2084                max_src_lines_per_dst_line = 4;
2085        else
2086                max_src_lines_per_dst_line = 2;
2087
2088        a.full = dfixed_const(available_bandwidth);
2089        b.full = dfixed_const(wm->num_heads);
2090        a.full = dfixed_div(a, b);
2091
2092        lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2093
2094        a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2095        b.full = dfixed_const(1000);
2096        c.full = dfixed_const(lb_fill_bw);
2097        b.full = dfixed_div(c, b);
2098        a.full = dfixed_div(a, b);
2099        line_fill_time = dfixed_trunc(a);
2100
2101        if (line_fill_time < wm->active_time)
2102                return latency;
2103        else
2104                return latency + (line_fill_time - wm->active_time);
2105
2106}
2107
2108static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2109{
2110        if (evergreen_average_bandwidth(wm) <=
2111            (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2112                return true;
2113        else
2114                return false;
2115};
2116
2117static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2118{
2119        if (evergreen_average_bandwidth(wm) <=
2120            (evergreen_available_bandwidth(wm) / wm->num_heads))
2121                return true;
2122        else
2123                return false;
2124};
2125
2126static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2127{
2128        u32 lb_partitions = wm->lb_size / wm->src_width;
2129        u32 line_time = wm->active_time + wm->blank_time;
2130        u32 latency_tolerant_lines;
2131        u32 latency_hiding;
2132        fixed20_12 a;
2133
2134        a.full = dfixed_const(1);
2135        if (wm->vsc.full > a.full)
2136                latency_tolerant_lines = 1;
2137        else {
2138                if (lb_partitions <= (wm->vtaps + 1))
2139                        latency_tolerant_lines = 1;
2140                else
2141                        latency_tolerant_lines = 2;
2142        }
2143
2144        latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2145
2146        if (evergreen_latency_watermark(wm) <= latency_hiding)
2147                return true;
2148        else
2149                return false;
2150}
2151
2152static void evergreen_program_watermarks(struct radeon_device *rdev,
2153                                         struct radeon_crtc *radeon_crtc,
2154                                         u32 lb_size, u32 num_heads)
2155{
2156        struct drm_display_mode *mode = &radeon_crtc->base.mode;
2157        struct evergreen_wm_params wm_low, wm_high;
2158        u32 dram_channels;
2159        u32 active_time;
2160        u32 line_time = 0;
2161        u32 latency_watermark_a = 0, latency_watermark_b = 0;
2162        u32 priority_a_mark = 0, priority_b_mark = 0;
2163        u32 priority_a_cnt = PRIORITY_OFF;
2164        u32 priority_b_cnt = PRIORITY_OFF;
2165        u32 pipe_offset = radeon_crtc->crtc_id * 16;
2166        u32 tmp, arb_control3;
2167        fixed20_12 a, b, c;
2168
2169        if (radeon_crtc->base.enabled && num_heads && mode) {
2170                active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2171                                            (u32)mode->clock);
2172                line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2173                                          (u32)mode->clock);
2174                line_time = min(line_time, (u32)65535);
2175                priority_a_cnt = 0;
2176                priority_b_cnt = 0;
2177                dram_channels = evergreen_get_number_of_dram_channels(rdev);
2178
2179                /* watermark for high clocks */
2180                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2181                        wm_high.yclk =
2182                                radeon_dpm_get_mclk(rdev, false) * 10;
2183                        wm_high.sclk =
2184                                radeon_dpm_get_sclk(rdev, false) * 10;
2185                } else {
2186                        wm_high.yclk = rdev->pm.current_mclk * 10;
2187                        wm_high.sclk = rdev->pm.current_sclk * 10;
2188                }
2189
2190                wm_high.disp_clk = mode->clock;
2191                wm_high.src_width = mode->crtc_hdisplay;
2192                wm_high.active_time = active_time;
2193                wm_high.blank_time = line_time - wm_high.active_time;
2194                wm_high.interlaced = false;
2195                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2196                        wm_high.interlaced = true;
2197                wm_high.vsc = radeon_crtc->vsc;
2198                wm_high.vtaps = 1;
2199                if (radeon_crtc->rmx_type != RMX_OFF)
2200                        wm_high.vtaps = 2;
2201                wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2202                wm_high.lb_size = lb_size;
2203                wm_high.dram_channels = dram_channels;
2204                wm_high.num_heads = num_heads;
2205
2206                /* watermark for low clocks */
2207                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2208                        wm_low.yclk =
2209                                radeon_dpm_get_mclk(rdev, true) * 10;
2210                        wm_low.sclk =
2211                                radeon_dpm_get_sclk(rdev, true) * 10;
2212                } else {
2213                        wm_low.yclk = rdev->pm.current_mclk * 10;
2214                        wm_low.sclk = rdev->pm.current_sclk * 10;
2215                }
2216
2217                wm_low.disp_clk = mode->clock;
2218                wm_low.src_width = mode->crtc_hdisplay;
2219                wm_low.active_time = active_time;
2220                wm_low.blank_time = line_time - wm_low.active_time;
2221                wm_low.interlaced = false;
2222                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2223                        wm_low.interlaced = true;
2224                wm_low.vsc = radeon_crtc->vsc;
2225                wm_low.vtaps = 1;
2226                if (radeon_crtc->rmx_type != RMX_OFF)
2227                        wm_low.vtaps = 2;
2228                wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2229                wm_low.lb_size = lb_size;
2230                wm_low.dram_channels = dram_channels;
2231                wm_low.num_heads = num_heads;
2232
2233                /* set for high clocks */
2234                latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2235                /* set for low clocks */
2236                latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2237
2238                /* possibly force display priority to high */
2239                /* should really do this at mode validation time... */
2240                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2241                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2242                    !evergreen_check_latency_hiding(&wm_high) ||
2243                    (rdev->disp_priority == 2)) {
2244                        DRM_DEBUG_KMS("force priority a to high\n");
2245                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
2246                }
2247                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2248                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2249                    !evergreen_check_latency_hiding(&wm_low) ||
2250                    (rdev->disp_priority == 2)) {
2251                        DRM_DEBUG_KMS("force priority b to high\n");
2252                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
2253                }
2254
2255                a.full = dfixed_const(1000);
2256                b.full = dfixed_const(mode->clock);
2257                b.full = dfixed_div(b, a);
2258                c.full = dfixed_const(latency_watermark_a);
2259                c.full = dfixed_mul(c, b);
2260                c.full = dfixed_mul(c, radeon_crtc->hsc);
2261                c.full = dfixed_div(c, a);
2262                a.full = dfixed_const(16);
2263                c.full = dfixed_div(c, a);
2264                priority_a_mark = dfixed_trunc(c);
2265                priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2266
2267                a.full = dfixed_const(1000);
2268                b.full = dfixed_const(mode->clock);
2269                b.full = dfixed_div(b, a);
2270                c.full = dfixed_const(latency_watermark_b);
2271                c.full = dfixed_mul(c, b);
2272                c.full = dfixed_mul(c, radeon_crtc->hsc);
2273                c.full = dfixed_div(c, a);
2274                a.full = dfixed_const(16);
2275                c.full = dfixed_div(c, a);
2276                priority_b_mark = dfixed_trunc(c);
2277                priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2278
2279                /* Save number of lines the linebuffer leads before the scanout */
2280                radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2281        }
2282
2283        /* select wm A */
2284        arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2285        tmp = arb_control3;
2286        tmp &= ~LATENCY_WATERMARK_MASK(3);
2287        tmp |= LATENCY_WATERMARK_MASK(1);
2288        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2289        WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2290               (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2291                LATENCY_HIGH_WATERMARK(line_time)));
2292        /* select wm B */
2293        tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2294        tmp &= ~LATENCY_WATERMARK_MASK(3);
2295        tmp |= LATENCY_WATERMARK_MASK(2);
2296        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2297        WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2298               (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2299                LATENCY_HIGH_WATERMARK(line_time)));
2300        /* restore original selection */
2301        WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2302
2303        /* write the priority marks */
2304        WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2305        WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2306
2307        /* save values for DPM */
2308        radeon_crtc->line_time = line_time;
2309        radeon_crtc->wm_high = latency_watermark_a;
2310        radeon_crtc->wm_low = latency_watermark_b;
2311}
2312
2313/**
2314 * evergreen_bandwidth_update - update display watermarks callback.
2315 *
2316 * @rdev: radeon_device pointer
2317 *
2318 * Update the display watermarks based on the requested mode(s)
2319 * (evergreen+).
2320 */
2321void evergreen_bandwidth_update(struct radeon_device *rdev)
2322{
2323        struct drm_display_mode *mode0 = NULL;
2324        struct drm_display_mode *mode1 = NULL;
2325        u32 num_heads = 0, lb_size;
2326        int i;
2327
2328        if (!rdev->mode_info.mode_config_initialized)
2329                return;
2330
2331        radeon_update_display_priority(rdev);
2332
2333        for (i = 0; i < rdev->num_crtc; i++) {
2334                if (rdev->mode_info.crtcs[i]->base.enabled)
2335                        num_heads++;
2336        }
2337        for (i = 0; i < rdev->num_crtc; i += 2) {
2338                mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2339                mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2340                lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2341                evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2342                lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2343                evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2344        }
2345}
2346
2347/**
2348 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2349 *
2350 * @rdev: radeon_device pointer
2351 *
2352 * Wait for the MC (memory controller) to be idle.
2353 * (evergreen+).
2354 * Returns 0 if the MC is idle, -1 if not.
2355 */
2356int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2357{
2358        unsigned i;
2359        u32 tmp;
2360
2361        for (i = 0; i < rdev->usec_timeout; i++) {
2362                /* read MC_STATUS */
2363                tmp = RREG32(SRBM_STATUS) & 0x1F00;
2364                if (!tmp)
2365                        return 0;
2366                udelay(1);
2367        }
2368        return -1;
2369}
2370
2371/*
2372 * GART
2373 */
2374void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2375{
2376        unsigned i;
2377        u32 tmp;
2378
2379        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2380
2381        WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2382        for (i = 0; i < rdev->usec_timeout; i++) {
2383                /* read MC_STATUS */
2384                tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2385                tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2386                if (tmp == 2) {
2387                        pr_warn("[drm] r600 flush TLB failed\n");
2388                        return;
2389                }
2390                if (tmp) {
2391                        return;
2392                }
2393                udelay(1);
2394        }
2395}
2396
2397static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2398{
2399        u32 tmp;
2400        int r;
2401
2402        if (rdev->gart.robj == NULL) {
2403                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2404                return -EINVAL;
2405        }
2406        r = radeon_gart_table_vram_pin(rdev);
2407        if (r)
2408                return r;
2409        /* Setup L2 cache */
2410        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2411                                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2412                                EFFECTIVE_L2_QUEUE_SIZE(7));
2413        WREG32(VM_L2_CNTL2, 0);
2414        WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2415        /* Setup TLB control */
2416        tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2417                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2418                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2419                EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2420        if (rdev->flags & RADEON_IS_IGP) {
2421                WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2422                WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2423                WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2424        } else {
2425                WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2426                WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2427                WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2428                if ((rdev->family == CHIP_JUNIPER) ||
2429                    (rdev->family == CHIP_CYPRESS) ||
2430                    (rdev->family == CHIP_HEMLOCK) ||
2431                    (rdev->family == CHIP_BARTS))
2432                        WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2433        }
2434        WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2435        WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2436        WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2437        WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2438        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2439        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2440        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2441        WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2442                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2443        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2444                        (u32)(rdev->dummy_page.addr >> 12));
2445        WREG32(VM_CONTEXT1_CNTL, 0);
2446
2447        evergreen_pcie_gart_tlb_flush(rdev);
2448        DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2449                 (unsigned)(rdev->mc.gtt_size >> 20),
2450                 (unsigned long long)rdev->gart.table_addr);
2451        rdev->gart.ready = true;
2452        return 0;
2453}
2454
2455static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2456{
2457        u32 tmp;
2458
2459        /* Disable all tables */
2460        WREG32(VM_CONTEXT0_CNTL, 0);
2461        WREG32(VM_CONTEXT1_CNTL, 0);
2462
2463        /* Setup L2 cache */
2464        WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2465                                EFFECTIVE_L2_QUEUE_SIZE(7));
2466        WREG32(VM_L2_CNTL2, 0);
2467        WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2468        /* Setup TLB control */
2469        tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2470        WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2471        WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2472        WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2473        WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2474        WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2475        WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2476        WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2477        radeon_gart_table_vram_unpin(rdev);
2478}
2479
2480static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2481{
2482        evergreen_pcie_gart_disable(rdev);
2483        radeon_gart_table_vram_free(rdev);
2484        radeon_gart_fini(rdev);
2485}
2486
2487
2488static void evergreen_agp_enable(struct radeon_device *rdev)
2489{
2490        u32 tmp;
2491
2492        /* Setup L2 cache */
2493        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2494                                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2495                                EFFECTIVE_L2_QUEUE_SIZE(7));
2496        WREG32(VM_L2_CNTL2, 0);
2497        WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2498        /* Setup TLB control */
2499        tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2500                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2501                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2502                EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2503        WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2504        WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2505        WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2506        WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2507        WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2508        WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2509        WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2510        WREG32(VM_CONTEXT0_CNTL, 0);
2511        WREG32(VM_CONTEXT1_CNTL, 0);
2512}
2513
2514static const unsigned ni_dig_offsets[] =
2515{
2516        NI_DIG0_REGISTER_OFFSET,
2517        NI_DIG1_REGISTER_OFFSET,
2518        NI_DIG2_REGISTER_OFFSET,
2519        NI_DIG3_REGISTER_OFFSET,
2520        NI_DIG4_REGISTER_OFFSET,
2521        NI_DIG5_REGISTER_OFFSET
2522};
2523
2524static const unsigned ni_tx_offsets[] =
2525{
2526        NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2527        NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2528        NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2529        NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2530        NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2531        NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2532};
2533
2534static const unsigned evergreen_dp_offsets[] =
2535{
2536        EVERGREEN_DP0_REGISTER_OFFSET,
2537        EVERGREEN_DP1_REGISTER_OFFSET,
2538        EVERGREEN_DP2_REGISTER_OFFSET,
2539        EVERGREEN_DP3_REGISTER_OFFSET,
2540        EVERGREEN_DP4_REGISTER_OFFSET,
2541        EVERGREEN_DP5_REGISTER_OFFSET
2542};
2543
2544static const unsigned evergreen_disp_int_status[] =
2545{
2546        DISP_INTERRUPT_STATUS,
2547        DISP_INTERRUPT_STATUS_CONTINUE,
2548        DISP_INTERRUPT_STATUS_CONTINUE2,
2549        DISP_INTERRUPT_STATUS_CONTINUE3,
2550        DISP_INTERRUPT_STATUS_CONTINUE4,
2551        DISP_INTERRUPT_STATUS_CONTINUE5
2552};
2553
2554/*
2555 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2556 * We go from crtc to connector and it is not relible  since it
2557 * should be an opposite direction .If crtc is enable then
2558 * find the dig_fe which selects this crtc and insure that it enable.
2559 * if such dig_fe is found then find dig_be which selects found dig_be and
2560 * insure that it enable and in DP_SST mode.
2561 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2562 * from dp symbols clocks .
2563 */
2564static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2565                                               unsigned crtc_id, unsigned *ret_dig_fe)
2566{
2567        unsigned i;
2568        unsigned dig_fe;
2569        unsigned dig_be;
2570        unsigned dig_en_be;
2571        unsigned uniphy_pll;
2572        unsigned digs_fe_selected;
2573        unsigned dig_be_mode;
2574        unsigned dig_fe_mask;
2575        bool is_enabled = false;
2576        bool found_crtc = false;
2577
2578        /* loop through all running dig_fe to find selected crtc */
2579        for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2580                dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2581                if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2582                    crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2583                        /* found running pipe */
2584                        found_crtc = true;
2585                        dig_fe_mask = 1 << i;
2586                        dig_fe = i;
2587                        break;
2588                }
2589        }
2590
2591        if (found_crtc) {
2592                /* loop through all running dig_be to find selected dig_fe */
2593                for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2594                        dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2595                        /* if dig_fe_selected by dig_be? */
2596                        digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2597                        dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2598                        if (dig_fe_mask &  digs_fe_selected &&
2599                            /* if dig_be in sst mode? */
2600                            dig_be_mode == NI_DIG_BE_DPSST) {
2601                                dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2602                                                   ni_dig_offsets[i]);
2603                                uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2604                                                    ni_tx_offsets[i]);
2605                                /* dig_be enable and tx is running */
2606                                if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2607                                    dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2608                                    uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2609                                        is_enabled = true;
2610                                        *ret_dig_fe = dig_fe;
2611                                        break;
2612                                }
2613                        }
2614                }
2615        }
2616
2617        return is_enabled;
2618}
2619
2620/*
2621 * Blank dig when in dp sst mode
2622 * Dig ignores crtc timing
2623 */
2624static void evergreen_blank_dp_output(struct radeon_device *rdev,
2625                                      unsigned dig_fe)
2626{
2627        unsigned stream_ctrl;
2628        unsigned fifo_ctrl;
2629        unsigned counter = 0;
2630
2631        if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2632                DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2633                return;
2634        }
2635
2636        stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2637                             evergreen_dp_offsets[dig_fe]);
2638        if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2639                DRM_ERROR("dig %d , should be enable\n", dig_fe);
2640                return;
2641        }
2642
2643        stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2644        WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2645               evergreen_dp_offsets[dig_fe], stream_ctrl);
2646
2647        stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648                             evergreen_dp_offsets[dig_fe]);
2649        while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2650                msleep(1);
2651                counter++;
2652                stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2653                                     evergreen_dp_offsets[dig_fe]);
2654        }
2655        if (counter >= 32 )
2656                DRM_ERROR("counter exceeds %d\n", counter);
2657
2658        fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2659        fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2660        WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2661
2662}
2663
2664void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2665{
2666        u32 crtc_enabled, tmp, frame_count, blackout;
2667        int i, j;
2668        unsigned dig_fe;
2669
2670        if (!ASIC_IS_NODCE(rdev)) {
2671                save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2672                save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2673
2674                /* disable VGA render */
2675                WREG32(VGA_RENDER_CONTROL, 0);
2676        }
2677        /* blank the display controllers */
2678        for (i = 0; i < rdev->num_crtc; i++) {
2679                crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2680                if (crtc_enabled) {
2681                        save->crtc_enabled[i] = true;
2682                        if (ASIC_IS_DCE6(rdev)) {
2683                                tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2684                                if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2685                                        radeon_wait_for_vblank(rdev, i);
2686                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2687                                        tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2688                                        WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2689                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2690                                }
2691                        } else {
2692                                tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2693                                if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2694                                        radeon_wait_for_vblank(rdev, i);
2695                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2696                                        tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2697                                        WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2698                                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2699                                }
2700                        }
2701                        /* wait for the next frame */
2702                        frame_count = radeon_get_vblank_counter(rdev, i);
2703                        for (j = 0; j < rdev->usec_timeout; j++) {
2704                                if (radeon_get_vblank_counter(rdev, i) != frame_count)
2705                                        break;
2706                                udelay(1);
2707                        }
2708                        /*we should disable dig if it drives dp sst*/
2709                        /*but we are in radeon_device_init and the topology is unknown*/
2710                        /*and it is available after radeon_modeset_init*/
2711                        /*the following method radeon_atom_encoder_dpms_dig*/
2712                        /*does the job if we initialize it properly*/
2713                        /*for now we do it this manually*/
2714                        /**/
2715                        if (ASIC_IS_DCE5(rdev) &&
2716                            evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2717                                evergreen_blank_dp_output(rdev, dig_fe);
2718                        /*we could remove 6 lines below*/
2719                        /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2720                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2721                        tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2722                        tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2723                        WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2724                        WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2725                        save->crtc_enabled[i] = false;
2726                        /* ***** */
2727                } else {
2728                        save->crtc_enabled[i] = false;
2729                }
2730        }
2731
2732        radeon_mc_wait_for_idle(rdev);
2733
2734        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2735        if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2736                /* Block CPU access */
2737                WREG32(BIF_FB_EN, 0);
2738                /* blackout the MC */
2739                blackout &= ~BLACKOUT_MODE_MASK;
2740                WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2741        }
2742        /* wait for the MC to settle */
2743        udelay(100);
2744
2745        /* lock double buffered regs */
2746        for (i = 0; i < rdev->num_crtc; i++) {
2747                if (save->crtc_enabled[i]) {
2748                        tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2749                        if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2750                                tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2751                                WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2752                        }
2753                        tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2754                        if (!(tmp & 1)) {
2755                                tmp |= 1;
2756                                WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2757                        }
2758                }
2759        }
2760}
2761
2762void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2763{
2764        u32 tmp, frame_count;
2765        int i, j;
2766
2767        /* update crtc base addresses */
2768        for (i = 0; i < rdev->num_crtc; i++) {
2769                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2770                       upper_32_bits(rdev->mc.vram_start));
2771                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2772                       upper_32_bits(rdev->mc.vram_start));
2773                WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2774                       (u32)rdev->mc.vram_start);
2775                WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2776                       (u32)rdev->mc.vram_start);
2777        }
2778
2779        if (!ASIC_IS_NODCE(rdev)) {
2780                WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2781                WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2782        }
2783
2784        /* unlock regs and wait for update */
2785        for (i = 0; i < rdev->num_crtc; i++) {
2786                if (save->crtc_enabled[i]) {
2787                        tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2788                        if ((tmp & 0x7) != 0) {
2789                                tmp &= ~0x7;
2790                                WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2791                        }
2792                        tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2793                        if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2794                                tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2795                                WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2796                        }
2797                        tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2798                        if (tmp & 1) {
2799                                tmp &= ~1;
2800                                WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2801                        }
2802                        for (j = 0; j < rdev->usec_timeout; j++) {
2803                                tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2804                                if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2805                                        break;
2806                                udelay(1);
2807                        }
2808                }
2809        }
2810
2811        /* unblackout the MC */
2812        tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2813        tmp &= ~BLACKOUT_MODE_MASK;
2814        WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2815        /* allow CPU access */
2816        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2817
2818        for (i = 0; i < rdev->num_crtc; i++) {
2819                if (save->crtc_enabled[i]) {
2820                        if (ASIC_IS_DCE6(rdev)) {
2821                                tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2822                                tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2823                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2824                                WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2825                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2826                        } else {
2827                                tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2828                                tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2829                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2830                                WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2831                                WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2832                        }
2833                        /* wait for the next frame */
2834                        frame_count = radeon_get_vblank_counter(rdev, i);
2835                        for (j = 0; j < rdev->usec_timeout; j++) {
2836                                if (radeon_get_vblank_counter(rdev, i) != frame_count)
2837                                        break;
2838                                udelay(1);
2839                        }
2840                }
2841        }
2842        if (!ASIC_IS_NODCE(rdev)) {
2843                /* Unlock vga access */
2844                WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2845                mdelay(1);
2846                WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2847        }
2848}
2849
2850void evergreen_mc_program(struct radeon_device *rdev)
2851{
2852        struct evergreen_mc_save save;
2853        u32 tmp;
2854        int i, j;
2855
2856        /* Initialize HDP */
2857        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2858                WREG32((0x2c14 + j), 0x00000000);
2859                WREG32((0x2c18 + j), 0x00000000);
2860                WREG32((0x2c1c + j), 0x00000000);
2861                WREG32((0x2c20 + j), 0x00000000);
2862                WREG32((0x2c24 + j), 0x00000000);
2863        }
2864        WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2865
2866        evergreen_mc_stop(rdev, &save);
2867        if (evergreen_mc_wait_for_idle(rdev)) {
2868                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2869        }
2870        /* Lockout access through VGA aperture*/
2871        WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2872        /* Update configuration */
2873        if (rdev->flags & RADEON_IS_AGP) {
2874                if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2875                        /* VRAM before AGP */
2876                        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2877                                rdev->mc.vram_start >> 12);
2878                        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2879                                rdev->mc.gtt_end >> 12);
2880                } else {
2881                        /* VRAM after AGP */
2882                        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2883                                rdev->mc.gtt_start >> 12);
2884                        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2885                                rdev->mc.vram_end >> 12);
2886                }
2887        } else {
2888                WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2889                        rdev->mc.vram_start >> 12);
2890                WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2891                        rdev->mc.vram_end >> 12);
2892        }
2893        WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2894        /* llano/ontario only */
2895        if ((rdev->family == CHIP_PALM) ||
2896            (rdev->family == CHIP_SUMO) ||
2897            (rdev->family == CHIP_SUMO2)) {
2898                tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2899                tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2900                tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2901                WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2902        }
2903        tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2904        tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2905        WREG32(MC_VM_FB_LOCATION, tmp);
2906        WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2907        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2908        WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2909        if (rdev->flags & RADEON_IS_AGP) {
2910                WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2911                WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2912                WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2913        } else {
2914                WREG32(MC_VM_AGP_BASE, 0);
2915                WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2916                WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2917        }
2918        if (evergreen_mc_wait_for_idle(rdev)) {
2919                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2920        }
2921        evergreen_mc_resume(rdev, &save);
2922        /* we need to own VRAM, so turn off the VGA renderer here
2923         * to stop it overwriting our objects */
2924        rv515_vga_render_disable(rdev);
2925}
2926
2927/*
2928 * CP.
2929 */
2930void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2931{
2932        struct radeon_ring *ring = &rdev->ring[ib->ring];
2933        u32 next_rptr;
2934
2935        /* set to DX10/11 mode */
2936        radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2937        radeon_ring_write(ring, 1);
2938
2939        if (ring->rptr_save_reg) {
2940                next_rptr = ring->wptr + 3 + 4;
2941                radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2942                radeon_ring_write(ring, ((ring->rptr_save_reg - 
2943                                          PACKET3_SET_CONFIG_REG_START) >> 2));
2944                radeon_ring_write(ring, next_rptr);
2945        } else if (rdev->wb.enabled) {
2946                next_rptr = ring->wptr + 5 + 4;
2947                radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2948                radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2949                radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2950                radeon_ring_write(ring, next_rptr);
2951                radeon_ring_write(ring, 0);
2952        }
2953
2954        radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2955        radeon_ring_write(ring,
2956#ifdef __BIG_ENDIAN
2957                          (2 << 0) |
2958#endif
2959                          (ib->gpu_addr & 0xFFFFFFFC));
2960        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2961        radeon_ring_write(ring, ib->length_dw);
2962}
2963
2964
2965static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2966{
2967        const __be32 *fw_data;
2968        int i;
2969
2970        if (!rdev->me_fw || !rdev->pfp_fw)
2971                return -EINVAL;
2972
2973        r700_cp_stop(rdev);
2974        WREG32(CP_RB_CNTL,
2975#ifdef __BIG_ENDIAN
2976               BUF_SWAP_32BIT |
2977#endif
2978               RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2979
2980        fw_data = (const __be32 *)rdev->pfp_fw->data;
2981        WREG32(CP_PFP_UCODE_ADDR, 0);
2982        for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2983                WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2984        WREG32(CP_PFP_UCODE_ADDR, 0);
2985
2986        fw_data = (const __be32 *)rdev->me_fw->data;
2987        WREG32(CP_ME_RAM_WADDR, 0);
2988        for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2989                WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2990
2991        WREG32(CP_PFP_UCODE_ADDR, 0);
2992        WREG32(CP_ME_RAM_WADDR, 0);
2993        WREG32(CP_ME_RAM_RADDR, 0);
2994        return 0;
2995}
2996
2997static int evergreen_cp_start(struct radeon_device *rdev)
2998{
2999        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3000        int r, i;

3001        uint32_t cp_me;
3002
3003        r = radeon_ring_lock(rdev, ring, 7);
3004        if (r) {
3005                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3006                return r;
3007        }
3008        radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3009        radeon_ring_write(ring, 0x1);
3010        radeon_ring_write(ring, 0x0);
3011        radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3012        radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3013        radeon_ring_write(ring, 0);
3014        radeon_ring_write(ring, 0);
3015        radeon_ring_unlock_commit(rdev, ring, false);
3016
3017        cp_me = 0xff;
3018        WREG32(CP_ME_CNTL, cp_me);
3019
3020        r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3021        if (r) {
3022                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3023                return r;
3024        }
3025
3026        /* setup clear context state */
3027        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3028        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3029
3030        for (i = 0; i < evergreen_default_size; i++)
3031                radeon_ring_write(ring, evergreen_default_state[i]);
3032
3033        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3034        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3035
3036        /* set clear context state */
3037        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3038        radeon_ring_write(ring, 0);
3039
3040        /* SQ_VTX_BASE_VTX_LOC */
3041        radeon_ring_write(ring, 0xc0026f00);
3042        radeon_ring_write(ring, 0x00000000);
3043        radeon_ring_write(ring, 0x00000000);
3044        radeon_ring_write(ring, 0x00000000);
3045
3046        /* Clear consts */
3047        radeon_ring_write(ring, 0xc0036f00);
3048        radeon_ring_write(ring, 0x00000bc4);
3049        radeon_ring_write(ring, 0xffffffff);
3050        radeon_ring_write(ring, 0xffffffff);
3051        radeon_ring_write(ring, 0xffffffff);
3052
3053        radeon_ring_write(ring, 0xc0026900);
3054        radeon_ring_write(ring, 0x00000316);
3055        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3056        radeon_ring_write(ring, 0x00000010); /*  */
3057
3058        radeon_ring_unlock_commit(rdev, ring, false);
3059
3060        return 0;
3061}
3062
3063static int evergreen_cp_resume(struct radeon_device *rdev)
3064{
3065        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3066        u32 tmp;
3067        u32 rb_bufsz;
3068        int r;
3069
3070        /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3071        WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3072                                 SOFT_RESET_PA |
3073                                 SOFT_RESET_SH |
3074                                 SOFT_RESET_VGT |
3075                                 SOFT_RESET_SPI |
3076                                 SOFT_RESET_SX));
3077        RREG32(GRBM_SOFT_RESET);
3078        mdelay(15);
3079        WREG32(GRBM_SOFT_RESET, 0);
3080        RREG32(GRBM_SOFT_RESET);
3081
3082        /* Set ring buffer size */
3083        rb_bufsz = order_base_2(ring->ring_size / 8);
3084        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3085#ifdef __BIG_ENDIAN
3086        tmp |= BUF_SWAP_32BIT;
3087#endif
3088        WREG32(CP_RB_CNTL, tmp);
3089        WREG32(CP_SEM_WAIT_TIMER, 0x0);
3090        WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3091
3092        /* Set the write pointer delay */
3093        WREG32(CP_RB_WPTR_DELAY, 0);
3094
3095        /* Initialize the ring buffer's read and write pointers */
3096        WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3097        WREG32(CP_RB_RPTR_WR, 0);
3098        ring->wptr = 0;
3099        WREG32(CP_RB_WPTR, ring->wptr);
3100
3101        /* set the wb address whether it's enabled or not */
3102        WREG32(CP_RB_RPTR_ADDR,
3103               ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3104        WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3105        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3106
3107        if (rdev->wb.enabled)
3108                WREG32(SCRATCH_UMSK, 0xff);
3109        else {
3110                tmp |= RB_NO_UPDATE;
3111                WREG32(SCRATCH_UMSK, 0);
3112        }
3113
3114        mdelay(1);
3115        WREG32(CP_RB_CNTL, tmp);
3116
3117        WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3118        WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3119
3120        evergreen_cp_start(rdev);
3121        ring->ready = true;
3122        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3123        if (r) {
3124                ring->ready = false;
3125                return r;
3126        }
3127        return 0;
3128}
3129
3130/*
3131 * Core functions
3132 */
3133static void evergreen_gpu_init(struct radeon_device *rdev)
3134{
3135        u32 gb_addr_config;
3136        u32 mc_shared_chmap, mc_arb_ramcfg;
3137        u32 sx_debug_1;
3138        u32 smx_dc_ctl0;
3139        u32 sq_config;
3140        u32 sq_lds_resource_mgmt;
3141        u32 sq_gpr_resource_mgmt_1;
3142        u32 sq_gpr_resource_mgmt_2;
3143        u32 sq_gpr_resource_mgmt_3;
3144        u32 sq_thread_resource_mgmt;
3145        u32 sq_thread_resource_mgmt_2;
3146        u32 sq_stack_resource_mgmt_1;
3147        u32 sq_stack_resource_mgmt_2;
3148        u32 sq_stack_resource_mgmt_3;
3149        u32 vgt_cache_invalidation;
3150        u32 hdp_host_path_cntl, tmp;
3151        u32 disabled_rb_mask;
3152        int i, j, ps_thread_count;
3153
3154        switch (rdev->family) {
3155        case CHIP_CYPRESS:
3156        case CHIP_HEMLOCK:
3157                rdev->config.evergreen.num_ses = 2;
3158                rdev->config.evergreen.max_pipes = 4;
3159                rdev->config.evergreen.max_tile_pipes = 8;
3160                rdev->config.evergreen.max_simds = 10;
3161                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3162                rdev->config.evergreen.max_gprs = 256;
3163                rdev->config.evergreen.max_threads = 248;
3164                rdev->config.evergreen.max_gs_threads = 32;
3165                rdev->config.evergreen.max_stack_entries = 512;
3166                rdev->config.evergreen.sx_num_of_sets = 4;
3167                rdev->config.evergreen.sx_max_export_size = 256;
3168                rdev->config.evergreen.sx_max_export_pos_size = 64;
3169                rdev->config.evergreen.sx_max_export_smx_size = 192;
3170                rdev->config.evergreen.max_hw_contexts = 8;
3171                rdev->config.evergreen.sq_num_cf_insts = 2;
3172
3173                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3174                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3175                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3176                gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3177                break;
3178        case CHIP_JUNIPER:
3179                rdev->config.evergreen.num_ses = 1;
3180                rdev->config.evergreen.max_pipes = 4;
3181                rdev->config.evergreen.max_tile_pipes = 4;
3182                rdev->config.evergreen.max_simds = 10;
3183                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3184                rdev->config.evergreen.max_gprs = 256;
3185                rdev->config.evergreen.max_threads = 248;
3186                rdev->config.evergreen.max_gs_threads = 32;
3187                rdev->config.evergreen.max_stack_entries = 512;
3188                rdev->config.evergreen.sx_num_of_sets = 4;
3189                rdev->config.evergreen.sx_max_export_size = 256;
3190                rdev->config.evergreen.sx_max_export_pos_size = 64;
3191                rdev->config.evergreen.sx_max_export_smx_size = 192;
3192                rdev->config.evergreen.max_hw_contexts = 8;
3193                rdev->config.evergreen.sq_num_cf_insts = 2;
3194
3195                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3196                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3197                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3198                gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3199                break;
3200        case CHIP_REDWOOD:
3201                rdev->config.evergreen.num_ses = 1;
3202                rdev->config.evergreen.max_pipes = 4;
3203                rdev->config.evergreen.max_tile_pipes = 4;
3204                rdev->config.evergreen.max_simds = 5;
3205                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3206                rdev->config.evergreen.max_gprs = 256;
3207                rdev->config.evergreen.max_threads = 248;
3208                rdev->config.evergreen.max_gs_threads = 32;
3209                rdev->config.evergreen.max_stack_entries = 256;
3210                rdev->config.evergreen.sx_num_of_sets = 4;
3211                rdev->config.evergreen.sx_max_export_size = 256;
3212                rdev->config.evergreen.sx_max_export_pos_size = 64;
3213                rdev->config.evergreen.sx_max_export_smx_size = 192;
3214                rdev->config.evergreen.max_hw_contexts = 8;
3215                rdev->config.evergreen.sq_num_cf_insts = 2;
3216
3217                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3218                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3219                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3220                gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3221                break;
3222        case CHIP_CEDAR:
3223        default:
3224                rdev->config.evergreen.num_ses = 1;
3225                rdev->config.evergreen.max_pipes = 2;
3226                rdev->config.evergreen.max_tile_pipes = 2;
3227                rdev->config.evergreen.max_simds = 2;
3228                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3229                rdev->config.evergreen.max_gprs = 256;
3230                rdev->config.evergreen.max_threads = 192;
3231                rdev->config.evergreen.max_gs_threads = 16;
3232                rdev->config.evergreen.max_stack_entries = 256;
3233                rdev->config.evergreen.sx_num_of_sets = 4;
3234                rdev->config.evergreen.sx_max_export_size = 128;
3235                rdev->config.evergreen.sx_max_export_pos_size = 32;
3236                rdev->config.evergreen.sx_max_export_smx_size = 96;
3237                rdev->config.evergreen.max_hw_contexts = 4;
3238                rdev->config.evergreen.sq_num_cf_insts = 1;
3239
3240                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3241                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3242                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3243                gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3244                break;
3245        case CHIP_PALM:
3246                rdev->config.evergreen.num_ses = 1;
3247                rdev->config.evergreen.max_pipes = 2;
3248                rdev->config.evergreen.max_tile_pipes = 2;
3249                rdev->config.evergreen.max_simds = 2;
3250                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3251                rdev->config.evergreen.max_gprs = 256;
3252                rdev->config.evergreen.max_threads = 192;
3253                rdev->config.evergreen.max_gs_threads = 16;
3254                rdev->config.evergreen.max_stack_entries = 256;
3255                rdev->config.evergreen.sx_num_of_sets = 4;
3256                rdev->config.evergreen.sx_max_export_size = 128;
3257                rdev->config.evergreen.sx_max_export_pos_size = 32;
3258                rdev->config.evergreen.sx_max_export_smx_size = 96;
3259                rdev->config.evergreen.max_hw_contexts = 4;
3260                rdev->config.evergreen.sq_num_cf_insts = 1;
3261
3262                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3263                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3264                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3265                gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3266                break;
3267        case CHIP_SUMO:
3268                rdev->config.evergreen.num_ses = 1;
3269                rdev->config.evergreen.max_pipes = 4;
3270                rdev->config.evergreen.max_tile_pipes = 4;
3271                if (rdev->pdev->device == 0x9648)
3272                        rdev->config.evergreen.max_simds = 3;
3273                else if ((rdev->pdev->device == 0x9647) ||
3274                         (rdev->pdev->device == 0x964a))
3275                        rdev->config.evergreen.max_simds = 4;
3276                else
3277                        rdev->config.evergreen.max_simds = 5;
3278                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3279                rdev->config.evergreen.max_gprs = 256;
3280                rdev->config.evergreen.max_threads = 248;
3281                rdev->config.evergreen.max_gs_threads = 32;
3282                rdev->config.evergreen.max_stack_entries = 256;
3283                rdev->config.evergreen.sx_num_of_sets = 4;
3284                rdev->config.evergreen.sx_max_export_size = 256;
3285                rdev->config.evergreen.sx_max_export_pos_size = 64;
3286                rdev->config.evergreen.sx_max_export_smx_size = 192;
3287                rdev->config.evergreen.max_hw_contexts = 8;
3288                rdev->config.evergreen.sq_num_cf_insts = 2;
3289
3290                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3291                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3292                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3293                gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3294                break;
3295        case CHIP_SUMO2:
3296                rdev->config.evergreen.num_ses = 1;
3297                rdev->config.evergreen.max_pipes = 4;
3298                rdev->config.evergreen.max_tile_pipes = 4;
3299                rdev->config.evergreen.max_simds = 2;
3300                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3301                rdev->config.evergreen.max_gprs = 256;
3302                rdev->config.evergreen.max_threads = 248;
3303                rdev->config.evergreen.max_gs_threads = 32;
3304                rdev->config.evergreen.max_stack_entries = 512;
3305                rdev->config.evergreen.sx_num_of_sets = 4;
3306                rdev->config.evergreen.sx_max_export_size = 256;
3307                rdev->config.evergreen.sx_max_export_pos_size = 64;
3308                rdev->config.evergreen.sx_max_export_smx_size = 192;
3309                rdev->config.evergreen.max_hw_contexts = 4;
3310                rdev->config.evergreen.sq_num_cf_insts = 2;
3311
3312                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3313                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3314                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3315                gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3316                break;
3317        case CHIP_BARTS:
3318                rdev->config.evergreen.num_ses = 2;
3319                rdev->config.evergreen.max_pipes = 4;
3320                rdev->config.evergreen.max_tile_pipes = 8;
3321                rdev->config.evergreen.max_simds = 7;
3322                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3323                rdev->config.evergreen.max_gprs = 256;
3324                rdev->config.evergreen.max_threads = 248;
3325                rdev->config.evergreen.max_gs_threads = 32;
3326                rdev->config.evergreen.max_stack_entries = 512;
3327                rdev->config.evergreen.sx_num_of_sets = 4;
3328                rdev->config.evergreen.sx_max_export_size = 256;
3329                rdev->config.evergreen.sx_max_export_pos_size = 64;
3330                rdev->config.evergreen.sx_max_export_smx_size = 192;
3331                rdev->config.evergreen.max_hw_contexts = 8;
3332                rdev->config.evergreen.sq_num_cf_insts = 2;
3333
3334                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3335                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3336                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3337                gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3338                break;
3339        case CHIP_TURKS:
3340                rdev->config.evergreen.num_ses = 1;
3341                rdev->config.evergreen.max_pipes = 4;
3342                rdev->config.evergreen.max_tile_pipes = 4;
3343                rdev->config.evergreen.max_simds = 6;
3344                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3345                rdev->config.evergreen.max_gprs = 256;
3346                rdev->config.evergreen.max_threads = 248;
3347                rdev->config.evergreen.max_gs_threads = 32;
3348                rdev->config.evergreen.max_stack_entries = 256;
3349                rdev->config.evergreen.sx_num_of_sets = 4;
3350                rdev->config.evergreen.sx_max_export_size = 256;
3351                rdev->config.evergreen.sx_max_export_pos_size = 64;
3352                rdev->config.evergreen.sx_max_export_smx_size = 192;
3353                rdev->config.evergreen.max_hw_contexts = 8;
3354                rdev->config.evergreen.sq_num_cf_insts = 2;
3355
3356                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3357                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3358                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3359                gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3360                break;
3361        case CHIP_CAICOS:
3362                rdev->config.evergreen.num_ses = 1;
3363                rdev->config.evergreen.max_pipes = 2;
3364                rdev->config.evergreen.max_tile_pipes = 2;
3365                rdev->config.evergreen.max_simds = 2;
3366                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3367                rdev->config.evergreen.max_gprs = 256;
3368                rdev->config.evergreen.max_threads = 192;
3369                rdev->config.evergreen.max_gs_threads = 16;
3370                rdev->config.evergreen.max_stack_entries = 256;
3371                rdev->config.evergreen.sx_num_of_sets = 4;
3372                rdev->config.evergreen.sx_max_export_size = 128;
3373                rdev->config.evergreen.sx_max_export_pos_size = 32;
3374                rdev->config.evergreen.sx_max_export_smx_size = 96;
3375                rdev->config.evergreen.max_hw_contexts = 4;
3376                rdev->config.evergreen.sq_num_cf_insts = 1;
3377
3378                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3379                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3380                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3381                gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3382                break;
3383        }
3384
3385        /* Initialize HDP */
3386        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3387                WREG32((0x2c14 + j), 0x00000000);
3388                WREG32((0x2c18 + j), 0x00000000);
3389                WREG32((0x2c1c + j), 0x00000000);
3390                WREG32((0x2c20 + j), 0x00000000);
3391                WREG32((0x2c24 + j), 0x00000000);
3392        }
3393
3394        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3395        WREG32(SRBM_INT_CNTL, 0x1);
3396        WREG32(SRBM_INT_ACK, 0x1);
3397
3398        evergreen_fix_pci_max_read_req_size(rdev);
3399
3400        mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3401        if ((rdev->family == CHIP_PALM) ||
3402            (rdev->family == CHIP_SUMO) ||
3403            (rdev->family == CHIP_SUMO2))
3404                mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3405        else
3406                mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3407
3408        /* setup tiling info dword.  gb_addr_config is not adequate since it does
3409         * not have bank info, so create a custom tiling dword.
3410         * bits 3:0   num_pipes
3411         * bits 7:4   num_banks
3412         * bits 11:8  group_size
3413         * bits 15:12 row_size
3414         */
3415        rdev->config.evergreen.tile_config = 0;
3416        switch (rdev->config.evergreen.max_tile_pipes) {
3417        case 1:
3418        default:
3419                rdev->config.evergreen.tile_config |= (0 << 0);
3420                break;
3421        case 2:
3422                rdev->config.evergreen.tile_config |= (1 << 0);
3423                break;
3424        case 4:
3425                rdev->config.evergreen.tile_config |= (2 << 0);
3426                break;
3427        case 8:
3428                rdev->config.evergreen.tile_config |= (3 << 0);
3429                break;
3430        }
3431        /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3432        if (rdev->flags & RADEON_IS_IGP)
3433                rdev->config.evergreen.tile_config |= 1 << 4;
3434        else {
3435                switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3436                case 0: /* four banks */
3437                        rdev->config.evergreen.tile_config |= 0 << 4;
3438                        break;
3439                case 1: /* eight banks */
3440                        rdev->config.evergreen.tile_config |= 1 << 4;
3441                        break;
3442                case 2: /* sixteen banks */
3443                default:
3444                        rdev->config.evergreen.tile_config |= 2 << 4;
3445                        break;
3446                }
3447        }
3448        rdev->config.evergreen.tile_config |= 0 << 8;
3449        rdev->config.evergreen.tile_config |=
3450                ((gb_addr_config & 0x30000000) >> 28) << 12;
3451
3452        if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3453                u32 efuse_straps_4;
3454                u32 efuse_straps_3;
3455
3456                efuse_straps_4 = RREG32_RCU(0x204);
3457                efuse_straps_3 = RREG32_RCU(0x203);
3458                tmp = (((efuse_straps_4 & 0xf) << 4) |
3459                      ((efuse_straps_3 & 0xf0000000) >> 28));
3460        } else {
3461                tmp = 0;
3462                for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3463                        u32 rb_disable_bitmap;
3464
3465                        WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466                        WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3467                        rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3468                        tmp <<= 4;
3469                        tmp |= rb_disable_bitmap;
3470                }
3471        }
3472        /* enabled rb are just the one not disabled :) */
3473        disabled_rb_mask = tmp;
3474        tmp = 0;
3475        for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3476                tmp |= (1 << i);
3477        /* if all the backends are disabled, fix it up here */
3478        if ((disabled_rb_mask & tmp) == tmp) {
3479                for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3480                        disabled_rb_mask &= ~(1 << i);
3481        }
3482
3483        for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3484                u32 simd_disable_bitmap;
3485
3486                WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487                WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3488                simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3489                simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3490                tmp <<= 16;
3491                tmp |= simd_disable_bitmap;
3492        }
3493        rdev->config.evergreen.active_simds = hweight32(~tmp);
3494
3495        WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3496        WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497
3498        WREG32(GB_ADDR_CONFIG, gb_addr_config);
3499        WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3500        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3501        WREG32(DMA_TILING_CONFIG, gb_addr_config);
3502        WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3503        WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3504        WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3505
3506        if ((rdev->config.evergreen.max_backends == 1) &&
3507            (rdev->flags & RADEON_IS_IGP)) {
3508                if ((disabled_rb_mask & 3) == 1) {
3509                        /* RB0 disabled, RB1 enabled */
3510                        tmp = 0x11111111;
3511                } else {
3512                        /* RB1 disabled, RB0 enabled */
3513                        tmp = 0x00000000;
3514                }
3515        } else {
3516                tmp = gb_addr_config & NUM_PIPES_MASK;
3517                tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3518                                                EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3519        }
3520        rdev->config.evergreen.backend_map = tmp;
3521        WREG32(GB_BACKEND_MAP, tmp);
3522
3523        WREG32(CGTS_SYS_TCC_DISABLE, 0);
3524        WREG32(CGTS_TCC_DISABLE, 0);
3525        WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3526        WREG32(CGTS_USER_TCC_DISABLE, 0);
3527
3528        /* set HW defaults for 3D engine */
3529        WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3530                                     ROQ_IB2_START(0x2b)));
3531
3532        WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3533
3534        WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3535                             SYNC_GRADIENT |
3536                             SYNC_WALKER |
3537                             SYNC_ALIGNER));
3538
3539        sx_debug_1 = RREG32(SX_DEBUG_1);
3540        sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3541        WREG32(SX_DEBUG_1, sx_debug_1);
3542
3543
3544        smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3545        smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3546        smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3547        WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3548
3549        if (rdev->family <= CHIP_SUMO2)
3550                WREG32(SMX_SAR_CTL0, 0x00010000);
3551
3552        WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3553                                        POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3554                                        SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3555
3556        WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3557                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3558                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3559
3560        WREG32(VGT_NUM_INSTANCES, 1);
3561        WREG32(SPI_CONFIG_CNTL, 0);
3562        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3563        WREG32(CP_PERFMON_CNTL, 0);
3564
3565        WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3566                                  FETCH_FIFO_HIWATER(0x4) |
3567                                  DONE_FIFO_HIWATER(0xe0) |
3568                                  ALU_UPDATE_FIFO_HIWATER(0x8)));
3569
3570        sq_config = RREG32(SQ_CONFIG);
3571        sq_config &= ~(PS_PRIO(3) |
3572                       VS_PRIO(3) |
3573                       GS_PRIO(3) |
3574                       ES_PRIO(3));
3575        sq_config |= (VC_ENABLE |
3576                      EXPORT_SRC_C |
3577                      PS_PRIO(0) |
3578                      VS_PRIO(1) |
3579                      GS_PRIO(2) |
3580                      ES_PRIO(3));
3581
3582        switch (rdev->family) {
3583        case CHIP_CEDAR:
3584        case CHIP_PALM:
3585        case CHIP_SUMO:
3586        case CHIP_SUMO2:
3587        case CHIP_CAICOS:
3588                /* no vertex cache */
3589                sq_config &= ~VC_ENABLE;
3590                break;
3591        default:
3592                break;
3593        }
3594
3595        sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3596
3597        sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3598        sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3599        sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3600        sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601        sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3602        sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3603        sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604
3605        switch (rdev->family) {
3606        case CHIP_CEDAR:
3607        case CHIP_PALM:
3608        case CHIP_SUMO:
3609        case CHIP_SUMO2:
3610                ps_thread_count = 96;
3611                break;
3612        default:
3613                ps_thread_count = 128;
3614                break;
3615        }
3616
3617        sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3618        sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619        sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620        sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621        sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622        sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623
3624        sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625        sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626        sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627        sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628        sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629        sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630
3631        WREG32(SQ_CONFIG, sq_config);
3632        WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3633        WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3634        WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3635        WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3636        WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3637        WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3638        WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3639        WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3640        WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3641        WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3642
3643        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3644                                          FORCE_EOV_MAX_REZ_CNT(255)));
3645
3646        switch (rdev->family) {
3647        case CHIP_CEDAR:
3648        case CHIP_PALM:
3649        case CHIP_SUMO:
3650        case CHIP_SUMO2:
3651        case CHIP_CAICOS:
3652                vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3653                break;
3654        default:
3655                vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3656                break;
3657        }
3658        vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3659        WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3660
3661        WREG32(VGT_GS_VERTEX_REUSE, 16);
3662        WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3663        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3664
3665        WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3666        WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3667
3668        WREG32(CB_PERF_CTR0_SEL_0, 0);
3669        WREG32(CB_PERF_CTR0_SEL_1, 0);
3670        WREG32(CB_PERF_CTR1_SEL_0, 0);
3671        WREG32(CB_PERF_CTR1_SEL_1, 0);
3672        WREG32(CB_PERF_CTR2_SEL_0, 0);
3673        WREG32(CB_PERF_CTR2_SEL_1, 0);
3674        WREG32(CB_PERF_CTR3_SEL_0, 0);
3675        WREG32(CB_PERF_CTR3_SEL_1, 0);
3676
3677        /* clear render buffer base addresses */
3678        WREG32(CB_COLOR0_BASE, 0);
3679        WREG32(CB_COLOR1_BASE, 0);
3680        WREG32(CB_COLOR2_BASE, 0);
3681        WREG32(CB_COLOR3_BASE, 0);
3682        WREG32(CB_COLOR4_BASE, 0);
3683        WREG32(CB_COLOR5_BASE, 0);
3684        WREG32(CB_COLOR6_BASE, 0);
3685        WREG32(CB_COLOR7_BASE, 0);
3686        WREG32(CB_COLOR8_BASE, 0);
3687        WREG32(CB_COLOR9_BASE, 0);
3688        WREG32(CB_COLOR10_BASE, 0);
3689        WREG32(CB_COLOR11_BASE, 0);
3690
3691        /* set the shader const cache sizes to 0 */
3692        for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3693                WREG32(i, 0);
3694        for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3695                WREG32(i, 0);
3696
3697        tmp = RREG32(HDP_MISC_CNTL);
3698        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3699        WREG32(HDP_MISC_CNTL, tmp);
3700
3701        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3702        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3703
3704        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3705
3706        udelay(50);
3707
3708}
3709
3710int evergreen_mc_init(struct radeon_device *rdev)
3711{
3712        u32 tmp;
3713        int chansize, numchan;
3714
3715        /* Get VRAM informations */
3716        rdev->mc.vram_is_ddr = true;
3717        if ((rdev->family == CHIP_PALM) ||
3718            (rdev->family == CHIP_SUMO) ||
3719            (rdev->family == CHIP_SUMO2))
3720                tmp = RREG32(FUS_MC_ARB_RAMCFG);
3721        else
3722                tmp = RREG32(MC_ARB_RAMCFG);
3723        if (tmp & CHANSIZE_OVERRIDE) {
3724                chansize = 16;
3725        } else if (tmp & CHANSIZE_MASK) {
3726                chansize = 64;
3727        } else {
3728                chansize = 32;
3729        }
3730        tmp = RREG32(MC_SHARED_CHMAP);
3731        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3732        case 0:
3733        default:
3734                numchan = 1;
3735                break;
3736        case 1:
3737                numchan = 2;
3738                break;
3739        case 2:
3740                numchan = 4;
3741                break;
3742        case 3:
3743                numchan = 8;
3744                break;
3745        }
3746        rdev->mc.vram_width = numchan * chansize;
3747        /* Could aper size report 0 ? */
3748        rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3749        rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3750        /* Setup GPU memory space */
3751        if ((rdev->family == CHIP_PALM) ||
3752            (rdev->family == CHIP_SUMO) ||
3753            (rdev->family == CHIP_SUMO2)) {
3754                /* size in bytes on fusion */
3755                rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3756                rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3757        } else {
3758                /* size in MB on evergreen/cayman/tn */
3759                rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3760                rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761        }
3762        rdev->mc.visible_vram_size = rdev->mc.aper_size;
3763        r700_vram_gtt_location(rdev, &rdev->mc);
3764        radeon_update_bandwidth_info(rdev);
3765
3766        return 0;
3767}
3768
3769void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3770{
3771        dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3772                RREG32(GRBM_STATUS));
3773        dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3774                RREG32(GRBM_STATUS_SE0));
3775        dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3776                RREG32(GRBM_STATUS_SE1));
3777        dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3778                RREG32(SRBM_STATUS));
3779        dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3780                RREG32(SRBM_STATUS2));
3781        dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3782                RREG32(CP_STALLED_STAT1));
3783        dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3784                RREG32(CP_STALLED_STAT2));
3785        dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3786                RREG32(CP_BUSY_STAT));
3787        dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3788                RREG32(CP_STAT));
3789        dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3790                RREG32(DMA_STATUS_REG));
3791        if (rdev->family >= CHIP_CAYMAN) {
3792                dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3793                         RREG32(DMA_STATUS_REG + 0x800));
3794        }
3795}
3796
3797bool evergreen_is_display_hung(struct radeon_device *rdev)
3798{
3799        u32 crtc_hung = 0;
3800        u32 crtc_status[6];
3801        u32 i, j, tmp;
3802
3803        for (i = 0; i < rdev->num_crtc; i++) {
3804                if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3805                        crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3806                        crtc_hung |= (1 << i);
3807                }
3808        }
3809
3810        for (j = 0; j < 10; j++) {
3811                for (i = 0; i < rdev->num_crtc; i++) {
3812                        if (crtc_hung & (1 << i)) {
3813                                tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3814                                if (tmp != crtc_status[i])
3815                                        crtc_hung &= ~(1 << i);
3816                        }
3817                }
3818                if (crtc_hung == 0)
3819                        return false;
3820                udelay(100);
3821        }
3822
3823        return true;
3824}
3825
3826u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3827{
3828        u32 reset_mask = 0;
3829        u32 tmp;
3830
3831        /* GRBM_STATUS */
3832        tmp = RREG32(GRBM_STATUS);
3833        if (tmp & (PA_BUSY | SC_BUSY |
3834                   SH_BUSY | SX_BUSY |
3835                   TA_BUSY | VGT_BUSY |
3836                   DB_BUSY | CB_BUSY |
3837                   SPI_BUSY | VGT_BUSY_NO_DMA))
3838                reset_mask |= RADEON_RESET_GFX;
3839
3840        if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3841                   CP_BUSY | CP_COHERENCY_BUSY))
3842                reset_mask |= RADEON_RESET_CP;
3843
3844        if (tmp & GRBM_EE_BUSY)
3845                reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3846
3847        /* DMA_STATUS_REG */
3848        tmp = RREG32(DMA_STATUS_REG);
3849        if (!(tmp & DMA_IDLE))
3850                reset_mask |= RADEON_RESET_DMA;
3851
3852        /* SRBM_STATUS2 */
3853        tmp = RREG32(SRBM_STATUS2);
3854        if (tmp & DMA_BUSY)
3855                reset_mask |= RADEON_RESET_DMA;
3856
3857        /* SRBM_STATUS */
3858        tmp = RREG32(SRBM_STATUS);
3859        if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3860                reset_mask |= RADEON_RESET_RLC;
3861
3862        if (tmp & IH_BUSY)
3863                reset_mask |= RADEON_RESET_IH;
3864
3865        if (tmp & SEM_BUSY)
3866                reset_mask |= RADEON_RESET_SEM;
3867
3868        if (tmp & GRBM_RQ_PENDING)
3869                reset_mask |= RADEON_RESET_GRBM;
3870
3871        if (tmp & VMC_BUSY)
3872                reset_mask |= RADEON_RESET_VMC;
3873
3874        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3875                   MCC_BUSY | MCD_BUSY))
3876                reset_mask |= RADEON_RESET_MC;
3877
3878        if (evergreen_is_display_hung(rdev))
3879                reset_mask |= RADEON_RESET_DISPLAY;
3880
3881        /* VM_L2_STATUS */
3882        tmp = RREG32(VM_L2_STATUS);
3883        if (tmp & L2_BUSY)
3884                reset_mask |= RADEON_RESET_VMC;
3885
3886        /* Skip MC reset as it's mostly likely not hung, just busy */
3887        if (reset_mask & RADEON_RESET_MC) {
3888                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3889                reset_mask &= ~RADEON_RESET_MC;
3890        }
3891
3892        return reset_mask;
3893}
3894
3895static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3896{
3897        struct evergreen_mc_save save;
3898        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3899        u32 tmp;
3900
3901        if (reset_mask == 0)
3902                return;
3903
3904        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3905
3906        evergreen_print_gpu_status_regs(rdev);
3907
3908        /* Disable CP parsing/prefetching */
3909        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3910
3911        if (reset_mask & RADEON_RESET_DMA) {
3912                /* Disable DMA */
3913                tmp = RREG32(DMA_RB_CNTL);
3914                tmp &= ~DMA_RB_ENABLE;
3915                WREG32(DMA_RB_CNTL, tmp);
3916        }
3917
3918        udelay(50);
3919
3920        evergreen_mc_stop(rdev, &save);
3921        if (evergreen_mc_wait_for_idle(rdev)) {
3922                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3923        }
3924
3925        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3926                grbm_soft_reset |= SOFT_RESET_DB |
3927                        SOFT_RESET_CB |
3928                        SOFT_RESET_PA |
3929                        SOFT_RESET_SC |
3930                        SOFT_RESET_SPI |
3931                        SOFT_RESET_SX |
3932                        SOFT_RESET_SH |
3933                        SOFT_RESET_TC |
3934                        SOFT_RESET_TA |
3935                        SOFT_RESET_VC |
3936                        SOFT_RESET_VGT;
3937        }
3938
3939        if (reset_mask & RADEON_RESET_CP) {
3940                grbm_soft_reset |= SOFT_RESET_CP |
3941                        SOFT_RESET_VGT;
3942
3943                srbm_soft_reset |= SOFT_RESET_GRBM;
3944        }
3945
3946        if (reset_mask & RADEON_RESET_DMA)
3947                srbm_soft_reset |= SOFT_RESET_DMA;
3948
3949        if (reset_mask & RADEON_RESET_DISPLAY)
3950                srbm_soft_reset |= SOFT_RESET_DC;
3951
3952        if (reset_mask & RADEON_RESET_RLC)
3953                srbm_soft_reset |= SOFT_RESET_RLC;
3954
3955        if (reset_mask & RADEON_RESET_SEM)
3956                srbm_soft_reset |= SOFT_RESET_SEM;
3957
3958        if (reset_mask & RADEON_RESET_IH)
3959                srbm_soft_reset |= SOFT_RESET_IH;
3960
3961        if (reset_mask & RADEON_RESET_GRBM)
3962                srbm_soft_reset |= SOFT_RESET_GRBM;
3963
3964        if (reset_mask & RADEON_RESET_VMC)
3965                srbm_soft_reset |= SOFT_RESET_VMC;
3966
3967        if (!(rdev->flags & RADEON_IS_IGP)) {
3968                if (reset_mask & RADEON_RESET_MC)
3969                        srbm_soft_reset |= SOFT_RESET_MC;
3970        }
3971
3972        if (grbm_soft_reset) {
3973                tmp = RREG32(GRBM_SOFT_RESET);
3974                tmp |= grbm_soft_reset;
3975                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3976                WREG32(GRBM_SOFT_RESET, tmp);
3977                tmp = RREG32(GRBM_SOFT_RESET);
3978
3979                udelay(50);
3980
3981                tmp &= ~grbm_soft_reset;
3982                WREG32(GRBM_SOFT_RESET, tmp);
3983                tmp = RREG32(GRBM_SOFT_RESET);
3984        }
3985
3986        if (srbm_soft_reset) {
3987                tmp = RREG32(SRBM_SOFT_RESET);
3988                tmp |= srbm_soft_reset;
3989                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3990                WREG32(SRBM_SOFT_RESET, tmp);
3991                tmp = RREG32(SRBM_SOFT_RESET);
3992
3993                udelay(50);
3994
3995                tmp &= ~srbm_soft_reset;
3996                WREG32(SRBM_SOFT_RESET, tmp);
3997                tmp = RREG32(SRBM_SOFT_RESET);
3998        }
3999
4000        /* Wait a little for things to settle down */

4001        udelay(50);
4002
4003        evergreen_mc_resume(rdev, &save);
4004        udelay(50);
4005
4006        evergreen_print_gpu_status_regs(rdev);
4007}
4008
4009void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4010{
4011        struct evergreen_mc_save save;
4012        u32 tmp, i;
4013
4014        dev_info(rdev->dev, "GPU pci config reset\n");
4015
4016        /* disable dpm? */
4017
4018        /* Disable CP parsing/prefetching */
4019        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4020        udelay(50);
4021        /* Disable DMA */
4022        tmp = RREG32(DMA_RB_CNTL);
4023        tmp &= ~DMA_RB_ENABLE;
4024        WREG32(DMA_RB_CNTL, tmp);
4025        /* XXX other engines? */
4026
4027        /* halt the rlc */
4028        r600_rlc_stop(rdev);
4029
4030        udelay(50);
4031
4032        /* set mclk/sclk to bypass */
4033        rv770_set_clk_bypass_mode(rdev);
4034        /* disable BM */
4035        pci_clear_master(rdev->pdev);
4036        /* disable mem access */
4037        evergreen_mc_stop(rdev, &save);
4038        if (evergreen_mc_wait_for_idle(rdev)) {
4039                dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4040        }
4041        /* reset */
4042        radeon_pci_config_reset(rdev);
4043        /* wait for asic to come out of reset */
4044        for (i = 0; i < rdev->usec_timeout; i++) {
4045                if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4046                        break;
4047                udelay(1);
4048        }
4049}
4050
4051int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4052{
4053        u32 reset_mask;
4054
4055        if (hard) {
4056                evergreen_gpu_pci_config_reset(rdev);
4057                return 0;
4058        }
4059
4060        reset_mask = evergreen_gpu_check_soft_reset(rdev);
4061
4062        if (reset_mask)
4063                r600_set_bios_scratch_engine_hung(rdev, true);
4064
4065        /* try soft reset */
4066        evergreen_gpu_soft_reset(rdev, reset_mask);
4067
4068        reset_mask = evergreen_gpu_check_soft_reset(rdev);
4069
4070        /* try pci config reset */
4071        if (reset_mask && radeon_hard_reset)
4072                evergreen_gpu_pci_config_reset(rdev);
4073
4074        reset_mask = evergreen_gpu_check_soft_reset(rdev);
4075
4076        if (!reset_mask)
4077                r600_set_bios_scratch_engine_hung(rdev, false);
4078
4079        return 0;
4080}
4081
4082/**
4083 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4084 *
4085 * @rdev: radeon_device pointer
4086 * @ring: radeon_ring structure holding ring information
4087 *
4088 * Check if the GFX engine is locked up.
4089 * Returns true if the engine appears to be locked up, false if not.
4090 */
4091bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4092{
4093        u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4094
4095        if (!(reset_mask & (RADEON_RESET_GFX |
4096                            RADEON_RESET_COMPUTE |
4097                            RADEON_RESET_CP))) {
4098                radeon_ring_lockup_update(rdev, ring);
4099                return false;
4100        }
4101        return radeon_ring_test_lockup(rdev, ring);
4102}
4103
4104/*
4105 * RLC
4106 */
4107#define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4108#define RLC_CLEAR_STATE_END_MARKER          0x00000001
4109
4110void sumo_rlc_fini(struct radeon_device *rdev)
4111{
4112        int r;
4113
4114        /* save restore block */
4115        if (rdev->rlc.save_restore_obj) {
4116                r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4117                if (unlikely(r != 0))
4118                        dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4119                radeon_bo_unpin(rdev->rlc.save_restore_obj);
4120                radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4121
4122                radeon_bo_unref(&rdev->rlc.save_restore_obj);
4123                rdev->rlc.save_restore_obj = NULL;
4124        }
4125
4126        /* clear state block */
4127        if (rdev->rlc.clear_state_obj) {
4128                r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4129                if (unlikely(r != 0))
4130                        dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4131                radeon_bo_unpin(rdev->rlc.clear_state_obj);
4132                radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4133
4134                radeon_bo_unref(&rdev->rlc.clear_state_obj);
4135                rdev->rlc.clear_state_obj = NULL;
4136        }
4137
4138        /* clear state block */
4139        if (rdev->rlc.cp_table_obj) {
4140                r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4141                if (unlikely(r != 0))
4142                        dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4143                radeon_bo_unpin(rdev->rlc.cp_table_obj);
4144                radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4145
4146                radeon_bo_unref(&rdev->rlc.cp_table_obj);
4147                rdev->rlc.cp_table_obj = NULL;
4148        }
4149}
4150
4151#define CP_ME_TABLE_SIZE    96
4152
4153int sumo_rlc_init(struct radeon_device *rdev)
4154{
4155        const u32 *src_ptr;
4156        volatile u32 *dst_ptr;
4157        u32 dws, data, i, j, k, reg_num;
4158        u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4159        u64 reg_list_mc_addr;
4160        const struct cs_section_def *cs_data;
4161        int r;
4162
4163        src_ptr = rdev->rlc.reg_list;
4164        dws = rdev->rlc.reg_list_size;
4165        if (rdev->family >= CHIP_BONAIRE) {
4166                dws += (5 * 16) + 48 + 48 + 64;
4167        }
4168        cs_data = rdev->rlc.cs_data;
4169
4170        if (src_ptr) {
4171                /* save restore block */
4172                if (rdev->rlc.save_restore_obj == NULL) {
4173                        r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4174                                             RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4175                                             NULL, &rdev->rlc.save_restore_obj);
4176                        if (r) {
4177                                dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4178                                return r;
4179                        }
4180                }
4181
4182                r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4183                if (unlikely(r != 0)) {
4184                        sumo_rlc_fini(rdev);
4185                        return r;
4186                }
4187                r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4188                                  &rdev->rlc.save_restore_gpu_addr);
4189                if (r) {
4190                        radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4191                        dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4192                        sumo_rlc_fini(rdev);
4193                        return r;
4194                }
4195
4196                r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4197                if (r) {
4198                        dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4199                        sumo_rlc_fini(rdev);
4200                        return r;
4201                }
4202                /* write the sr buffer */
4203                dst_ptr = rdev->rlc.sr_ptr;
4204                if (rdev->family >= CHIP_TAHITI) {
4205                        /* SI */
4206                        for (i = 0; i < rdev->rlc.reg_list_size; i++)
4207                                dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4208                } else {
4209                        /* ON/LN/TN */
4210                        /* format:
4211                         * dw0: (reg2 << 16) | reg1
4212                         * dw1: reg1 save space
4213                         * dw2: reg2 save space
4214                         */
4215                        for (i = 0; i < dws; i++) {
4216                                data = src_ptr[i] >> 2;
4217                                i++;
4218                                if (i < dws)
4219                                        data |= (src_ptr[i] >> 2) << 16;
4220                                j = (((i - 1) * 3) / 2);
4221                                dst_ptr[j] = cpu_to_le32(data);
4222                        }
4223                        j = ((i * 3) / 2);
4224                        dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4225                }
4226                radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4227                radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4228        }
4229
4230        if (cs_data) {
4231                /* clear state block */
4232                if (rdev->family >= CHIP_BONAIRE) {
4233                        rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4234                } else if (rdev->family >= CHIP_TAHITI) {
4235                        rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4236                        dws = rdev->rlc.clear_state_size + (256 / 4);
4237                } else {
4238                        reg_list_num = 0;
4239                        dws = 0;
4240                        for (i = 0; cs_data[i].section != NULL; i++) {
4241                                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4242                                        reg_list_num++;
4243                                        dws += cs_data[i].section[j].reg_count;
4244                                }
4245                        }
4246                        reg_list_blk_index = (3 * reg_list_num + 2);
4247                        dws += reg_list_blk_index;
4248                        rdev->rlc.clear_state_size = dws;
4249                }
4250
4251                if (rdev->rlc.clear_state_obj == NULL) {
4252                        r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4253                                             RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4254                                             NULL, &rdev->rlc.clear_state_obj);
4255                        if (r) {
4256                                dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4257                                sumo_rlc_fini(rdev);
4258                                return r;
4259                        }
4260                }
4261                r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4262                if (unlikely(r != 0)) {
4263                        sumo_rlc_fini(rdev);
4264                        return r;
4265                }
4266                r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4267                                  &rdev->rlc.clear_state_gpu_addr);
4268                if (r) {
4269                        radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4270                        dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4271                        sumo_rlc_fini(rdev);
4272                        return r;
4273                }
4274
4275                r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4276                if (r) {
4277                        dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4278                        sumo_rlc_fini(rdev);
4279                        return r;
4280                }
4281                /* set up the cs buffer */
4282                dst_ptr = rdev->rlc.cs_ptr;
4283                if (rdev->family >= CHIP_BONAIRE) {
4284                        cik_get_csb_buffer(rdev, dst_ptr);
4285                } else if (rdev->family >= CHIP_TAHITI) {
4286                        reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4287                        dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4288                        dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4289                        dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4290                        si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4291                } else {
4292                        reg_list_hdr_blk_index = 0;
4293                        reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4294                        data = upper_32_bits(reg_list_mc_addr);
4295                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4296                        reg_list_hdr_blk_index++;
4297                        for (i = 0; cs_data[i].section != NULL; i++) {
4298                                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4299                                        reg_num = cs_data[i].section[j].reg_count;
4300                                        data = reg_list_mc_addr & 0xffffffff;
4301                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4302                                        reg_list_hdr_blk_index++;
4303
4304                                        data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4305                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4306                                        reg_list_hdr_blk_index++;
4307
4308                                        data = 0x08000000 | (reg_num * 4);
4309                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4310                                        reg_list_hdr_blk_index++;
4311
4312                                        for (k = 0; k < reg_num; k++) {
4313                                                data = cs_data[i].section[j].extent[k];
4314                                                dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4315                                        }
4316                                        reg_list_mc_addr += reg_num * 4;
4317                                        reg_list_blk_index += reg_num;
4318                                }
4319                        }
4320                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4321                }
4322                radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4323                radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4324        }
4325
4326        if (rdev->rlc.cp_table_size) {
4327                if (rdev->rlc.cp_table_obj == NULL) {
4328                        r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4329                                             PAGE_SIZE, true,
4330                                             RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4331                                             NULL, &rdev->rlc.cp_table_obj);
4332                        if (r) {
4333                                dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4334                                sumo_rlc_fini(rdev);
4335                                return r;
4336                        }
4337                }
4338
4339                r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4340                if (unlikely(r != 0)) {
4341                        dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4342                        sumo_rlc_fini(rdev);
4343                        return r;
4344                }
4345                r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4346                                  &rdev->rlc.cp_table_gpu_addr);
4347                if (r) {
4348                        radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4349                        dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4350                        sumo_rlc_fini(rdev);
4351                        return r;
4352                }
4353                r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4354                if (r) {
4355                        dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4356                        sumo_rlc_fini(rdev);
4357                        return r;
4358                }
4359
4360                cik_init_cp_pg_table(rdev);
4361
4362                radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4363                radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4364
4365        }
4366
4367        return 0;
4368}
4369
4370static void evergreen_rlc_start(struct radeon_device *rdev)
4371{
4372        u32 mask = RLC_ENABLE;
4373
4374        if (rdev->flags & RADEON_IS_IGP) {
4375                mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4376        }
4377
4378        WREG32(RLC_CNTL, mask);
4379}
4380
4381int evergreen_rlc_resume(struct radeon_device *rdev)
4382{
4383        u32 i;
4384        const __be32 *fw_data;
4385
4386        if (!rdev->rlc_fw)
4387                return -EINVAL;
4388
4389        r600_rlc_stop(rdev);
4390
4391        WREG32(RLC_HB_CNTL, 0);
4392
4393        if (rdev->flags & RADEON_IS_IGP) {
4394                if (rdev->family == CHIP_ARUBA) {
4395                        u32 always_on_bitmap =
4396                                3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4397                        /* find out the number of active simds */
4398                        u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4399                        tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4400                        tmp = hweight32(~tmp);
4401                        if (tmp == rdev->config.cayman.max_simds_per_se) {
4402                                WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4403                                WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4404                                WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4405                                WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4406                                WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4407                        }
4408                } else {
4409                        WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4410                        WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4411                }
4412                WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4413                WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4414        } else {
4415                WREG32(RLC_HB_BASE, 0);
4416                WREG32(RLC_HB_RPTR, 0);
4417                WREG32(RLC_HB_WPTR, 0);
4418                WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4419                WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4420        }
4421        WREG32(RLC_MC_CNTL, 0);
4422        WREG32(RLC_UCODE_CNTL, 0);
4423
4424        fw_data = (const __be32 *)rdev->rlc_fw->data;
4425        if (rdev->family >= CHIP_ARUBA) {
4426                for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4427                        WREG32(RLC_UCODE_ADDR, i);
4428                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4429                }
4430        } else if (rdev->family >= CHIP_CAYMAN) {
4431                for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4432                        WREG32(RLC_UCODE_ADDR, i);
4433                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4434                }
4435        } else {
4436                for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4437                        WREG32(RLC_UCODE_ADDR, i);
4438                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4439                }
4440        }
4441        WREG32(RLC_UCODE_ADDR, 0);
4442
4443        evergreen_rlc_start(rdev);
4444
4445        return 0;
4446}
4447
4448/* Interrupts */
4449
4450u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4451{
4452        if (crtc >= rdev->num_crtc)
4453                return 0;
4454        else
4455                return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4456}
4457
4458void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4459{
4460        int i;
4461        u32 tmp;
4462
4463        if (rdev->family >= CHIP_CAYMAN) {
4464                cayman_cp_int_cntl_setup(rdev, 0,
4465                                         CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4466                cayman_cp_int_cntl_setup(rdev, 1, 0);
4467                cayman_cp_int_cntl_setup(rdev, 2, 0);
4468                tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4469                WREG32(CAYMAN_DMA1_CNTL, tmp);
4470        } else
4471                WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4472        tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4473        WREG32(DMA_CNTL, tmp);
4474        WREG32(GRBM_INT_CNTL, 0);
4475        WREG32(SRBM_INT_CNTL, 0);
4476        for (i = 0; i < rdev->num_crtc; i++)
4477                WREG32(INT_MASK + crtc_offsets[i], 0);
4478        for (i = 0; i < rdev->num_crtc; i++)
4479                WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4480
4481        /* only one DAC on DCE5 */
4482        if (!ASIC_IS_DCE5(rdev))
4483                WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4484        WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4485
4486        for (i = 0; i < 6; i++)
4487                WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4488}
4489
4490/* Note that the order we write back regs here is important */
4491int evergreen_irq_set(struct radeon_device *rdev)
4492{
4493        int i;
4494        u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4495        u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4496        u32 grbm_int_cntl = 0;
4497        u32 dma_cntl, dma_cntl1 = 0;
4498        u32 thermal_int = 0;
4499
4500        if (!rdev->irq.installed) {
4501                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4502                return -EINVAL;
4503        }
4504        /* don't enable anything if the ih is disabled */
4505        if (!rdev->ih.enabled) {
4506                r600_disable_interrupts(rdev);
4507                /* force the active interrupt state to all disabled */
4508                evergreen_disable_interrupt_state(rdev);
4509                return 0;
4510        }
4511
4512        if (rdev->family == CHIP_ARUBA)
4513                thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4514                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4515        else
4516                thermal_int = RREG32(CG_THERMAL_INT) &
4517                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4518
4519        dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4520
4521        if (rdev->family >= CHIP_CAYMAN) {
4522                /* enable CP interrupts on all rings */
4523                if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4524                        DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4525                        cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4526                }
4527                if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4528                        DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4529                        cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4530                }
4531                if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4532                        DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4533                        cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4534                }
4535        } else {
4536                if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4537                        DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4538                        cp_int_cntl |= RB_INT_ENABLE;
4539                        cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4540                }
4541        }
4542
4543        if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4544                DRM_DEBUG("r600_irq_set: sw int dma\n");
4545                dma_cntl |= TRAP_ENABLE;
4546        }
4547
4548        if (rdev->family >= CHIP_CAYMAN) {
4549                dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4550                if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4551                        DRM_DEBUG("r600_irq_set: sw int dma1\n");
4552                        dma_cntl1 |= TRAP_ENABLE;
4553                }
4554        }
4555
4556        if (rdev->irq.dpm_thermal) {
4557                DRM_DEBUG("dpm thermal\n");
4558                thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4559        }
4560
4561        if (rdev->family >= CHIP_CAYMAN) {
4562                cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4563                cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4564                cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4565        } else
4566                WREG32(CP_INT_CNTL, cp_int_cntl);
4567
4568        WREG32(DMA_CNTL, dma_cntl);
4569
4570        if (rdev->family >= CHIP_CAYMAN)
4571                WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4572
4573        WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4574
4575        for (i = 0; i < rdev->num_crtc; i++) {
4576                radeon_irq_kms_set_irq_n_enabled(
4577                    rdev, INT_MASK + crtc_offsets[i],
4578                    VBLANK_INT_MASK,
4579                    rdev->irq.crtc_vblank_int[i] ||
4580                    atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4581        }
4582
4583        for (i = 0; i < rdev->num_crtc; i++)
4584                WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4585
4586        for (i = 0; i < 6; i++) {
4587                radeon_irq_kms_set_irq_n_enabled(
4588                    rdev, DC_HPDx_INT_CONTROL(i),
4589                    DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4590                    rdev->irq.hpd[i], "HPD", i);
4591        }
4592
4593        if (rdev->family == CHIP_ARUBA)
4594                WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4595        else
4596                WREG32(CG_THERMAL_INT, thermal_int);
4597
4598        for (i = 0; i < 6; i++) {
4599                radeon_irq_kms_set_irq_n_enabled(
4600                    rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4601                    AFMT_AZ_FORMAT_WTRIG_MASK,
4602                    rdev->irq.afmt[i], "HDMI", i);
4603        }
4604
4605        /* posting read */
4606        RREG32(SRBM_STATUS);
4607
4608        return 0;
4609}
4610
4611/* Note that the order we write back regs here is important */
4612static void evergreen_irq_ack(struct radeon_device *rdev)
4613{
4614        int i, j;
4615        u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4616        u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4617        u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4618
4619        for (i = 0; i < 6; i++) {
4620                disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4621                afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4622                if (i < rdev->num_crtc)
4623                        grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4624        }
4625
4626        /* We write back each interrupt register in pairs of two */
4627        for (i = 0; i < rdev->num_crtc; i += 2) {
4628                for (j = i; j < (i + 2); j++) {
4629                        if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4630                                WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4631                                       GRPH_PFLIP_INT_CLEAR);
4632                }
4633
4634                for (j = i; j < (i + 2); j++) {
4635                        if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4636                                WREG32(VBLANK_STATUS + crtc_offsets[j],
4637                                       VBLANK_ACK);
4638                        if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4639                                WREG32(VLINE_STATUS + crtc_offsets[j],
4640                                       VLINE_ACK);
4641                }
4642        }
4643
4644        for (i = 0; i < 6; i++) {
4645                if (disp_int[i] & DC_HPD1_INTERRUPT)
4646                        WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4647        }
4648
4649        for (i = 0; i < 6; i++) {
4650                if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4651                        WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4652        }
4653
4654        for (i = 0; i < 6; i++) {
4655                if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4656                        WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4657                                  AFMT_AZ_FORMAT_WTRIG_ACK);
4658        }
4659}
4660
4661static void evergreen_irq_disable(struct radeon_device *rdev)
4662{
4663        r600_disable_interrupts(rdev);
4664        /* Wait and acknowledge irq */
4665        mdelay(1);
4666        evergreen_irq_ack(rdev);
4667        evergreen_disable_interrupt_state(rdev);
4668}
4669
4670void evergreen_irq_suspend(struct radeon_device *rdev)
4671{
4672        evergreen_irq_disable(rdev);
4673        r600_rlc_stop(rdev);
4674}
4675
4676static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4677{
4678        u32 wptr, tmp;
4679
4680        if (rdev->wb.enabled)
4681                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4682        else
4683                wptr = RREG32(IH_RB_WPTR);
4684
4685        if (wptr & RB_OVERFLOW) {
4686                wptr &= ~RB_OVERFLOW;
4687                /* When a ring buffer overflow happen start parsing interrupt
4688                 * from the last not overwritten vector (wptr + 16). Hopefully
4689                 * this should allow us to catchup.
4690                 */
4691                dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4692                         wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4693                rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4694                tmp = RREG32(IH_RB_CNTL);
4695                tmp |= IH_WPTR_OVERFLOW_CLEAR;
4696                WREG32(IH_RB_CNTL, tmp);
4697        }
4698        return (wptr & rdev->ih.ptr_mask);
4699}
4700
4701int evergreen_irq_process(struct radeon_device *rdev)
4702{
4703        u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4704        u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4705        u32 crtc_idx, hpd_idx, afmt_idx;
4706        u32 mask;
4707        u32 wptr;
4708        u32 rptr;
4709        u32 src_id, src_data;
4710        u32 ring_index;
4711        bool queue_hotplug = false;
4712        bool queue_hdmi = false;
4713        bool queue_dp = false;
4714        bool queue_thermal = false;
4715        u32 status, addr;
4716        const char *event_name;
4717
4718        if (!rdev->ih.enabled || rdev->shutdown)
4719                return IRQ_NONE;
4720
4721        wptr = evergreen_get_ih_wptr(rdev);
4722
4723restart_ih:
4724        /* is somebody else already processing irqs? */
4725        if (atomic_xchg(&rdev->ih.lock, 1))
4726                return IRQ_NONE;
4727
4728        rptr = rdev->ih.rptr;
4729        DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4730
4731        /* Order reading of wptr vs. reading of IH ring data */
4732        rmb();
4733
4734        /* display interrupts */
4735        evergreen_irq_ack(rdev);
4736
4737        while (rptr != wptr) {
4738                /* wptr/rptr are in bytes! */
4739                ring_index = rptr / 4;
4740                src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4741                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4742
4743                switch (src_id) {
4744                case 1: /* D1 vblank/vline */
4745                case 2: /* D2 vblank/vline */
4746                case 3: /* D3 vblank/vline */
4747                case 4: /* D4 vblank/vline */
4748                case 5: /* D5 vblank/vline */
4749                case 6: /* D6 vblank/vline */
4750                        crtc_idx = src_id - 1;
4751
4752                        if (src_data == 0) { /* vblank */
4753                                mask = LB_D1_VBLANK_INTERRUPT;
4754                                event_name = "vblank";
4755
4756                                if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4757                                        drm_handle_vblank(rdev->ddev, crtc_idx);
4758                                        rdev->pm.vblank_sync = true;
4759                                        wake_up(&rdev->irq.vblank_queue);
4760                                }
4761                                if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4762                                        radeon_crtc_handle_vblank(rdev,
4763                                                                  crtc_idx);
4764                                }
4765
4766                        } else if (src_data == 1) { /* vline */
4767                                mask = LB_D1_VLINE_INTERRUPT;
4768                                event_name = "vline";
4769                        } else {
4770                                DRM_DEBUG("Unhandled interrupt: %d %d\n",
4771                                          src_id, src_data);
4772                                break;
4773                        }
4774
4775                        if (!(disp_int[crtc_idx] & mask)) {
4776                                DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4777                                          crtc_idx + 1, event_name);
4778                        }
4779
4780                        disp_int[crtc_idx] &= ~mask;
4781                        DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4782
4783                        break;
4784                case 8: /* D1 page flip */
4785                case 10: /* D2 page flip */
4786                case 12: /* D3 page flip */
4787                case 14: /* D4 page flip */
4788                case 16: /* D5 page flip */
4789                case 18: /* D6 page flip */
4790                        DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4791                        if (radeon_use_pflipirq > 0)
4792                                radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4793                        break;
4794                case 42: /* HPD hotplug */
4795                        if (src_data <= 5) {
4796                                hpd_idx = src_data;
4797                                mask = DC_HPD1_INTERRUPT;
4798                                queue_hotplug = true;
4799                                event_name = "HPD";
4800
4801                        } else if (src_data <= 11) {
4802                                hpd_idx = src_data - 6;
4803                                mask = DC_HPD1_RX_INTERRUPT;
4804                                queue_dp = true;
4805                                event_name = "HPD_RX";
4806
4807                        } else {
4808                                DRM_DEBUG("Unhandled interrupt: %d %d\n",
4809                                          src_id, src_data);
4810                                break;
4811                        }
4812
4813                        if (!(disp_int[hpd_idx] & mask))
4814                                DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4815
4816                        disp_int[hpd_idx] &= ~mask;
4817                        DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4818
4819                        break;
4820                case 44: /* hdmi */
4821                        afmt_idx = src_data;
4822                        if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4823                                DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4824
4825                        if (afmt_idx > 5) {
4826                                DRM_ERROR("Unhandled interrupt: %d %d\n",
4827                                          src_id, src_data);
4828                                break;
4829                        }
4830                        afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4831                        queue_hdmi = true;
4832                        DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4833                        break;
4834                case 96:
4835                        DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836                        WREG32(SRBM_INT_ACK, 0x1);
4837                        break;
4838                case 124: /* UVD */
4839                        DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840                        radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4841                        break;
4842                case 146:
4843                case 147:
4844                        addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845                        status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846                        /* reset addr and status */
4847                        WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848                        if (addr == 0x0 && status == 0x0)
4849                                break;
4850                        dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4852                                addr);
4853                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4854                                status);
4855                        cayman_vm_decode_fault(rdev, status, addr);
4856                        break;
4857                case 176: /* CP_INT in ring buffer */
4858                case 177: /* CP_INT in IB1 */
4859                case 178: /* CP_INT in IB2 */
4860                        DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862                        break;
4863                case 181: /* CP EOP event */
4864                        DRM_DEBUG("IH: CP EOP\n");
4865                        if (rdev->family >= CHIP_CAYMAN) {
4866                                switch (src_data) {
4867                                case 0:
4868                                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869                                        break;
4870                                case 1:
4871                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872                                        break;
4873                                case 2:
4874                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875                                        break;
4876                                }
4877                        } else
4878                                radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879                        break;
4880                case 224: /* DMA trap event */
4881                        DRM_DEBUG("IH: DMA trap\n");
4882                        radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883                        break;
4884                case 230: /* thermal low to high */
4885                        DRM_DEBUG("IH: thermal low to high\n");
4886                        rdev->pm.dpm.thermal.high_to_low = false;
4887                        queue_thermal = true;
4888                        break;
4889                case 231: /* thermal high to low */
4890                        DRM_DEBUG("IH: thermal high to low\n");
4891                        rdev->pm.dpm.thermal.high_to_low = true;
4892                        queue_thermal = true;
4893                        break;
4894                case 233: /* GUI IDLE */
4895                        DRM_DEBUG("IH: GUI idle\n");
4896                        break;
4897                case 244: /* DMA trap event */
4898                        if (rdev->family >= CHIP_CAYMAN) {
4899                                DRM_DEBUG("IH: DMA1 trap\n");
4900                                radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901                        }
4902                        break;
4903                default:
4904                        DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905                        break;
4906                }
4907
4908                /* wptr/rptr are in bytes! */
4909                rptr += 16;
4910                rptr &= rdev->ih.ptr_mask;
4911                WREG32(IH_RB_RPTR, rptr);
4912        }
4913        if (queue_dp)
4914                schedule_work(&rdev->dp_work);
4915        if (queue_hotplug)
4916                schedule_delayed_work(&rdev->hotplug_work, 0);
4917        if (queue_hdmi)
4918                schedule_work(&rdev->audio_work);
4919        if (queue_thermal && rdev->pm.dpm_enabled)
4920                schedule_work(&rdev->pm.dpm.thermal.work);
4921        rdev->ih.rptr = rptr;
4922        atomic_set(&rdev->ih.lock, 0);
4923
4924        /* make sure wptr hasn't changed while processing */
4925        wptr = evergreen_get_ih_wptr(rdev);
4926        if (wptr != rptr)
4927                goto restart_ih;
4928
4929        return IRQ_HANDLED;
4930}
4931
4932static void evergreen_uvd_init(struct radeon_device *rdev)
4933{
4934        int r;
4935
4936        if (!rdev->has_uvd)
4937                return;
4938
4939        r = radeon_uvd_init(rdev);
4940        if (r) {
4941                dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4942                /*
4943                 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944                 * to early fails uvd_v2_2_resume() and thus nothing happens
4945                 * there. So it is pointless to try to go through that code
4946                 * hence why we disable uvd here.
4947                 */
4948                rdev->has_uvd = false;
4949                return;
4950        }
4951        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952        r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4953}
4954
4955static void evergreen_uvd_start(struct radeon_device *rdev)
4956{
4957        int r;
4958
4959        if (!rdev->has_uvd)
4960                return;
4961
4962        r = uvd_v2_2_resume(rdev);
4963        if (r) {
4964                dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4965                goto error;
4966        }
4967        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4968        if (r) {
4969                dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4970                goto error;
4971        }
4972        return;
4973
4974error:
4975        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4976}
4977
4978static void evergreen_uvd_resume(struct radeon_device *rdev)
4979{
4980        struct radeon_ring *ring;
4981        int r;
4982
4983        if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4984                return;
4985
4986        ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4988        if (r) {
4989                dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4990                return;
4991        }
4992        r = uvd_v1_0_init(rdev);
4993        if (r) {
4994                dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4995                return;
4996        }
4997}
4998
4999static int evergreen_startup(struct radeon_device *rdev)
5000{

5001        struct radeon_ring *ring;
5002        int r;
5003
5004        /* enable pcie gen2 link */
5005        evergreen_pcie_gen2_enable(rdev);
5006        /* enable aspm */
5007        evergreen_program_aspm(rdev);
5008
5009        /* scratch needs to be initialized before MC */
5010        r = r600_vram_scratch_init(rdev);
5011        if (r)
5012                return r;
5013
5014        evergreen_mc_program(rdev);
5015
5016        if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017                r = ni_mc_load_microcode(rdev);
5018                if (r) {
5019                        DRM_ERROR("Failed to load MC firmware!\n");
5020                        return r;
5021                }
5022        }
5023
5024        if (rdev->flags & RADEON_IS_AGP) {
5025                evergreen_agp_enable(rdev);
5026        } else {
5027                r = evergreen_pcie_gart_enable(rdev);
5028                if (r)
5029                        return r;
5030        }
5031        evergreen_gpu_init(rdev);
5032
5033        /* allocate rlc buffers */
5034        if (rdev->flags & RADEON_IS_IGP) {
5035                rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036                rdev->rlc.reg_list_size =
5037                        (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038                rdev->rlc.cs_data = evergreen_cs_data;
5039                r = sumo_rlc_init(rdev);
5040                if (r) {
5041                        DRM_ERROR("Failed to init rlc BOs!\n");
5042                        return r;
5043                }
5044        }
5045
5046        /* allocate wb buffer */
5047        r = radeon_wb_init(rdev);
5048        if (r)
5049                return r;
5050
5051        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5052        if (r) {
5053                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5054                return r;
5055        }
5056
5057        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5058        if (r) {
5059                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5060                return r;
5061        }
5062
5063        evergreen_uvd_start(rdev);
5064
5065        /* Enable IRQ */
5066        if (!rdev->irq.installed) {
5067                r = radeon_irq_kms_init(rdev);
5068                if (r)
5069                        return r;
5070        }
5071
5072        r = r600_irq_init(rdev);
5073        if (r) {
5074                DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075                radeon_irq_kms_fini(rdev);
5076                return r;
5077        }
5078        evergreen_irq_set(rdev);
5079
5080        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5082                             RADEON_CP_PACKET2);
5083        if (r)
5084                return r;
5085
5086        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087        r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088                             DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5089        if (r)
5090                return r;
5091
5092        r = evergreen_cp_load_microcode(rdev);
5093        if (r)
5094                return r;
5095        r = evergreen_cp_resume(rdev);
5096        if (r)
5097                return r;
5098        r = r600_dma_resume(rdev);
5099        if (r)
5100                return r;
5101
5102        evergreen_uvd_resume(rdev);
5103
5104        r = radeon_ib_pool_init(rdev);
5105        if (r) {
5106                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5107                return r;
5108        }
5109
5110        r = radeon_audio_init(rdev);
5111        if (r) {
5112                DRM_ERROR("radeon: audio init failed\n");
5113                return r;
5114        }
5115
5116        return 0;
5117}
5118
5119int evergreen_resume(struct radeon_device *rdev)
5120{
5121        int r;
5122
5123        /* reset the asic, the gfx blocks are often in a bad state
5124         * after the driver is unloaded or after a resume
5125         */
5126        if (radeon_asic_reset(rdev))
5127                dev_warn(rdev->dev, "GPU reset failed !\n");
5128        /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129         * posting will perform necessary task to bring back GPU into good
5130         * shape.
5131         */
5132        /* post card */
5133        atom_asic_init(rdev->mode_info.atom_context);
5134
5135        /* init golden registers */
5136        evergreen_init_golden_registers(rdev);
5137
5138        if (rdev->pm.pm_method == PM_METHOD_DPM)
5139                radeon_pm_resume(rdev);
5140
5141        rdev->accel_working = true;
5142        r = evergreen_startup(rdev);
5143        if (r) {
5144                DRM_ERROR("evergreen startup failed on resume\n");
5145                rdev->accel_working = false;
5146                return r;
5147        }
5148
5149        return r;
5150
5151}
5152
5153int evergreen_suspend(struct radeon_device *rdev)
5154{
5155        radeon_pm_suspend(rdev);
5156        radeon_audio_fini(rdev);
5157        if (rdev->has_uvd) {
5158                uvd_v1_0_fini(rdev);
5159                radeon_uvd_suspend(rdev);
5160        }
5161        r700_cp_stop(rdev);
5162        r600_dma_stop(rdev);
5163        evergreen_irq_suspend(rdev);
5164        radeon_wb_disable(rdev);
5165        evergreen_pcie_gart_disable(rdev);
5166
5167        return 0;
5168}
5169
5170/* Plan is to move initialization in that function and use
5171 * helper function so that radeon_device_init pretty much
5172 * do nothing more than calling asic specific function. This
5173 * should also allow to remove a bunch of callback function
5174 * like vram_info.
5175 */
5176int evergreen_init(struct radeon_device *rdev)
5177{
5178        int r;
5179
5180        /* Read BIOS */
5181        if (!radeon_get_bios(rdev)) {
5182                if (ASIC_IS_AVIVO(rdev))
5183                        return -EINVAL;
5184        }
5185        /* Must be an ATOMBIOS */
5186        if (!rdev->is_atom_bios) {
5187                dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5188                return -EINVAL;
5189        }
5190        r = radeon_atombios_init(rdev);
5191        if (r)
5192                return r;
5193        /* reset the asic, the gfx blocks are often in a bad state
5194         * after the driver is unloaded or after a resume
5195         */
5196        if (radeon_asic_reset(rdev))
5197                dev_warn(rdev->dev, "GPU reset failed !\n");
5198        /* Post card if necessary */
5199        if (!radeon_card_posted(rdev)) {
5200                if (!rdev->bios) {
5201                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5202                        return -EINVAL;
5203                }
5204                DRM_INFO("GPU not posted. posting now...\n");
5205                atom_asic_init(rdev->mode_info.atom_context);
5206        }
5207        /* init golden registers */
5208        evergreen_init_golden_registers(rdev);
5209        /* Initialize scratch registers */
5210        r600_scratch_init(rdev);
5211        /* Initialize surface registers */
5212        radeon_surface_init(rdev);
5213        /* Initialize clocks */
5214        radeon_get_clock_info(rdev->ddev);
5215        /* Fence driver */
5216        r = radeon_fence_driver_init(rdev);
5217        if (r)
5218                return r;
5219        /* initialize AGP */
5220        if (rdev->flags & RADEON_IS_AGP) {
5221                r = radeon_agp_init(rdev);
5222                if (r)
5223                        radeon_agp_disable(rdev);
5224        }
5225        /* initialize memory controller */
5226        r = evergreen_mc_init(rdev);
5227        if (r)
5228                return r;
5229        /* Memory manager */
5230        r = radeon_bo_init(rdev);
5231        if (r)
5232                return r;
5233
5234        if (ASIC_IS_DCE5(rdev)) {
5235                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5236                        r = ni_init_microcode(rdev);
5237                        if (r) {
5238                                DRM_ERROR("Failed to load firmware!\n");
5239                                return r;
5240                        }
5241                }
5242        } else {
5243                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5244                        r = r600_init_microcode(rdev);
5245                        if (r) {
5246                                DRM_ERROR("Failed to load firmware!\n");
5247                                return r;
5248                        }
5249                }
5250        }
5251
5252        /* Initialize power management */
5253        radeon_pm_init(rdev);
5254
5255        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5256        r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5257
5258        rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5259        r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5260
5261        evergreen_uvd_init(rdev);
5262
5263        rdev->ih.ring_obj = NULL;
5264        r600_ih_ring_init(rdev, 64 * 1024);
5265
5266        r = r600_pcie_gart_init(rdev);
5267        if (r)
5268                return r;
5269
5270        rdev->accel_working = true;
5271        r = evergreen_startup(rdev);
5272        if (r) {
5273                dev_err(rdev->dev, "disabling GPU acceleration\n");
5274                r700_cp_fini(rdev);
5275                r600_dma_fini(rdev);
5276                r600_irq_fini(rdev);
5277                if (rdev->flags & RADEON_IS_IGP)
5278                        sumo_rlc_fini(rdev);
5279                radeon_wb_fini(rdev);
5280                radeon_ib_pool_fini(rdev);
5281                radeon_irq_kms_fini(rdev);
5282                evergreen_pcie_gart_fini(rdev);
5283                rdev->accel_working = false;
5284        }
5285
5286        /* Don't start up if the MC ucode is missing on BTC parts.
5287         * The default clocks and voltages before the MC ucode
5288         * is loaded are not suffient for advanced operations.
5289         */
5290        if (ASIC_IS_DCE5(rdev)) {
5291                if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5292                        DRM_ERROR("radeon: MC ucode required for NI+.\n");
5293                        return -EINVAL;
5294                }
5295        }
5296
5297        return 0;
5298}
5299
5300void evergreen_fini(struct radeon_device *rdev)
5301{
5302        radeon_pm_fini(rdev);
5303        radeon_audio_fini(rdev);
5304        r700_cp_fini(rdev);
5305        r600_dma_fini(rdev);
5306        r600_irq_fini(rdev);
5307        if (rdev->flags & RADEON_IS_IGP)
5308                sumo_rlc_fini(rdev);
5309        radeon_wb_fini(rdev);
5310        radeon_ib_pool_fini(rdev);
5311        radeon_irq_kms_fini(rdev);
5312        uvd_v1_0_fini(rdev);
5313        radeon_uvd_fini(rdev);
5314        evergreen_pcie_gart_fini(rdev);
5315        r600_vram_scratch_fini(rdev);
5316        radeon_gem_fini(rdev);
5317        radeon_fence_driver_fini(rdev);
5318        radeon_agp_fini(rdev);
5319        radeon_bo_fini(rdev);
5320        radeon_atombios_fini(rdev);
5321        kfree(rdev->bios);
5322        rdev->bios = NULL;
5323}
5324
5325void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5326{
5327        u32 link_width_cntl, speed_cntl;
5328
5329        if (radeon_pcie_gen2 == 0)
5330                return;
5331
5332        if (rdev->flags & RADEON_IS_IGP)
5333                return;
5334
5335        if (!(rdev->flags & RADEON_IS_PCIE))
5336                return;
5337
5338        /* x2 cards have a special sequence */
5339        if (ASIC_IS_X2(rdev))
5340                return;
5341
5342        if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5343                (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5344                return;
5345
5346        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5347        if (speed_cntl & LC_CURRENT_DATA_RATE) {
5348                DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5349                return;
5350        }
5351
5352        DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5353
5354        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5355            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5356
5357                link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5358                link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5359                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5360
5361                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5362                speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5363                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5364
5365                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5366                speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5367                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5368
5369                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5370                speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5371                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5372
5373                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5374                speed_cntl |= LC_GEN2_EN_STRAP;
5375                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5376
5377        } else {
5378                link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5379                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5380                if (1)
5381                        link_width_cntl |= LC_UPCONFIGURE_DIS;
5382                else
5383                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5384                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5385        }
5386}
5387
5388void evergreen_program_aspm(struct radeon_device *rdev)
5389{
5390        u32 data, orig;
5391        u32 pcie_lc_cntl, pcie_lc_cntl_old;
5392        bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5393        /* fusion_platform = true
5394         * if the system is a fusion system
5395         * (APU or DGPU in a fusion system).
5396         * todo: check if the system is a fusion platform.
5397         */
5398        bool fusion_platform = false;
5399
5400        if (radeon_aspm == 0)
5401                return;
5402
5403        if (!(rdev->flags & RADEON_IS_PCIE))
5404                return;
5405
5406        switch (rdev->family) {
5407        case CHIP_CYPRESS:
5408        case CHIP_HEMLOCK:
5409        case CHIP_JUNIPER:
5410        case CHIP_REDWOOD:
5411        case CHIP_CEDAR:
5412        case CHIP_SUMO:
5413        case CHIP_SUMO2:
5414        case CHIP_PALM:
5415        case CHIP_ARUBA:
5416                disable_l0s = true;
5417                break;
5418        default:
5419                disable_l0s = false;
5420                break;
5421        }
5422
5423        if (rdev->flags & RADEON_IS_IGP)
5424                fusion_platform = true; /* XXX also dGPUs in a fusion system */
5425
5426        data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5427        if (fusion_platform)
5428                data &= ~MULTI_PIF;
5429        else
5430                data |= MULTI_PIF;
5431        if (data != orig)
5432                WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5433
5434        data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5435        if (fusion_platform)
5436                data &= ~MULTI_PIF;
5437        else
5438                data |= MULTI_PIF;
5439        if (data != orig)
5440                WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5441
5442        pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5443        pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5444        if (!disable_l0s) {
5445                if (rdev->family >= CHIP_BARTS)
5446                        pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5447                else
5448                        pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5449        }
5450
5451        if (!disable_l1) {
5452                if (rdev->family >= CHIP_BARTS)
5453                        pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5454                else
5455                        pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5456
5457                if (!disable_plloff_in_l1) {
5458                        data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5459                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5460                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5461                        if (data != orig)
5462                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5463
5464                        data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5465                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5466                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5467                        if (data != orig)
5468                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5469
5470                        data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5471                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5472                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5473                        if (data != orig)
5474                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5475
5476                        data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5477                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5478                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5479                        if (data != orig)
5480                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5481
5482                        if (rdev->family >= CHIP_BARTS) {
5483                                data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5484                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
5485                                data |= PLL_RAMP_UP_TIME_0(4);
5486                                if (data != orig)
5487                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5488
5489                                data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5490                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
5491                                data |= PLL_RAMP_UP_TIME_1(4);
5492                                if (data != orig)
5493                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5494
5495                                data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5496                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
5497                                data |= PLL_RAMP_UP_TIME_0(4);
5498                                if (data != orig)
5499                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5500
5501                                data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5502                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
5503                                data |= PLL_RAMP_UP_TIME_1(4);
5504                                if (data != orig)
5505                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5506                        }
5507
5508                        data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5509                        data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5510                        data |= LC_DYN_LANES_PWR_STATE(3);
5511                        if (data != orig)
5512                                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5513
5514                        if (rdev->family >= CHIP_BARTS) {
5515                                data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5516                                data &= ~LS2_EXIT_TIME_MASK;
5517                                data |= LS2_EXIT_TIME(1);
5518                                if (data != orig)
5519                                        WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5520
5521                                data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5522                                data &= ~LS2_EXIT_TIME_MASK;
5523                                data |= LS2_EXIT_TIME(1);
5524                                if (data != orig)
5525                                        WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5526                        }
5527                }
5528        }
5529
5530        /* evergreen parts only */
5531        if (rdev->family < CHIP_BARTS)
5532                pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5533
5534        if (pcie_lc_cntl != pcie_lc_cntl_old)
5535                WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5536}
5537