linux/drivers/gpu/drm/amd/amdgpu/cik.c
<<
>>
Prefs
   1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <drm/drmP.h>
  28#include "amdgpu.h"
  29#include "amdgpu_atombios.h"
  30#include "amdgpu_ih.h"
  31#include "amdgpu_uvd.h"
  32#include "amdgpu_vce.h"
  33#include "cikd.h"
  34#include "atom.h"
  35#include "amd_pcie.h"
  36
  37#include "cik.h"
  38#include "gmc_v7_0.h"
  39#include "cik_ih.h"
  40#include "dce_v8_0.h"
  41#include "gfx_v7_0.h"
  42#include "cik_sdma.h"
  43#include "uvd_v4_2.h"
  44#include "vce_v2_0.h"
  45#include "cik_dpm.h"
  46
  47#include "uvd/uvd_4_2_d.h"
  48
  49#include "smu/smu_7_0_1_d.h"
  50#include "smu/smu_7_0_1_sh_mask.h"
  51
  52#include "dce/dce_8_0_d.h"
  53#include "dce/dce_8_0_sh_mask.h"
  54
  55#include "bif/bif_4_1_d.h"
  56#include "bif/bif_4_1_sh_mask.h"
  57
  58#include "gca/gfx_7_2_d.h"
  59#include "gca/gfx_7_2_enum.h"
  60#include "gca/gfx_7_2_sh_mask.h"
  61
  62#include "gmc/gmc_7_1_d.h"
  63#include "gmc/gmc_7_1_sh_mask.h"
  64
  65#include "oss/oss_2_0_d.h"
  66#include "oss/oss_2_0_sh_mask.h"
  67
  68#include "amdgpu_dm.h"
  69#include "amdgpu_amdkfd.h"
  70#include "amdgpu_powerplay.h"
  71#include "dce_virtual.h"
  72
  73/*
  74 * Indirect registers accessor
  75 */
  76static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  77{
  78        unsigned long flags;
  79        u32 r;
  80
  81        spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  82        WREG32(mmPCIE_INDEX, reg);
  83        (void)RREG32(mmPCIE_INDEX);
  84        r = RREG32(mmPCIE_DATA);
  85        spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  86        return r;
  87}
  88
  89static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  90{
  91        unsigned long flags;
  92
  93        spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  94        WREG32(mmPCIE_INDEX, reg);
  95        (void)RREG32(mmPCIE_INDEX);
  96        WREG32(mmPCIE_DATA, v);
  97        (void)RREG32(mmPCIE_DATA);
  98        spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  99}
 100
 101static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 102{
 103        unsigned long flags;
 104        u32 r;
 105
 106        spin_lock_irqsave(&adev->smc_idx_lock, flags);
 107        WREG32(mmSMC_IND_INDEX_0, (reg));
 108        r = RREG32(mmSMC_IND_DATA_0);
 109        spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 110        return r;
 111}
 112
 113static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 114{
 115        unsigned long flags;
 116
 117        spin_lock_irqsave(&adev->smc_idx_lock, flags);
 118        WREG32(mmSMC_IND_INDEX_0, (reg));
 119        WREG32(mmSMC_IND_DATA_0, (v));
 120        spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 121}
 122
 123static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 124{
 125        unsigned long flags;
 126        u32 r;
 127
 128        spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 129        WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 130        r = RREG32(mmUVD_CTX_DATA);
 131        spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 132        return r;
 133}
 134
 135static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 136{
 137        unsigned long flags;
 138
 139        spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 140        WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 141        WREG32(mmUVD_CTX_DATA, (v));
 142        spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 143}
 144
 145static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 146{
 147        unsigned long flags;
 148        u32 r;
 149
 150        spin_lock_irqsave(&adev->didt_idx_lock, flags);
 151        WREG32(mmDIDT_IND_INDEX, (reg));
 152        r = RREG32(mmDIDT_IND_DATA);
 153        spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 154        return r;
 155}
 156
 157static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 158{
 159        unsigned long flags;
 160
 161        spin_lock_irqsave(&adev->didt_idx_lock, flags);
 162        WREG32(mmDIDT_IND_INDEX, (reg));
 163        WREG32(mmDIDT_IND_DATA, (v));
 164        spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 165}
 166
 167static const u32 bonaire_golden_spm_registers[] =
 168{
 169        0xc200, 0xe0ffffff, 0xe0000000
 170};
 171
 172static const u32 bonaire_golden_common_registers[] =
 173{
 174        0x31dc, 0xffffffff, 0x00000800,
 175        0x31dd, 0xffffffff, 0x00000800,
 176        0x31e6, 0xffffffff, 0x00007fbf,
 177        0x31e7, 0xffffffff, 0x00007faf
 178};
 179
 180static const u32 bonaire_golden_registers[] =
 181{
 182        0xcd5, 0x00000333, 0x00000333,
 183        0xcd4, 0x000c0fc0, 0x00040200,
 184        0x2684, 0x00010000, 0x00058208,
 185        0xf000, 0xffff1fff, 0x00140000,
 186        0xf080, 0xfdfc0fff, 0x00000100,
 187        0xf08d, 0x40000000, 0x40000200,
 188        0x260c, 0xffffffff, 0x00000000,
 189        0x260d, 0xf00fffff, 0x00000400,
 190        0x260e, 0x0002021c, 0x00020200,
 191        0x31e, 0x00000080, 0x00000000,
 192        0x16ec, 0x000000f0, 0x00000070,
 193        0x16f0, 0xf0311fff, 0x80300000,
 194        0x263e, 0x73773777, 0x12010001,
 195        0xd43, 0x00810000, 0x408af000,
 196        0x1c0c, 0x31000111, 0x00000011,
 197        0xbd2, 0x73773777, 0x12010001,
 198        0x883, 0x00007fb6, 0x0021a1b1,
 199        0x884, 0x00007fb6, 0x002021b1,
 200        0x860, 0x00007fb6, 0x00002191,
 201        0x886, 0x00007fb6, 0x002121b1,
 202        0x887, 0x00007fb6, 0x002021b1,
 203        0x877, 0x00007fb6, 0x00002191,
 204        0x878, 0x00007fb6, 0x00002191,
 205        0xd8a, 0x0000003f, 0x0000000a,
 206        0xd8b, 0x0000003f, 0x0000000a,
 207        0xab9, 0x00073ffe, 0x000022a2,
 208        0x903, 0x000007ff, 0x00000000,
 209        0x2285, 0xf000003f, 0x00000007,
 210        0x22fc, 0x00002001, 0x00000001,
 211        0x22c9, 0xffffffff, 0x00ffffff,
 212        0xc281, 0x0000ff0f, 0x00000000,
 213        0xa293, 0x07ffffff, 0x06000000,
 214        0x136, 0x00000fff, 0x00000100,
 215        0xf9e, 0x00000001, 0x00000002,
 216        0x2440, 0x03000000, 0x0362c688,
 217        0x2300, 0x000000ff, 0x00000001,
 218        0x390, 0x00001fff, 0x00001fff,
 219        0x2418, 0x0000007f, 0x00000020,
 220        0x2542, 0x00010000, 0x00010000,
 221        0x2b05, 0x000003ff, 0x000000f3,
 222        0x2b03, 0xffffffff, 0x00001032
 223};
 224
 225static const u32 bonaire_mgcg_cgcg_init[] =
 226{
 227        0x3108, 0xffffffff, 0xfffffffc,
 228        0xc200, 0xffffffff, 0xe0000000,
 229        0xf0a8, 0xffffffff, 0x00000100,
 230        0xf082, 0xffffffff, 0x00000100,
 231        0xf0b0, 0xffffffff, 0xc0000100,
 232        0xf0b2, 0xffffffff, 0xc0000100,
 233        0xf0b1, 0xffffffff, 0xc0000100,
 234        0x1579, 0xffffffff, 0x00600100,
 235        0xf0a0, 0xffffffff, 0x00000100,
 236        0xf085, 0xffffffff, 0x06000100,
 237        0xf088, 0xffffffff, 0x00000100,
 238        0xf086, 0xffffffff, 0x06000100,
 239        0xf081, 0xffffffff, 0x00000100,
 240        0xf0b8, 0xffffffff, 0x00000100,
 241        0xf089, 0xffffffff, 0x00000100,
 242        0xf080, 0xffffffff, 0x00000100,
 243        0xf08c, 0xffffffff, 0x00000100,
 244        0xf08d, 0xffffffff, 0x00000100,
 245        0xf094, 0xffffffff, 0x00000100,
 246        0xf095, 0xffffffff, 0x00000100,
 247        0xf096, 0xffffffff, 0x00000100,
 248        0xf097, 0xffffffff, 0x00000100,
 249        0xf098, 0xffffffff, 0x00000100,
 250        0xf09f, 0xffffffff, 0x00000100,
 251        0xf09e, 0xffffffff, 0x00000100,
 252        0xf084, 0xffffffff, 0x06000100,
 253        0xf0a4, 0xffffffff, 0x00000100,
 254        0xf09d, 0xffffffff, 0x00000100,
 255        0xf0ad, 0xffffffff, 0x00000100,
 256        0xf0ac, 0xffffffff, 0x00000100,
 257        0xf09c, 0xffffffff, 0x00000100,
 258        0xc200, 0xffffffff, 0xe0000000,
 259        0xf008, 0xffffffff, 0x00010000,
 260        0xf009, 0xffffffff, 0x00030002,
 261        0xf00a, 0xffffffff, 0x00040007,
 262        0xf00b, 0xffffffff, 0x00060005,
 263        0xf00c, 0xffffffff, 0x00090008,
 264        0xf00d, 0xffffffff, 0x00010000,
 265        0xf00e, 0xffffffff, 0x00030002,
 266        0xf00f, 0xffffffff, 0x00040007,
 267        0xf010, 0xffffffff, 0x00060005,
 268        0xf011, 0xffffffff, 0x00090008,
 269        0xf012, 0xffffffff, 0x00010000,
 270        0xf013, 0xffffffff, 0x00030002,
 271        0xf014, 0xffffffff, 0x00040007,
 272        0xf015, 0xffffffff, 0x00060005,
 273        0xf016, 0xffffffff, 0x00090008,
 274        0xf017, 0xffffffff, 0x00010000,
 275        0xf018, 0xffffffff, 0x00030002,
 276        0xf019, 0xffffffff, 0x00040007,
 277        0xf01a, 0xffffffff, 0x00060005,
 278        0xf01b, 0xffffffff, 0x00090008,
 279        0xf01c, 0xffffffff, 0x00010000,
 280        0xf01d, 0xffffffff, 0x00030002,
 281        0xf01e, 0xffffffff, 0x00040007,
 282        0xf01f, 0xffffffff, 0x00060005,
 283        0xf020, 0xffffffff, 0x00090008,
 284        0xf021, 0xffffffff, 0x00010000,
 285        0xf022, 0xffffffff, 0x00030002,
 286        0xf023, 0xffffffff, 0x00040007,
 287        0xf024, 0xffffffff, 0x00060005,
 288        0xf025, 0xffffffff, 0x00090008,
 289        0xf026, 0xffffffff, 0x00010000,
 290        0xf027, 0xffffffff, 0x00030002,
 291        0xf028, 0xffffffff, 0x00040007,
 292        0xf029, 0xffffffff, 0x00060005,
 293        0xf02a, 0xffffffff, 0x00090008,
 294        0xf000, 0xffffffff, 0x96e00200,
 295        0x21c2, 0xffffffff, 0x00900100,
 296        0x3109, 0xffffffff, 0x0020003f,
 297        0xe, 0xffffffff, 0x0140001c,
 298        0xf, 0x000f0000, 0x000f0000,
 299        0x88, 0xffffffff, 0xc060000c,
 300        0x89, 0xc0000fff, 0x00000100,
 301        0x3e4, 0xffffffff, 0x00000100,
 302        0x3e6, 0x00000101, 0x00000000,
 303        0x82a, 0xffffffff, 0x00000104,
 304        0x1579, 0xff000fff, 0x00000100,
 305        0xc33, 0xc0000fff, 0x00000104,
 306        0x3079, 0x00000001, 0x00000001,
 307        0x3403, 0xff000ff0, 0x00000100,
 308        0x3603, 0xff000ff0, 0x00000100
 309};
 310
 311static const u32 spectre_golden_spm_registers[] =
 312{
 313        0xc200, 0xe0ffffff, 0xe0000000
 314};
 315
 316static const u32 spectre_golden_common_registers[] =
 317{
 318        0x31dc, 0xffffffff, 0x00000800,
 319        0x31dd, 0xffffffff, 0x00000800,
 320        0x31e6, 0xffffffff, 0x00007fbf,
 321        0x31e7, 0xffffffff, 0x00007faf
 322};
 323
 324static const u32 spectre_golden_registers[] =
 325{
 326        0xf000, 0xffff1fff, 0x96940200,
 327        0xf003, 0xffff0001, 0xff000000,
 328        0xf080, 0xfffc0fff, 0x00000100,
 329        0x1bb6, 0x00010101, 0x00010000,
 330        0x260d, 0xf00fffff, 0x00000400,
 331        0x260e, 0xfffffffc, 0x00020200,
 332        0x16ec, 0x000000f0, 0x00000070,
 333        0x16f0, 0xf0311fff, 0x80300000,
 334        0x263e, 0x73773777, 0x12010001,
 335        0x26df, 0x00ff0000, 0x00fc0000,
 336        0xbd2, 0x73773777, 0x12010001,
 337        0x2285, 0xf000003f, 0x00000007,
 338        0x22c9, 0xffffffff, 0x00ffffff,
 339        0xa0d4, 0x3f3f3fff, 0x00000082,
 340        0xa0d5, 0x0000003f, 0x00000000,
 341        0xf9e, 0x00000001, 0x00000002,
 342        0x244f, 0xffff03df, 0x00000004,
 343        0x31da, 0x00000008, 0x00000008,
 344        0x2300, 0x000008ff, 0x00000800,
 345        0x2542, 0x00010000, 0x00010000,
 346        0x2b03, 0xffffffff, 0x54763210,
 347        0x853e, 0x01ff01ff, 0x00000002,
 348        0x8526, 0x007ff800, 0x00200000,
 349        0x8057, 0xffffffff, 0x00000f40,
 350        0xc24d, 0xffffffff, 0x00000001
 351};
 352
 353static const u32 spectre_mgcg_cgcg_init[] =
 354{
 355        0x3108, 0xffffffff, 0xfffffffc,
 356        0xc200, 0xffffffff, 0xe0000000,
 357        0xf0a8, 0xffffffff, 0x00000100,
 358        0xf082, 0xffffffff, 0x00000100,
 359        0xf0b0, 0xffffffff, 0x00000100,
 360        0xf0b2, 0xffffffff, 0x00000100,
 361        0xf0b1, 0xffffffff, 0x00000100,
 362        0x1579, 0xffffffff, 0x00600100,
 363        0xf0a0, 0xffffffff, 0x00000100,
 364        0xf085, 0xffffffff, 0x06000100,
 365        0xf088, 0xffffffff, 0x00000100,
 366        0xf086, 0xffffffff, 0x06000100,
 367        0xf081, 0xffffffff, 0x00000100,
 368        0xf0b8, 0xffffffff, 0x00000100,
 369        0xf089, 0xffffffff, 0x00000100,
 370        0xf080, 0xffffffff, 0x00000100,
 371        0xf08c, 0xffffffff, 0x00000100,
 372        0xf08d, 0xffffffff, 0x00000100,
 373        0xf094, 0xffffffff, 0x00000100,
 374        0xf095, 0xffffffff, 0x00000100,
 375        0xf096, 0xffffffff, 0x00000100,
 376        0xf097, 0xffffffff, 0x00000100,
 377        0xf098, 0xffffffff, 0x00000100,
 378        0xf09f, 0xffffffff, 0x00000100,
 379        0xf09e, 0xffffffff, 0x00000100,
 380        0xf084, 0xffffffff, 0x06000100,
 381        0xf0a4, 0xffffffff, 0x00000100,
 382        0xf09d, 0xffffffff, 0x00000100,
 383        0xf0ad, 0xffffffff, 0x00000100,
 384        0xf0ac, 0xffffffff, 0x00000100,
 385        0xf09c, 0xffffffff, 0x00000100,
 386        0xc200, 0xffffffff, 0xe0000000,
 387        0xf008, 0xffffffff, 0x00010000,
 388        0xf009, 0xffffffff, 0x00030002,
 389        0xf00a, 0xffffffff, 0x00040007,
 390        0xf00b, 0xffffffff, 0x00060005,
 391        0xf00c, 0xffffffff, 0x00090008,
 392        0xf00d, 0xffffffff, 0x00010000,
 393        0xf00e, 0xffffffff, 0x00030002,
 394        0xf00f, 0xffffffff, 0x00040007,
 395        0xf010, 0xffffffff, 0x00060005,
 396        0xf011, 0xffffffff, 0x00090008,
 397        0xf012, 0xffffffff, 0x00010000,
 398        0xf013, 0xffffffff, 0x00030002,
 399        0xf014, 0xffffffff, 0x00040007,
 400        0xf015, 0xffffffff, 0x00060005,
 401        0xf016, 0xffffffff, 0x00090008,
 402        0xf017, 0xffffffff, 0x00010000,
 403        0xf018, 0xffffffff, 0x00030002,
 404        0xf019, 0xffffffff, 0x00040007,
 405        0xf01a, 0xffffffff, 0x00060005,
 406        0xf01b, 0xffffffff, 0x00090008,
 407        0xf01c, 0xffffffff, 0x00010000,
 408        0xf01d, 0xffffffff, 0x00030002,
 409        0xf01e, 0xffffffff, 0x00040007,
 410        0xf01f, 0xffffffff, 0x00060005,
 411        0xf020, 0xffffffff, 0x00090008,
 412        0xf021, 0xffffffff, 0x00010000,
 413        0xf022, 0xffffffff, 0x00030002,
 414        0xf023, 0xffffffff, 0x00040007,
 415        0xf024, 0xffffffff, 0x00060005,
 416        0xf025, 0xffffffff, 0x00090008,
 417        0xf026, 0xffffffff, 0x00010000,
 418        0xf027, 0xffffffff, 0x00030002,
 419        0xf028, 0xffffffff, 0x00040007,
 420        0xf029, 0xffffffff, 0x00060005,
 421        0xf02a, 0xffffffff, 0x00090008,
 422        0xf02b, 0xffffffff, 0x00010000,
 423        0xf02c, 0xffffffff, 0x00030002,
 424        0xf02d, 0xffffffff, 0x00040007,
 425        0xf02e, 0xffffffff, 0x00060005,
 426        0xf02f, 0xffffffff, 0x00090008,
 427        0xf000, 0xffffffff, 0x96e00200,
 428        0x21c2, 0xffffffff, 0x00900100,
 429        0x3109, 0xffffffff, 0x0020003f,
 430        0xe, 0xffffffff, 0x0140001c,
 431        0xf, 0x000f0000, 0x000f0000,
 432        0x88, 0xffffffff, 0xc060000c,
 433        0x89, 0xc0000fff, 0x00000100,
 434        0x3e4, 0xffffffff, 0x00000100,
 435        0x3e6, 0x00000101, 0x00000000,
 436        0x82a, 0xffffffff, 0x00000104,
 437        0x1579, 0xff000fff, 0x00000100,
 438        0xc33, 0xc0000fff, 0x00000104,
 439        0x3079, 0x00000001, 0x00000001,
 440        0x3403, 0xff000ff0, 0x00000100,
 441        0x3603, 0xff000ff0, 0x00000100
 442};
 443
 444static const u32 kalindi_golden_spm_registers[] =
 445{
 446        0xc200, 0xe0ffffff, 0xe0000000
 447};
 448
 449static const u32 kalindi_golden_common_registers[] =
 450{
 451        0x31dc, 0xffffffff, 0x00000800,
 452        0x31dd, 0xffffffff, 0x00000800,
 453        0x31e6, 0xffffffff, 0x00007fbf,
 454        0x31e7, 0xffffffff, 0x00007faf
 455};
 456
 457static const u32 kalindi_golden_registers[] =
 458{
 459        0xf000, 0xffffdfff, 0x6e944040,
 460        0x1579, 0xff607fff, 0xfc000100,
 461        0xf088, 0xff000fff, 0x00000100,
 462        0xf089, 0xff000fff, 0x00000100,
 463        0xf080, 0xfffc0fff, 0x00000100,
 464        0x1bb6, 0x00010101, 0x00010000,
 465        0x260c, 0xffffffff, 0x00000000,
 466        0x260d, 0xf00fffff, 0x00000400,
 467        0x16ec, 0x000000f0, 0x00000070,
 468        0x16f0, 0xf0311fff, 0x80300000,
 469        0x263e, 0x73773777, 0x12010001,
 470        0x263f, 0xffffffff, 0x00000010,
 471        0x26df, 0x00ff0000, 0x00fc0000,
 472        0x200c, 0x00001f0f, 0x0000100a,
 473        0xbd2, 0x73773777, 0x12010001,
 474        0x902, 0x000fffff, 0x000c007f,
 475        0x2285, 0xf000003f, 0x00000007,
 476        0x22c9, 0x3fff3fff, 0x00ffcfff,
 477        0xc281, 0x0000ff0f, 0x00000000,
 478        0xa293, 0x07ffffff, 0x06000000,
 479        0x136, 0x00000fff, 0x00000100,
 480        0xf9e, 0x00000001, 0x00000002,
 481        0x31da, 0x00000008, 0x00000008,
 482        0x2300, 0x000000ff, 0x00000003,
 483        0x853e, 0x01ff01ff, 0x00000002,
 484        0x8526, 0x007ff800, 0x00200000,
 485        0x8057, 0xffffffff, 0x00000f40,
 486        0x2231, 0x001f3ae3, 0x00000082,
 487        0x2235, 0x0000001f, 0x00000010,
 488        0xc24d, 0xffffffff, 0x00000000
 489};
 490
 491static const u32 kalindi_mgcg_cgcg_init[] =
 492{
 493        0x3108, 0xffffffff, 0xfffffffc,
 494        0xc200, 0xffffffff, 0xe0000000,
 495        0xf0a8, 0xffffffff, 0x00000100,
 496        0xf082, 0xffffffff, 0x00000100,
 497        0xf0b0, 0xffffffff, 0x00000100,
 498        0xf0b2, 0xffffffff, 0x00000100,
 499        0xf0b1, 0xffffffff, 0x00000100,
 500        0x1579, 0xffffffff, 0x00600100,
 501        0xf0a0, 0xffffffff, 0x00000100,
 502        0xf085, 0xffffffff, 0x06000100,
 503        0xf088, 0xffffffff, 0x00000100,
 504        0xf086, 0xffffffff, 0x06000100,
 505        0xf081, 0xffffffff, 0x00000100,
 506        0xf0b8, 0xffffffff, 0x00000100,
 507        0xf089, 0xffffffff, 0x00000100,
 508        0xf080, 0xffffffff, 0x00000100,
 509        0xf08c, 0xffffffff, 0x00000100,
 510        0xf08d, 0xffffffff, 0x00000100,
 511        0xf094, 0xffffffff, 0x00000100,
 512        0xf095, 0xffffffff, 0x00000100,
 513        0xf096, 0xffffffff, 0x00000100,
 514        0xf097, 0xffffffff, 0x00000100,
 515        0xf098, 0xffffffff, 0x00000100,
 516        0xf09f, 0xffffffff, 0x00000100,
 517        0xf09e, 0xffffffff, 0x00000100,
 518        0xf084, 0xffffffff, 0x06000100,
 519        0xf0a4, 0xffffffff, 0x00000100,
 520        0xf09d, 0xffffffff, 0x00000100,
 521        0xf0ad, 0xffffffff, 0x00000100,
 522        0xf0ac, 0xffffffff, 0x00000100,
 523        0xf09c, 0xffffffff, 0x00000100,
 524        0xc200, 0xffffffff, 0xe0000000,
 525        0xf008, 0xffffffff, 0x00010000,
 526        0xf009, 0xffffffff, 0x00030002,
 527        0xf00a, 0xffffffff, 0x00040007,
 528        0xf00b, 0xffffffff, 0x00060005,
 529        0xf00c, 0xffffffff, 0x00090008,
 530        0xf00d, 0xffffffff, 0x00010000,
 531        0xf00e, 0xffffffff, 0x00030002,
 532        0xf00f, 0xffffffff, 0x00040007,
 533        0xf010, 0xffffffff, 0x00060005,
 534        0xf011, 0xffffffff, 0x00090008,
 535        0xf000, 0xffffffff, 0x96e00200,
 536        0x21c2, 0xffffffff, 0x00900100,
 537        0x3109, 0xffffffff, 0x0020003f,
 538        0xe, 0xffffffff, 0x0140001c,
 539        0xf, 0x000f0000, 0x000f0000,
 540        0x88, 0xffffffff, 0xc060000c,
 541        0x89, 0xc0000fff, 0x00000100,
 542        0x82a, 0xffffffff, 0x00000104,
 543        0x1579, 0xff000fff, 0x00000100,
 544        0xc33, 0xc0000fff, 0x00000104,
 545        0x3079, 0x00000001, 0x00000001,
 546        0x3403, 0xff000ff0, 0x00000100,
 547        0x3603, 0xff000ff0, 0x00000100
 548};
 549
 550static const u32 hawaii_golden_spm_registers[] =
 551{
 552        0xc200, 0xe0ffffff, 0xe0000000
 553};
 554
 555static const u32 hawaii_golden_common_registers[] =
 556{
 557        0xc200, 0xffffffff, 0xe0000000,
 558        0xa0d4, 0xffffffff, 0x3a00161a,
 559        0xa0d5, 0xffffffff, 0x0000002e,
 560        0x2684, 0xffffffff, 0x00018208,
 561        0x263e, 0xffffffff, 0x12011003
 562};
 563
 564static const u32 hawaii_golden_registers[] =
 565{
 566        0xcd5, 0x00000333, 0x00000333,
 567        0x2684, 0x00010000, 0x00058208,
 568        0x260c, 0xffffffff, 0x00000000,
 569        0x260d, 0xf00fffff, 0x00000400,
 570        0x260e, 0x0002021c, 0x00020200,
 571        0x31e, 0x00000080, 0x00000000,
 572        0x16ec, 0x000000f0, 0x00000070,
 573        0x16f0, 0xf0311fff, 0x80300000,
 574        0xd43, 0x00810000, 0x408af000,
 575        0x1c0c, 0x31000111, 0x00000011,
 576        0xbd2, 0x73773777, 0x12010001,
 577        0x848, 0x0000007f, 0x0000001b,
 578        0x877, 0x00007fb6, 0x00002191,
 579        0xd8a, 0x0000003f, 0x0000000a,
 580        0xd8b, 0x0000003f, 0x0000000a,
 581        0xab9, 0x00073ffe, 0x000022a2,
 582        0x903, 0x000007ff, 0x00000000,
 583        0x22fc, 0x00002001, 0x00000001,
 584        0x22c9, 0xffffffff, 0x00ffffff,
 585        0xc281, 0x0000ff0f, 0x00000000,
 586        0xa293, 0x07ffffff, 0x06000000,
 587        0xf9e, 0x00000001, 0x00000002,
 588        0x31da, 0x00000008, 0x00000008,
 589        0x31dc, 0x00000f00, 0x00000800,
 590        0x31dd, 0x00000f00, 0x00000800,
 591        0x31e6, 0x00ffffff, 0x00ff7fbf,
 592        0x31e7, 0x00ffffff, 0x00ff7faf,
 593        0x2300, 0x000000ff, 0x00000800,
 594        0x390, 0x00001fff, 0x00001fff,
 595        0x2418, 0x0000007f, 0x00000020,
 596        0x2542, 0x00010000, 0x00010000,
 597        0x2b80, 0x00100000, 0x000ff07c,
 598        0x2b05, 0x000003ff, 0x0000000f,
 599        0x2b04, 0xffffffff, 0x7564fdec,
 600        0x2b03, 0xffffffff, 0x3120b9a8,
 601        0x2b02, 0x20000000, 0x0f9c0000
 602};
 603
 604static const u32 hawaii_mgcg_cgcg_init[] =
 605{
 606        0x3108, 0xffffffff, 0xfffffffd,
 607        0xc200, 0xffffffff, 0xe0000000,
 608        0xf0a8, 0xffffffff, 0x00000100,
 609        0xf082, 0xffffffff, 0x00000100,
 610        0xf0b0, 0xffffffff, 0x00000100,
 611        0xf0b2, 0xffffffff, 0x00000100,
 612        0xf0b1, 0xffffffff, 0x00000100,
 613        0x1579, 0xffffffff, 0x00200100,
 614        0xf0a0, 0xffffffff, 0x00000100,
 615        0xf085, 0xffffffff, 0x06000100,
 616        0xf088, 0xffffffff, 0x00000100,
 617        0xf086, 0xffffffff, 0x06000100,
 618        0xf081, 0xffffffff, 0x00000100,
 619        0xf0b8, 0xffffffff, 0x00000100,
 620        0xf089, 0xffffffff, 0x00000100,
 621        0xf080, 0xffffffff, 0x00000100,
 622        0xf08c, 0xffffffff, 0x00000100,
 623        0xf08d, 0xffffffff, 0x00000100,
 624        0xf094, 0xffffffff, 0x00000100,
 625        0xf095, 0xffffffff, 0x00000100,
 626        0xf096, 0xffffffff, 0x00000100,
 627        0xf097, 0xffffffff, 0x00000100,
 628        0xf098, 0xffffffff, 0x00000100,
 629        0xf09f, 0xffffffff, 0x00000100,
 630        0xf09e, 0xffffffff, 0x00000100,
 631        0xf084, 0xffffffff, 0x06000100,
 632        0xf0a4, 0xffffffff, 0x00000100,
 633        0xf09d, 0xffffffff, 0x00000100,
 634        0xf0ad, 0xffffffff, 0x00000100,
 635        0xf0ac, 0xffffffff, 0x00000100,
 636        0xf09c, 0xffffffff, 0x00000100,
 637        0xc200, 0xffffffff, 0xe0000000,
 638        0xf008, 0xffffffff, 0x00010000,
 639        0xf009, 0xffffffff, 0x00030002,
 640        0xf00a, 0xffffffff, 0x00040007,
 641        0xf00b, 0xffffffff, 0x00060005,
 642        0xf00c, 0xffffffff, 0x00090008,
 643        0xf00d, 0xffffffff, 0x00010000,
 644        0xf00e, 0xffffffff, 0x00030002,
 645        0xf00f, 0xffffffff, 0x00040007,
 646        0xf010, 0xffffffff, 0x00060005,
 647        0xf011, 0xffffffff, 0x00090008,
 648        0xf012, 0xffffffff, 0x00010000,
 649        0xf013, 0xffffffff, 0x00030002,
 650        0xf014, 0xffffffff, 0x00040007,
 651        0xf015, 0xffffffff, 0x00060005,
 652        0xf016, 0xffffffff, 0x00090008,
 653        0xf017, 0xffffffff, 0x00010000,
 654        0xf018, 0xffffffff, 0x00030002,
 655        0xf019, 0xffffffff, 0x00040007,
 656        0xf01a, 0xffffffff, 0x00060005,
 657        0xf01b, 0xffffffff, 0x00090008,
 658        0xf01c, 0xffffffff, 0x00010000,
 659        0xf01d, 0xffffffff, 0x00030002,
 660        0xf01e, 0xffffffff, 0x00040007,
 661        0xf01f, 0xffffffff, 0x00060005,
 662        0xf020, 0xffffffff, 0x00090008,
 663        0xf021, 0xffffffff, 0x00010000,
 664        0xf022, 0xffffffff, 0x00030002,
 665        0xf023, 0xffffffff, 0x00040007,
 666        0xf024, 0xffffffff, 0x00060005,
 667        0xf025, 0xffffffff, 0x00090008,
 668        0xf026, 0xffffffff, 0x00010000,
 669        0xf027, 0xffffffff, 0x00030002,
 670        0xf028, 0xffffffff, 0x00040007,
 671        0xf029, 0xffffffff, 0x00060005,
 672        0xf02a, 0xffffffff, 0x00090008,
 673        0xf02b, 0xffffffff, 0x00010000,
 674        0xf02c, 0xffffffff, 0x00030002,
 675        0xf02d, 0xffffffff, 0x00040007,
 676        0xf02e, 0xffffffff, 0x00060005,
 677        0xf02f, 0xffffffff, 0x00090008,
 678        0xf030, 0xffffffff, 0x00010000,
 679        0xf031, 0xffffffff, 0x00030002,
 680        0xf032, 0xffffffff, 0x00040007,
 681        0xf033, 0xffffffff, 0x00060005,
 682        0xf034, 0xffffffff, 0x00090008,
 683        0xf035, 0xffffffff, 0x00010000,
 684        0xf036, 0xffffffff, 0x00030002,
 685        0xf037, 0xffffffff, 0x00040007,
 686        0xf038, 0xffffffff, 0x00060005,
 687        0xf039, 0xffffffff, 0x00090008,
 688        0xf03a, 0xffffffff, 0x00010000,
 689        0xf03b, 0xffffffff, 0x00030002,
 690        0xf03c, 0xffffffff, 0x00040007,
 691        0xf03d, 0xffffffff, 0x00060005,
 692        0xf03e, 0xffffffff, 0x00090008,
 693        0x30c6, 0xffffffff, 0x00020200,
 694        0xcd4, 0xffffffff, 0x00000200,
 695        0x570, 0xffffffff, 0x00000400,
 696        0x157a, 0xffffffff, 0x00000000,
 697        0xbd4, 0xffffffff, 0x00000902,
 698        0xf000, 0xffffffff, 0x96940200,
 699        0x21c2, 0xffffffff, 0x00900100,
 700        0x3109, 0xffffffff, 0x0020003f,
 701        0xe, 0xffffffff, 0x0140001c,
 702        0xf, 0x000f0000, 0x000f0000,
 703        0x88, 0xffffffff, 0xc060000c,
 704        0x89, 0xc0000fff, 0x00000100,
 705        0x3e4, 0xffffffff, 0x00000100,
 706        0x3e6, 0x00000101, 0x00000000,
 707        0x82a, 0xffffffff, 0x00000104,
 708        0x1579, 0xff000fff, 0x00000100,
 709        0xc33, 0xc0000fff, 0x00000104,
 710        0x3079, 0x00000001, 0x00000001,
 711        0x3403, 0xff000ff0, 0x00000100,
 712        0x3603, 0xff000ff0, 0x00000100
 713};
 714
 715static const u32 godavari_golden_registers[] =
 716{
 717        0x1579, 0xff607fff, 0xfc000100,
 718        0x1bb6, 0x00010101, 0x00010000,
 719        0x260c, 0xffffffff, 0x00000000,
 720        0x260c0, 0xf00fffff, 0x00000400,
 721        0x184c, 0xffffffff, 0x00010000,
 722        0x16ec, 0x000000f0, 0x00000070,
 723        0x16f0, 0xf0311fff, 0x80300000,
 724        0x263e, 0x73773777, 0x12010001,
 725        0x263f, 0xffffffff, 0x00000010,
 726        0x200c, 0x00001f0f, 0x0000100a,
 727        0xbd2, 0x73773777, 0x12010001,
 728        0x902, 0x000fffff, 0x000c007f,
 729        0x2285, 0xf000003f, 0x00000007,
 730        0x22c9, 0xffffffff, 0x00ff0fff,
 731        0xc281, 0x0000ff0f, 0x00000000,
 732        0xa293, 0x07ffffff, 0x06000000,
 733        0x136, 0x00000fff, 0x00000100,
 734        0x3405, 0x00010000, 0x00810001,
 735        0x3605, 0x00010000, 0x00810001,
 736        0xf9e, 0x00000001, 0x00000002,
 737        0x31da, 0x00000008, 0x00000008,
 738        0x31dc, 0x00000f00, 0x00000800,
 739        0x31dd, 0x00000f00, 0x00000800,
 740        0x31e6, 0x00ffffff, 0x00ff7fbf,
 741        0x31e7, 0x00ffffff, 0x00ff7faf,
 742        0x2300, 0x000000ff, 0x00000001,
 743        0x853e, 0x01ff01ff, 0x00000002,
 744        0x8526, 0x007ff800, 0x00200000,
 745        0x8057, 0xffffffff, 0x00000f40,
 746        0x2231, 0x001f3ae3, 0x00000082,
 747        0x2235, 0x0000001f, 0x00000010,
 748        0xc24d, 0xffffffff, 0x00000000
 749};
 750
 751static void cik_init_golden_registers(struct amdgpu_device *adev)
 752{
 753        /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 754        mutex_lock(&adev->grbm_idx_mutex);
 755
 756        switch (adev->asic_type) {
 757        case CHIP_BONAIRE:
 758                amdgpu_device_program_register_sequence(adev,
 759                                                        bonaire_mgcg_cgcg_init,
 760                                                        ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 761                amdgpu_device_program_register_sequence(adev,
 762                                                        bonaire_golden_registers,
 763                                                        ARRAY_SIZE(bonaire_golden_registers));
 764                amdgpu_device_program_register_sequence(adev,
 765                                                        bonaire_golden_common_registers,
 766                                                        ARRAY_SIZE(bonaire_golden_common_registers));
 767                amdgpu_device_program_register_sequence(adev,
 768                                                        bonaire_golden_spm_registers,
 769                                                        ARRAY_SIZE(bonaire_golden_spm_registers));
 770                break;
 771        case CHIP_KABINI:
 772                amdgpu_device_program_register_sequence(adev,
 773                                                        kalindi_mgcg_cgcg_init,
 774                                                        ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 775                amdgpu_device_program_register_sequence(adev,
 776                                                        kalindi_golden_registers,
 777                                                        ARRAY_SIZE(kalindi_golden_registers));
 778                amdgpu_device_program_register_sequence(adev,
 779                                                        kalindi_golden_common_registers,
 780                                                        ARRAY_SIZE(kalindi_golden_common_registers));
 781                amdgpu_device_program_register_sequence(adev,
 782                                                        kalindi_golden_spm_registers,
 783                                                        ARRAY_SIZE(kalindi_golden_spm_registers));
 784                break;
 785        case CHIP_MULLINS:
 786                amdgpu_device_program_register_sequence(adev,
 787                                                        kalindi_mgcg_cgcg_init,
 788                                                        ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 789                amdgpu_device_program_register_sequence(adev,
 790                                                        godavari_golden_registers,
 791                                                        ARRAY_SIZE(godavari_golden_registers));
 792                amdgpu_device_program_register_sequence(adev,
 793                                                        kalindi_golden_common_registers,
 794                                                        ARRAY_SIZE(kalindi_golden_common_registers));
 795                amdgpu_device_program_register_sequence(adev,
 796                                                        kalindi_golden_spm_registers,
 797                                                        ARRAY_SIZE(kalindi_golden_spm_registers));
 798                break;
 799        case CHIP_KAVERI:
 800                amdgpu_device_program_register_sequence(adev,
 801                                                        spectre_mgcg_cgcg_init,
 802                                                        ARRAY_SIZE(spectre_mgcg_cgcg_init));
 803                amdgpu_device_program_register_sequence(adev,
 804                                                        spectre_golden_registers,
 805                                                        ARRAY_SIZE(spectre_golden_registers));
 806                amdgpu_device_program_register_sequence(adev,
 807                                                        spectre_golden_common_registers,
 808                                                        ARRAY_SIZE(spectre_golden_common_registers));
 809                amdgpu_device_program_register_sequence(adev,
 810                                                        spectre_golden_spm_registers,
 811                                                        ARRAY_SIZE(spectre_golden_spm_registers));
 812                break;
 813        case CHIP_HAWAII:
 814                amdgpu_device_program_register_sequence(adev,
 815                                                        hawaii_mgcg_cgcg_init,
 816                                                        ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 817                amdgpu_device_program_register_sequence(adev,
 818                                                        hawaii_golden_registers,
 819                                                        ARRAY_SIZE(hawaii_golden_registers));
 820                amdgpu_device_program_register_sequence(adev,
 821                                                        hawaii_golden_common_registers,
 822                                                        ARRAY_SIZE(hawaii_golden_common_registers));
 823                amdgpu_device_program_register_sequence(adev,
 824                                                        hawaii_golden_spm_registers,
 825                                                        ARRAY_SIZE(hawaii_golden_spm_registers));
 826                break;
 827        default:
 828                break;
 829        }
 830        mutex_unlock(&adev->grbm_idx_mutex);
 831}
 832
 833/**
 834 * cik_get_xclk - get the xclk
 835 *
 836 * @adev: amdgpu_device pointer
 837 *
 838 * Returns the reference clock used by the gfx engine
 839 * (CIK).
 840 */
 841static u32 cik_get_xclk(struct amdgpu_device *adev)
 842{
 843        u32 reference_clock = adev->clock.spll.reference_freq;
 844
 845        if (adev->flags & AMD_IS_APU) {
 846                if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 847                        return reference_clock / 2;
 848        } else {
 849                if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 850                        return reference_clock / 4;
 851        }
 852        return reference_clock;
 853}
 854
 855/**
 856 * cik_srbm_select - select specific register instances
 857 *
 858 * @adev: amdgpu_device pointer
 859 * @me: selected ME (micro engine)
 860 * @pipe: pipe
 861 * @queue: queue
 862 * @vmid: VMID
 863 *
 864 * Switches the currently active registers instances.  Some
 865 * registers are instanced per VMID, others are instanced per
 866 * me/pipe/queue combination.
 867 */
 868void cik_srbm_select(struct amdgpu_device *adev,
 869                     u32 me, u32 pipe, u32 queue, u32 vmid)
 870{
 871        u32 srbm_gfx_cntl =
 872                (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 873                ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 874                ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 875                ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 876        WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 877}
 878
 879static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 880{
 881        uint32_t tmp;
 882
 883        tmp = RREG32(mmCONFIG_CNTL);
 884        if (!state)
 885                tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 886        else
 887                tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 888        WREG32(mmCONFIG_CNTL, tmp);
 889}
 890
 891static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 892{
 893        u32 bus_cntl;
 894        u32 d1vga_control = 0;
 895        u32 d2vga_control = 0;
 896        u32 vga_render_control = 0;
 897        u32 rom_cntl;
 898        bool r;
 899
 900        bus_cntl = RREG32(mmBUS_CNTL);
 901        if (adev->mode_info.num_crtc) {
 902                d1vga_control = RREG32(mmD1VGA_CONTROL);
 903                d2vga_control = RREG32(mmD2VGA_CONTROL);
 904                vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 905        }
 906        rom_cntl = RREG32_SMC(ixROM_CNTL);
 907
 908        /* enable the rom */
 909        WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 910        if (adev->mode_info.num_crtc) {
 911                /* Disable VGA mode */
 912                WREG32(mmD1VGA_CONTROL,
 913                       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 914                                          D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 915                WREG32(mmD2VGA_CONTROL,
 916                       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 917                                          D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 918                WREG32(mmVGA_RENDER_CONTROL,
 919                       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 920        }
 921        WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 922
 923        r = amdgpu_read_bios(adev);
 924
 925        /* restore regs */
 926        WREG32(mmBUS_CNTL, bus_cntl);
 927        if (adev->mode_info.num_crtc) {
 928                WREG32(mmD1VGA_CONTROL, d1vga_control);
 929                WREG32(mmD2VGA_CONTROL, d2vga_control);
 930                WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 931        }
 932        WREG32_SMC(ixROM_CNTL, rom_cntl);
 933        return r;
 934}
 935
 936static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 937                                   u8 *bios, u32 length_bytes)
 938{
 939        u32 *dw_ptr;
 940        unsigned long flags;
 941        u32 i, length_dw;
 942
 943        if (bios == NULL)
 944                return false;
 945        if (length_bytes == 0)
 946                return false;
 947        /* APU vbios image is part of sbios image */
 948        if (adev->flags & AMD_IS_APU)
 949                return false;
 950
 951        dw_ptr = (u32 *)bios;
 952        length_dw = ALIGN(length_bytes, 4) / 4;
 953        /* take the smc lock since we are using the smc index */
 954        spin_lock_irqsave(&adev->smc_idx_lock, flags);
 955        /* set rom index to 0 */
 956        WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 957        WREG32(mmSMC_IND_DATA_0, 0);
 958        /* set index to data for continous read */
 959        WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 960        for (i = 0; i < length_dw; i++)
 961                dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 962        spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 963
 964        return true;
 965}
 966
 967static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 968        {mmGRBM_STATUS},
 969        {mmGB_ADDR_CONFIG},
 970        {mmMC_ARB_RAMCFG},
 971        {mmGB_TILE_MODE0},
 972        {mmGB_TILE_MODE1},
 973        {mmGB_TILE_MODE2},
 974        {mmGB_TILE_MODE3},
 975        {mmGB_TILE_MODE4},
 976        {mmGB_TILE_MODE5},
 977        {mmGB_TILE_MODE6},
 978        {mmGB_TILE_MODE7},
 979        {mmGB_TILE_MODE8},
 980        {mmGB_TILE_MODE9},
 981        {mmGB_TILE_MODE10},
 982        {mmGB_TILE_MODE11},
 983        {mmGB_TILE_MODE12},
 984        {mmGB_TILE_MODE13},
 985        {mmGB_TILE_MODE14},
 986        {mmGB_TILE_MODE15},
 987        {mmGB_TILE_MODE16},
 988        {mmGB_TILE_MODE17},
 989        {mmGB_TILE_MODE18},
 990        {mmGB_TILE_MODE19},
 991        {mmGB_TILE_MODE20},
 992        {mmGB_TILE_MODE21},
 993        {mmGB_TILE_MODE22},
 994        {mmGB_TILE_MODE23},
 995        {mmGB_TILE_MODE24},
 996        {mmGB_TILE_MODE25},
 997        {mmGB_TILE_MODE26},
 998        {mmGB_TILE_MODE27},
 999        {mmGB_TILE_MODE28},
1000        {mmGB_TILE_MODE29},
1001        {mmGB_TILE_MODE30},
1002        {mmGB_TILE_MODE31},
1003        {mmGB_MACROTILE_MODE0},
1004        {mmGB_MACROTILE_MODE1},
1005        {mmGB_MACROTILE_MODE2},
1006        {mmGB_MACROTILE_MODE3},
1007        {mmGB_MACROTILE_MODE4},
1008        {mmGB_MACROTILE_MODE5},
1009        {mmGB_MACROTILE_MODE6},
1010        {mmGB_MACROTILE_MODE7},
1011        {mmGB_MACROTILE_MODE8},
1012        {mmGB_MACROTILE_MODE9},
1013        {mmGB_MACROTILE_MODE10},
1014        {mmGB_MACROTILE_MODE11},
1015        {mmGB_MACROTILE_MODE12},
1016        {mmGB_MACROTILE_MODE13},
1017        {mmGB_MACROTILE_MODE14},
1018        {mmGB_MACROTILE_MODE15},
1019        {mmCC_RB_BACKEND_DISABLE, true},
1020        {mmGC_USER_RB_BACKEND_DISABLE, true},
1021        {mmGB_BACKEND_MAP, false},
1022        {mmPA_SC_RASTER_CONFIG, true},
1023        {mmPA_SC_RASTER_CONFIG_1, true},
1024};
1025
1026
1027static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1028                                       bool indexed, u32 se_num,
1029                                       u32 sh_num, u32 reg_offset)
1030{
1031        if (indexed) {
1032                uint32_t val;
1033                unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1034                unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1035
1036                switch (reg_offset) {
1037                case mmCC_RB_BACKEND_DISABLE:
1038                        return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1039                case mmGC_USER_RB_BACKEND_DISABLE:
1040                        return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1041                case mmPA_SC_RASTER_CONFIG:
1042                        return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1043                case mmPA_SC_RASTER_CONFIG_1:
1044                        return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1045                }
1046
1047                mutex_lock(&adev->grbm_idx_mutex);
1048                if (se_num != 0xffffffff || sh_num != 0xffffffff)
1049                        amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1050
1051                val = RREG32(reg_offset);
1052
1053                if (se_num != 0xffffffff || sh_num != 0xffffffff)
1054                        amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1055                mutex_unlock(&adev->grbm_idx_mutex);
1056                return val;
1057        } else {
1058                unsigned idx;
1059
1060                switch (reg_offset) {
1061                case mmGB_ADDR_CONFIG:
1062                        return adev->gfx.config.gb_addr_config;
1063                case mmMC_ARB_RAMCFG:
1064                        return adev->gfx.config.mc_arb_ramcfg;
1065                case mmGB_TILE_MODE0:
1066                case mmGB_TILE_MODE1:
1067                case mmGB_TILE_MODE2:
1068                case mmGB_TILE_MODE3:
1069                case mmGB_TILE_MODE4:
1070                case mmGB_TILE_MODE5:
1071                case mmGB_TILE_MODE6:
1072                case mmGB_TILE_MODE7:
1073                case mmGB_TILE_MODE8:
1074                case mmGB_TILE_MODE9:
1075                case mmGB_TILE_MODE10:
1076                case mmGB_TILE_MODE11:
1077                case mmGB_TILE_MODE12:
1078                case mmGB_TILE_MODE13:
1079                case mmGB_TILE_MODE14:
1080                case mmGB_TILE_MODE15:
1081                case mmGB_TILE_MODE16:
1082                case mmGB_TILE_MODE17:
1083                case mmGB_TILE_MODE18:
1084                case mmGB_TILE_MODE19:
1085                case mmGB_TILE_MODE20:
1086                case mmGB_TILE_MODE21:
1087                case mmGB_TILE_MODE22:
1088                case mmGB_TILE_MODE23:
1089                case mmGB_TILE_MODE24:
1090                case mmGB_TILE_MODE25:
1091                case mmGB_TILE_MODE26:
1092                case mmGB_TILE_MODE27:
1093                case mmGB_TILE_MODE28:
1094                case mmGB_TILE_MODE29:
1095                case mmGB_TILE_MODE30:
1096                case mmGB_TILE_MODE31:
1097                        idx = (reg_offset - mmGB_TILE_MODE0);
1098                        return adev->gfx.config.tile_mode_array[idx];
1099                case mmGB_MACROTILE_MODE0:
1100                case mmGB_MACROTILE_MODE1:
1101                case mmGB_MACROTILE_MODE2:
1102                case mmGB_MACROTILE_MODE3:
1103                case mmGB_MACROTILE_MODE4:
1104                case mmGB_MACROTILE_MODE5:
1105                case mmGB_MACROTILE_MODE6:
1106                case mmGB_MACROTILE_MODE7:
1107                case mmGB_MACROTILE_MODE8:
1108                case mmGB_MACROTILE_MODE9:
1109                case mmGB_MACROTILE_MODE10:
1110                case mmGB_MACROTILE_MODE11:
1111                case mmGB_MACROTILE_MODE12:
1112                case mmGB_MACROTILE_MODE13:
1113                case mmGB_MACROTILE_MODE14:
1114                case mmGB_MACROTILE_MODE15:
1115                        idx = (reg_offset - mmGB_MACROTILE_MODE0);
1116                        return adev->gfx.config.macrotile_mode_array[idx];
1117                default:
1118                        return RREG32(reg_offset);
1119                }
1120        }
1121}
1122
1123static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1124                             u32 sh_num, u32 reg_offset, u32 *value)
1125{
1126        uint32_t i;
1127
1128        *value = 0;
1129        for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1130                bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1131
1132                if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1133                        continue;
1134
1135                *value = cik_get_register_value(adev, indexed, se_num, sh_num,
1136                                                reg_offset);
1137                return 0;
1138        }
1139        return -EINVAL;
1140}
1141
1142struct kv_reset_save_regs {
1143        u32 gmcon_reng_execute;
1144        u32 gmcon_misc;
1145        u32 gmcon_misc3;
1146};
1147
1148static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1149                                   struct kv_reset_save_regs *save)
1150{
1151        save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1152        save->gmcon_misc = RREG32(mmGMCON_MISC);
1153        save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1154
1155        WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1156                ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1157        WREG32(mmGMCON_MISC, save->gmcon_misc &
1158                ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1159                        GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1160}
1161
1162static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1163                                      struct kv_reset_save_regs *save)
1164{
1165        int i;
1166
1167        WREG32(mmGMCON_PGFSM_WRITE, 0);
1168        WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1169
1170        for (i = 0; i < 5; i++)
1171                WREG32(mmGMCON_PGFSM_WRITE, 0);
1172
1173        WREG32(mmGMCON_PGFSM_WRITE, 0);
1174        WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1175
1176        for (i = 0; i < 5; i++)
1177                WREG32(mmGMCON_PGFSM_WRITE, 0);
1178
1179        WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1180        WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1181
1182        for (i = 0; i < 5; i++)
1183                WREG32(mmGMCON_PGFSM_WRITE, 0);
1184
1185        WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1186        WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1187
1188        for (i = 0; i < 5; i++)
1189                WREG32(mmGMCON_PGFSM_WRITE, 0);
1190
1191        WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1192        WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1193
1194        for (i = 0; i < 5; i++)
1195                WREG32(mmGMCON_PGFSM_WRITE, 0);
1196
1197        WREG32(mmGMCON_PGFSM_WRITE, 0);
1198        WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1199
1200        for (i = 0; i < 5; i++)
1201                WREG32(mmGMCON_PGFSM_WRITE, 0);
1202
1203        WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1204        WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1205
1206        for (i = 0; i < 5; i++)
1207                WREG32(mmGMCON_PGFSM_WRITE, 0);
1208
1209        WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1210        WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1211
1212        for (i = 0; i < 5; i++)
1213                WREG32(mmGMCON_PGFSM_WRITE, 0);
1214
1215        WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1216        WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1217
1218        for (i = 0; i < 5; i++)
1219                WREG32(mmGMCON_PGFSM_WRITE, 0);
1220
1221        WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1222        WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1223
1224        for (i = 0; i < 5; i++)
1225                WREG32(mmGMCON_PGFSM_WRITE, 0);
1226
1227        WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1228        WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1229
1230        WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1231        WREG32(mmGMCON_MISC, save->gmcon_misc);
1232        WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1233}
1234
1235static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1236{
1237        struct kv_reset_save_regs kv_save = { 0 };
1238        u32 i;
1239        int r = -EINVAL;
1240
1241        dev_info(adev->dev, "GPU pci config reset\n");
1242
1243        if (adev->flags & AMD_IS_APU)
1244                kv_save_regs_for_reset(adev, &kv_save);
1245
1246        /* disable BM */
1247        pci_clear_master(adev->pdev);
1248        /* reset */
1249        amdgpu_device_pci_config_reset(adev);
1250
1251        udelay(100);
1252
1253        /* wait for asic to come out of reset */
1254        for (i = 0; i < adev->usec_timeout; i++) {
1255                if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1256                        /* enable BM */
1257                        pci_set_master(adev->pdev);
1258                        adev->has_hw_reset = true;
1259                        r = 0;
1260                        break;
1261                }
1262                udelay(1);
1263        }
1264
1265        /* does asic init need to be run first??? */
1266        if (adev->flags & AMD_IS_APU)
1267                kv_restore_regs_for_reset(adev, &kv_save);
1268
1269        return r;
1270}
1271
1272/**
1273 * cik_asic_reset - soft reset GPU
1274 *
1275 * @adev: amdgpu_device pointer
1276 *
1277 * Look up which blocks are hung and attempt
1278 * to reset them.
1279 * Returns 0 for success.
1280 */
1281static int cik_asic_reset(struct amdgpu_device *adev)
1282{
1283        int r;
1284
1285        amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1286
1287        r = cik_gpu_pci_config_reset(adev);
1288
1289        amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1290
1291        return r;
1292}
1293
1294static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1295{
1296        return RREG32(mmCONFIG_MEMSIZE);
1297}
1298
1299static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1300                              u32 cntl_reg, u32 status_reg)
1301{
1302        int r, i;
1303        struct atom_clock_dividers dividers;
1304        uint32_t tmp;
1305
1306        r = amdgpu_atombios_get_clock_dividers(adev,
1307                                               COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1308                                               clock, false, &dividers);
1309        if (r)
1310                return r;
1311
1312        tmp = RREG32_SMC(cntl_reg);
1313        tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1314                CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1315        tmp |= dividers.post_divider;
1316        WREG32_SMC(cntl_reg, tmp);
1317
1318        for (i = 0; i < 100; i++) {
1319                if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1320                        break;
1321                mdelay(10);
1322        }
1323        if (i == 100)
1324                return -ETIMEDOUT;
1325
1326        return 0;
1327}
1328
1329static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1330{
1331        int r = 0;
1332
1333        r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1334        if (r)
1335                return r;
1336
1337        r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1338        return r;
1339}
1340
1341static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1342{
1343        int r, i;
1344        struct atom_clock_dividers dividers;
1345        u32 tmp;
1346
1347        r = amdgpu_atombios_get_clock_dividers(adev,
1348                                               COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1349                                               ecclk, false, &dividers);
1350        if (r)
1351                return r;
1352
1353        for (i = 0; i < 100; i++) {
1354                if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1355                        break;
1356                mdelay(10);
1357        }
1358        if (i == 100)
1359                return -ETIMEDOUT;
1360
1361        tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1362        tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1363                CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1364        tmp |= dividers.post_divider;
1365        WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1366
1367        for (i = 0; i < 100; i++) {
1368                if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1369                        break;
1370                mdelay(10);
1371        }
1372        if (i == 100)
1373                return -ETIMEDOUT;
1374
1375        return 0;
1376}
1377
1378static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1379{
1380        struct pci_dev *root = adev->pdev->bus->self;
1381        int bridge_pos, gpu_pos;
1382        u32 speed_cntl, current_data_rate;
1383        int i;
1384        u16 tmp16;
1385
1386        if (pci_is_root_bus(adev->pdev->bus))
1387                return;
1388
1389        if (amdgpu_pcie_gen2 == 0)
1390                return;
1391
1392        if (adev->flags & AMD_IS_APU)
1393                return;
1394
1395        if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1396                                        CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1397                return;
1398
1399        speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1400        current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1401                PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1402        if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1403                if (current_data_rate == 2) {
1404                        DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1405                        return;
1406                }
1407                DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1408        } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1409                if (current_data_rate == 1) {
1410                        DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1411                        return;
1412                }
1413                DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1414        }
1415
1416        bridge_pos = pci_pcie_cap(root);
1417        if (!bridge_pos)
1418                return;
1419
1420        gpu_pos = pci_pcie_cap(adev->pdev);
1421        if (!gpu_pos)
1422                return;
1423
1424        if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1425                /* re-try equalization if gen3 is not already enabled */
1426                if (current_data_rate != 2) {
1427                        u16 bridge_cfg, gpu_cfg;
1428                        u16 bridge_cfg2, gpu_cfg2;
1429                        u32 max_lw, current_lw, tmp;
1430
1431                        pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1432                        pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1433
1434                        tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1435                        pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1436
1437                        tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1438                        pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1439
1440                        tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1441                        max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1442                                PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1443                        current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1444                                >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1445
1446                        if (current_lw < max_lw) {
1447                                tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1448                                if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1449                                        tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1450                                                PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1451                                        tmp |= (max_lw <<
1452                                                PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1453                                        tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1454                                        PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1455                                        PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1456                                        WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1457                                }
1458                        }
1459
1460                        for (i = 0; i < 10; i++) {
1461                                /* check status */
1462                                pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1463                                if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1464                                        break;
1465
1466                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1467                                pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1468
1469                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1470                                pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1471
1472                                tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1473                                tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1474                                WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1475
1476                                tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1477                                tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1478                                WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1479
1480                                mdelay(100);
1481
1482                                /* linkctl */
1483                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1484                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1485                                tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1486                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1487
1488                                pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1489                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1490                                tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1491                                pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1492
1493                                /* linkctl2 */
1494                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1495                                tmp16 &= ~((1 << 4) | (7 << 9));
1496                                tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1497                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1498
1499                                pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1500                                tmp16 &= ~((1 << 4) | (7 << 9));
1501                                tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1502                                pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1503
1504                                tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1505                                tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1506                                WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1507                        }
1508                }
1509        }
1510
1511        /* set the link speed */
1512        speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1513                PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1514        speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1515        WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1516
1517        pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1518        tmp16 &= ~0xf;
1519        if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1520                tmp16 |= 3; /* gen3 */
1521        else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1522                tmp16 |= 2; /* gen2 */
1523        else
1524                tmp16 |= 1; /* gen1 */
1525        pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1526
1527        speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1528        speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1529        WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1530
1531        for (i = 0; i < adev->usec_timeout; i++) {
1532                speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1533                if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1534                        break;
1535                udelay(1);
1536        }
1537}
1538
1539static void cik_program_aspm(struct amdgpu_device *adev)
1540{
1541        u32 data, orig;
1542        bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1543        bool disable_clkreq = false;
1544
1545        if (amdgpu_aspm == 0)
1546                return;
1547
1548        if (pci_is_root_bus(adev->pdev->bus))
1549                return;
1550
1551        /* XXX double check APUs */
1552        if (adev->flags & AMD_IS_APU)
1553                return;
1554
1555        orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1556        data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1557        data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1558                PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1559        if (orig != data)
1560                WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1561
1562        orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1563        data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1564        if (orig != data)
1565                WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1566
1567        orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1568        data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1569        if (orig != data)
1570                WREG32_PCIE(ixPCIE_P_CNTL, data);
1571
1572        orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1573        data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1574                PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1575        data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1576        if (!disable_l0s)
1577                data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1578
1579        if (!disable_l1) {
1580                data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1581                data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1582                if (orig != data)
1583                        WREG32_PCIE(ixPCIE_LC_CNTL, data);
1584
1585                if (!disable_plloff_in_l1) {
1586                        bool clk_req_support;
1587
1588                        orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1589                        data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1590                                PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1591                        data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1592                                (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1593                        if (orig != data)
1594                                WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1595
1596                        orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1597                        data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1598                                PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1599                        data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1600                                (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1601                        if (orig != data)
1602                                WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1603
1604                        orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1605                        data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1606                                PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1607                        data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1608                                (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1609                        if (orig != data)
1610                                WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1611
1612                        orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1613                        data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1614                                PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1615                        data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1616                                (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1617                        if (orig != data)
1618                                WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1619
1620                        orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1621                        data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1622                        data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1623                        if (orig != data)
1624                                WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1625
1626                        if (!disable_clkreq) {
1627                                struct pci_dev *root = adev->pdev->bus->self;
1628                                u32 lnkcap;
1629
1630                                clk_req_support = false;
1631                                pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1632                                if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1633                                        clk_req_support = true;
1634                        } else {
1635                                clk_req_support = false;
1636                        }
1637
1638                        if (clk_req_support) {
1639                                orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1640                                data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1641                                        PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1642                                if (orig != data)
1643                                        WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1644
1645                                orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1646                                data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1647                                        THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1648                                data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1649                                        (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1650                                if (orig != data)
1651                                        WREG32_SMC(ixTHM_CLK_CNTL, data);
1652
1653                                orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1654                                data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1655                                        MISC_CLK_CTRL__ZCLK_SEL_MASK);
1656                                data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1657                                        (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1658                                if (orig != data)
1659                                        WREG32_SMC(ixMISC_CLK_CTRL, data);
1660
1661                                orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1662                                data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1663                                if (orig != data)
1664                                        WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1665
1666                                orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1667                                data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1668                                if (orig != data)
1669                                        WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1670
1671                                orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1672                                data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1673                                data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1674                                if (orig != data)
1675                                        WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1676                        }
1677                }
1678        } else {
1679                if (orig != data)
1680                        WREG32_PCIE(ixPCIE_LC_CNTL, data);
1681        }
1682
1683        orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1684        data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1685                PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1686                PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1687        if (orig != data)
1688                WREG32_PCIE(ixPCIE_CNTL2, data);
1689
1690        if (!disable_l0s) {
1691                data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1692                if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1693                                PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1694                        data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1695                        if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1696                        (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1697                                orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1698                                data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1699                                if (orig != data)
1700                                        WREG32_PCIE(ixPCIE_LC_CNTL, data);
1701                        }
1702                }
1703        }
1704}
1705
1706static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1707{
1708        return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1709                >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1710}
1711
1712static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
1713{
1714        if (is_virtual_machine()) /* passthrough mode */
1715                adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1716}
1717
1718static const struct amdgpu_asic_funcs cik_asic_funcs =
1719{
1720        .read_disabled_bios = &cik_read_disabled_bios,
1721        .read_bios_from_rom = &cik_read_bios_from_rom,
1722        .read_register = &cik_read_register,
1723        .reset = &cik_asic_reset,
1724        .set_vga_state = &cik_vga_set_state,
1725        .get_xclk = &cik_get_xclk,
1726        .set_uvd_clocks = &cik_set_uvd_clocks,
1727        .set_vce_clocks = &cik_set_vce_clocks,
1728        .get_config_memsize = &cik_get_config_memsize,
1729};
1730
1731static int cik_common_early_init(void *handle)
1732{
1733        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1734
1735        adev->smc_rreg = &cik_smc_rreg;
1736        adev->smc_wreg = &cik_smc_wreg;
1737        adev->pcie_rreg = &cik_pcie_rreg;
1738        adev->pcie_wreg = &cik_pcie_wreg;
1739        adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1740        adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1741        adev->didt_rreg = &cik_didt_rreg;
1742        adev->didt_wreg = &cik_didt_wreg;
1743
1744        adev->asic_funcs = &cik_asic_funcs;
1745
1746        adev->rev_id = cik_get_rev_id(adev);
1747        adev->external_rev_id = 0xFF;
1748        switch (adev->asic_type) {
1749        case CHIP_BONAIRE:
1750                adev->cg_flags =
1751                        AMD_CG_SUPPORT_GFX_MGCG |
1752                        AMD_CG_SUPPORT_GFX_MGLS |
1753                        /*AMD_CG_SUPPORT_GFX_CGCG |*/
1754                        AMD_CG_SUPPORT_GFX_CGLS |
1755                        AMD_CG_SUPPORT_GFX_CGTS |
1756                        AMD_CG_SUPPORT_GFX_CGTS_LS |
1757                        AMD_CG_SUPPORT_GFX_CP_LS |
1758                        AMD_CG_SUPPORT_MC_LS |
1759                        AMD_CG_SUPPORT_MC_MGCG |
1760                        AMD_CG_SUPPORT_SDMA_MGCG |
1761                        AMD_CG_SUPPORT_SDMA_LS |
1762                        AMD_CG_SUPPORT_BIF_LS |
1763                        AMD_CG_SUPPORT_VCE_MGCG |
1764                        AMD_CG_SUPPORT_UVD_MGCG |
1765                        AMD_CG_SUPPORT_HDP_LS |
1766                        AMD_CG_SUPPORT_HDP_MGCG;
1767                adev->pg_flags = 0;
1768                adev->external_rev_id = adev->rev_id + 0x14;
1769                break;
1770        case CHIP_HAWAII:
1771                adev->cg_flags =
1772                        AMD_CG_SUPPORT_GFX_MGCG |
1773                        AMD_CG_SUPPORT_GFX_MGLS |
1774                        /*AMD_CG_SUPPORT_GFX_CGCG |*/
1775                        AMD_CG_SUPPORT_GFX_CGLS |
1776                        AMD_CG_SUPPORT_GFX_CGTS |
1777                        AMD_CG_SUPPORT_GFX_CP_LS |
1778                        AMD_CG_SUPPORT_MC_LS |
1779                        AMD_CG_SUPPORT_MC_MGCG |
1780                        AMD_CG_SUPPORT_SDMA_MGCG |
1781                        AMD_CG_SUPPORT_SDMA_LS |
1782                        AMD_CG_SUPPORT_BIF_LS |
1783                        AMD_CG_SUPPORT_VCE_MGCG |
1784                        AMD_CG_SUPPORT_UVD_MGCG |
1785                        AMD_CG_SUPPORT_HDP_LS |
1786                        AMD_CG_SUPPORT_HDP_MGCG;
1787                adev->pg_flags = 0;
1788                adev->external_rev_id = 0x28;
1789                break;
1790        case CHIP_KAVERI:
1791                adev->cg_flags =
1792                        AMD_CG_SUPPORT_GFX_MGCG |
1793                        AMD_CG_SUPPORT_GFX_MGLS |
1794                        /*AMD_CG_SUPPORT_GFX_CGCG |*/
1795                        AMD_CG_SUPPORT_GFX_CGLS |
1796                        AMD_CG_SUPPORT_GFX_CGTS |
1797                        AMD_CG_SUPPORT_GFX_CGTS_LS |
1798                        AMD_CG_SUPPORT_GFX_CP_LS |
1799                        AMD_CG_SUPPORT_SDMA_MGCG |
1800                        AMD_CG_SUPPORT_SDMA_LS |
1801                        AMD_CG_SUPPORT_BIF_LS |
1802                        AMD_CG_SUPPORT_VCE_MGCG |
1803                        AMD_CG_SUPPORT_UVD_MGCG |
1804                        AMD_CG_SUPPORT_HDP_LS |
1805                        AMD_CG_SUPPORT_HDP_MGCG;
1806                adev->pg_flags =
1807                        /*AMD_PG_SUPPORT_GFX_PG |
1808                          AMD_PG_SUPPORT_GFX_SMG |
1809                          AMD_PG_SUPPORT_GFX_DMG |*/
1810                        AMD_PG_SUPPORT_UVD |
1811                        AMD_PG_SUPPORT_VCE |
1812                        /*  AMD_PG_SUPPORT_CP |
1813                          AMD_PG_SUPPORT_GDS |
1814                          AMD_PG_SUPPORT_RLC_SMU_HS |
1815                          AMD_PG_SUPPORT_ACP |
1816                          AMD_PG_SUPPORT_SAMU |*/
1817                        0;
1818                if (adev->pdev->device == 0x1312 ||
1819                        adev->pdev->device == 0x1316 ||
1820                        adev->pdev->device == 0x1317)
1821                        adev->external_rev_id = 0x41;
1822                else
1823                        adev->external_rev_id = 0x1;
1824                break;
1825        case CHIP_KABINI:
1826        case CHIP_MULLINS:
1827                adev->cg_flags =
1828                        AMD_CG_SUPPORT_GFX_MGCG |
1829                        AMD_CG_SUPPORT_GFX_MGLS |
1830                        /*AMD_CG_SUPPORT_GFX_CGCG |*/
1831                        AMD_CG_SUPPORT_GFX_CGLS |
1832                        AMD_CG_SUPPORT_GFX_CGTS |
1833                        AMD_CG_SUPPORT_GFX_CGTS_LS |
1834                        AMD_CG_SUPPORT_GFX_CP_LS |
1835                        AMD_CG_SUPPORT_SDMA_MGCG |
1836                        AMD_CG_SUPPORT_SDMA_LS |
1837                        AMD_CG_SUPPORT_BIF_LS |
1838                        AMD_CG_SUPPORT_VCE_MGCG |
1839                        AMD_CG_SUPPORT_UVD_MGCG |
1840                        AMD_CG_SUPPORT_HDP_LS |
1841                        AMD_CG_SUPPORT_HDP_MGCG;
1842                adev->pg_flags =
1843                        /*AMD_PG_SUPPORT_GFX_PG |
1844                          AMD_PG_SUPPORT_GFX_SMG | */
1845                        AMD_PG_SUPPORT_UVD |
1846                        /*AMD_PG_SUPPORT_VCE |
1847                          AMD_PG_SUPPORT_CP |
1848                          AMD_PG_SUPPORT_GDS |
1849                          AMD_PG_SUPPORT_RLC_SMU_HS |
1850                          AMD_PG_SUPPORT_SAMU |*/
1851                        0;
1852                if (adev->asic_type == CHIP_KABINI) {
1853                        if (adev->rev_id == 0)
1854                                adev->external_rev_id = 0x81;
1855                        else if (adev->rev_id == 1)
1856                                adev->external_rev_id = 0x82;
1857                        else if (adev->rev_id == 2)
1858                                adev->external_rev_id = 0x85;
1859                } else
1860                        adev->external_rev_id = adev->rev_id + 0xa1;
1861                break;
1862        default:
1863                /* FIXME: not supported yet */
1864                return -EINVAL;
1865        }
1866
1867        adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1868
1869        amdgpu_device_get_pcie_info(adev);
1870
1871        return 0;
1872}
1873
1874static int cik_common_sw_init(void *handle)
1875{
1876        return 0;
1877}
1878
1879static int cik_common_sw_fini(void *handle)
1880{
1881        return 0;
1882}
1883
1884static int cik_common_hw_init(void *handle)
1885{
1886        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1887
1888        /* move the golden regs per IP block */
1889        cik_init_golden_registers(adev);
1890        /* enable pcie gen2/3 link */
1891        cik_pcie_gen3_enable(adev);
1892        /* enable aspm */
1893        cik_program_aspm(adev);
1894
1895        return 0;
1896}
1897
1898static int cik_common_hw_fini(void *handle)
1899{
1900        return 0;
1901}
1902
1903static int cik_common_suspend(void *handle)
1904{
1905        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1906
1907        return cik_common_hw_fini(adev);
1908}
1909
1910static int cik_common_resume(void *handle)
1911{
1912        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1913
1914        return cik_common_hw_init(adev);
1915}
1916
1917static bool cik_common_is_idle(void *handle)
1918{
1919        return true;
1920}
1921
1922static int cik_common_wait_for_idle(void *handle)
1923{
1924        return 0;
1925}
1926
1927static int cik_common_soft_reset(void *handle)
1928{
1929        /* XXX hard reset?? */
1930        return 0;
1931}
1932
1933static int cik_common_set_clockgating_state(void *handle,
1934                                            enum amd_clockgating_state state)
1935{
1936        return 0;
1937}
1938
1939static int cik_common_set_powergating_state(void *handle,
1940                                            enum amd_powergating_state state)
1941{
1942        return 0;
1943}
1944
1945static const struct amd_ip_funcs cik_common_ip_funcs = {
1946        .name = "cik_common",
1947        .early_init = cik_common_early_init,
1948        .late_init = NULL,
1949        .sw_init = cik_common_sw_init,
1950        .sw_fini = cik_common_sw_fini,
1951        .hw_init = cik_common_hw_init,
1952        .hw_fini = cik_common_hw_fini,
1953        .suspend = cik_common_suspend,
1954        .resume = cik_common_resume,
1955        .is_idle = cik_common_is_idle,
1956        .wait_for_idle = cik_common_wait_for_idle,
1957        .soft_reset = cik_common_soft_reset,
1958        .set_clockgating_state = cik_common_set_clockgating_state,
1959        .set_powergating_state = cik_common_set_powergating_state,
1960};
1961
1962static const struct amdgpu_ip_block_version cik_common_ip_block =
1963{
1964        .type = AMD_IP_BLOCK_TYPE_COMMON,
1965        .major = 1,
1966        .minor = 0,
1967        .rev = 0,
1968        .funcs = &cik_common_ip_funcs,
1969};
1970
1971int cik_set_ip_blocks(struct amdgpu_device *adev)
1972{
1973        cik_detect_hw_virtualization(adev);
1974
1975        switch (adev->asic_type) {
1976        case CHIP_BONAIRE:
1977                amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
1978                amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
1979                amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
1980                amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
1981                if (adev->enable_virtual_display)
1982                        amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
1983#if defined(CONFIG_DRM_AMD_DC)
1984                else if (amdgpu_device_has_dc_support(adev))
1985                        amdgpu_device_ip_block_add(adev, &dm_ip_block);
1986#endif
1987                else
1988                        amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
1989                amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
1990                amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
1991                amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
1992                amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
1993                break;
1994        case CHIP_HAWAII:
1995                amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
1996                amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
1997                amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
1998                amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
1999                if (adev->enable_virtual_display)
2000                        amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2001#if defined(CONFIG_DRM_AMD_DC)
2002                else if (amdgpu_device_has_dc_support(adev))
2003                        amdgpu_device_ip_block_add(adev, &dm_ip_block);
2004#endif
2005                else
2006                        amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2007                amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2008                amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2009                amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2010                amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2011                break;
2012        case CHIP_KAVERI:
2013                amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2014                amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2015                amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2016                amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
2017                if (adev->enable_virtual_display)
2018                        amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2019#if defined(CONFIG_DRM_AMD_DC)
2020                else if (amdgpu_device_has_dc_support(adev))
2021                        amdgpu_device_ip_block_add(adev, &dm_ip_block);
2022#endif
2023                else
2024                        amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2025                amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2026                amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2027                amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2028                amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2029                break;
2030        case CHIP_KABINI:
2031        case CHIP_MULLINS:
2032                amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2033                amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2034                amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2035                amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
2036                if (adev->enable_virtual_display)
2037                        amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2038#if defined(CONFIG_DRM_AMD_DC)
2039                else if (amdgpu_device_has_dc_support(adev))
2040                        amdgpu_device_ip_block_add(adev, &dm_ip_block);
2041#endif
2042                else
2043                        amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2044                amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2045                amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2046                amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2047                amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2048                break;
2049        default:
2050                /* FIXME: not supported yet */
2051                return -EINVAL;
2052        }
2053        return 0;
2054}
2055